diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4f10fceae29..ab7c849f122 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,31 +8,58 @@ env: BASENAME: "vorestation" jobs: - file_tests: + + run_linters: name: Run Linters runs-on: ubuntu-20.04 + steps: - uses: actions/checkout@v4 - - name: Ensure +x on CI directory - run: | - chmod -R +x ./tools/ci - - name: Install Tools - run: | - bash tools/ci/install_build_deps.sh + # Caches - name: Restore Yarn cache - if: "${{ contains(github.event.pull_request.labels.*.name, 'Type: TGUI Bundle') }}" uses: actions/cache@v4 with: path: tgui/.yarn/cache - key: ${{ runner.os }}-yarn-${{ secrets.CACHE_PURGE_KEY }}-${{ hashFiles('tgui/yarn.lock') }} + key: ${{ runner.os }}-yarn-${{ hashFiles('tgui/yarn.lock') }} restore-keys: | - ${{ runner.os }}-build-${{ secrets.CACHE_PURGE_KEY }}- - ${{ runner.os }}-build- - ${{ runner.os }}- - - name: Run Tests + ${{ runner.os }}-yarn- + - name: Restore Node cache + uses: actions/cache@v4 + with: + path: ~/.nvm + key: ${{ runner.os }}-node-${{ hashFiles('dependencies.sh') }} + restore-keys: | + ${{ runner.os }}-node- + - name: Restore Bootstrap cache + uses: actions/cache@v4 + with: + path: tools/bootstrap/.cache + key: ${{ runner.os }}-bootstrap-${{ hashFiles('tools/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-bootstrap- + - name: Restore Rust cache + uses: actions/cache@v4 + with: + path: ~/.cargo + key: ${{ runner.os }}-rust-${{ hashFiles('tools/ci/ci_dependencies.sh')}} + restore-keys: | + ${{ runner.os }}-rust- + - name: Restore Cutter cache + uses: actions/cache@v4 + with: + path: tools/icon_cutter/cache + key: ${{ runner.os }}-cutter-${{ hashFiles('dependencies.sh') }} + # End Caches + - name: Install Tools run: | - tools/ci/validate_files.sh - tools/ci/build_tgui.sh + pip3 install setuptools + bash tools/ci/install_node.sh + bash tools/ci/install_ripgrep.sh + tools/bootstrap/python -c '' + - name: Run Grep Checks + run: tools/ci/validate_files.sh + - name: Run TGUI Checks + run: tools/build/build --ci lint tgui-test dreamchecker: name: DreamChecker @@ -68,7 +95,7 @@ jobs: map: ['tether', 'stellar_delight', 'groundbase'] # name: Integration Tests (${{ matrix.map }}) name: Integration Tests - # needs: ['file_tests', 'dreamchecker'] + # needs: ['run_linters', 'dreamchecker'] runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v4 @@ -119,7 +146,7 @@ jobs: tests_successful: name: Integration Tests - needs: ['file_tests', 'dreamchecker', 'unit_tests'] + needs: ['run_linters', 'dreamchecker', 'unit_tests'] runs-on: ubuntu-20.04 steps: - name: Report Success diff --git a/_build_dependencies.sh b/_build_dependencies.sh index a66b3a5a867..2edcacda857 100644 --- a/_build_dependencies.sh +++ b/_build_dependencies.sh @@ -8,4 +8,4 @@ export BYOND_MAJOR=515 # Byond Minor export BYOND_MINOR=1637 # Macro Count -export MACRO_COUNT=4 +export MACRO_COUNT=8 diff --git a/dependencies.sh b/dependencies.sh index cba44327e5c..57e14e9c8cd 100644 --- a/dependencies.sh +++ b/dependencies.sh @@ -19,7 +19,7 @@ export NODE_VERSION_COMPAT=20.2.0 export SPACEMAN_DMM_VERSION=suite-1.8 # Python version for mapmerge and other tools -export PYTHON_VERSION=3.9.0 +export PYTHON_VERSION=3.8.10 #dreamluau repo export DREAMLUAU_REPO="tgstation/dreamluau" diff --git a/tools/bootstrap/node b/tools/bootstrap/node old mode 100644 new mode 100755 diff --git a/tools/bootstrap/python b/tools/bootstrap/python old mode 100644 new mode 100755 diff --git a/tools/build/build b/tools/build/build old mode 100644 new mode 100755 diff --git a/tools/build/build.bat b/tools/build/build.bat old mode 100644 new mode 100755 diff --git a/tools/build/build.js b/tools/build/build.js index fb3f93d5ab5..5f1a1019400 100644 --- a/tools/build/build.js +++ b/tools/build/build.js @@ -182,12 +182,13 @@ export const IconCutterTarget = new Juke.Target({ .map((file) => file.replace(`.dmi.toml`, '.png')); }, executes: async () => { - await Juke.exec(cutter_path, [ - '--dont-wait', - '--templates', - 'cutter_templates', - 'icons', - ]); + // Don't run icon cutter, we don't use it and it's not compiled for ubuntu 20.04 + // await Juke.exec(cutter_path, [ + // '--dont-wait', + // '--templates', + // 'cutter_templates', + // 'icons', + // ]); }, }); diff --git a/tools/ci/build_tgui.sh b/tools/ci/build_tgui.sh old mode 100644 new mode 100755 diff --git a/tools/ci/ci_dependencies.sh b/tools/ci/ci_dependencies.sh new file mode 100644 index 00000000000..fd1bee5ea88 --- /dev/null +++ b/tools/ci/ci_dependencies.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +#Project dependencies file +#Contains versions of programs that we might need to install for CI purposes - do not add anything here that is REQUIRED to run the project, this is just for CI. + +export RIPGREP_VERSION=14.0.3 diff --git a/tools/ci/compile_and_run.sh b/tools/ci/compile_and_run.sh old mode 100644 new mode 100755 index d991773ec71..ae58fa18074 --- a/tools/ci/compile_and_run.sh +++ b/tools/ci/compile_and_run.sh @@ -25,12 +25,11 @@ elif grep -q '#include[[:space:]]\"maps\\groundbase\\groundbase.dm\"' $BASENAME. fi # Compile a copy of the codebase, and print errors as Github Actions annotations -DreamMaker $BASENAME.dme > compile.log +tools/build/build --ci dm -DCIBUILDING -DCITESTING exitVal=$? -cat compile.log -if [ $exitVal -gt 0 ]; then - sed -E -n 's/^(.+?\.dm):([0-9]+):(error|warning): (.+)$/::\3 file=\1,line=\2::\4/gp' < compile.log -fi +# if [ $exitVal -gt 0 ]; then +# sed -E -n 's/^(.+?\.dm):([0-9]+):(error|warning): (.+)$/::\3 file=\1,line=\2::\4/gp' +# fi # Compile failed on map_test if [ $exitVal -gt 0 ] && [ $TEST_DEFINE = "MAP_TEST" ]; then diff --git a/tools/ci/install_build_deps.sh b/tools/ci/install_build_deps.sh deleted file mode 100644 index e59934c1e2f..00000000000 --- a/tools/ci/install_build_deps.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -set -euo pipefail - -source _build_dependencies.sh - -source ~/.nvm/nvm.sh -nvm install $NODE_VERSION -nvm use $NODE_VERSION -npm install --location=global yarn - -pip install --user PyYaml -q -pip install --user beautifulsoup4 -q diff --git a/tools/ci/install_byond.sh b/tools/ci/install_byond.sh old mode 100644 new mode 100755 diff --git a/tools/ci/install_node.sh b/tools/ci/install_node.sh new file mode 100755 index 00000000000..906984ed3fa --- /dev/null +++ b/tools/ci/install_node.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -euo pipefail + +source dependencies.sh + +if [[ -e ~/.nvm/nvm.sh ]]; then + source ~/.nvm/nvm.sh + nvm install $NODE_VERSION_COMPAT + nvm use $NODE_VERSION_COMPAT +fi diff --git a/tools/ci/install_ripgrep.sh b/tools/ci/install_ripgrep.sh new file mode 100644 index 00000000000..455027d93a6 --- /dev/null +++ b/tools/ci/install_ripgrep.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -euo pipefail + +source tools/ci/ci_dependencies.sh + +cargo install ripgrep --features pcre2 --version $RIPGREP_VERSION diff --git a/tools/ci/validate_files.sh b/tools/ci/validate_files.sh old mode 100644 new mode 100755 index 034bd222c8f..a97d28fe782 --- a/tools/ci/validate_files.sh +++ b/tools/ci/validate_files.sh @@ -1,51 +1,86 @@ #!/bin/bash +set -euo pipefail + +# nb: must be bash to support shopt globstar +shopt -s globstar extglob source _build_dependencies.sh +# ANSI Colors RED='\033[0;31m' -NC='\033[0m' +GREEN="\033[0;32m" +BLUE="\033[0;34m" +NC='\033[0m' # No color + FAILED=0 +# check for ripgrep +if command -v rg >/dev/null 2>&1; then + grep=rg + pcre2_support=1 + if [ ! rg -P '' >/dev/null 2>&1 ] ; then + pcre2_support=0 + fi + code_files="code/**/**.dm" + map_files="maps/**/**.dmm" + # shuttle_map_files="_maps/shuttles/**.dmm" + # code_x_515="code/**/!(__byond_version_compat).dm" +else + pcre2_support=0 + grep=grep + code_files="-r --include=code/**/**.dm" + map_files="-r --include=maps/**/**.dmm" + # shuttle_map_files="-r --include=_maps/shuttles/**.dmm" + # code_x_515="-r --include=code/**/!(__byond_version_compat).dm" +fi + +echo -e "${BLUE}Using grep provider at $(which $grep)${NC}" + +part=0 +section() { + echo -e "${BLUE}Checking for $1${NC}..." + part=0 +} + +part() { + part=$((part+1)) + padded=$(printf "%02d" $part) + echo -e "${GREEN} $padded- $1${NC}" +} + +section "map issues" + +part "step_[xy]" #Checking for step_x/step_y defined in any maps anywhere. -(! grep 'step_[xy]' maps/**/*.dmm) +(! $grep 'step_[xy]' $map_files) retVal=$? if [ $retVal -ne 0 ]; then echo -e "${RED}The variables 'step_x' and 'step_y' are present on a map, and they 'break' movement ingame.${NC}" FAILED=1 fi -#Checking for 'tag' set to something on maps -(! grep -Pn '( |\t|;|{)tag( ?)=' maps/**/*.dmm) -retVal=$? -if [ $retVal -ne 0 ]; then - echo -e "${RED}A map has 'tag' set on an atom. It may cause problems and should be removed.${NC}" - FAILED=1 -fi - -#Checking for broken HTML tags (didn't close the quote for class) -(! grep -En "<\s*span\s+class\s*=\s*('[^'>]+|[^'>]+')\s*>" **/*.dm) -retVal=$? -if [ $retVal -ne 0 ]; then - echo -e "${RED}A broken span tag class is present (check quotes).${NC}" - FAILED=1 -fi - +part "test map included" #Checking for any 'checked' maps that include 'test' -(! grep 'maps\\.*test.*' *.dme) +(! $grep 'maps\\.*test.*' *.dme) retVal=$? if [ $retVal -ne 0 ]; then echo -e "${RED}A map containing the word 'test' is included. This is not allowed to be committed.${NC}" FAILED=1 fi -#Check for weird indentation in any .dm files -awk -f tools/indentation.awk **/*.dm -retVal=$? -if [ $retVal -ne 0 ]; then - echo -e "${RED}Indention testing failed. Please see results and fix indentation.${NC}" - FAILED=1 -fi +section "code issues" +part "indentation" +echo -e "${RED}DISABLED" +#Check for weird indentation in any .dm files +# awk -f tools/indentation.awk $code_files +# retVal=$? +# if [ $retVal -ne 0 ]; then +# echo -e "${RED}Indention testing failed. Please see results and fix indentation.${NC}" +# FAILED=1 +# fi + +part "changelog" #Checking for a change to html/changelogs/example.yml md5sum -c - <<< "ea467b7b75774b41ecdf35e07091d96f *html/changelogs/example.yml" retVal=$? @@ -54,20 +89,57 @@ if [ $retVal -ne 0 ]; then FAILED=1 fi +part "color macros" #Checking for color macros -(num=`grep -E '\\\\(red|blue|green|black|b|i[^mc])' **/*.dm | wc -l`; echo "$num escapes (expecting ${MACRO_COUNT} or less)"; [ $num -le ${MACRO_COUNT} ]) +(num=`$grep -n '\\\\(red|blue|green|black|b|i[^mc])' $code_files | wc -l`; echo "$num escapes (expecting ${MACRO_COUNT} or less)"; [ $num -le ${MACRO_COUNT} ]) retVal=$? if [ $retVal -ne 0 ]; then echo -e "${RED}Do not use any byond color macros (such as \blue), they are deprecated.${NC}" FAILED=1 fi +part "html tag matching" #Checking for missed tags -python tools/TagMatcher/tag-matcher.py ../.. -retVal=$? -if [ $retVal -ne 0 ]; then - echo -e "${RED}Some HTML tags are missing their opening/closing partners. Please correct this.${NC}" - FAILED=1 +echo -e "${RED}DISABLED" +# python tools/TagMatcher/tag-matcher.py ../.. +# retVal=$? +# if [ $retVal -ne 0 ]; then +# echo -e "${RED}Some HTML tags are missing their opening/closing partners. Please correct this.${NC}" +# FAILED=1 +# fi + +if [ "$pcre2_support" -eq 1 ]; then + section "regexes requiring PCRE2" + + part "tag" + #Checking for 'tag' set to something on maps + (! $grep -Pn '( |\t|;|{)tag( ?)=' $map_files) + retVal=$? + if [ $retVal -ne 0 ]; then + echo -e "${RED}A map has 'tag' set on an atom. It may cause problems and should be removed.${NC}" + FAILED=1 + fi + + part "broken html" + # echo -e "${RED}DISABLED" + #Checking for broken HTML tags (didn't close the quote for class) + (! $grep -Pn "<\s*span\s+class\s*=\s*('[^'>]+|[^'>]+')\s*>" $code_files) + retVal=$? + if [ $retVal -ne 0 ]; then + echo -e "${RED}A broken span tag class is present (check quotes).${NC}" + FAILED=1 + fi +else + echo -e "${RED}pcre2 not supported, skipping checks requiring pcre2" + echo -e "if you want to run these checks install ripgrep with pcre2 support.${NC}" +fi + +if [ $FAILED = 0 ]; then + echo + echo -e "${GREEN}No errors found using $grep!${NC}" +else + echo + echo -e "${RED}Errors found, please fix them and try again.${NC}" fi # Quit with our status code diff --git a/tools/requirements.txt b/tools/requirements.txt new file mode 100644 index 00000000000..2d9bf2fbcdb --- /dev/null +++ b/tools/requirements.txt @@ -0,0 +1,13 @@ +pygit2==1.7.2 +bidict==0.22.0 +Pillow==9.3.0 + +# changelogs +PyYaml==6.0.1 +beautifulsoup4==4.9.3 + +# ezdb +mysql-connector-python==8.0.33 + +# icon cutter +numpy==1.24.4