From e5ccafd0b002f8eb04fc022618a0b20942bffcf7 Mon Sep 17 00:00:00 2001 From: Tad Hardesty Date: Wed, 3 Jul 2019 00:37:06 -0700 Subject: [PATCH] Split Travis shell scripts by job (#44845) Part one of some needful Travis cleanup. Split the different tasks into their own .sh files rather than doing many things per .sh file Use .travis.yml jobs list to select which tasks are run rather than setting environment variables which the shell scripts then check Moves the grep lints from running in both "Maps" and "Tests" jobs, and runs them in "Lints" job (formerly "Tools") instead Makes the check_filedirs.sh lint only run in the "Lints" job rather than in all three jobs. Removes the check that dependencies.sh matches Dockerfile since the former now pulls from the latter Uncache node_modules - npm ci is expected to trash this every time and warns when it already exists Uncache and remove the unnecessary fake MariaDB folder used when building BSQL, give it the real path instead Attempt to cache the BYOND-provided libmariadb (not sure if this works, will re-examine it in part two) For part two I hope to switch to using rust_g and BSQL binaries from GitHub releases, and add SpacemanDMM's dreamchecker to the Lints job. --- .editorconfig | 4 + .travis.yml | 74 +++++++++-------- dependencies.sh | 3 + tools/travis/before_build_tools.sh | 7 -- tools/travis/build_bsql.sh | 21 +++++ tools/travis/build_byond.sh | 80 ------------------- tools/travis/build_dependencies.sh | 55 ------------- tools/travis/build_rust_g.sh | 18 +++++ tools/travis/build_tools.sh | 21 ++--- tools/travis/install_build_tools.sh | 15 ++-- ...before_build_byond.sh => install_byond.sh} | 12 +-- tools/travis/install_libmariadb.sh | 15 ++++ tools/travis/lint_grep.sh | 49 ++++++++++++ tools/travis/run_server.sh | 14 ++++ 14 files changed, 180 insertions(+), 208 deletions(-) delete mode 100755 tools/travis/before_build_tools.sh create mode 100755 tools/travis/build_bsql.sh delete mode 100755 tools/travis/build_byond.sh delete mode 100755 tools/travis/build_dependencies.sh create mode 100755 tools/travis/build_rust_g.sh rename tools/travis/{before_build_byond.sh => install_byond.sh} (67%) create mode 100755 tools/travis/install_libmariadb.sh create mode 100755 tools/travis/lint_grep.sh create mode 100755 tools/travis/run_server.sh diff --git a/.editorconfig b/.editorconfig index a25dbdfc85c..df93ae3a161 100644 --- a/.editorconfig +++ b/.editorconfig @@ -2,3 +2,7 @@ insert_final_newline = true indent_style = tab indent_size = 4 + +[*.yml] +indent_style = space +indent_size = 2 diff --git a/.travis.yml b/.travis.yml index f09e4557b9c..9dd1c43b51e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,29 +1,29 @@ language: generic -dist: trusty -sudo: false dist: xenial +sudo: false + branches: except: - ___TGS3TempBranch - ___TGSTempBranch + matrix: include: - - env: - - BUILD_TOOLS=true - name: "Build Tools" + - name: "Run Linters" addons: apt: packages: - python3 - python3-pip - python3-setuptools - cache: - directories: - - tgui/node_modules - - env: - - BUILD_TESTING=true - - BUILD_TOOLS=false - name: "Build All Maps" + install: + - tools/travis/install_build_tools.sh + script: + - tools/travis/check_filedirs.sh tgstation.dme + - tools/travis/build_tools.sh + - tools/travis/lint_grep.sh + + - name: "Compile All Maps" addons: apt: packages: @@ -31,10 +31,15 @@ matrix: cache: directories: - $HOME/BYOND - - env: - - BUILD_TESTING=false - - BUILD_TOOLS=false - name: "Build and Run Unit Tests" + install: + - tools/travis/install_byond.sh + - source $HOME/BYOND/byond/bin/byondsetup + before_script: + - tools/travis/template_dm_generator.py + script: + - tools/travis/dm.sh -DTRAVISBUILDING -DTRAVISTESTING -DALL_MAPS tgstation.dme + + - name: "Compile and Run Tests" addons: mariadb: '10.2' apt: @@ -51,23 +56,22 @@ matrix: cache: directories: - $HOME/.cargo - - $HOME/BYOND - - $HOME/MariaDB - $HOME/.rustup - -install: - - tools/travis/install_build_tools.sh - - if [ $BUILD_TOOLS = false ] && [ $BUILD_TESTING = false ]; then mysql -u root -e 'CREATE DATABASE tg_travis;'; fi - - if [ $BUILD_TOOLS = false ] && [ $BUILD_TESTING = false ]; then mysql -u root tg_travis < SQL/tgstation_schema.sql; fi - - if [ $BUILD_TOOLS = false ] && [ $BUILD_TESTING = false ]; then mysql -u root -e 'CREATE DATABASE tg_travis_prefixed;'; fi - - if [ $BUILD_TOOLS = false ] && [ $BUILD_TESTING = false ]; then mysql -u root tg_travis_prefixed < SQL/tgstation_schema_prefixed.sql; fi - -before_script: - - tools/travis/before_build_tools.sh - - tools/travis/before_build_byond.sh - -script: - - tools/travis/check_filedirs.sh tgstation.dme - - tools/travis/build_tools.sh || travis_terminate 1 - - tools/travis/build_dependencies.sh || travis_terminate 1 - - tools/travis/build_byond.sh + - $HOME/BYOND + - $HOME/libmariadb + install: + - tools/travis/install_byond.sh + - source $HOME/BYOND/byond/bin/byondsetup + - tools/travis/install_libmariadb.sh + - curl https://sh.rustup.rs -sSf | sh -s -- -y --default-host i686-unknown-linux-gnu + - source ~/.cargo/env + before_script: + - mysql -u root -e 'CREATE DATABASE tg_travis;' + - mysql -u root tg_travis < SQL/tgstation_schema.sql + - mysql -u root -e 'CREATE DATABASE tg_travis_prefixed;' + - mysql -u root tg_travis_prefixed < SQL/tgstation_schema_prefixed.sql + - tools/travis/build_rust_g.sh + - tools/travis/build_bsql.sh + script: + - tools/travis/dm.sh -DTRAVISBUILDING tgstation.dme || travis_terminate 1 + - tools/travis/run_server.sh diff --git a/dependencies.sh b/dependencies.sh index a75940088f6..830da48aff9 100755 --- a/dependencies.sh +++ b/dependencies.sh @@ -18,3 +18,6 @@ export BSQL_VERSION=v1.4.0.0 #node version export NODE_VERSION=8 + +# PHP version +export PHP_VERSION=5.6 diff --git a/tools/travis/before_build_tools.sh b/tools/travis/before_build_tools.sh deleted file mode 100755 index 2329de82b2c..00000000000 --- a/tools/travis/before_build_tools.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -set -e - -if [ "$BUILD_TOOLS" = true ]; then - cd tgui && source ~/.nvm/nvm.sh && npm install && cd .. -fi; - diff --git a/tools/travis/build_bsql.sh b/tools/travis/build_bsql.sh new file mode 100755 index 00000000000..e2b281efc64 --- /dev/null +++ b/tools/travis/build_bsql.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -euo pipefail + +source dependencies.sh + +mkdir -p BSQL +cd BSQL +git init +git remote add origin https://github.com/tgstation/BSQL +git fetch --depth 1 origin $BSQL_VERSION +git checkout FETCH_HEAD + +mkdir -p artifacts +cd artifacts +export CXX=g++-7 +# The -D will be unnecessary past BSQL v1.4.0.0 +cmake .. -DMARIA_LIBRARY=/usr/lib/i386-linux-gnu/libmariadb.so +make + +mkdir -p ~/.byond/bin +ln -s $PWD/src/BSQL/libBSQL.so ../../libBSQL.so diff --git a/tools/travis/build_byond.sh b/tools/travis/build_byond.sh deleted file mode 100755 index 3a7e00df298..00000000000 --- a/tools/travis/build_byond.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/bash - -#nb: must be bash to support shopt globstar -set -e -shopt -s globstar - -if [ "$BUILD_TOOLS" = false ]; then - if grep -E '^\".+\" = \(.+\)' _maps/**/*.dmm; then - echo "Non-TGM formatted map detected. Please convert it using Map Merger!" - exit 1 - fi; - if grep -P '^\ttag = \"icon' _maps/**/*.dmm; then - echo "tag vars from icon state generation detected in maps, please remove them." - exit 1 - fi; - if grep 'step_[xy]' _maps/**/*.dmm; then - echo "step_x/step_y variables detected in maps, please remove them." - exit 1 - fi; - if grep 'pixel_[xy] = 0' _maps/**/*.dmm; then - echo "pixel_x/pixel_y = 0 variables detected in maps, please review to ensure they are not dirty varedits." - fi; - if grep -P '\td[1-2] =' _maps/**/*.dmm; then - echo "d1/d2 cable variables detected in maps, please remove them." - exit 1 - fi; - if grep '^/area/.+[\{]' _maps/**/*.dmm; then - echo "Vareditted /area path use detected in maps, please replace with proper paths." - exit 1 - fi; - if grep '\W\/turf\s*[,\){]' _maps/**/*.dmm; then - echo "base /turf path use detected in maps, please replace with proper paths." - exit 1 - fi; - if grep '^/*var/' code/**/*.dm; then - echo "Unmanaged global var use detected in code, please use the helpers." - exit 1 - fi; - if grep -i 'centcomm' code/**/*.dm; then - echo "Misspelling(s) of CENTCOM detected in code, please remove the extra M(s)." - exit 1 - fi; - if grep -i 'centcomm' _maps/**/*.dmm; then - echo "Misspelling(s) of CENTCOM detected in maps, please remove the extra M(s)." - exit 1 - fi; - - source $HOME/BYOND/byond/bin/byondsetup - if [ "$BUILD_TESTING" = true ]; then - tools/travis/dm.sh -DTRAVISBUILDING -DTRAVISTESTING -DALL_MAPS tgstation.dme - else - tools/travis/dm.sh -DTRAVISBUILDING tgstation.dme - - tools/deploy.sh travis_test - mkdir travis_test/config - - #test config - cp tools/travis/travis_config.txt travis_test/config/config.txt - - # get libmariadb, cache it so limmex doesn't get angery - if [ -f $HOME/libmariadb ]; then - #travis likes to interpret the cache command as it being a file for some reason - rm $HOME/libmariadb - fi - mkdir -p $HOME/libmariadb - if [ ! -f $HOME/libmariadb/libmariadb.so ]; then - wget http://www.byond.com/download/db/mariadb_client-2.0.0-linux.tgz - tar -xvf mariadb_client-2.0.0-linux.tgz - mv mariadb_client-2.0.0-linux/libmariadb.so $HOME/libmariadb/libmariadb.so - rm -rf mariadb_client-2.0.0-linux.tgz mariadb_client-2.0.0-linux - fi - - cd travis_test - ln -s $HOME/libmariadb/libmariadb.so libmariadb.so - DreamDaemon tgstation.dmb -close -trusted -verbose -params "test-run&log-directory=travis" - cd .. - cat travis_test/data/logs/travis/clean_run.lk - - fi; -fi; diff --git a/tools/travis/build_dependencies.sh b/tools/travis/build_dependencies.sh deleted file mode 100755 index 2c24bdbaaf2..00000000000 --- a/tools/travis/build_dependencies.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash - -set -e - -source dependencies.sh - -#ensure the Dockerfile version matches the dependencies.sh version -line=$(head -n 1 Dockerfile) -if [[ $line != *"$BYOND_MAJOR.$BYOND_MINOR"* ]]; then - echo "Dockerfile BYOND version in FROM command does not match dependencies.sh (Or it's not on line 1)!" - exit 1 -fi - -if [ $BUILD_TOOLS = false ] && [ $BUILD_TESTING = false ]; then - curl https://sh.rustup.rs -sSf | sh -s -- -y --default-host i686-unknown-linux-gnu - source ~/.profile - - mkdir rust-g - cd rust-g - git init - git remote add origin https://github.com/tgstation/rust-g - git fetch --depth 1 origin $RUST_G_VERSION - git checkout FETCH_HEAD - cargo build --release - cmp target/rust_g.dm ../code/__DEFINES/rust_g.dm - - mkdir -p ~/.byond/bin - ln -s $PWD/target/release/librust_g.so ~/.byond/bin/rust_g - - mkdir -p ../BSQL/artifacts - cd ../BSQL - git init - git remote add origin https://github.com/tgstation/BSQL - git fetch --depth 1 origin $BSQL_VERSION - git checkout FETCH_HEAD - - if [ -f "$HOME/MariaDB/libmariadb.so.2" ] && [ -f "$HOME/MariaDB/libmariadb.so" ] && [ -d "$HOME/MariaDB/include" ]; - then - echo "Using cached MariaDB library." - else - echo "Setting up MariaDB." - rm -rf "$HOME/MariaDB" - mkdir -p "$HOME/MariaDB" - mkdir -p "$HOME/MariaDB/include" - cp /usr/lib/i386-linux-gnu/libmariadb.so.2 $HOME/MariaDB/ - ln -s $HOME/MariaDB/libmariadb.so.2 $HOME/MariaDB/libmariadb.so - cp -r /usr/include/mariadb $HOME/MariaDB/include/ - fi - - cd artifacts - export CXX=g++-7 - cmake .. -DMARIA_INCLUDE_DIR=$HOME/MariaDB/include - make - mv src/BSQL/libBSQL.so ../../ -fi diff --git a/tools/travis/build_rust_g.sh b/tools/travis/build_rust_g.sh new file mode 100755 index 00000000000..556c6f5842a --- /dev/null +++ b/tools/travis/build_rust_g.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail + +source dependencies.sh + +mkdir -p rust-g +cd rust-g +git init +git remote add origin https://github.com/tgstation/rust-g +git fetch --depth 1 origin $RUST_G_VERSION +git checkout FETCH_HEAD + +cargo build --release + +cmp target/rust_g.dm ../code/__DEFINES/rust_g.dm + +mkdir -p ~/.byond/bin +ln -s $PWD/target/release/librust_g.so ~/.byond/bin/rust_g diff --git a/tools/travis/build_tools.sh b/tools/travis/build_tools.sh index eaccd64c8c1..a6a78e02a4b 100755 --- a/tools/travis/build_tools.sh +++ b/tools/travis/build_tools.sh @@ -1,16 +1,11 @@ #!/bin/bash +set -euo pipefail -#must also be bash for the md5sum commands -set -e +md5sum -c - <<< "49bc6b1b9ed56c83cceb6674bd97cb34 *html/changelogs/example.yml" +python3 tools/ss13_genchangelog.py html/changelog.html html/changelogs -if [ "$BUILD_TOOLS" = true ]; -then - md5sum -c - <<< "49bc6b1b9ed56c83cceb6674bd97cb34 *html/changelogs/example.yml"; - (cd tgui && source ~/.nvm/nvm.sh && npm ci && node node_modules/gulp/bin/gulp.js --min) - phpenv global 5.6 - php -l tools/WebhookProcessor/github_webhook_processor.php; - php -l tools/TGUICompiler.php; - echo "Checking for JSON errors"; - find . -name "*.json" -not -path "./tgui/node_modules/*" | xargs -0 python3 ./tools/json_verifier.py; - python3 tools/ss13_genchangelog.py html/changelog.html html/changelogs; -fi; +find . -name "*.php" -print0 | xargs -0 -n1 php -l + +find . -name "*.json" -not -path "./tgui/node_modules/*" | xargs -0 python3 ./tools/json_verifier.py + +(cd tgui && source ~/.nvm/nvm.sh && npm ci && node node_modules/gulp/bin/gulp.js --min) diff --git a/tools/travis/install_build_tools.sh b/tools/travis/install_build_tools.sh index 0ea2a605ab1..224687f0882 100755 --- a/tools/travis/install_build_tools.sh +++ b/tools/travis/install_build_tools.sh @@ -1,11 +1,12 @@ #!/bin/bash -set -e +set -euo pipefail source dependencies.sh -if [ "$BUILD_TOOLS" = true ]; then - source ~/.nvm/nvm.sh - nvm install $NODE_VERSION - pip3 install --user PyYaml - pip3 install --user beautifulsoup4 -fi; +source ~/.nvm/nvm.sh +nvm install $NODE_VERSION + +pip3 install --user PyYaml +pip3 install --user beautifulsoup4 + +phpenv global $PHP_VERSION diff --git a/tools/travis/before_build_byond.sh b/tools/travis/install_byond.sh similarity index 67% rename from tools/travis/before_build_byond.sh rename to tools/travis/install_byond.sh index 55afb07ffe5..4a688755d3d 100755 --- a/tools/travis/before_build_byond.sh +++ b/tools/travis/install_byond.sh @@ -1,15 +1,5 @@ #!/bin/bash -set -e - -#If this is the build tools step, we do not bother to install/build byond -if [ "$BUILD_TOOLS" = true ]; then - exit 0 -fi - -echo "Combining maps for building" -if [ $BUILD_TESTING = true ]; then - python tools/travis/template_dm_generator.py -fi +set -euo pipefail source dependencies.sh diff --git a/tools/travis/install_libmariadb.sh b/tools/travis/install_libmariadb.sh new file mode 100755 index 00000000000..d0ce4adc145 --- /dev/null +++ b/tools/travis/install_libmariadb.sh @@ -0,0 +1,15 @@ +#!/bin/bash +set -euo pipefail + +# get libmariadb, cache it so limmex doesn't get angery +if [ -f $HOME/libmariadb ]; then + #travis likes to interpret the cache command as it being a file for some reason + rm $HOME/libmariadb +fi +mkdir -p $HOME/libmariadb +if [ ! -f $HOME/libmariadb/libmariadb.so ]; then + wget http://www.byond.com/download/db/mariadb_client-2.0.0-linux.tgz + tar -xvf mariadb_client-2.0.0-linux.tgz + mv mariadb_client-2.0.0-linux/libmariadb.so $HOME/libmariadb/libmariadb.so + rm -rf mariadb_client-2.0.0-linux.tgz mariadb_client-2.0.0-linux +fi diff --git a/tools/travis/lint_grep.sh b/tools/travis/lint_grep.sh new file mode 100755 index 00000000000..fe94f39a2f0 --- /dev/null +++ b/tools/travis/lint_grep.sh @@ -0,0 +1,49 @@ +#!/bin/bash +set -euo pipefail + +#nb: must be bash to support shopt globstar +shopt -s globstar + +st=0 + +if grep -E '^\".+\" = \(.+\)' _maps/**/*.dmm; then + echo "Non-TGM formatted map detected. Please convert it using Map Merger!" + st=1 +fi; +if grep -P '^\ttag = \"icon' _maps/**/*.dmm; then + echo "tag vars from icon state generation detected in maps, please remove them." + st=1 +fi; +if grep 'step_[xy]' _maps/**/*.dmm; then + echo "step_x/step_y variables detected in maps, please remove them." + st=1 +fi; +if grep 'pixel_[xy] = 0' _maps/**/*.dmm; then + echo "pixel_x/pixel_y = 0 variables detected in maps, please review to ensure they are not dirty varedits." +fi; +if grep -P '\td[1-2] =' _maps/**/*.dmm; then + echo "d1/d2 cable variables detected in maps, please remove them." + st=1 +fi; +if grep '^/area/.+[\{]' _maps/**/*.dmm; then + echo "Vareditted /area path use detected in maps, please replace with proper paths." + st=1 +fi; +if grep '\W\/turf\s*[,\){]' _maps/**/*.dmm; then + echo "base /turf path use detected in maps, please replace with proper paths." + st=1 +fi; +if grep '^/*var/' code/**/*.dm; then + echo "Unmanaged global var use detected in code, please use the helpers." + st=1 +fi; +if grep -i 'centcomm' code/**/*.dm; then + echo "Misspelling(s) of CENTCOM detected in code, please remove the extra M(s)." + st=1 +fi; +if grep -i 'centcomm' _maps/**/*.dmm; then + echo "Misspelling(s) of CENTCOM detected in maps, please remove the extra M(s)." + st=1 +fi; + +exit $st diff --git a/tools/travis/run_server.sh b/tools/travis/run_server.sh new file mode 100755 index 00000000000..975e8f68849 --- /dev/null +++ b/tools/travis/run_server.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -euo pipefail + +tools/deploy.sh travis_test +mkdir travis_test/config + +#test config +cp tools/travis/travis_config.txt travis_test/config/config.txt + +cd travis_test +ln -s $HOME/libmariadb/libmariadb.so libmariadb.so +DreamDaemon tgstation.dmb -close -trusted -verbose -params "test-run&log-directory=travis" +cd .. +cat travis_test/data/logs/travis/clean_run.lk