From d8d29f6701987af2de3117373baf936c5441f4ef Mon Sep 17 00:00:00 2001 From: Tastyfish Date: Thu, 12 May 2022 03:12:02 -0400 Subject: [PATCH] Test all maps in parallel integration tests (#66864) --- .github/workflows/ci_suite.yml | 26 +++++++++++++++++++++++++- code/__HELPERS/_logging.dm | 3 +++ code/modules/unit_tests/_unit_tests.dm | 2 ++ code/modules/unit_tests/mapping.dm | 19 +++++++++++++++++++ code/modules/unit_tests/unit_test.dm | 5 ++++- tools/ci/run_server.sh | 8 ++++++++ 6 files changed, 61 insertions(+), 2 deletions(-) create mode 100644 code/modules/unit_tests/mapping.dm diff --git a/.github/workflows/ci_suite.yml b/.github/workflows/ci_suite.yml index f07684010e4..fee04d66517 100644 --- a/.github/workflows/ci_suite.yml +++ b/.github/workflows/ci_suite.yml @@ -67,10 +67,31 @@ jobs: source $HOME/BYOND/byond/bin/byondsetup tools/build/build --ci dm -DCIBUILDING -DCITESTING -DALL_MAPS + find_all_maps: + if: "!contains(github.event.head_commit.message, '[ci skip]')" + name: Find Maps to Test + runs-on: ubuntu-20.04 + outputs: + maps: ${{ steps.map_finder.outputs.maps }} + steps: + - uses: actions/checkout@v2 + - name: Find Maps + id: map_finder + run: | + echo "$(ls -mw0 _maps/*.json)" > maps_output.txt + sed -i -e s+_maps/+\"+g -e s+.json+\"+g maps_output.txt + echo "Maps: $(cat maps_output.txt)" + echo "::set-output name=maps::{\"paths\":[$(cat maps_output.txt)]}" + run_all_tests: if: "!contains(github.event.head_commit.message, '[ci skip]')" name: Integration Tests runs-on: ubuntu-20.04 + needs: [find_all_maps] + strategy: + fail-fast: false + matrix: + map: ${{ fromJSON(needs.find_all_maps.outputs.maps).paths }} services: mysql: image: mysql:latest @@ -79,6 +100,9 @@ jobs: ports: - 3306 options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 + concurrency: + group: ci-${{ github.ref }}-${{ matrix.map }} + cancel-in-progress: true steps: - uses: actions/checkout@v2 - name: Restore BYOND cache @@ -107,7 +131,7 @@ jobs: - name: Run Tests run: | source $HOME/BYOND/byond/bin/byondsetup - bash tools/ci/run_server.sh + bash tools/ci/run_server.sh ${{ matrix.map }} test_windows: if: "!contains(github.event.head_commit.message, '[ci skip]')" diff --git a/code/__HELPERS/_logging.dm b/code/__HELPERS/_logging.dm index c5b763f0830..bca8cd9e344 100644 --- a/code/__HELPERS/_logging.dm +++ b/code/__HELPERS/_logging.dm @@ -292,6 +292,9 @@ GLOBAL_LIST_INIT(testing_global_profiler, list("_PROFILE_NAME" = "Global")) SEND_TEXT(world.log, text) /proc/log_mapping(text, skip_world_log) +#ifdef UNIT_TESTS + GLOB.unit_test_mapping_logs += text +#endif WRITE_LOG(GLOB.world_map_error_log, text) if(skip_world_log) return diff --git a/code/modules/unit_tests/_unit_tests.dm b/code/modules/unit_tests/_unit_tests.dm index bb5224f48a5..2b81fe89f6d 100644 --- a/code/modules/unit_tests/_unit_tests.dm +++ b/code/modules/unit_tests/_unit_tests.dm @@ -46,6 +46,7 @@ #define UNIT_TEST_FAILED 1 #define UNIT_TEST_SKIPPED 2 +#define TEST_PRE 0 #define TEST_DEFAULT 1 #define TEST_DEL_WORLD INFINITY @@ -100,6 +101,7 @@ #include "keybinding_init.dm" #include "load_map_security.dm" #include "machine_disassembly.dm" +#include "mapping.dm" #include "medical_wounds.dm" #include "merge_type.dm" #include "metabolizing.dm" diff --git a/code/modules/unit_tests/mapping.dm b/code/modules/unit_tests/mapping.dm new file mode 100644 index 00000000000..ff6a2577917 --- /dev/null +++ b/code/modules/unit_tests/mapping.dm @@ -0,0 +1,19 @@ +/// Conveys all log_mapping messages as unit test failures, as they all indicate mapping problems. +/datum/unit_test/log_mapping + // Happen before all other tests, to make sure we only capture normal mapping logs. + priority = TEST_PRE + +/datum/unit_test/log_mapping/Run() + var/static/regex/test_areacoord_regex = regex(@"\(-?\d+,-?\d+,(-?\d+)\)") + + for(var/log_entry in GLOB.unit_test_mapping_logs) + // Only fail if AREACOORD was conveyed, and it's a station or mining z-level. + // This is due to mapping errors don't have coords being impossible to diagnose as a unit test, + // and various ruins frequently intentionally doing non-standard things. + if(!test_areacoord_regex.Find(log_entry)) + continue + var/z = text2num(test_areacoord_regex.group[1]) + if(!is_station_level(z) && !is_mining_level(z)) + continue + + TEST_FAIL(log_entry) diff --git a/code/modules/unit_tests/unit_test.dm b/code/modules/unit_tests/unit_test.dm index 3a25d85ac72..4359d2f1de0 100644 --- a/code/modules/unit_tests/unit_test.dm +++ b/code/modules/unit_tests/unit_test.dm @@ -14,6 +14,8 @@ You can use the run_loc_floor_bottom_left and run_loc_floor_top_right to get tur GLOBAL_DATUM(current_test, /datum/unit_test) GLOBAL_VAR_INIT(failed_any_test, FALSE) GLOBAL_VAR(test_log) +/// When unit testing, all logs sent to log_mapping are stored here and retrieved in log_mapping unit test. +GLOBAL_LIST_EMPTY(unit_test_mapping_logs) /datum/unit_test //Bit of metadata for the future maybe @@ -98,6 +100,7 @@ GLOBAL_VAR(test_log) "[test.succeeded ? TEST_OUTPUT_GREEN("PASS") : TEST_OUTPUT_RED("FAIL")]: [test_path] [duration / 10]s", ) var/list/fail_reasons = test.fail_reasons + var/map_name = SSmapping.config.map_name for(var/reasonID in 1 to LAZYLEN(fail_reasons)) var/text = fail_reasons[reasonID][1] @@ -111,7 +114,7 @@ GLOBAL_VAR(test_log) var/annotation_text = replacetext(text, "%", "%25") annotation_text = replacetext(annotation_text, "\n", "%0A") - log_world("::error file=[file],line=[line],title=[test_path]::[annotation_text]") + log_world("::error file=[file],line=[line],title=[map_name]: [test_path]::[annotation_text]") // Normal log message log_entry += "\tREASON #[reasonID]: [text] at [file]:[line]" diff --git a/tools/ci/run_server.sh b/tools/ci/run_server.sh index 4d943846f04..f9eaa31e566 100644 --- a/tools/ci/run_server.sh +++ b/tools/ci/run_server.sh @@ -1,12 +1,20 @@ #!/bin/bash set -euo pipefail +MAP=$1 + +echo Testing $MAP + tools/deploy.sh ci_test mkdir ci_test/config +mkdir ci_test/data #test config cp tools/ci/ci_config.txt ci_test/config/config.txt +#set the map +cp _maps/$MAP.json ci_test/data/next_map.json + cd ci_test DreamDaemon tgstation.dmb -close -trusted -verbose -params "log-directory=ci" cd ..