From dfac9f7ef8d711a9a3a0762cf4245fd820dea3b4 Mon Sep 17 00:00:00 2001 From: Jeremiah <42397676+jlsnow301@users.noreply.github.com> Date: Thu, 20 Mar 2025 06:41:35 -0700 Subject: [PATCH] Extracts the steps inside ci suite into files (#90077) ## About The Pull Request Moves almost every step of the CI suite into its own file in workflows. It should function exactly the same. ## Why It's Good For The Game Makes our CI suite more readable at a glance, more modular and scalable ## Changelog N/A --- .github/workflows/ci_suite.yml | 218 ++---------------- .github/workflows/collect_data.yml | 45 ++++ .github/workflows/compare_screenshots.yml | 40 ++++ .github/workflows/compile_all_maps.yml | 29 +++ .../perform_alternate_version_tests.yml | 1 + .../perform_regular_version_tests.yml | 1 + .github/workflows/run_integration_tests.yml | 1 + .github/workflows/run_linters.yml | 106 +++++++++ .github/workflows/test_windows.yml | 29 +++ 9 files changed, 267 insertions(+), 203 deletions(-) create mode 100644 .github/workflows/collect_data.yml create mode 100644 .github/workflows/compare_screenshots.yml create mode 100644 .github/workflows/compile_all_maps.yml create mode 100644 .github/workflows/run_linters.yml create mode 100644 .github/workflows/test_windows.yml diff --git a/.github/workflows/ci_suite.yml b/.github/workflows/ci_suite.yml index 5f6ceb6ff8b..11e532cc732 100644 --- a/.github/workflows/ci_suite.yml +++ b/.github/workflows/ci_suite.yml @@ -1,4 +1,5 @@ name: CI Suite + on: push: branches: @@ -30,174 +31,32 @@ jobs: run_linters: name: Run Linters needs: start_gate - runs-on: ubuntu-22.04 - timeout-minutes: 5 - - steps: - - uses: actions/checkout@v4 - - name: Restore SpacemanDMM cache - uses: actions/cache@v4 - with: - path: ~/SpacemanDMM - key: ${{ runner.os }}-spacemandmm-${{ hashFiles('dependencies.sh') }} - restore-keys: | - ${{ runner.os }}-spacemandmm- - - name: Setup Node - uses: ./.github/actions/setup_node - with: - restore-yarn-cache: true - - name: Restore Bootstrap cache - uses: actions/cache@v4 - with: - path: tools/bootstrap/.cache - key: ${{ runner.os }}-bootstrap-${{ hashFiles('tools/requirements.txt') }} - restore-keys: | - ${{ runner.os }}-bootstrap- - - name: Restore Rust cache - uses: actions/cache@v4 - with: - path: ~/.cargo - key: ${{ runner.os }}-rust-${{ hashFiles('tools/ci/ci_dependencies.sh')}} - restore-keys: | - ${{ runner.os }}-rust- - - name: Restore Cutter cache - uses: actions/cache@v4 - with: - path: tools/icon_cutter/cache - key: ${{ runner.os }}-cutter-${{ hashFiles('dependencies.sh') }} - - name: Setup .NET SDK - uses: actions/setup-dotnet@v4.2.0 - with: - dotnet-version: 9.x - - name: Install OpenDream - uses: robinraju/release-downloader@v1.12 - with: - repository: "OpenDreamProject/OpenDream" - tag: "latest" - fileName: "DMCompiler_linux-x64.tar.gz" - extract: true - - name: Install Tools - run: | - pip3 install setuptools - bash tools/ci/install_spaceman_dmm.sh dreamchecker - bash tools/ci/install_ripgrep.sh - tools/bootstrap/python -c '' - - name: Give Linters A Go - id: linter-setup - run: ':' - - name: Run Grep Checks - if: steps.linter-setup.conclusion == 'success' && !cancelled() - run: bash tools/ci/check_grep.sh - - name: Ticked File Enforcement - if: steps.linter-setup.conclusion == 'success' && !cancelled() - run: | - tools/bootstrap/python tools/ticked_file_enforcement/ticked_file_enforcement.py < tools/ticked_file_enforcement/schemas/tgstation_dme.json - tools/bootstrap/python tools/ticked_file_enforcement/ticked_file_enforcement.py < tools/ticked_file_enforcement/schemas/unit_tests.json - - name: Check Define Sanity - if: steps.linter-setup.conclusion == 'success' && !cancelled() - run: tools/bootstrap/python -m define_sanity.check - - name: Check Trait Validity - if: steps.linter-setup.conclusion == 'success' && !cancelled() - run: tools/bootstrap/python -m trait_validity.check - - name: Run DreamChecker - if: steps.linter-setup.conclusion == 'success' && !cancelled() - shell: bash - run: ~/dreamchecker 2>&1 | bash tools/ci/annotate_dm.sh - - name: Run OpenDream - if: steps.linter-setup.conclusion == 'success' && !cancelled() - run: ./DMCompiler_linux-x64/DMCompiler tgstation.dme --suppress-unimplemented --define=CIBUILDING | bash tools/ci/annotate_od.sh - - name: Run Map Checks - if: steps.linter-setup.conclusion == 'success' && !cancelled() - run: | - tools/bootstrap/python -m mapmerge2.dmm_test - tools/bootstrap/python -m tools.maplint.source - - name: Check Cutter - if: steps.linter-setup.conclusion == 'success' && !cancelled() - run: tools/bootstrap/python -m tools.icon_cutter.check - - name: Run DMI Tests - if: steps.linter-setup.conclusion == 'success' && !cancelled() - run: tools/bootstrap/python -m dmi.test - - name: Check File Directories - if: steps.linter-setup.conclusion == 'success' && !cancelled() - run: bash tools/ci/check_filedirs.sh tgstation.dme - - name: Check Changelogs - if: steps.linter-setup.conclusion == 'success' && !cancelled() - run: bash tools/ci/check_changelogs.sh - - name: Check Miscellaneous Files - if: steps.linter-setup.conclusion == 'success' && !cancelled() - run: bash tools/ci/check_misc.sh - - name: Run TGUI Checks - if: steps.linter-setup.conclusion == 'success' && !cancelled() - run: tools/build/build --ci lint tgui-test - - compile_all_maps: - name: Compile Maps - needs: collect_data - runs-on: ubuntu-22.04 - timeout-minutes: 5 - - steps: - - uses: actions/checkout@v4 - - name: Setup Node - uses: ./.github/actions/setup_node - - name: Restore BYOND from Cache - uses: ./.github/actions/restore_or_install_byond - - name: Compile All Maps - run: | - source $HOME/BYOND/byond/bin/byondsetup - tools/build/build --ci dm -DCIBUILDING -DCITESTING -DALL_MAPS - - name: Check client Compatibility - uses: tgstation/byond-client-compatibility-check@v3 - with: - dmb-location: tgstation.dmb - max-required-client-version: ${{needs.collect_data.outputs.max_required_byond_client}} + uses: ./.github/workflows/run_linters.yml collect_data: name: Collect data and setup caches for other tasks needs: start_gate - runs-on: ubuntu-22.04 - timeout-minutes: 5 - outputs: - maps: ${{ steps.map_finder.outputs.maps }} - alternate_tests: ${{ steps.alternate_test_finder.outputs.alternate_tests }} - max_required_byond_client: ${{ steps.max_required_byond_client.outputs.max_required_byond_client }} + uses: ./.github/workflows/collect_data.yml - steps: - - uses: actions/checkout@v4 - - name: Find Maps - id: map_finder - run: | - echo "$(ls -mw0 _maps/*.json)" > maps_output.txt - sed -i -e s+_maps/+\"+g -e s+.json+\"+g maps_output.txt - echo "Maps: $(cat maps_output.txt)" - echo "maps={\"paths\":[$(cat maps_output.txt)]}" >> $GITHUB_OUTPUT - - name: Find Alternate Tests - id: alternate_test_finder - run: | - ALTERNATE_TESTS_JSON=$(jq -nRc '[inputs | capture("^(?[0-9]+)\\.(?[0-9]+): (?[^;]+);?(?[0-9]+)?$")]' .github/alternate_byond_versions.txt) - echo "alternate_tests=$ALTERNATE_TESTS_JSON" >> $GITHUB_OUTPUT - - name: Collect byond client version configuration - id: max_required_byond_client - #the regex here does not filter out non-numbers because error messages about no input are less helpful then error messages about bad input (which includes the bad input) - run: | - echo "max_required_byond_client=$(grep -Ev '^[[:blank:]]{0,}#{1,}|^[[:blank:]]{0,}$' .github/max_required_byond_client.txt | tail -n1)" >> $GITHUB_OUTPUT - - name: Set up BYOND cache - uses: ./.github/actions/restore_or_install_byond + compile_all_maps: + name: Compile Maps + needs: collect_data + uses: ./.github/workflows/compile_all_maps.yml + with: + max_required_byond_client: ${{ needs.collect_data.outputs.max_required_byond_client }} run_all_tests: name: Integration Tests needs: collect_data - uses: ./.github/workflows/perform_regular_version_tests.yml with: maps: ${{ needs.collect_data.outputs.maps }} - max_required_byond_client: ${{needs.collect_data.outputs.max_required_byond_client}} + max_required_byond_client: ${{ needs.collect_data.outputs.max_required_byond_client }} run_alternate_tests: if: needs.collect_data.outputs.alternate_tests != '[]' name: Alternate Tests needs: collect_data - uses: ./.github/workflows/perform_alternate_version_tests.yml with: alternate_tests: ${{ needs.collect_data.outputs.alternate_tests }} @@ -207,61 +66,14 @@ jobs: if: needs.collect_data.outputs.alternate_tests == '[]' || needs.run_alternate_tests.result == 'success' needs: [ collect_data, run_all_tests, run_alternate_tests ] name: Compare Screenshot Tests - timeout-minutes: 15 - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v4 - - name: Setup directory - run: mkdir -p artifacts - # If we ever add more artifacts, this is going to break, but it'll be obvious. - - name: Download screenshot tests - uses: actions/download-artifact@v4 - with: - path: artifacts - - name: ls -R - run: ls -R artifacts - - name: Setup screenshot comparison - run: npm i - working-directory: tools/screenshot-test-comparison - - name: Run screenshot comparison - run: node tools/screenshot-test-comparison/index.js artifacts code/modules/unit_tests/screenshots artifacts/screenshot_comparisons - # workflow_run does not give you the PR it ran on, - # even through the thing literally named "matching pull requests". - # However, in GraphQL, you can check if the check suite was ran - # by a specific PR, so trusting the (user controlled) action here is okay, - # as long as we check it later in show_screenshot_test_results - - name: Save PR ID - if: failure() && github.event.pull_request - run: | - echo ${{ github.event.pull_request.number }} > artifacts/screenshot_comparisons/pull_request_number.txt - - name: Upload bad screenshots - if: failure() - uses: actions/upload-artifact@v4 - with: - name: bad-screenshots - path: artifacts/screenshot_comparisons + uses: ./.github/workflows/compare_screenshots.yml test_windows: name: Windows Build - needs: [collect_data] - runs-on: windows-latest - timeout-minutes: 5 - - steps: - - uses: actions/checkout@v4 - - name: Setup Node - uses: ./.github/actions/setup_node - with: - restore-yarn-cache: true - - name: Compile - run: pwsh tools/ci/build.ps1 - env: - DM_EXE: "C:\\byond\\bin\\dm.exe" - - name: Check client Compatibility - uses: tgstation/byond-client-compatibility-check@v3 - with: - dmb-location: tgstation.dmb - max-required-client-version: ${{needs.collect_data.outputs.max_required_byond_client}} + needs: collect_data + uses: ./.github/workflows/test_windows.yml + with: + max_required_byond_client: ${{needs.collect_data.outputs.max_required_byond_client}} completion_gate: # Serves as a non-moving target for branch rulesets if: always() && !cancelled() diff --git a/.github/workflows/collect_data.yml b/.github/workflows/collect_data.yml new file mode 100644 index 00000000000..ad93bbeed0f --- /dev/null +++ b/.github/workflows/collect_data.yml @@ -0,0 +1,45 @@ +name: Collect Data + +on: + workflow_call: + outputs: + maps: + description: "The maps that were found" + value: ${{ jobs.collect_data.outputs.maps }} + alternate_tests: + description: "The alternate tests that were found" + value: ${{ jobs.collect_data.outputs.alternate_tests }} + max_required_byond_client: + description: "The max required byond client version" + value: ${{ jobs.collect_data.outputs.max_required_byond_client }} + +jobs: + collect_data: + runs-on: ubuntu-22.04 + timeout-minutes: 5 + outputs: + maps: ${{ steps.map_finder.outputs.maps }} + alternate_tests: ${{ steps.alternate_test_finder.outputs.alternate_tests }} + max_required_byond_client: ${{ steps.max_required_byond_client.outputs.max_required_byond_client }} + + steps: + - uses: actions/checkout@v4 + - name: Find Maps + id: map_finder + run: | + echo "$(ls -mw0 _maps/*.json)" > maps_output.txt + sed -i -e s+_maps/+\"+g -e s+.json+\"+g maps_output.txt + echo "Maps: $(cat maps_output.txt)" + echo "maps={\"paths\":[$(cat maps_output.txt)]}" >> $GITHUB_OUTPUT + - name: Find Alternate Tests + id: alternate_test_finder + run: | + ALTERNATE_TESTS_JSON=$(jq -nRc '[inputs | capture("^(?[0-9]+)\\.(?[0-9]+): (?[^;]+);?(?[0-9]+)?$")]' .github/alternate_byond_versions.txt) + echo "alternate_tests=$ALTERNATE_TESTS_JSON" >> $GITHUB_OUTPUT + - name: Collect byond client version configuration + id: max_required_byond_client + #the regex here does not filter out non-numbers because error messages about no input are less helpful then error messages about bad input (which includes the bad input) + run: | + echo "max_required_byond_client=$(grep -Ev '^[[:blank:]]{0,}#{1,}|^[[:blank:]]{0,}$' .github/max_required_byond_client.txt | tail -n1)" >> $GITHUB_OUTPUT + - name: Set up BYOND cache + uses: ./.github/actions/restore_or_install_byond diff --git a/.github/workflows/compare_screenshots.yml b/.github/workflows/compare_screenshots.yml new file mode 100644 index 00000000000..b0c02450829 --- /dev/null +++ b/.github/workflows/compare_screenshots.yml @@ -0,0 +1,40 @@ +name: Compare Screenshots + +on: + workflow_call: + +jobs: + compare_screenshots: + timeout-minutes: 15 + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - name: Setup directory + run: mkdir -p artifacts + # If we ever add more artifacts, this is going to break, but it'll be obvious. + - name: Download screenshot tests + uses: actions/download-artifact@v4 + with: + path: artifacts + - name: ls -R + run: ls -R artifacts + - name: Setup screenshot comparison + run: npm i + working-directory: tools/screenshot-test-comparison + - name: Run screenshot comparison + run: node tools/screenshot-test-comparison/index.js artifacts code/modules/unit_tests/screenshots artifacts/screenshot_comparisons + # workflow_run does not give you the PR it ran on, + # even through the thing literally named "matching pull requests". + # However, in GraphQL, you can check if the check suite was ran + # by a specific PR, so trusting the (user controlled) action here is okay, + # as long as we check it later in show_screenshot_test_results + - name: Save PR ID + if: failure() && github.event.pull_request + run: | + echo ${{ github.event.pull_request.number }} > artifacts/screenshot_comparisons/pull_request_number.txt + - name: Upload bad screenshots + if: failure() + uses: actions/upload-artifact@v4 + with: + name: bad-screenshots + path: artifacts/screenshot_comparisons diff --git a/.github/workflows/compile_all_maps.yml b/.github/workflows/compile_all_maps.yml new file mode 100644 index 00000000000..94bdafdfc70 --- /dev/null +++ b/.github/workflows/compile_all_maps.yml @@ -0,0 +1,29 @@ +name: Compile All Maps + +on: + workflow_call: + inputs: + max_required_byond_client: + required: true + type: string + +jobs: + compile_all_maps: + runs-on: ubuntu-22.04 + timeout-minutes: 5 + + steps: + - uses: actions/checkout@v4 + - name: Setup Node + uses: ./.github/actions/setup_node + - name: Restore BYOND from Cache + uses: ./.github/actions/restore_or_install_byond + - name: Compile All Maps + run: | + source $HOME/BYOND/byond/bin/byondsetup + tools/build/build --ci dm -DCIBUILDING -DCITESTING -DALL_MAPS + - name: Check client Compatibility + uses: tgstation/byond-client-compatibility-check@v3 + with: + dmb-location: tgstation.dmb + max-required-client-version: ${{inputs.max_required_byond_client}} diff --git a/.github/workflows/perform_alternate_version_tests.yml b/.github/workflows/perform_alternate_version_tests.yml index de2abfe3e81..1718cb84d30 100644 --- a/.github/workflows/perform_alternate_version_tests.yml +++ b/.github/workflows/perform_alternate_version_tests.yml @@ -1,4 +1,5 @@ name: Run Alternate BYOND Version Tests + on: workflow_call: inputs: diff --git a/.github/workflows/perform_regular_version_tests.yml b/.github/workflows/perform_regular_version_tests.yml index bc515d85002..724db11f7df 100644 --- a/.github/workflows/perform_regular_version_tests.yml +++ b/.github/workflows/perform_regular_version_tests.yml @@ -1,4 +1,5 @@ name: Run Regular BYOND Version Tests + on: workflow_call: inputs: diff --git a/.github/workflows/run_integration_tests.yml b/.github/workflows/run_integration_tests.yml index 3354b4ccf3e..5f7b39059e8 100644 --- a/.github/workflows/run_integration_tests.yml +++ b/.github/workflows/run_integration_tests.yml @@ -1,6 +1,7 @@ # This is a reusable workflow to run integration tests on a single map. # This is run for every single map in ci_suite.yml. You might want to edit that instead. name: Run Integration Tests + on: workflow_call: inputs: diff --git a/.github/workflows/run_linters.yml b/.github/workflows/run_linters.yml new file mode 100644 index 00000000000..c00ecaf6db5 --- /dev/null +++ b/.github/workflows/run_linters.yml @@ -0,0 +1,106 @@ +name: Run Linters + +on: + workflow_call: + +jobs: + linters: + runs-on: ubuntu-22.04 + timeout-minutes: 5 + + steps: + - uses: actions/checkout@v4 + - name: Restore SpacemanDMM cache + uses: actions/cache@v4 + with: + path: ~/SpacemanDMM + key: ${{ runner.os }}-spacemandmm-${{ hashFiles('dependencies.sh') }} + restore-keys: | + ${{ runner.os }}-spacemandmm- + - name: Setup Node + uses: ./.github/actions/setup_node + with: + restore-yarn-cache: true + - name: Restore Bootstrap cache + uses: actions/cache@v4 + with: + path: tools/bootstrap/.cache + key: ${{ runner.os }}-bootstrap-${{ hashFiles('tools/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-bootstrap- + - name: Restore Rust cache + uses: actions/cache@v4 + with: + path: ~/.cargo + key: ${{ runner.os }}-rust-${{ hashFiles('tools/ci/ci_dependencies.sh')}} + restore-keys: | + ${{ runner.os }}-rust- + - name: Restore Cutter cache + uses: actions/cache@v4 + with: + path: tools/icon_cutter/cache + key: ${{ runner.os }}-cutter-${{ hashFiles('dependencies.sh') }} + - name: Setup .NET SDK + uses: actions/setup-dotnet@v4.2.0 + with: + dotnet-version: 9.x + - name: Install OpenDream + uses: robinraju/release-downloader@v1.12 + with: + repository: "OpenDreamProject/OpenDream" + tag: "latest" + fileName: "DMCompiler_linux-x64.tar.gz" + extract: true + - name: Install Tools + run: | + pip3 install setuptools + bash tools/ci/install_spaceman_dmm.sh dreamchecker + bash tools/ci/install_ripgrep.sh + tools/bootstrap/python -c '' + - name: Give Linters A Go + id: linter-setup + run: ":" + - name: Run Grep Checks + if: steps.linter-setup.conclusion == 'success' && !cancelled() + run: bash tools/ci/check_grep.sh + - name: Ticked File Enforcement + if: steps.linter-setup.conclusion == 'success' && !cancelled() + run: | + tools/bootstrap/python tools/ticked_file_enforcement/ticked_file_enforcement.py < tools/ticked_file_enforcement/schemas/tgstation_dme.json + tools/bootstrap/python tools/ticked_file_enforcement/ticked_file_enforcement.py < tools/ticked_file_enforcement/schemas/unit_tests.json + - name: Check Define Sanity + if: steps.linter-setup.conclusion == 'success' && !cancelled() + run: tools/bootstrap/python -m define_sanity.check + - name: Check Trait Validity + if: steps.linter-setup.conclusion == 'success' && !cancelled() + run: tools/bootstrap/python -m trait_validity.check + - name: Run DreamChecker + if: steps.linter-setup.conclusion == 'success' && !cancelled() + shell: bash + run: ~/dreamchecker 2>&1 | bash tools/ci/annotate_dm.sh + - name: Run OpenDream + if: steps.linter-setup.conclusion == 'success' && !cancelled() + run: ./DMCompiler_linux-x64/DMCompiler tgstation.dme --suppress-unimplemented --define=CIBUILDING | bash tools/ci/annotate_od.sh + - name: Run Map Checks + if: steps.linter-setup.conclusion == 'success' && !cancelled() + run: | + tools/bootstrap/python -m mapmerge2.dmm_test + tools/bootstrap/python -m tools.maplint.source + - name: Check Cutter + if: steps.linter-setup.conclusion == 'success' && !cancelled() + run: tools/bootstrap/python -m tools.icon_cutter.check + - name: Run DMI Tests + if: steps.linter-setup.conclusion == 'success' && !cancelled() + run: tools/bootstrap/python -m dmi.test + - name: Check File Directories + if: steps.linter-setup.conclusion == 'success' && !cancelled() + run: bash tools/ci/check_filedirs.sh tgstation.dme + - name: Check Changelogs + if: steps.linter-setup.conclusion == 'success' && !cancelled() + run: bash tools/ci/check_changelogs.sh + - name: Check Miscellaneous Files + if: steps.linter-setup.conclusion == 'success' && !cancelled() + run: bash tools/ci/check_misc.sh + - name: Run TGUI Checks + if: steps.linter-setup.conclusion == 'success' && !cancelled() + run: tools/build/build --ci lint tgui-test diff --git a/.github/workflows/test_windows.yml b/.github/workflows/test_windows.yml new file mode 100644 index 00000000000..a437a067b71 --- /dev/null +++ b/.github/workflows/test_windows.yml @@ -0,0 +1,29 @@ +name: Test Windows + +on: + workflow_call: + inputs: + max_required_byond_client: + required: true + type: string + +jobs: + test_windows: + runs-on: windows-latest + timeout-minutes: 5 + + steps: + - uses: actions/checkout@v4 + - name: Setup Node + uses: ./.github/actions/setup_node + with: + restore-yarn-cache: true + - name: Compile + run: pwsh tools/ci/build.ps1 + env: + DM_EXE: "C:\\byond\\bin\\dm.exe" + - name: Check client Compatibility + uses: tgstation/byond-client-compatibility-check@v3 + with: + dmb-location: tgstation.dmb + max-required-client-version: ${{inputs.max_required_byond_client}}