Add support for running tests on alternate versions (#71177)

Adds support for additional byond versions tests

.github/alternate_byond_versions.txt

@@ -0,0 +1,10 @@
+# This file contains extra tests to run for specific BYOND versions.
+# This is useful for making sure we maintain compatibility with both older and newer versions,
+# while still having our main tests run on a guaranteed pinned version.
+
+# Format is version: map
+# Example:
+# 500.1337: runtimestation
+
+# AnturK uncomment this and then remove this line
+# 515.1594: runtimestation

.github/workflows/ci_suite.yml

@@ -79,6 +79,7 @@ jobs:
     runs-on: ubuntu-20.04
     outputs:
       maps: ${{ steps.map_finder.outputs.maps }}
+      alternate_tests: ${{ steps.alternate_test_finder.outputs.alternate_tests }}
     concurrency:
       group: find_all_maps-${{ github.ref }}
       cancel-in-progress: true
@@ -91,70 +92,44 @@ jobs:
           sed -i -e s+_maps/+\"+g -e s+.json+\"+g maps_output.txt
           echo "Maps: $(cat maps_output.txt)"
           echo "maps={\"paths\":[$(cat maps_output.txt)]}" >> $GITHUB_OUTPUT
-
+      - name: Find Alternate Tests
+        id: alternate_test_finder
+        run: |
+          ALTERNATE_TESTS_JSON=$(jq -nRc '[inputs | capture("^(?<major>[0-9]+)\\.(?<minor>[0-9]+): (?<map>.+)$")]' .github/alternate_byond_versions.txt)
+          echo "alternate_tests=$ALTERNATE_TESTS_JSON" >> $GITHUB_OUTPUT
   run_all_tests:
     if: "!contains(github.event.head_commit.message, '[ci skip]')"
     name: Integration Tests
-    runs-on: ubuntu-20.04
     needs: [find_all_maps]
     strategy:
       fail-fast: false
       matrix:
         map: ${{ fromJSON(needs.find_all_maps.outputs.maps).paths }}
-    services:
-      mysql:
-        image: mysql:latest
-        env:
-          MYSQL_ROOT_PASSWORD: root
-        ports:
-          - 3306
-        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
     concurrency:
       group: run_all_tests-${{ github.ref }}-${{ matrix.map }}
       cancel-in-progress: true
-    steps:
-      - uses: actions/checkout@v3
-      - name: Restore BYOND cache
-        uses: actions/cache@v3
-        with:
-          path: ~/BYOND
-          key: ${{ runner.os }}-byond-${{ secrets.CACHE_PURGE_KEY }}
-      - name: Setup database
-        run: |
-          sudo systemctl start mysql
-          mysql -u root -proot -e 'CREATE DATABASE tg_ci;'
-          mysql -u root -proot tg_ci < SQL/tgstation_schema.sql
-          mysql -u root -proot -e 'CREATE DATABASE tg_ci_prefixed;'
-          mysql -u root -proot tg_ci_prefixed < SQL/tgstation_schema_prefixed.sql
-      - name: Install rust-g
-        run: |
-          sudo dpkg --add-architecture i386
-          sudo apt update || true
-          sudo apt install -o APT::Immediate-Configure=false libssl1.1:i386
-          bash tools/ci/install_rust_g.sh
-      - name: Install auxlua
-        run: |
-          bash tools/ci/install_auxlua.sh
-      - name: Compile Tests
-        run: |
-          bash tools/ci/install_byond.sh
-          source $HOME/BYOND/byond/bin/byondsetup
-          tools/build/build --ci dm -DCIBUILDING -DANSICOLORS
-      - name: Run Tests
-        run: |
-          source $HOME/BYOND/byond/bin/byondsetup
-          bash tools/ci/run_server.sh ${{ matrix.map }}
-      - name: Upload screenshot tests
-        if: always()
-        uses: actions/upload-artifact@v3
-        with:
-          name: test_artifacts_${{ matrix.map }}
-          path: data/screenshots_new/
-          retention-days: 1
-
+    uses: ./.github/workflows/run_integration_tests.yml
+    with:
+      map: ${{ matrix.map }}
+  run_alternate_tests:
+    if: "!contains(github.event.head_commit.message, '[ci skip]') && needs.find_all_maps.outputs.alternate_tests != '[]'"
+    name: Alternate Tests
+    needs: [find_all_maps]
+    strategy:
+      fail-fast: false
+      matrix:
+        setup: ${{ fromJSON(needs.find_all_maps.outputs.alternate_tests) }}
+    concurrency:
+      group: run_all_tests-${{ github.ref }}-${{ matrix.setup.major }}.${{ matrix.setup.minor }}-${{ matrix.setup.map }}
+      cancel-in-progress: true
+    uses: ./.github/workflows/run_integration_tests.yml
+    with:
+      map: ${{ matrix.setup.map }}
+      major: ${{ matrix.setup.major }}
+      minor: ${{ matrix.setup.minor }}
   compare_screenshots:
     if: "!contains(github.event.head_commit.message, '[ci skip]') && always()"
-    needs: [run_all_tests]
+    needs: [run_all_tests, run_alternate_tests]
     name: Compare Screenshot Tests
     runs-on: ubuntu-20.04
     steps:

.github/workflows/run_integration_tests.yml

@@ -0,0 +1,70 @@
+# This is a reusable workflow to run integration tests on a single map.
+# This is run for every single map in ci_suite.yml. You might want to edit that instead.
+name: Run Integration Tests
+on:
+  workflow_call:
+    inputs:
+      map:
+        required: true
+        type: string
+      major:
+        required: false
+        type: string
+      minor:
+        required: false
+        type: string
+jobs:
+  run_integration_tests:
+    runs-on: ubuntu-20.04
+    services:
+      mysql:
+        image: mysql:latest
+        env:
+          MYSQL_ROOT_PASSWORD: root
+        ports:
+          - 3306
+        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
+    steps:
+      - uses: actions/checkout@v3
+      - name: Restore BYOND cache
+        uses: actions/cache@v3
+        with:
+          path: ~/BYOND
+          key: ${{ runner.os }}-byond-${{ secrets.CACHE_PURGE_KEY }}
+      - name: Setup database
+        run: |
+          sudo systemctl start mysql
+          mysql -u root -proot -e 'CREATE DATABASE tg_ci;'
+          mysql -u root -proot tg_ci < SQL/tgstation_schema.sql
+          mysql -u root -proot -e 'CREATE DATABASE tg_ci_prefixed;'
+          mysql -u root -proot tg_ci_prefixed < SQL/tgstation_schema_prefixed.sql
+      - name: Install rust-g
+        run: |
+          sudo dpkg --add-architecture i386
+          sudo apt update || true
+          sudo apt install -o APT::Immediate-Configure=false libssl1.1:i386
+          bash tools/ci/install_rust_g.sh
+      - name: Install auxlua
+        run: |
+          bash tools/ci/install_auxlua.sh
+      - name: Configure version
+        run: |
+          echo "BYOND_MAJOR=${{ inputs.major }}" >> $GITHUB_ENV
+          echo "BYOND_MINOR=${{ inputs.minor }}" >> $GITHUB_ENV
+        if: ${{ inputs.major }}
+      - name: Compile Tests
+        run: |
+          bash tools/ci/install_byond.sh
+          source $HOME/BYOND/byond/bin/byondsetup
+          tools/build/build --ci dm -DCIBUILDING -DANSICOLORS
+      - name: Run Tests
+        run: |
+          source $HOME/BYOND/byond/bin/byondsetup
+          bash tools/ci/run_server.sh ${{ inputs.map }}
+      - name: Upload screenshot tests
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: test_artifacts_${{ inputs.map }}
+          path: data/screenshots_new/
+          retention-days: 1

tools/ci/install_byond.sh

@@ -1,7 +1,10 @@
 #!/bin/bash
 set -euo pipefail
 
-source dependencies.sh
+# BYOND_MAJOR and BYOND_MINOR can be explicitly set, such as in alt_byond_versions.txt
+if [ -z "${BYOND_MAJOR+x}" ]; then
+  source dependencies.sh
+fi
 
 if [ -d "$HOME/BYOND/byond/bin" ] && grep -Fxq "${BYOND_MAJOR}.${BYOND_MINOR}" $HOME/BYOND/version.txt;
 then