CI Cache Optimization (#16626)

* Let's see what we can do about it

* test

* fake

* fake2

* sdfas

* python cache?

* guh?

* sadff

* oopsie daisy + node

* better?

* maybe?

* fake

* change to trigger

* hope

* sadf

* uhhh what?

* *sigh*

* afsdfa

* dsadfasd

* [linter skip] fucking hell

* [linter skip] fr

* [linter skip] linter skip comeon

* fuck's sake

* alright

* maybe we can do it

* sadfas

* changelog

* safasdf

* sdfas

* sadfsdafa

* guh?

* sdfa

.github/workflows/byond.yml

@@ -15,11 +15,18 @@ env:
   MACRO_COUNT: 0
   GENDER_COUNT: 6
   TO_WORLD_COUNT: 185
-  TGM_CHECK: "//MAP CONVERTED BY dmm2tgm.py THIS HEADER COMMENT PREVENTS RECONVERSION, DO NOT REMOVE"
+  BYOND_MAJOR: ""
+  BYOND_MINOR: ""
+  RUST_G_VERSION: ""
+  FLYWAY_BUILD: ""
+  SPACEMAN_DMM_VERSION: ""
+  NODE_VERSION: ""
+  NODE_VERSION_PRECISE: ""
+  PYTHON_VERSION: ""
 
 jobs:
   lint:
-    if: "!contains(github.event.head_commit.message, '[ci skip]')"
+    if: ${{ !(contains(github.event.head_commit.message, '[ci skip]')) }}
     runs-on: ubuntu-22.04
     name: Run Linters
     concurrency:
@@ -37,7 +44,7 @@ jobs:
       - name: Restore OpenDream cache
         uses: actions/cache@v3
         with:
-          path: ~/OpenDream
+          path: ~/OpenDream/*
           key: ${{ runner.os }}-opendream
       - name: Restore Yarn cache
         uses: actions/cache@v3
@@ -48,13 +55,27 @@ jobs:
             ${{ runner.os }}-build-
             ${{ runner.os }}-
       # Install Dependencies
+      - name: "Set ENV variables"
+        run: bash dependencies.sh
+      - name: Restore Python Cache
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: 'pip'
+      - name: Setup Node
+        uses: actions/setup-node@v3
+        with:
+          cache-dependency-path: tgui/yarn.lock
+          node-version: ${{ env.NODE_VERSION_PRECISE }}
+          cache: 'yarn'
+      - name: Install Python Packages
+        run: |
+          pip install -r tools/requirements.txt
       - name: Install Tools
         run: |
           pip3 install setuptools
-          bash tools/ci/install_node.sh
           bash tools/ci/install_spaceman_dmm.sh dreamchecker
           bash tools/ci/install_od.sh
-          tools/bootstrap/python -c ''
       - name: Check Filedirs
         run: |
           bash tools/ci/check_filedirs.sh aurorastation.dme
@@ -97,7 +118,7 @@ jobs:
           outputFile: output-annotations.txt
 
   unit-test-linux:
-    if: "!contains(github.event.head_commit.message, '[ci skip]')"
+    if: ${{ !(contains(github.event.head_commit.message, '[ci skip]')) }}
     strategy:
       matrix:
         map: [runtime, aurora, sccv_horizon, away_sites_testing, exoplanet_testing]
@@ -122,7 +143,7 @@ jobs:
       - name: Restore BYOND cache
         uses: actions/cache@v3
         with:
-          path: ~/BYOND
+          path: ~/BYOND/*
           key: ${{ runner.os }}-byond
 
       - name: Restore FlyWay cache
@@ -130,24 +151,31 @@ jobs:
         with:
           path: ~/flyway
           key: ${{ runner.os }}-flyway
+      - name: Prepare OS Environtment
+        run: |
+          sudo dpkg --add-architecture i386
+          sudo apt-get update
+      - name: Install OS Packages
+        uses: awalsh128/cache-apt-pkgs-action@latest
+        with:
+          packages: gcc-multilib zlib1g-dev:i386 libssl-dev:i386 libgcc-s1:i386 libc6:i386
+          version: 1.0
+      - name: Install OS Packages (Native)
+        run: |
+          sudo apt-get install libgcc-s1:i386 libssl-dev:i386
       - name: Install BYOND
         run: |
           source dependencies.sh
-          sudo dpkg --add-architecture i386
-          sudo apt-get update
-          sudo apt-get install -y gcc-multilib zlib1g-dev:i386 libssl-dev:i386 libgcc-s1:i386 libc6:i386
           bash tools/ci/install_byond.sh
           source $HOME/BYOND/byond/bin/byondsetup
           bash tools/ci/install_rust_g.sh
       - name: Install Flyway
         run: |
-          source dependencies.sh
           tools/ci/install_flyway.sh
       - name: Migrate database
         env:
           PORT: ${{ job.services.mariadb.ports[3306] }}
         run: |
-          source dependencies.sh
           $HOME/flyway/flyway-$FLYWAY_BUILD/flyway migrate -user=root -password=root -url="jdbc:mariadb://localhost:$PORT/game"
       - name: Build
         run: |

dependencies.sh

@@ -1,15 +1,23 @@
 #!/bin/sh
 
 export BYOND_MAJOR=514
+echo "BYOND_MAJOR=$BYOND_MAJOR" >> "$GITHUB_ENV"
 export BYOND_MINOR=1589
+echo "BYOND_MINOR=$BYOND_MINOR" >> "$GITHUB_ENV"
 
 export RUST_G_VERSION=v1.2.0+a4
+echo "RUST_G_VERSION=$RUST_G_VERSION" >> "$GITHUB_ENV"
 export FLYWAY_BUILD=9.9.0
+echo "FLYWAY_BUILD=$FLYWAY_BUILD" >> "$GITHUB_ENV"
 export SPACEMAN_DMM_VERSION=suite-1.7.3
+echo "SPACEMAN_DMM_VERSION=$SPACEMAN_DMM_VERSION" >> "$GITHUB_ENV"
 
 #node version
 export NODE_VERSION=14
+echo "NODE_VERSION=$NODE_VERSION" >> "$GITHUB_ENV"
 export NODE_VERSION_PRECISE=14.16.1
+echo "NODE_VERSION_PRECISE=$NODE_VERSION_PRECISE" >> "$GITHUB_ENV"
 
 # Python version for mapmerge and other tools
-export PYTHON_VERSION=3.7.9
+export PYTHON_VERSION=3.9
+echo "PYTHON_VERSION=$PYTHON_VERSION" >> "$GITHUB_ENV"

html/changelogs/fluffyghost-cicacheoptimization.yml

@@ -0,0 +1,42 @@
+################################
+# Example Changelog File
+#
+# Note: This file, and files beginning with ".", and files that don't end in ".yml" will not be read. If you change this file, you will look really dumb.
+#
+# Your changelog will be merged with a master changelog. (New stuff added only, and only on the date entry for the day it was merged.)
+# When it is, any changes listed below will disappear.
+#
+# Valid Prefixes:
+#   bugfix
+#   wip (For works in progress)
+#   tweak
+#   soundadd
+#   sounddel
+#   rscadd (general adding of nice things)
+#   rscdel (general deleting of nice things)
+#   imageadd
+#   imagedel
+#   maptweak
+#   spellcheck (typo fixes)
+#   experiment
+#   balance
+#   admin
+#   backend
+#   security
+#   refactor
+#################################
+
+# Your name.
+author: FluffyGhost
+
+# Optional: Remove this file after generating master changelog.  Useful for PR changelogs that won't get used again.
+delete-after: True
+
+# Any changes you've made.  See valid prefix list above.
+# INDENT WITH TWO SPACES.  NOT TABS.  SPACES.
+# SCREW THIS UP AND IT WON'T WORK.
+# Also, all entries are changed into a single [] after a master changelog generation. Just remove the brackets when you add new entries.
+# Please surround your changes in  double quotes ("), as certain characters otherwise screws up compiling. The quotes will not show up in the changelog.
+changes:
+  - backend: "Changed the CI to make better and more use of various caching systems in github."
+  - backend: "CI now uses python 3.9."

tools/ci/install_od.sh

@@ -5,10 +5,17 @@ if [ -d "$HOME/OpenDream/" ];
 then
   echo "Using cached OpenDream directory."
 
-  git -C $HOME/OpenDream fetch origin
-  git -C $HOME/OpenDream reset --hard origin/master
+  if [ -a "$HOME/OpenDream/OpenDream.sln" ];
+  then
+    echo "OpenDream is already compiled."
+    exit
+  else
+    echo "OpenDream is not compiled, compiling it now..."
+    git -C $HOME/OpenDream fetch origin
+    git -C $HOME/OpenDream reset --hard origin/master
 
-  git -C $HOME/OpenDream submodule update --init --recursive
+    git -C $HOME/OpenDream submodule update --init --recursive
+  fi
 
 
 else