From 8c0f3400a388539179bc22740fecb39a49524b25 Mon Sep 17 00:00:00 2001 From: Fluffy <65877598+FluffyGhoster@users.noreply.github.com> Date: Tue, 18 Jul 2023 13:29:19 +0200 Subject: [PATCH] CI Cache Optimization (#16626) * Let's see what we can do about it * test * fake * fake2 * sdfas * python cache? * guh? * sadff * oopsie daisy + node * better? * maybe? * fake * change to trigger * hope * sadf * uhhh what? * *sigh* * afsdfa * dsadfasd * [linter skip] fucking hell * [linter skip] fr * [linter skip] linter skip comeon * fuck's sake * alright * maybe we can do it * sadfas * changelog * safasdf * sdfas * sadfsdafa * guh? * sdfa --- .github/workflows/byond.yml | 52 ++++++++++++++----- dependencies.sh | 10 +++- .../fluffyghost-cicacheoptimization.yml | 42 +++++++++++++++ tools/ci/install_od.sh | 13 +++-- 4 files changed, 101 insertions(+), 16 deletions(-) create mode 100644 html/changelogs/fluffyghost-cicacheoptimization.yml diff --git a/.github/workflows/byond.yml b/.github/workflows/byond.yml index c31d5e93cae..9dfde70c749 100644 --- a/.github/workflows/byond.yml +++ b/.github/workflows/byond.yml @@ -15,11 +15,18 @@ env: MACRO_COUNT: 0 GENDER_COUNT: 6 TO_WORLD_COUNT: 185 - TGM_CHECK: "//MAP CONVERTED BY dmm2tgm.py THIS HEADER COMMENT PREVENTS RECONVERSION, DO NOT REMOVE" + BYOND_MAJOR: "" + BYOND_MINOR: "" + RUST_G_VERSION: "" + FLYWAY_BUILD: "" + SPACEMAN_DMM_VERSION: "" + NODE_VERSION: "" + NODE_VERSION_PRECISE: "" + PYTHON_VERSION: "" jobs: lint: - if: "!contains(github.event.head_commit.message, '[ci skip]')" + if: ${{ !(contains(github.event.head_commit.message, '[ci skip]')) }} runs-on: ubuntu-22.04 name: Run Linters concurrency: @@ -37,7 +44,7 @@ jobs: - name: Restore OpenDream cache uses: actions/cache@v3 with: - path: ~/OpenDream + path: ~/OpenDream/* key: ${{ runner.os }}-opendream - name: Restore Yarn cache uses: actions/cache@v3 @@ -48,13 +55,27 @@ jobs: ${{ runner.os }}-build- ${{ runner.os }}- # Install Dependencies + - name: "Set ENV variables" + run: bash dependencies.sh + - name: Restore Python Cache + uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: 'pip' + - name: Setup Node + uses: actions/setup-node@v3 + with: + cache-dependency-path: tgui/yarn.lock + node-version: ${{ env.NODE_VERSION_PRECISE }} + cache: 'yarn' + - name: Install Python Packages + run: | + pip install -r tools/requirements.txt - name: Install Tools run: | pip3 install setuptools - bash tools/ci/install_node.sh bash tools/ci/install_spaceman_dmm.sh dreamchecker bash tools/ci/install_od.sh - tools/bootstrap/python -c '' - name: Check Filedirs run: | bash tools/ci/check_filedirs.sh aurorastation.dme @@ -97,7 +118,7 @@ jobs: outputFile: output-annotations.txt unit-test-linux: - if: "!contains(github.event.head_commit.message, '[ci skip]')" + if: ${{ !(contains(github.event.head_commit.message, '[ci skip]')) }} strategy: matrix: map: [runtime, aurora, sccv_horizon, away_sites_testing, exoplanet_testing] @@ -122,7 +143,7 @@ jobs: - name: Restore BYOND cache uses: actions/cache@v3 with: - path: ~/BYOND + path: ~/BYOND/* key: ${{ runner.os }}-byond - name: Restore FlyWay cache @@ -130,24 +151,31 @@ jobs: with: path: ~/flyway key: ${{ runner.os }}-flyway + - name: Prepare OS Environtment + run: | + sudo dpkg --add-architecture i386 + sudo apt-get update + - name: Install OS Packages + uses: awalsh128/cache-apt-pkgs-action@latest + with: + packages: gcc-multilib zlib1g-dev:i386 libssl-dev:i386 libgcc-s1:i386 libc6:i386 + version: 1.0 + - name: Install OS Packages (Native) + run: | + sudo apt-get install libgcc-s1:i386 libssl-dev:i386 - name: Install BYOND run: | source dependencies.sh - sudo dpkg --add-architecture i386 - sudo apt-get update - sudo apt-get install -y gcc-multilib zlib1g-dev:i386 libssl-dev:i386 libgcc-s1:i386 libc6:i386 bash tools/ci/install_byond.sh source $HOME/BYOND/byond/bin/byondsetup bash tools/ci/install_rust_g.sh - name: Install Flyway run: | - source dependencies.sh tools/ci/install_flyway.sh - name: Migrate database env: PORT: ${{ job.services.mariadb.ports[3306] }} run: | - source dependencies.sh $HOME/flyway/flyway-$FLYWAY_BUILD/flyway migrate -user=root -password=root -url="jdbc:mariadb://localhost:$PORT/game" - name: Build run: | diff --git a/dependencies.sh b/dependencies.sh index 813264b2f10..c7465cd06cf 100755 --- a/dependencies.sh +++ b/dependencies.sh @@ -1,15 +1,23 @@ #!/bin/sh export BYOND_MAJOR=514 +echo "BYOND_MAJOR=$BYOND_MAJOR" >> "$GITHUB_ENV" export BYOND_MINOR=1589 +echo "BYOND_MINOR=$BYOND_MINOR" >> "$GITHUB_ENV" export RUST_G_VERSION=v1.2.0+a4 +echo "RUST_G_VERSION=$RUST_G_VERSION" >> "$GITHUB_ENV" export FLYWAY_BUILD=9.9.0 +echo "FLYWAY_BUILD=$FLYWAY_BUILD" >> "$GITHUB_ENV" export SPACEMAN_DMM_VERSION=suite-1.7.3 +echo "SPACEMAN_DMM_VERSION=$SPACEMAN_DMM_VERSION" >> "$GITHUB_ENV" #node version export NODE_VERSION=14 +echo "NODE_VERSION=$NODE_VERSION" >> "$GITHUB_ENV" export NODE_VERSION_PRECISE=14.16.1 +echo "NODE_VERSION_PRECISE=$NODE_VERSION_PRECISE" >> "$GITHUB_ENV" # Python version for mapmerge and other tools -export PYTHON_VERSION=3.7.9 +export PYTHON_VERSION=3.9 +echo "PYTHON_VERSION=$PYTHON_VERSION" >> "$GITHUB_ENV" diff --git a/html/changelogs/fluffyghost-cicacheoptimization.yml b/html/changelogs/fluffyghost-cicacheoptimization.yml new file mode 100644 index 00000000000..542d5df342e --- /dev/null +++ b/html/changelogs/fluffyghost-cicacheoptimization.yml @@ -0,0 +1,42 @@ +################################ +# Example Changelog File +# +# Note: This file, and files beginning with ".", and files that don't end in ".yml" will not be read. If you change this file, you will look really dumb. +# +# Your changelog will be merged with a master changelog. (New stuff added only, and only on the date entry for the day it was merged.) +# When it is, any changes listed below will disappear. +# +# Valid Prefixes: +# bugfix +# wip (For works in progress) +# tweak +# soundadd +# sounddel +# rscadd (general adding of nice things) +# rscdel (general deleting of nice things) +# imageadd +# imagedel +# maptweak +# spellcheck (typo fixes) +# experiment +# balance +# admin +# backend +# security +# refactor +################################# + +# Your name. +author: FluffyGhost + +# Optional: Remove this file after generating master changelog. Useful for PR changelogs that won't get used again. +delete-after: True + +# Any changes you've made. See valid prefix list above. +# INDENT WITH TWO SPACES. NOT TABS. SPACES. +# SCREW THIS UP AND IT WON'T WORK. +# Also, all entries are changed into a single [] after a master changelog generation. Just remove the brackets when you add new entries. +# Please surround your changes in double quotes ("), as certain characters otherwise screws up compiling. The quotes will not show up in the changelog. +changes: + - backend: "Changed the CI to make better and more use of various caching systems in github." + - backend: "CI now uses python 3.9." diff --git a/tools/ci/install_od.sh b/tools/ci/install_od.sh index f2ac4613044..3249692f731 100644 --- a/tools/ci/install_od.sh +++ b/tools/ci/install_od.sh @@ -5,10 +5,17 @@ if [ -d "$HOME/OpenDream/" ]; then echo "Using cached OpenDream directory." - git -C $HOME/OpenDream fetch origin - git -C $HOME/OpenDream reset --hard origin/master + if [ -a "$HOME/OpenDream/OpenDream.sln" ]; + then + echo "OpenDream is already compiled." + exit + else + echo "OpenDream is not compiled, compiling it now..." + git -C $HOME/OpenDream fetch origin + git -C $HOME/OpenDream reset --hard origin/master - git -C $HOME/OpenDream submodule update --init --recursive + git -C $HOME/OpenDream submodule update --init --recursive + fi else