From 283b548aa405f60b6adaf77f1b69a9d04cedfdd4 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Tue, 17 Jul 2018 23:49:39 -0400 Subject: [PATCH] Adds deploy script. CI artifacts. Dependencies file (#39040) Adds a simple file to create a work tree with only what's needed to RUN the game. .dmb/.rsc, _maps, strings, and some icons and sounds. Makes the Dockerfile use it. Also adds dependencies.sh which can serve as a single version source used by all CI and utils. Enables appveyor and artifacts, you can now download a .dmb deployment straight from a PR or master commit. --- .dockerignore | 3 +- .travis.yml | 11 ++++- Dockerfile | 70 ++++++++++++----------------- appveyor.yml | 11 +++++ dependencies.sh | 19 ++++++++ tgui/build.sh | 3 +- tools/appveyor/build.ps1 | 10 +++++ tools/appveyor/download_byond.sh | 5 +++ tools/travis/before_build_byond.sh | 39 ++++++++-------- tools/travis/build_byond.sh | 39 ++++++++++++++-- tools/travis/build_dependencies.sh | 9 ++++ tools/travis/install_build_tools.sh | 2 + 12 files changed, 153 insertions(+), 68 deletions(-) create mode 100644 appveyor.yml create mode 100755 dependencies.sh create mode 100644 tools/appveyor/build.ps1 create mode 100755 tools/appveyor/download_byond.sh diff --git a/.dockerignore b/.dockerignore index fc4b7caec6aa..400701794cf0 100644 --- a/.dockerignore +++ b/.dockerignore @@ -14,7 +14,6 @@ TGS3.json .git/objects .git/refs .vs* -tools cfg data SQL @@ -26,4 +25,6 @@ tgstation.lk tgstation.dyn.rsc libmariadb.dll rust_g.dll +BSQL.dll +appveyor.yml Dockerfile diff --git a/.travis.yml b/.travis.yml index 1ca9d24fc022..33cccc0c3e29 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,13 +3,15 @@ sudo: false branches: except: - ___TGS3TempBranch +<<<<<<< HEAD env: global: - BYOND_MAJOR="512" - BYOND_MINOR="1427" - NODE_VERSION="4" - RUST_G_VERSION="0.3.0" - - BSQL_VERSION="v1.3.0.2" +======= +>>>>>>> c20be496a8... Adds deploy script. CI artifacts. Dependencies file (#39040) matrix: include: - env: @@ -31,7 +33,7 @@ matrix: - libstdc++6:i386 cache: directories: - - $HOME/BYOND-${BYOND_MAJOR}.${BYOND_MINOR} + - $HOME/BYOND - env: - BUILD_TESTING=false - BUILD_TOOLS=false @@ -50,8 +52,13 @@ matrix: cache: directories: - $HOME/.cargo +<<<<<<< HEAD - $HOME/BYOND-${BYOND_MAJOR}.${BYOND_MINOR} + - $HOME/libmariadb +======= + - $HOME/BYOND - $HOME/MariaDB +>>>>>>> c20be496a8... Adds deploy script. CI artifacts. Dependencies file (#39040) - $HOME/.rustup install: diff --git a/Dockerfile b/Dockerfile index 4d24e938fe38..a14e183a7360 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,19 +1,21 @@ FROM tgstation/byond:512.1427 as base +#above version must be the same as the one in dependencies.sh -FROM base as rustg +FROM base as build_base -WORKDIR /rust_g - -RUN apt-get update && apt-get install -y \ +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ git \ - ca-certificates \ - libc6-dev +<<<<<<< HEAD +======= + ca-certificates -FROM build as rust_g +FROM build_base as rust_g WORKDIR /rust_g RUN apt-get install -y --no-install-recommends \ +>>>>>>> c20be496a8... Adds deploy script. CI artifacts. Dependencies file (#39040) libssl-dev \ rustc \ cargo \ @@ -22,14 +24,16 @@ RUN apt-get install -y --no-install-recommends \ RUN git init \ && git remote add origin https://github.com/tgstation/rust-g -#TODO: find a way to read these from .travis.yml or a common source eventually -ENV RUST_G_VERSION=0.3.0 +COPY dependencies.sh . -RUN git fetch --depth 1 origin $RUST_G_VERSION \ +RUN /bin/bash -c "source dependencies.sh \ + && git fetch --depth 1 origin \$RUST_G_VERSION" \ && git checkout FETCH_HEAD \ && cargo build --release -FROM base as bsql +<<<<<<< HEAD +======= +FROM build_base as bsql WORKDIR /bsql @@ -40,16 +44,14 @@ RUN apt-get install -y --no-install-recommends software-properties-common \ cmake \ make \ g++-7 \ - libstdc++6 \ - libmariadb-client-lgpl-dev - -RUN git init \ + libmariadb-client-lgpl-dev \ + && git init \ && git remote add origin https://github.com/tgstation/BSQL -#TODO: find a way to read these from .travis.yml or a common source eventually -ENV BSQL_VERSION=v1.3.0.2 +COPY dependencies.sh . -RUN git fetch --depth 1 origin $BSQL_VERSION \ +RUN /bin/bash -c "source dependencies.sh \ + && git fetch --depth 1 origin \$BSQL_VERSION" \ && git checkout FETCH_HEAD WORKDIR /bsql/artifacts @@ -61,6 +63,7 @@ RUN ln -s /usr/include/mariadb /usr/include/mysql \ && cmake .. \ && make +>>>>>>> c20be496a8... Adds deploy script. CI artifacts. Dependencies file (#39040) FROM base as dm_base WORKDIR /tgstation @@ -69,42 +72,25 @@ FROM dm_base as build COPY . . -RUN DreamMaker -max_errors 0 tgstation.dme - -WORKDIR /deploy - -RUN mkdir -p \ - .git/logs \ - _maps \ - config \ - icons/minimaps \ - sound/chatter \ - sound/voice/complionator \ - sound/instruments \ - strings \ - && cp /tgstation/tgstation.dmb /tgstation/tgstation.rsc ./ \ - && cp -r /tgstation/.git/logs/* .git/logs/ \ - && cp -r /tgstation/_maps/* _maps/ \ - && cp -r /tgstation/config/* config/ \ - && cp /tgstation/icons/default_title.dmi icons/ \ - && cp -r /tgstation/icons/minimaps/* icons/minimaps/ \ - && cp -r /tgstation/sound/chatter/* sound/chatter/ \ - && cp -r /tgstation/sound/voice/complionator/* sound/voice/complionator/ \ - && cp -r /tgstation/sound/instruments/* sound/instruments/ \ - && cp -r /tgstation/strings/* strings/ +RUN DreamMaker -max_errors 0 tgstation.dme && tools/deploy.sh /deploy FROM dm_base EXPOSE 1337 -RUN apt-get update && apt-get install -y \ +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ mariadb-client \ libssl1.0.0 \ && rm -rf /var/lib/apt/lists/* \ && mkdir -p /root/.byond/bin +<<<<<<< HEAD COPY --from=rustg /rust_g/target/release/librust_g.so /root/.byond/bin/rust_g +======= +COPY --from=rust_g /rust_g/target/release/librust_g.so /root/.byond/bin/rust_g COPY --from=bsql /bsql/artifacts/src/BSQL/libBSQL.so ./ +>>>>>>> c20be496a8... Adds deploy script. CI artifacts. Dependencies file (#39040) COPY --from=build /deploy ./ #bsql fexists memes diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 000000000000..ce5e87a217bf --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,11 @@ +version: '{build}' +skip_branch_with_pr: true +shallow_clone: true +cache: + - C:\byond\ -> dependencies.sh +build_script: + - ps: tools/appveyor/build.ps1 + - ps: "$deployPath = $env:APPVEYOR_BUILD_FOLDER + '/deploy'; bash tools/deploy.sh $deployPath" + - ps: "[System.IO.Compression.ZipFile]::CreateFromDirectory($env:APPVEYOR_BUILD_FOLDER + '/deploy', $env:APPVEYOR_BUILD_FOLDER + '/deploy.zip')" +artifacts: + - path: deploy.zip diff --git a/dependencies.sh b/dependencies.sh new file mode 100755 index 000000000000..e286a4ed9020 --- /dev/null +++ b/dependencies.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +#Project dependencies file +#Final authority on what's required to fully build the project + +#byond version +#note, this also needs to be changed in the Dockerfile's initial FROM command +#If someone has an idea for how to set that version within the Dockerfile itself without any other dependencies, feel free to PR it +export BYOND_MAJOR=512 +export BYOND_MINOR=1427 + +#rust_g git tag +export RUST_G_VERSION=0.3.0 + +#bsql git tag +export BSQL_VERSION=v1.3.0.2 + +#node version +export NODE_VERSION=4 diff --git a/tgui/build.sh b/tgui/build.sh index 421f4f8f22c2..ac773faa060d 100755 --- a/tgui/build.sh +++ b/tgui/build.sh @@ -1,7 +1,8 @@ #!/bin/bash #RUN THIS IN THE tgui/ folder set -e -export NODE_VERSION=4 + +source ../dependencies.sh if [ ! -d "/tmp/nvm" ]; then rm -rf /tmp/nvm && git clone https://github.com/creationix/nvm.git /tmp/nvm && (cd ~/.nvm && git checkout `git describe --abbrev=0 --tags`) && source /tmp/nvm/nvm.sh && nvm install $NODE_VERSION diff --git a/tools/appveyor/build.ps1 b/tools/appveyor/build.ps1 new file mode 100644 index 000000000000..6bda07997716 --- /dev/null +++ b/tools/appveyor/build.ps1 @@ -0,0 +1,10 @@ +if(!(Test-Path -Path "C:/byond")){ + bash tools/appveyor/download_byond.sh + [System.IO.Compression.ZipFile]::ExtractToDirectory("C:/byond.zip", "C:/") + Remove-Item C:/byond.zip +} + +Set-Location $env:APPVEYOR_BUILD_FOLDER + +&"C:/byond/bin/dm.exe" -max_errors 0 tgstation.dme +exit $LASTEXITCODE \ No newline at end of file diff --git a/tools/appveyor/download_byond.sh b/tools/appveyor/download_byond.sh new file mode 100755 index 000000000000..19b7f2017078 --- /dev/null +++ b/tools/appveyor/download_byond.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -e +source dependencies.sh +echo "Downloading BYOND version $BYOND_MAJOR.$BYOND_MINOR" +curl "http://www.byond.com/download/build/$BYOND_MAJOR/$BYOND_MAJOR.${BYOND_MINOR}_byond.zip" -o C:/byond.zip diff --git a/tools/travis/before_build_byond.sh b/tools/travis/before_build_byond.sh index 24a53ebe4eca..55afb07ffe5b 100755 --- a/tools/travis/before_build_byond.sh +++ b/tools/travis/before_build_byond.sh @@ -4,27 +4,28 @@ set -e #If this is the build tools step, we do not bother to install/build byond if [ "$BUILD_TOOLS" = true ]; then exit 0 -fi; +fi + echo "Combining maps for building" if [ $BUILD_TESTING = true ]; then python tools/travis/template_dm_generator.py -fi; - -if [ -d "$HOME/BYOND-${BYOND_MAJOR}.${BYOND_MINOR}/byond/bin" ]; -then - echo "Using cached directory." - exit 0 -else - echo "Setting up BYOND." - mkdir -p "$HOME/BYOND-${BYOND_MAJOR}.${BYOND_MINOR}" - cd "$HOME/BYOND-${BYOND_MAJOR}.${BYOND_MINOR}" - curl "http://www.byond.com/download/build/${BYOND_MAJOR}/${BYOND_MAJOR}.${BYOND_MINOR}_byond_linux.zip" -o byond.zip - unzip byond.zip - cd byond - make here - cd ~/ - exit 0 fi -#some variable not set correctly, panic -exit 1 +source dependencies.sh + +if [ -d "$HOME/BYOND/byond/bin" ] && grep -Fxq "${BYOND_MAJOR}.${BYOND_MINOR}" $HOME/BYOND/version.txt; +then + echo "Using cached directory." +else + echo "Setting up BYOND." + rm -rf "$HOME/BYOND" + mkdir -p "$HOME/BYOND" + cd "$HOME/BYOND" + curl "http://www.byond.com/download/build/${BYOND_MAJOR}/${BYOND_MAJOR}.${BYOND_MINOR}_byond_linux.zip" -o byond.zip + unzip byond.zip + rm byond.zip + cd byond + make here + echo "$BYOND_MAJOR.$BYOND_MINOR" > "$HOME/BYOND/version.txt" + cd ~/ +fi diff --git a/tools/travis/build_byond.sh b/tools/travis/build_byond.sh index 584ac55756dd..c7ba1c8274bf 100755 --- a/tools/travis/build_byond.sh +++ b/tools/travis/build_byond.sh @@ -41,19 +41,52 @@ if [ "$BUILD_TOOLS" = false ]; then exit 1 fi; - source $HOME/BYOND-${BYOND_MAJOR}.${BYOND_MINOR}/byond/bin/byondsetup + source $HOME/BYOND/byond/bin/byondsetup if [ "$BUILD_TESTING" = true ]; then tools/travis/dm.sh -DTRAVISBUILDING -DTRAVISTESTING -DALL_MAPS yogstation.dme else tools/travis/dm.sh -DTRAVISBUILDING yogstation.dme - #config folder should not be mandatory - rm -rf config/* + tools/deploy.sh travis_test + mkdir travis_test/config #test config +<<<<<<< HEAD cp tools/travis/travis_config.txt config/config.txt +======= + cp tools/travis/travis_config.txt travis_test/config/config.txt +>>>>>>> c20be496a8... Adds deploy script. CI artifacts. Dependencies file (#39040) + + # get libmariadb, cache it so limmex doesn't get angery + if [ -f $HOME/libmariadb ]; then + #travis likes to interpret the cache command as it being a file for some reason + rm $HOME/libmariadb +<<<<<<< HEAD + mkdir $HOME/libmariadb + fi +======= + fi + mkdir -p $HOME/libmariadb +>>>>>>> c20be496a8... Adds deploy script. CI artifacts. Dependencies file (#39040) + if [ ! -f $HOME/libmariadb/libmariadb.so ]; then + wget http://www.byond.com/download/db/mariadb_client-2.0.0-linux.tgz + tar -xvf mariadb_client-2.0.0-linux.tgz + mv mariadb_client-2.0.0-linux/libmariadb.so $HOME/libmariadb/libmariadb.so + rm -rf mariadb_client-2.0.0-linux.tgz mariadb_client-2.0.0-linux + fi +<<<<<<< HEAD + ln -s $HOME/libmariadb/libmariadb.so libmariadb.so DreamDaemon yogstation.dmb -close -trusted -verbose -params "test-run&log-directory=travis" cat data/logs/travis/clean_run.lk +======= + + cd travis_test + ln -s $HOME/libmariadb/libmariadb.so libmariadb.so + DreamDaemon tgstation.dmb -close -trusted -verbose -params "test-run&log-directory=travis" + cd .. + cat travis_test/data/logs/travis/clean_run.lk + +>>>>>>> c20be496a8... Adds deploy script. CI artifacts. Dependencies file (#39040) fi; fi; diff --git a/tools/travis/build_dependencies.sh b/tools/travis/build_dependencies.sh index 8de5ffa7db30..69f5e1e11918 100755 --- a/tools/travis/build_dependencies.sh +++ b/tools/travis/build_dependencies.sh @@ -2,6 +2,15 @@ set -e +source dependencies.sh + +#ensure the Dockerfile version matches the dependencies.sh version +line=$(head -n 1 Dockerfile) +if [[ $line != *"$BYOND_MAJOR.$BYOND_MINOR"* ]]; then + echo "Dockerfile BYOND version in FROM command does not match dependencies.sh (Or it's not on line 1)!" + exit 1 +fi + if [ $BUILD_TOOLS = false ] && [ $BUILD_TESTING = false ]; then curl https://sh.rustup.rs -sSf | sh -s -- -y --default-host i686-unknown-linux-gnu source ~/.profile diff --git a/tools/travis/install_build_tools.sh b/tools/travis/install_build_tools.sh index 76d0dbf69ddd..1d0006727928 100755 --- a/tools/travis/install_build_tools.sh +++ b/tools/travis/install_build_tools.sh @@ -1,6 +1,8 @@ #!/bin/bash set -e +source dependencies.sh + if [ "$BUILD_TOOLS" = true ]; then rm -rf ~/.nvm && git clone https://github.com/creationix/nvm.git ~/.nvm && (cd ~/.nvm && git checkout `git describe --abbrev=0 --tags`) && source ~/.nvm/nvm.sh && nvm install $NODE_VERSION npm install -g gulp-cli