diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 5717627962..b61a1e318e 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -47,8 +47,17 @@ concurrency: cancel-in-progress: true jobs: + start-gate: + name: CI Start Gate + runs-on: ubuntu-latest + if: (!contains(github.event.head_commit.message, '[TGSRelease]')) + steps: + - name: GitHub Requires at Least One Step for a Job + run: exit 0 + build-releasenotes: name: Build ReleaseNotes for Other Jobs + needs: start-gate runs-on: ubuntu-latest steps: - name: Install Native Dependencies @@ -83,6 +92,7 @@ jobs: code-scanning: name: Run CodeQL + needs: start-gate runs-on: ubuntu-latest permissions: security-events: write @@ -134,6 +144,7 @@ jobs: dmapi-build: name: Build DMAPI + needs: start-gate strategy: fail-fast: false matrix: @@ -222,6 +233,7 @@ jobs: opendream-build: name: Build DMAPI (OpenDream) + needs: start-gate strategy: fail-fast: false matrix: @@ -278,6 +290,7 @@ jobs: efcore-version-match: name: Check Nuget Versions Match Tools runs-on: ubuntu-latest + needs: start-gate steps: - name: Checkout (Branch) uses: actions/checkout@v4 @@ -412,6 +425,7 @@ jobs: docker-build: name: Build Docker Image runs-on: ubuntu-latest + needs: start-gate env: TGS_TELEMETRY_KEY_FILE: tgs_telemetry_key.txt steps: @@ -438,6 +452,7 @@ jobs: linux-unit-tests: name: Linux Tests + needs: start-gate strategy: fail-fast: false matrix: @@ -510,6 +525,7 @@ jobs: windows-unit-tests: name: Windows Tests + needs: start-gate strategy: fail-fast: false matrix: @@ -1428,6 +1444,7 @@ jobs: build-msi: name: Build Windows Installer .exe runs-on: windows-latest + needs: start-gate env: TGS_TELEMETRY_KEY_FILE: C:/tgs_telemetry_key.txt steps: @@ -1715,6 +1732,12 @@ jobs: body_path: release_notes.md commitish: ${{ github.event.head_commit.id }} + - name: Generate Artifact Attestation + uses: actions/attest-build-provenance@v1 + with: + subject-path: ./swagger/tgs_api.json + github-token: ${{ steps.app-token-generation.outputs.token }} + - name: Upload OpenApi Spec uses: actions/upload-release-asset@v1 env: @@ -1795,6 +1818,12 @@ jobs: commitish: ${{ github.event.head_commit.id }} prerelease: ${{ env.TGS_GRAPHQL_PRERELEASE }} + - name: Generate Artifact Attestation + uses: actions/attest-build-provenance@v1 + with: + subject-path: ./schema/tgs_api.graphql + github-token: ${{ steps.app-token-generation.outputs.token }} + - name: Upload GraphQL Schema uses: actions/upload-release-asset@v1 env: @@ -1867,6 +1896,12 @@ jobs: body_path: release_notes.md commitish: ${{ github.event.head_commit.id }} + - name: Generate Artifact Attestation + uses: actions/attest-build-provenance@v1 + with: + subject-path: ./DMAPI.zip + github-token: ${{ steps.app-token-generation.outputs.token }} + - name: Upload DMAPI Artifact uses: actions/upload-release-asset@v1 env: @@ -2132,7 +2167,19 @@ jobs: body_path: release_notes.md commitish: ${{ github.event.head_commit.id }} - - name: Upload Server Console Artifact + - name: Generate Server Console Artifact Attestation + uses: actions/attest-build-provenance@v1 + with: + subject-path: ./ServerConsole.zip + github-token: ${{ steps.app-token-generation.outputs.token }} + + - name: Upload Server Console Zip Artifact to Action + uses: actions/upload-artifact@v4 + with: + name: server-console-release + path: ./ServerConsole.zip + + - name: Upload Server Console Artifact to Release uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ steps.app-token-generation.outputs.token }} @@ -2142,6 +2189,12 @@ jobs: asset_name: ServerConsole.zip asset_content_type: application/zip + - name: Generate Server Service Artifact Attestation + uses: actions/attest-build-provenance@v1 + with: + subject-path: ./ServerService.zip + github-token: ${{ steps.app-token-generation.outputs.token }} + - name: Upload Server Service Artifact uses: actions/upload-release-asset@v1 env: @@ -2152,6 +2205,12 @@ jobs: asset_name: ServerService.zip asset_content_type: application/zip + - name: Generate DMAPI Artifact Attestation + uses: actions/attest-build-provenance@v1 + with: + subject-path: ./DMAPI.zip + github-token: ${{ steps.app-token-generation.outputs.token }} + - name: Upload DMAPI Artifact uses: actions/upload-release-asset@v1 env: @@ -2162,6 +2221,12 @@ jobs: asset_name: DMAPI.zip asset_content_type: application/zip + - name: Generate REST API Artifact Attestation + uses: actions/attest-build-provenance@v1 + with: + subject-path: ./swagger/tgs_api.json + github-token: ${{ steps.app-token-generation.outputs.token }} + - name: Upload REST API Artifact uses: actions/upload-release-asset@v1 env: @@ -2172,6 +2237,12 @@ jobs: asset_name: swagger.json asset_content_type: application/json + - name: Generate GraphQL API Artifact Attestation + uses: actions/attest-build-provenance@v1 + with: + subject-path: ./schema/tgs-api.graphql + github-token: ${{ steps.app-token-generation.outputs.token }} + - name: Upload GraphQL API Artifact uses: actions/upload-release-asset@v1 env: @@ -2182,6 +2253,12 @@ jobs: asset_name: tgs-api.graphql asset_content_type: text/plain + - name: Generate Server Update Package Artifact Attestation + uses: actions/attest-build-provenance@v1 + with: + subject-path: ./ServerUpdatePackage.zip + github-token: ${{ steps.app-token-generation.outputs.token }} + - name: Upload Server Update Package Artifact uses: actions/upload-release-asset@v1 env: @@ -2192,7 +2269,13 @@ jobs: asset_name: ServerUpdatePackage.zip asset_content_type: application/zip - - name: Upload Debian Pacakaging Artifact + - name: Generate Debian Packaging Artifact Attestation + uses: actions/attest-build-provenance@v1 + with: + subject-path: ./packaging-debian/tgstation-server-v${{ env.TGS_VERSION }}.debian.packaging.tar.xz + github-token: ${{ steps.app-token-generation.outputs.token }} + + - name: Upload Debian Packaging Artifact uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ steps.app-token-generation.outputs.token }} @@ -2202,6 +2285,12 @@ jobs: asset_name: tgstation-server-v${{ env.TGS_VERSION }}.debian.packaging.tar.xz asset_content_type: application/x-tar + - name: Generate MariaDB .msi Attestation + uses: actions/attest-build-provenance@v1 + with: + subject-path: ./build/package/winget/Tgstation.Server.Host.Service.Wix.Bundle/bin/Release/mariadb.msi + github-token: ${{ steps.app-token-generation.outputs.token }} + - name: Upload MariaDB .msi uses: actions/upload-release-asset@v1 env: @@ -2212,6 +2301,12 @@ jobs: asset_name: mariadb-${{ env.MARIADB_VERSION }}-winx64.msi asset_content_type: application/octet-stream + - name: Generate Installer .exe Attestation + uses: actions/attest-build-provenance@v1 + with: + subject-path: ./build/package/winget/tgstation-server-installer.exe + github-token: ${{ steps.app-token-generation.outputs.token }} + - name: Upload Installer .exe uses: actions/upload-release-asset@v1 env: @@ -2222,6 +2317,58 @@ jobs: asset_name: tgstation-server-installer.exe asset_content_type: application/octet-stream + update-nix: + name: Update Nix SHA + needs: deploy-tgs + runs-on: ubuntu-latest + if: (!(cancelled() || failure())) && needs.deploy-tgs.result == 'success' + steps: + - name: Install Native Packages # Name checked in rerunFlakyTests.js + run: | + sudo apt-get update + sudo apt-get install -y xmlstarlet + + - name: Setup Nix + uses: cachix/install-nix-action@v27 + with: + nix_path: nixpkgs=channel:nixos-unstable + + - name: Checkout + uses: actions/checkout@v4 + + - name: Parse TGS version + run: echo "TGS_VERSION=$(xmlstarlet sel -N X="http://schemas.microsoft.com/developer/msbuild/2003" --template --value-of /X:Project/X:PropertyGroup/X:TgsCoreVersion build/Version.props)" >> $GITHUB_ENV + + - name: Retrieve ServerConsole.zip Artifact + uses: actions/download-artifact@v4 + with: + name: server-console-release + path: server-console-release + + - name: Regenerate Nix Hash + run: | + nix hash path ./server-console-release > build/package/nix/ServerConsole.sha256 + cat build/package/nix/ServerConsole.sha256 + + - name: Commit + run: | + git config --global push.default simple + git config user.name "tgstation-server-ci[bot]" + git config user.email "161980869+tgstation-server-ci[bot]@users.noreply.github.com" + git add build/package/nix/ServerConsole.sha256 + git commit -m "Update nix SHA256 for [TGSRelease] v${{ env.TGS_VERSION }}" + + - name: Re-tag + run: | + git tag -d tgstation-server-v${{ env.TGS_VERSION }} + git tag tgstation-server-v${{ env.TGS_VERSION }} + + - name: Push Commit + run: git push + + - name: Force Push Tags + run: git push -f --tags + changelog-regen: name: Regenerate Changelog runs-on: ubuntu-latest diff --git a/README.md b/README.md index 24077a5a10..680f2f2a6b 100644 --- a/README.md +++ b/README.md @@ -137,6 +137,31 @@ sudo dpkg --add-architecture i386 \ The service will execute as the newly created user: `tgstation-server`. You should, ideally, store your instances somewhere under `/home/tgstation-server`. +##### Nix Flake + +TGS supports being setup on Nix starting with version 6.12.0. Add the [flake](./build/package/nix/flake.nix) to your own system by adding the following code to your flake inputs. +```nix + tgstation-server = { + url = "github:tgstation/tgstation-server/tgstation-server-v${version}?dir=build/package/nix"; + }; +``` + +Where `version` is the latest major TGS version you wish to use. + +Note that changing this version does not change the core version of TGS used after the first launch. Instead, have TGS self-update via its API. + +For maximum game server uptime, do NOT modify this version unless you are doing a major TGS version update in which case it is a requirement. + +Configure TGS by setting up its service definition: +```nix + services.tgstation-server = { + enable = true; + production-appsettings = (builtins.readFile ./path/to/your/appsettings.Production.yml); + }; +``` + +Refer to [tgstation-server.nix](./build/package/nix/tgstation-server.nix) for the full list of available configuration options. + ##### Manual Setup The following dependencies are required. @@ -243,6 +268,8 @@ Create an `appsettings.Production.yml` file next to `appsettings.yml`. This will - `General:SkipAddingByondFirewallException`: Set to `true` if you have Windows firewall disabled +- `General:AdditionalEventScriptsDirectories`: An array of directories that are considered to contain EventScripts alongside instance directories. Working directory for executed scripts will remain the instance EventScripts directory. + - `Session:HighPriorityLiveDreamDaemon`: Boolean controlling if live DreamDaemon instances get set to above normal priority processes. - `Session:LowPriorityDeploymentProcesses `: Boolean controlling if DreamMaker and API validation DreamDaemon instances get set to below normal priority processes. diff --git a/build/Version.props b/build/Version.props index 21e5da987b..02bbc0e30a 100644 --- a/build/Version.props +++ b/build/Version.props @@ -12,7 +12,7 @@ 19.3.0 7.3.0 5.10.0 - 1.5.0 + 1.6.0 8.0.0 1.2.1 2.0.0 diff --git a/build/package/nix/ServerConsole.sha256 b/build/package/nix/ServerConsole.sha256 new file mode 100644 index 0000000000..ae3d91ae7d --- /dev/null +++ b/build/package/nix/ServerConsole.sha256 @@ -0,0 +1 @@ +sha256-mHlRHPSeZxyJPqN3KUmc0ftYNZgh81LauIu+fCSKPUI= diff --git a/build/package/nix/flake.nix b/build/package/nix/flake.nix new file mode 100644 index 0000000000..f2bf8c14a9 --- /dev/null +++ b/build/package/nix/flake.nix @@ -0,0 +1,13 @@ +{ + description = "tgstation-server"; + + inputs = {}; + + outputs = { ... }: { + nixosModules = { + default = { ... }: { + imports = [ ./tgstation-server.nix ]; + }; + }; + }; +} diff --git a/build/package/nix/package.nix b/build/package/nix/package.nix new file mode 100644 index 0000000000..191e9f68ed --- /dev/null +++ b/build/package/nix/package.nix @@ -0,0 +1,119 @@ +{ + pkgs, + ... +}: + +let + inherit (pkgs) stdenv lib; + + versionParse = stdenv.mkDerivation { + pname = "tgstation-server-version-parse"; + version = "1.0.0"; + + meta = with pkgs.lib; { + description = "Version parser for tgstation-server"; + homepage = "https://github.com/tgstation/tgstation-server"; + changelog = "https://github.com/tgstation/tgstation-server/blob/gh-pages/changelog.yml"; + license = licenses.agpl3Plus; + platforms = platforms.x86_64; + }; + + nativeBuildInputs = with pkgs; [ + xmlstarlet + ]; + + src = ./../..; + + installPhase = '' + mkdir -p $out + xmlstarlet sel -N X="http://schemas.microsoft.com/developer/msbuild/2003" --template --value-of /X:Project/X:PropertyGroup/X:TgsCoreVersion ./Version.props > $out/tgs_version.txt + ''; + }; + + fixedOutput = stdenv.mkDerivation { + pname = "tgstation-server-release-server-console-zip"; + version = (builtins.readFile "${versionParse}/tgs_version.txt"); + + meta = with pkgs.lib; { + description = "Host watchdog binaries for tgstation-server"; + homepage = "https://github.com/tgstation/tgstation-server"; + changelog = "https://github.com/tgstation/tgstation-server/releases/tag/tgstation-server-v${version}"; + license = licenses.agpl3Plus; + platforms = platforms.x86_64; + }; + + nativeBuildInputs = with pkgs; [ + curl + cacert + versionParse + ]; + + src = ./.; + + buildPhase = '' + curl -L https://github.com/tgstation/tgstation-server/releases/download/tgstation-server-v${version}/ServerConsole.zip -o ServerConsole.zip + ''; + + installPhase = '' + mkdir -p $out + mv ServerConsole.zip $out/ServerConsole.zip + ''; + + outputHashAlgo = "sha256"; + outputHashMode = "recursive"; + outputHash = (builtins.readFile ./ServerConsole.sha256); + }; + rpath = lib.makeLibraryPath [ pkgs.stdenv_32bit.cc.cc.lib ]; +in +stdenv.mkDerivation { + pname = "tgstation-server"; + version = (builtins.readFile "${versionParse}/tgs_version.txt"); + + meta = with pkgs.lib; { + description = "A production scale tool for DreamMaker server management"; + homepage = "https://github.com/tgstation/tgstation-server"; + changelog = "https://github.com/tgstation/tgstation-server/releases/tag/tgstation-server-v${version}"; + license = licenses.agpl3Plus; + platforms = platforms.x86_64; + }; + + buildInputs = with pkgs; [ + dotnetCorePackages.aspnetcore_8_0 + gdb + systemd + zlib + gcc_multi + glibc + ]; + nativeBuildInputs = with pkgs; [ + makeWrapper + unzip + fixedOutput + versionParse + ]; + + src = ./.; + + installPhase = '' + mkdir -p $out/bin + unzip "${fixedOutput}/ServerConsole.zip" -d $out/bin + rm -rf $out/bin/lib + makeWrapper ${pkgs.dotnetCorePackages.aspnetcore_8_0}/dotnet $out/bin/tgstation-server --suffix PATH : ${ + lib.makeBinPath ( + with pkgs; + [ + dotnetCorePackages.aspnetcore_8_0 + gdb + ] + ) + } --suffix LD_LIBRARY_PATH : ${ + lib.makeLibraryPath ( + with pkgs; + [ + systemd + zlib + ] + ) + } --add-flags "$out/bin/Tgstation.Server.Host.Console.dll --bootstrap" + ''; +} diff --git a/build/package/nix/tgstation-server.nix b/build/package/nix/tgstation-server.nix new file mode 100644 index 0000000000..eb60c7c94c --- /dev/null +++ b/build/package/nix/tgstation-server.nix @@ -0,0 +1,133 @@ +inputs@{ + config, + lib, + nixpkgs, + pkgs, + writeShellScriptBin, + ... +}: + +let + pkgs-i686 = nixpkgs.legacyPackages.i686-linux; + + cfg = config.services.tgstation-server; + + package = import ./package.nix inputs; + + stdenv = pkgs-i686.stdenv_32bit; + + rpath = pkgs-i686.lib.makeLibraryPath [ + stdenv.cc.cc.lib + ]; + + byond-patcher = pkgs-i686.writeShellScriptBin "EngineInstallComplete-050-TgsPatchELFByond.sh" '' + BYOND_PATH=$(realpath ../../Byond/$1/byond/bin/) + + ${pkgs.patchelf}/bin/patchelf --set-interpreter "$(cat ${stdenv.cc}/nix-support/dynamic-linker)" \ + --set-rpath "$BYOND_PATH:${rpath}" \ + $BYOND_PATH/{DreamDaemon,DreamDownload,DreamMaker} + ''; + + tgs-wrapper = pkgs.writeShellScriptBin "tgs-path-wrapper" '' + export PATH=$PATH:${cfg.extra-path} + exec ${package}/bin/tgstation-server --appsettings-base-path=/etc/tgstation-server.d --General:SetupWizardMode=Never --General:AdditionalEventScriptsDirectories:0=/etc/tgstation-server.d/EventScripts --General:AdditionalEventScriptsDirectories:1=${byond-patcher}/bin + ''; +in +{ + ##### interface. here we define the options that users of our service can specify + options = { + # the options for our service will be located under services.foo + services.tgstation-server = { + enable = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Whether to enable tgstation-server. + ''; + }; + + username = lib.mkOption { + type = lib.types.str; + default = "tgstation-server"; + description = '' + The name of the user used to execute tgstation-server. + ''; + }; + + groupname = lib.mkOption { + type = lib.types.str; + default = "tgstation-server"; + description = '' + The name of group the user used to execute tgstation-server will belong to. + ''; + }; + + home-directory = lib.mkOption { + type = lib.types.str; + default = "/home/tgstation-server"; + description = '' + The home directory of TGS. Should be persistent. + ''; + }; + + production-appsettings = lib.mkOption { + type = lib.types.lines; + default = ''''; + description = '' + The contents of appsettings.Production.yml in the /etc/tgstation-server.d directory. + ''; + }; + + extra-path = lib.mkOption { + type = lib.types.str; + default = ""; + description = '' + Extra PATH entries to add to the TGS process + ''; + }; + }; + }; + + config = lib.mkIf cfg.enable { + users.groups."${cfg.groupname}" = { }; + + users.users."${cfg.username}" = { + isSystemUser = true; + createHome = true; + group = cfg.groupname; + home = cfg.home-directory; + }; + + environment.etc = { + "tgstation-server.d/appsettings.yml" = { + text = (builtins.readFile "${package}/bin/appsettings.yml"); + group = cfg.groupname; + mode = "0644"; + }; + "tgstation-server.d/appsettings.Production.yml" = { + text = cfg.production-appsettings; + group = cfg.groupname; + mode = "0640"; + }; + }; + + systemd.services.tgstation-server = { + description = "tgstation-server"; + serviceConfig = { + User = cfg.username; + Type = "notify-reload"; + NotifyAccess = "all"; + WorkingDirectory = "${package}/bin"; + ExecStart = "${tgs-wrapper}/bin/tgs-path-wrapper"; + Restart = "always"; + KillMode = "process"; + ReloadSignal = "SIGUSR2"; + AmbientCapabilities = "CAP_SYS_NICE CAP_SYS_PTRACE"; + WatchdogSec = "60"; + WatchdogSignal = "SIGTERM"; + LogsDirectory = "tgstation-server"; + }; + wantedBy = [ "multi-user.target" ]; + }; + }; +} diff --git a/src/Tgstation.Server.Host.Common/DotnetHelper.cs b/src/Tgstation.Server.Host.Common/DotnetHelper.cs index 5945eac25d..c4edb2d3bf 100644 --- a/src/Tgstation.Server.Host.Common/DotnetHelper.cs +++ b/src/Tgstation.Server.Host.Common/DotnetHelper.cs @@ -14,10 +14,10 @@ namespace Tgstation.Server.Host.Common public static class DotnetHelper { /// - /// Gets the path to the dotnet executable. + /// Gets potential paths to the dotnet executable. /// /// If the current system is a Windows OS. - /// The path to the dotnet executable. + /// An of potential paths to the dotnet executable. public static IEnumerable GetPotentialDotnetPaths(bool isWindows) { var enviromentPath = Environment.GetEnvironmentVariable("PATH"); diff --git a/src/Tgstation.Server.Host.Console/Tgstation.Server.Host.Console.csproj b/src/Tgstation.Server.Host.Console/Tgstation.Server.Host.Console.csproj index ff23533a16..9804e251e3 100644 --- a/src/Tgstation.Server.Host.Console/Tgstation.Server.Host.Console.csproj +++ b/src/Tgstation.Server.Host.Console/Tgstation.Server.Host.Console.csproj @@ -4,6 +4,7 @@ Exe $(TgsFrameworkVersion) + $(TgsCoreVersion) enable false diff --git a/src/Tgstation.Server.Host.Service/Tgstation.Server.Host.Service.csproj b/src/Tgstation.Server.Host.Service/Tgstation.Server.Host.Service.csproj index 417e03092c..e1f04e898b 100644 --- a/src/Tgstation.Server.Host.Service/Tgstation.Server.Host.Service.csproj +++ b/src/Tgstation.Server.Host.Service/Tgstation.Server.Host.Service.csproj @@ -7,6 +7,7 @@ enable $(TgsFrameworkVersion) + $(TgsCoreVersion) false diff --git a/src/Tgstation.Server.Host.Watchdog/BootstrapSettings.cs b/src/Tgstation.Server.Host.Watchdog/BootstrapSettings.cs new file mode 100644 index 0000000000..10c9ab11b4 --- /dev/null +++ b/src/Tgstation.Server.Host.Watchdog/BootstrapSettings.cs @@ -0,0 +1,38 @@ +using System; +using System.Reflection; + +using Tgstation.Server.Common.Extensions; + +namespace Tgstation.Server.Host.Watchdog +{ + /// + /// Settings for the bootstrapper feature. + /// + sealed class BootstrapSettings + { + /// + /// The current supported major version of . + /// + public const int FileMajorVersion = 1; + + /// + /// The token used to substitute . + /// + public const string VersionSubstitutionToken = "${version}"; + + /// + /// The version of the boostrapper file. + /// + public Version FileVersion { get; set; } = new Version(FileMajorVersion, 0, 0); + + /// + /// The of TGS last launched in the lib/Default directory. + /// + public Version TgsVersion { get; set; } = Assembly.GetEntryAssembly()!.GetName().Version!.Semver(); + + /// + /// The URL to format with to get the download URL. + /// + public string ServerUpdatePackageUrlFormatter { get; set; } = $"https://github.com/tgstation/tgstation-server/releases/download/tgstation-server-v{VersionSubstitutionToken}/ServerUpdatePackage.zip"; + } +} diff --git a/src/Tgstation.Server.Host.Watchdog/Watchdog.cs b/src/Tgstation.Server.Host.Watchdog/Watchdog.cs index 4813383440..09a985c9dd 100644 --- a/src/Tgstation.Server.Host.Watchdog/Watchdog.cs +++ b/src/Tgstation.Server.Host.Watchdog/Watchdog.cs @@ -3,9 +3,12 @@ using System.Collections.Generic; using System.Diagnostics; using System.Globalization; using System.IO; +using System.IO.Compression; using System.Linq; +using System.Net.Http; using System.Reflection; using System.Runtime.InteropServices; +using System.Text.Json; using System.Threading; using System.Threading.Tasks; @@ -89,7 +92,10 @@ namespace Tgstation.Server.Host.Watchdog return false; } - var assemblyStoragePath = Path.Combine(rootLocation, "lib"); // always always next to watchdog + var bootstrappable = args.Contains("--bootstrap", StringComparer.OrdinalIgnoreCase); + var homeDirectory = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".tgstation-server"); + + var assemblyStoragePath = Path.Combine(bootstrappable ? homeDirectory : rootLocation, "lib"); // always always next to watchdog var defaultAssemblyPath = Path.GetFullPath(Path.Combine(assemblyStoragePath, "Default")); @@ -119,6 +125,47 @@ namespace Tgstation.Server.Host.Watchdog else Directory.CreateDirectory(assemblyStoragePath); + var bootstrapperSettingsFile = Path.Combine(homeDirectory, "bootstrap.json"); + BootstrapSettings? bootstrapSettings = null; + if (!Directory.Exists(defaultAssemblyPath)) + { + if (!bootstrappable) + { + logger.LogCritical("Unable to locate host assembly directory!"); + return false; + } + + if (File.Exists(bootstrapperSettingsFile)) + { + logger.LogInformation("Loading bootstrap settings..."); + var bootstrapperSettingsJson = await File.ReadAllTextAsync(bootstrapperSettingsFile, cancellationToken); + bootstrapSettings = JsonSerializer.Deserialize(bootstrapperSettingsJson); + if (bootstrapSettings == null) + { + logger.LogCritical("Failed to deserialize {settingsFile}!", bootstrapperSettingsFile); + return false; + } + } + else + { + logger.LogInformation("Using default bootstrap settings..."); + bootstrapSettings = new BootstrapSettings(); // defaults + } + + if (bootstrapSettings.FileVersion.Major != BootstrapSettings.FileMajorVersion) + { + logger.LogCritical("Unable to parse bootstrapper file! Expected version: {expected}.X.X", BootstrapSettings.FileMajorVersion); + return false; + } + + string downloadUrl = bootstrapSettings.ServerUpdatePackageUrlFormatter.Replace(BootstrapSettings.VersionSubstitutionToken, bootstrapSettings.TgsVersion.ToString(), StringComparison.Ordinal); + logger.LogInformation("Bootstrapping from: {url}", downloadUrl); + using var httpClient = new HttpClient(); + await using var zipData = await httpClient.GetStreamAsync(new Uri(downloadUrl), cancellationToken); + using var archive = new ZipArchive(zipData, ZipArchiveMode.Read, true); + archive.ExtractToDirectory(defaultAssemblyPath); + } + var assemblyName = String.Join(".", nameof(Tgstation), nameof(Server), nameof(Host), "dll"); var assemblyPath = Path.Combine(defaultAssemblyPath, assemblyName); @@ -128,28 +175,59 @@ namespace Tgstation.Server.Host.Watchdog return false; } - if (!File.Exists(assemblyPath)) - { - logger.LogCritical("Unable to locate host assembly!"); - return false; - } - - var fileVersion = FileVersionInfo.GetVersionInfo(assemblyPath).FileVersion; - if (fileVersion == null) - { - logger.LogCritical("Failed to parse version info from {assemblyPath}!", assemblyPath); - return false; - } - - initialHostVersionTcs.SetResult( - Version.Parse( - fileVersion)); - var watchdogVersion = executingAssembly.GetName().Version?.Semver().ToString(); while (!cancellationToken.IsCancellationRequested) + { + if (!File.Exists(assemblyPath)) + { + logger.LogCritical("Unable to locate host assembly!"); + return false; + } + + var fileVersion = FileVersionInfo.GetVersionInfo(assemblyPath).FileVersion; + if (fileVersion == null) + { + logger.LogCritical("Failed to parse version info from {assemblyPath}!", assemblyPath); + return false; + } + + if (bootstrappable) + { + if (!Version.TryParse(fileVersion, out var bootstrappedVersion)) + { + logger.LogCritical("Failed to parse bootstrapped version prior to launch: {fileVersion}", fileVersion); + } + else + { + // save bootstrapper settings + var oldUrl = bootstrapSettings?.ServerUpdatePackageUrlFormatter; + bootstrapSettings = new BootstrapSettings + { + TgsVersion = bootstrappedVersion.Semver(), + }; + + bootstrapSettings.ServerUpdatePackageUrlFormatter = oldUrl ?? bootstrapSettings.ServerUpdatePackageUrlFormatter; + + Directory.CreateDirectory(homeDirectory); + await File.WriteAllTextAsync( + bootstrapperSettingsFile, + JsonSerializer.Serialize( + bootstrapSettings, + new JsonSerializerOptions + { + WriteIndented = true, + }), + cancellationToken); + } + } + using (logger.BeginScope("Host invocation")) { + initialHostVersionTcs.SetResult( + Version.Parse( + fileVersion)); + updateDirectory = Path.GetFullPath(Path.Combine(assemblyStoragePath, Guid.NewGuid().ToString())); logger.LogInformation("Update path set to {updateDirectory}", updateDirectory); using (var process = new Process()) @@ -358,6 +436,7 @@ namespace Tgstation.Server.Host.Watchdog } } } + } } catch (OperationCanceledException ex) { diff --git a/src/Tgstation.Server.Host/Components/Engine/EngineManager.cs b/src/Tgstation.Server.Host/Components/Engine/EngineManager.cs index 970e3ab89d..113a0d249b 100644 --- a/src/Tgstation.Server.Host/Components/Engine/EngineManager.cs +++ b/src/Tgstation.Server.Host/Components/Engine/EngineManager.cs @@ -456,6 +456,7 @@ namespace Tgstation.Server.Host.Components.Engine } // okay up to us to install it then + string? installPath = null; try { if (customVersionStream != null) @@ -475,15 +476,26 @@ namespace Tgstation.Server.Host.Components.Engine var versionString = version.ToString(); await eventConsumer.HandleEvent(EventType.EngineInstallStart, new List { versionString }, deploymentPipelineProcesses, cancellationToken); - await InstallVersionFiles(progressReporter, version, customVersionStream, deploymentPipelineProcesses, cancellationToken); + installPath = await InstallVersionFiles(progressReporter, version, customVersionStream, deploymentPipelineProcesses, cancellationToken); + await eventConsumer.HandleEvent(EventType.EngineInstallComplete, new List { versionString }, deploymentPipelineProcesses, cancellationToken); ourTcs.SetResult(); - - await eventConsumer.HandleEvent(EventType.EngineInstallComplete, new List { versionString }, deploymentPipelineProcesses, cancellationToken); } catch (Exception ex) { - if (ex is not OperationCanceledException) + if (installPath != null) + { + try + { + logger.LogDebug("Cleaning up failed installation at {path}...", installPath); + await ioManager.DeleteDirectory(installPath, cancellationToken); + } + catch (Exception ex2) + { + logger.LogError(ex2, "Error cleaning up failed installation!"); + } + } + else if (ex is not OperationCanceledException) await eventConsumer.HandleEvent(EventType.EngineInstallFail, new List { ex.Message }, deploymentPipelineProcesses, cancellationToken); lock (installedVersions) @@ -510,8 +522,8 @@ namespace Tgstation.Server.Host.Components.Engine /// Custom zip file to use. Will cause a number to be added. /// If processes should be launched as part of the deployment pipeline. /// The for the operation. - /// A representing the running operation. - async ValueTask InstallVersionFiles( + /// A resulting in the directory the engine was installed to. + async ValueTask InstallVersionFiles( JobProgressReporter progressReporter, EngineVersion version, Stream? customVersionStream, @@ -597,6 +609,8 @@ namespace Tgstation.Server.Host.Components.Engine await ioManager.DeleteDirectory(installFullPath, cancellationToken); throw; } + + return installFullPath; } /// diff --git a/src/Tgstation.Server.Host/Components/InstanceManager.cs b/src/Tgstation.Server.Host/Components/InstanceManager.cs index 12ebb1300e..d0f91c4c46 100644 --- a/src/Tgstation.Server.Host/Components/InstanceManager.cs +++ b/src/Tgstation.Server.Host/Components/InstanceManager.cs @@ -603,7 +603,7 @@ namespace Tgstation.Server.Host.Components logger.LogInformation("{versionString}", assemblyInformationProvider.VersionString); console.SetTitle(assemblyInformationProvider.VersionString); - CheckSystemCompatibility(); + PreflightChecks(); // To let the web server startup immediately before we do any intense work await Task.Yield(); @@ -673,7 +673,7 @@ namespace Tgstation.Server.Host.Components /// /// Check we have a valid system and configuration. /// - void CheckSystemCompatibility() + void PreflightChecks() { logger.LogDebug("Running as user: {username}", Environment.UserName); diff --git a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs index f0edb6fc09..e1b37b50b3 100644 --- a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs +++ b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs @@ -787,27 +787,48 @@ namespace Tgstation.Server.Host.Components.StaticFiles // always execute in serial using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken, logger)) { - var files = await ioManager.GetFilesWithExtension(EventScriptsSubdirectory, platformIdentifier.ScriptFileExtension, false, cancellationToken); - var resolvedScriptsDir = ioManager.ResolvePath(EventScriptsSubdirectory); + var directories = generalConfiguration.AdditionalEventScriptsDirectories?.ToList() ?? new List(); + directories.Add(EventScriptsSubdirectory); - var scriptFiles = files - .Select(x => ioManager.GetFileName(x)) - .Where(x => scriptNames.Any( - scriptName => x.StartsWith(scriptName, StringComparison.Ordinal))) + var allScripts = new List(); + var tasks = directories.Select( + async scriptDirectory => + { + var resolvedScriptsDir = ioManager.ResolvePath(scriptDirectory); + logger.LogTrace("Checking for scripts in {directory}...", scriptDirectory); + var files = await ioManager.GetFilesWithExtension(scriptDirectory, platformIdentifier.ScriptFileExtension, false, cancellationToken); + + var scriptFiles = files + .Select(ioManager.GetFileName) + .Where(x => scriptNames.Any( + scriptName => x.StartsWith(scriptName, StringComparison.Ordinal))) + .Select(x => + { + var fullScriptPath = ioManager.ConcatPath(resolvedScriptsDir, x); + logger.LogTrace("Found matching script: {scriptPath}", fullScriptPath); + return fullScriptPath; + }); + + lock (allScripts) + allScripts.AddRange(scriptFiles); + }) .ToList(); - if (scriptFiles.Count == 0) + await ValueTaskExtensions.WhenAll(tasks); + if (allScripts.Count == 0) { logger.LogTrace("No event scripts starting with \"{scriptName}\" detected", String.Join("\" or \"", scriptNames)); return; } - foreach (var scriptFile in scriptFiles) + var resolvedInstanceScriptsDir = ioManager.ResolvePath(EventScriptsSubdirectory); + + foreach (var scriptFile in allScripts.OrderBy(ioManager.GetFileName)) { logger.LogTrace("Running event script {scriptFile}...", scriptFile); await using (var script = await processExecutor.LaunchProcess( - ioManager.ConcatPath(resolvedScriptsDir, scriptFile), - resolvedScriptsDir, + scriptFile, + resolvedInstanceScriptsDir, String.Join( ' ', parameters.Select(arg => diff --git a/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs b/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs index 1790316e37..5ea16ae64e 100644 --- a/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs +++ b/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs @@ -1,4 +1,7 @@ using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; using Microsoft.Extensions.Logging; using Newtonsoft.Json; @@ -150,6 +153,11 @@ namespace Tgstation.Server.Host.Configuration /// public bool OpenDreamSuppressInstallOutput { get; set; } + /// + /// List of directories that have their contents merged with instance EventScripts directories when executing scripts. + /// + public List? AdditionalEventScriptsDirectories { get; set; } + /// /// Initializes a new instance of the class. /// @@ -190,6 +198,9 @@ namespace Tgstation.Server.Host.Configuration if (ByondTopicTimeout <= 1000) logger.LogWarning("The timeout for sending BYOND topics is very low ({ms}ms). Topic calls may fail to complete at all!", ByondTopicTimeout); + + if (AdditionalEventScriptsDirectories?.Any(path => !Path.IsPathRooted(path)) == true) + logger.LogWarning($"Config option \"{nameof(AdditionalEventScriptsDirectories)}\" contains non-rooted paths. These will be evaluated relative to each instances \"Configuration\" directory!"); } } } diff --git a/src/Tgstation.Server.Host/Server.cs b/src/Tgstation.Server.Host/Server.cs index 57a26f669b..640b0c8517 100644 --- a/src/Tgstation.Server.Host/Server.cs +++ b/src/Tgstation.Server.Host/Server.cs @@ -208,7 +208,23 @@ namespace Tgstation.Server.Host async Task RunUpdate() { - if (await updateExecutor.ExecuteUpdate(updatePath, criticalCancellationToken, criticalCancellationToken)) + var updateExecutedSuccessfully = false; + try + { + updateExecutedSuccessfully = await updateExecutor.ExecuteUpdate(updatePath, criticalCancellationToken, criticalCancellationToken); + } + catch (OperationCanceledException ex) + { + logger.LogDebug(ex, "Update cancelled!"); + UpdateInProgress = false; + } + catch (Exception ex) + { + logger.LogError(ex, "Update errored!"); + UpdateInProgress = false; + } + + if (updateExecutedSuccessfully) { logger.LogTrace("Update complete!"); await RestartImpl(newVersion, null, true, true); diff --git a/src/Tgstation.Server.Host/System/PosixProcessFeatures.cs b/src/Tgstation.Server.Host/System/PosixProcessFeatures.cs index f9894e9ac9..12a0164a1f 100644 --- a/src/Tgstation.Server.Host/System/PosixProcessFeatures.cs +++ b/src/Tgstation.Server.Host/System/PosixProcessFeatures.cs @@ -1,6 +1,8 @@ using System; +using System.Collections.Generic; using System.Globalization; using System.IO; +using System.Linq; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -62,6 +64,39 @@ namespace Tgstation.Server.Host.System this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); } + /// + /// Gets potential paths to the gcore executable. + /// + /// The potential paths to the gcore executable. + static IEnumerable GetPotentialGCorePaths() + { + var enviromentPath = Environment.GetEnvironmentVariable("PATH"); + IEnumerable enumerator; + if (enviromentPath == null) + enumerator = Enumerable.Empty(); + else + { + var paths = enviromentPath.Split(';'); + enumerator = paths + .Select(x => x.Split(':')) + .SelectMany(x => x); + } + + var exeName = "gcore"; + + enumerator = enumerator + .Concat(new List(2) + { + "/usr/bin", + "/usr/share/bin", + "/bin", + }); + + enumerator = enumerator.Select(x => Path.Combine(x, exeName)); + + return enumerator; + } + /// public void ResumeProcess(global::System.Diagnostics.Process process) { @@ -88,8 +123,15 @@ namespace Tgstation.Server.Host.System ArgumentNullException.ThrowIfNull(process); ArgumentNullException.ThrowIfNull(outputFile); - const string GCorePath = "/usr/bin/gcore"; - if (!await ioManager.FileExists(GCorePath, cancellationToken)) + string? gcorePath = null; + foreach (var path in GetPotentialGCorePaths()) + if (await ioManager.FileExists(path, cancellationToken)) + { + gcorePath = path; + break; + } + + if (gcorePath == null) throw new JobException(ErrorCode.MissingGCore); int pid; @@ -108,7 +150,7 @@ namespace Tgstation.Server.Host.System string? output; int exitCode; await using (var gcoreProc = await lazyLoadedProcessExecutor.Value.LaunchProcess( - GCorePath, + gcorePath, Environment.CurrentDirectory, $"{(!minidump ? "-a " : String.Empty)}-o {outputFile} {process.Id}", cancellationToken, diff --git a/src/Tgstation.Server.Host/appsettings.yml b/src/Tgstation.Server.Host/appsettings.yml index ac08b78c49..2978befbf9 100644 --- a/src/Tgstation.Server.Host/appsettings.yml +++ b/src/Tgstation.Server.Host/appsettings.yml @@ -19,6 +19,7 @@ General: OpenDreamGitUrl: https://github.com/OpenDreamProject/OpenDream # The repository to retrieve OpenDream from OpenDreamGitTagPrefix: v # The prefix to the OpenDream semver as tags appear in the git repository OpenDreamSuppressInstallOutput: false # Suppress the dotnet output of creating an OpenDream installation. Known to cause hangs in CI. + AdditionalEventScriptsDirectories: # An array of directories that are considered to contain EventScripts alongside instance directories. Working directory for exectued scripts will remain the instance EventScripts directory. Session: HighPriorityLiveDreamDaemon: false # If DreamDaemon instances should run as higher priority processes LowPriorityDeploymentProcesses: true # If TGS Deployments should run as lower priority processes diff --git a/tools/Tgstation.Server.ReleaseNotes/Program.cs b/tools/Tgstation.Server.ReleaseNotes/Program.cs index 3cceaf2237..66e6869508 100644 --- a/tools/Tgstation.Server.ReleaseNotes/Program.cs +++ b/tools/Tgstation.Server.ReleaseNotes/Program.cs @@ -315,6 +315,39 @@ namespace Tgstation.Server.ReleaseNotes Description = "Next patch version" }); + + async ValueTask RelocateOpenIssues(Milestone originalMilestone, int moveToMilestoneNumber) + { + if (originalMilestone.OpenIssues + originalMilestone.ClosedIssues > 0) + { + var issuesInUnusedMilestone = await client.Search.SearchIssues(new SearchIssuesRequest + { + Milestone = originalMilestone.Title, + Repos = { { RepoOwner, RepoName } } + }); + + var issueUpdateTasks = new List(); + foreach (var I in issuesInUnusedMilestone.Items) + { + if (I.State.Value != ItemState.Closed) + issueUpdateTasks.Add(client.Issue.Update(RepoOwner, RepoName, I.Number, new IssueUpdate + { + Milestone = moveToMilestoneNumber + })); + + if (I.PullRequest != null && I.PullRequest.Merged) + { + Console.WriteLine($"Adding additional merged PR #{I.Number}..."); + var task = GetReleaseNotesFromPR(client, I, doNotCloseMilestone, false, false); + noteTasks.Add(task); + allTasks.Add(task); + } + } + + await Task.WhenAll(issueUpdateTasks).ConfigureAwait(false); + } + } + if (version.Build == 0) { // close the patch milestone if it exists @@ -326,39 +359,10 @@ namespace Tgstation.Server.ReleaseNotes async ValueTask DeleteMilestone(Milestone milestoneToDelete, int moveToMilestoneNumber) { Console.WriteLine($"Moving {milestoneToDelete.OpenIssues} open issues and {milestoneToDelete.ClosedIssues} closed issues from unused patch milestone {milestoneToDelete.Title} to upcoming ones and deleting..."); - if (milestoneToDelete.OpenIssues + milestoneToDelete.ClosedIssues > 0) - { - var issuesInUnusedMilestone = await client.Search.SearchIssues(new SearchIssuesRequest - { - Milestone = milestoneToDelete.Title, - Repos = { { RepoOwner, RepoName } } - }); - - var issueUpdateTasks = new List(); - foreach (var I in issuesInUnusedMilestone.Items) - { - if (I.State.Value != ItemState.Closed) - issueUpdateTasks.Add(client.Issue.Update(RepoOwner, RepoName, I.Number, new IssueUpdate - { - Milestone = moveToMilestoneNumber - })); - - if (I.PullRequest != null) - { - Console.WriteLine($"Adding additional merged PR #{I.Number}..."); - var task = GetReleaseNotesFromPR(client, I, doNotCloseMilestone, false, false); - noteTasks.Add(task); - allTasks.Add(task); - } - } - - await Task.WhenAll(issueUpdateTasks).ConfigureAwait(false); - } - + await RelocateOpenIssues(milestoneToDelete, moveToMilestoneNumber); allTasks.Add(client.Issue.Milestone.Delete(RepoOwner, RepoName, milestoneToDelete.Number)); } - var unreleasedNextPatchMilestone = milestones.FirstOrDefault(x => x.Title.StartsWith($"v{highestReleaseVersion.Major}.{highestReleaseVersion.Minor}.")); if (unreleasedNextPatchMilestone != null) await DeleteMilestone(unreleasedNextPatchMilestone, nextPatchMilestone.Number); @@ -401,7 +405,11 @@ namespace Tgstation.Server.ReleaseNotes if (unreleasedNextMinorMilestone != null) await DeleteMilestone(unreleasedNextMinorMilestone, nextMinorMilestone.Number); } + else + await RelocateOpenIssues(milestone, nextMinorMilestone.Number); } + else + await RelocateOpenIssues(milestone, nextPatchMilestone.Number); } newNotes.Append(milestone.HtmlUrl);