From 41a70a7bc62e9793a43025cd72823ddbd98cacbf Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 29 Jan 2024 17:48:30 -0500 Subject: [PATCH 1/9] Update to actions/checkout@v4 for master merge workflow --- .github/workflows/stable-merge.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/stable-merge.yml b/.github/workflows/stable-merge.yml index 8b77f4d7d3..9135dc5f7a 100644 --- a/.github/workflows/stable-merge.yml +++ b/.github/workflows/stable-merge.yml @@ -12,7 +12,9 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v1 + uses: actions/checkout@v4 + with: + fetch-depth: 0 - name: Merge master into dev uses: robotology/gh-action-nightly-merge@22f5e45d028f22837d617fa07512925457eec184 #v1.3.3 From 62423db1c52b9fc8d308cff2267ce64c67333a3b Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 29 Jan 2024 17:49:02 -0500 Subject: [PATCH 2/9] Fix Code Scanning start conditional --- .github/workflows/ci-pipeline.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index c8612242b9..2df78f5bf2 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -85,6 +85,7 @@ jobs: name: Code Scanning needs: start-ci-run-gate runs-on: ubuntu-latest + if: (!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success') steps: - name: Setup dotnet uses: actions/setup-dotnet@v4 From ad43286ba88dfc3d173769b84c908e3be5f932ab Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 29 Jan 2024 22:52:56 -0500 Subject: [PATCH 3/9] Remove the need for the extra CI token --- .github/CONTRIBUTING.md | 1 - .github/workflows/ci-pipeline.yml | 13 ++++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 7c0274a445..48b19a8566 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -64,7 +64,6 @@ For the full CI gambit, the following repository configuration must be set: ![image](https://github.com/tgstation/tgstation-server/assets/8171642/ab17fa74-364f-4e66-b7c4-b9bb24c6a599) - Label `CI Cleared`: To allow PRs from forks to run CI with secrets after approval. - Integration [CodeCov](https://github.com/apps/codecov): Enables CodeCov status checks. -- Secret `CI_STATUSES_TOKEN`: A GitHub token with read access to the repository's contents/actions and write access to the repository's checks/security events. Used to create CI completion statuses. - Secret `CODECOV_TOKEN`: A CodeCov repo token to work around https://github.com/codecov/codecov-action/issues/837. - Secret `LIVE_TESTS_TOKEN`: A GitHub token with read access to the repository and write access to https://github.com/Cyberboss/common_core (TODO: Make the target repository here configurable). Despite it's name, it may be used across the entire test suite. - Secret `TGS_TEST_DISCORD_TOKEN`: See above note about test environment variables. diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 2df78f5bf2..4f653587be 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -85,6 +85,9 @@ jobs: name: Code Scanning needs: start-ci-run-gate runs-on: ubuntu-latest + permissions: + security-events: write + actions: read if: (!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success') steps: - name: Setup dotnet @@ -100,7 +103,6 @@ jobs: uses: github/codeql-action/init@v3 with: languages: csharp - token: ${{ secrets.CI_STATUSES_TOKEN }} - name: Build run: dotnet build -c ReleaseNoWindows -p:TGS_HOST_NO_WEBPANEL=true @@ -109,8 +111,6 @@ jobs: uses: github/codeql-action/analyze@v3 with: category: "/language:csharp" - token: ${{ secrets.CI_STATUSES_TOKEN }} - dmapi-build: name: Build DMAPI @@ -1374,12 +1374,15 @@ jobs: name: CI Completion Gate needs: [ pages-build, docker-build, build-deb, build-msi, validate-openapi-spec, upload-code-coverage, check-winget-pr-template, code-scanning ] runs-on: ubuntu-latest - if: (!(cancelled() || failure()) && needs.pages-build.result == 'success' && needs.docker-build.result == 'success' && needs.build-deb.result == 'success' && needs.build-msi.result == 'success' && needs.validate-openapi-spec.result == 'success' && needs.upload-code-coverage.result == 'success' && needs.check-winget-pr-template.result == 'success') + permissions: + checks: write + contents: read + if: (!(cancelled() || failure()) && needs.pages-build.result == 'success' && needs.docker-build.result == 'success' && needs.build-deb.result == 'success' && needs.build-msi.result == 'success' && needs.validate-openapi-spec.result == 'success' && needs.upload-code-coverage.result == 'success' && needs.check-winget-pr-template.result == 'success' && needs.code-scanning.result == 'success') steps: - name: Create Completion Check uses: LouisBrunner/checks-action@6b626ffbad7cc56fd58627f774b9067e6118af23 with: - token: ${{ secrets.CI_STATUSES_TOKEN }} + token: ${{ secrets.GITHUB_TOKEN }} name: CI Completion conclusion: success output: | From 4993304389d9607db9ec59f536792edbafe71d9e Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Tue, 30 Jan 2024 20:19:58 -0500 Subject: [PATCH 4/9] Add support for setting environment variables with `IProcessExecutor` --- .../Components/Engine/OpenDreamInstaller.cs | 1 + .../Session/SessionControllerFactory.cs | 1 + .../System/IProcessExecutor.cs | 6 +++++- .../System/ProcessExecutor.cs | 20 +++++++++++++++---- .../System/TestPosixSignalHandler.cs | 1 + .../Live/TestLiveServer.cs | 1 + .../TestSystemInteraction.cs | 4 ++-- tests/Tgstation.Server.Tests/TestVersions.cs | 1 + 8 files changed, 28 insertions(+), 7 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs b/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs index a2b74e1445..f0d39c2b34 100644 --- a/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs +++ b/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs @@ -258,6 +258,7 @@ namespace Tgstation.Server.Host.Components.Engine shortenedPath, $"run -c Release --project OpenDreamPackageTool -- --tgs -o {shortenedDeployPath}", null, + null, !GeneralConfiguration.OpenDreamSuppressInstallOutput, !GeneralConfiguration.OpenDreamSuppressInstallOutput); diff --git a/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs b/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs index 70bed0f5ba..316202a7ba 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs @@ -507,6 +507,7 @@ namespace Tgstation.Server.Host.Components.Session engineLock.ServerExePath, dmbProvider.Directory, arguments, + null, logFilePath, engineLock.HasStandardOutput, true); diff --git a/src/Tgstation.Server.Host/System/IProcessExecutor.cs b/src/Tgstation.Server.Host/System/IProcessExecutor.cs index aae807eec2..34962811e1 100644 --- a/src/Tgstation.Server.Host/System/IProcessExecutor.cs +++ b/src/Tgstation.Server.Host/System/IProcessExecutor.cs @@ -1,4 +1,6 @@ -namespace Tgstation.Server.Host.System +using System.Collections.Generic; + +namespace Tgstation.Server.Host.System { /// /// For launching '. @@ -11,6 +13,7 @@ /// The full path to the executable file. /// The working directory for the . /// The arguments for the . + /// A of environment variables to set. /// File to write process output and error streams to. Requires to be . /// If the process output and error streams should be read. /// If shell execute should not be used. Must be set if is set. @@ -19,6 +22,7 @@ string fileName, string workingDirectory, string arguments, + IReadOnlyDictionary? environment = null, string? fileRedirect = null, bool readStandardHandles = false, bool noShellExecute = false); diff --git a/src/Tgstation.Server.Host/System/ProcessExecutor.cs b/src/Tgstation.Server.Host/System/ProcessExecutor.cs index 7820ca11b3..445a333ffd 100644 --- a/src/Tgstation.Server.Host/System/ProcessExecutor.cs +++ b/src/Tgstation.Server.Host/System/ProcessExecutor.cs @@ -1,6 +1,8 @@ using System; +using System.Collections.Generic; using System.Diagnostics; using System.IO; +using System.Linq; using System.Text; using System.Threading; using System.Threading.Channels; @@ -107,6 +109,7 @@ namespace Tgstation.Server.Host.System string fileName, string workingDirectory, string arguments, + IReadOnlyDictionary? environment, string? fileRedirect, bool readStandardHandles, bool noShellExecute) @@ -115,24 +118,33 @@ namespace Tgstation.Server.Host.System ArgumentNullException.ThrowIfNull(workingDirectory); ArgumentNullException.ThrowIfNull(arguments); + var enviromentLogLines = environment == null + ? String.Empty + : String.Concat(environment.Select(kvp => $"{Environment.NewLine}\t- {kvp.Key}={kvp.Value}")); if (noShellExecute) logger.LogDebug( - "Launching process in {workingDirectory}: {exe} {arguments}", + "Launching process in {workingDirectory}: {exe} {arguments}{environment}", workingDirectory, fileName, - arguments); + arguments, + enviromentLogLines); else logger.LogDebug( - "Shell launching process in {workingDirectory}: {exe} {arguments}", + "Shell launching process in {workingDirectory}: {exe} {arguments}{environment}", workingDirectory, fileName, - arguments); + arguments, + enviromentLogLines); var handle = new global::System.Diagnostics.Process(); try { handle.StartInfo.FileName = fileName; handle.StartInfo.Arguments = arguments; + if (environment != null) + foreach (var kvp in environment) + handle.StartInfo.Environment.Add(kvp!); + handle.StartInfo.WorkingDirectory = workingDirectory; handle.StartInfo.UseShellExecute = !noShellExecute; diff --git a/tests/Tgstation.Server.Host.Tests/System/TestPosixSignalHandler.cs b/tests/Tgstation.Server.Host.Tests/System/TestPosixSignalHandler.cs index 083657a075..121df115e3 100644 --- a/tests/Tgstation.Server.Host.Tests/System/TestPosixSignalHandler.cs +++ b/tests/Tgstation.Server.Host.Tests/System/TestPosixSignalHandler.cs @@ -69,6 +69,7 @@ namespace Tgstation.Server.Host.System.Tests pathToSignalTestApp, $"run -c {CurrentConfig} --no-build", null, + null, true, true); diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index 91466db1d0..5e058d24eb 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -1105,6 +1105,7 @@ namespace Tgstation.Server.Tests.Live repoPath, args, null, + null, true, true); diff --git a/tests/Tgstation.Server.Tests/TestSystemInteraction.cs b/tests/Tgstation.Server.Tests/TestSystemInteraction.cs index 98b7c82f44..a2b43af6fd 100644 --- a/tests/Tgstation.Server.Tests/TestSystemInteraction.cs +++ b/tests/Tgstation.Server.Tests/TestSystemInteraction.cs @@ -28,7 +28,7 @@ namespace Tgstation.Server.Tests Mock.Of>(), loggerFactory); - await using var process = processExecutor.LaunchProcess("test." + platformIdentifier.ScriptFileExtension, ".", string.Empty, null, true, true); + await using var process = processExecutor.LaunchProcess("test." + platformIdentifier.ScriptFileExtension, ".", string.Empty, null, null, true, true); using var cts = new CancellationTokenSource(); cts.CancelAfter(3000); var exitCode = await process.Lifetime.WaitAsync(cts.Token); @@ -63,7 +63,7 @@ namespace Tgstation.Server.Tests File.Delete(tempFile); try { - await using (var process = processExecutor.LaunchProcess("test." + platformIdentifier.ScriptFileExtension, ".", string.Empty, tempFile, true, true)) + await using (var process = processExecutor.LaunchProcess("test." + platformIdentifier.ScriptFileExtension, ".", string.Empty, null, tempFile, true, true)) { using var cts = new CancellationTokenSource(); cts.CancelAfter(3000); diff --git a/tests/Tgstation.Server.Tests/TestVersions.cs b/tests/Tgstation.Server.Tests/TestVersions.cs index 7b52b79773..591517a46f 100644 --- a/tests/Tgstation.Server.Tests/TestVersions.cs +++ b/tests/Tgstation.Server.Tests/TestVersions.cs @@ -503,6 +503,7 @@ namespace Tgstation.Server.Tests Environment.CurrentDirectory, "fake.dmb -map-threads 3 -close", null, + null, true, true); From 3246ac93bb14de31ff91fd2a7d3a620befd213ed Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 29 Jan 2024 22:56:36 -0500 Subject: [PATCH 5/9] Use the correct checkout for Code Scanning --- .github/workflows/ci-pipeline.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 4f653587be..e7cc858520 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -96,8 +96,15 @@ jobs: dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - - name: Checkout + - name: Checkout (Branch) uses: actions/checkout@v4 + if: github.event_name == 'push' || github.event_name == 'schedule' + + - name: Checkout (PR Merge) + uses: actions/checkout@v4 + if: github.event_name != 'push' && github.event_name != 'schedule' + with: + ref: "refs/pull/${{ github.event.number }}/merge" - name: Initialize CodeQL uses: github/codeql-action/init@v3 From c4e7f0b04ac20fc68e186a77eb2640617b0d670f Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Tue, 30 Jan 2024 22:03:59 -0500 Subject: [PATCH 6/9] .env files for engine installations --- .../Components/Deployment/DreamMaker.cs | 2 + .../Components/Engine/ByondInstallation.cs | 4 ++ .../Components/Engine/ByondInstallerBase.cs | 15 +++--- .../Components/Engine/EngineExecutableLock.cs | 4 ++ .../Engine/EngineInstallationBase.cs | 53 +++++++++++++++++++ .../Components/Engine/IEngineInstallation.cs | 9 ++++ .../Engine/OpenDreamInstallation.cs | 15 +++--- .../Components/Engine/OpenDreamInstaller.cs | 2 +- .../Session/SessionControllerFactory.cs | 3 +- .../Tgstation.Server.Host.csproj | 2 + .../EngineActiveVersionChange-SetupEnv.bat | 4 ++ .../EngineActiveVersionChange-SetupEnv.sh | 7 +++ .../Live/Instance/ConfigurationTest.cs | 26 +++++---- tgstation-server.sln | 4 +- 14 files changed, 121 insertions(+), 29 deletions(-) create mode 100644 tests/DMAPI/LongRunning/EngineActiveVersionChange-SetupEnv.bat create mode 100644 tests/DMAPI/LongRunning/EngineActiveVersionChange-SetupEnv.sh diff --git a/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs b/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs index ac5c4576e3..408cc07bfb 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs @@ -852,6 +852,7 @@ namespace Tgstation.Server.Host.Components.Deployment /// A resulting in if compilation succeeded, otherwise. async ValueTask RunDreamMaker(IEngineExecutableLock engineLock, Models.CompileJob job, CancellationToken cancellationToken) { + var environment = await engineLock.LoadEnv(logger, true, cancellationToken); var arguments = engineLock.FormatCompilerArguments($"{job.DmeName}.{DmeExtension}"); await using var dm = processExecutor.LaunchProcess( @@ -859,6 +860,7 @@ namespace Tgstation.Server.Host.Components.Deployment ioManager.ResolvePath( job.DirectoryName!.Value.ToString()), arguments, + environment, readStandardHandles: true, noShellExecute: true); diff --git a/src/Tgstation.Server.Host/Components/Engine/ByondInstallation.cs b/src/Tgstation.Server.Host/Components/Engine/ByondInstallation.cs index ecfb965e58..781add748f 100644 --- a/src/Tgstation.Server.Host/Components/Engine/ByondInstallation.cs +++ b/src/Tgstation.Server.Host/Components/Engine/ByondInstallation.cs @@ -6,6 +6,7 @@ using System.Threading.Tasks; using Tgstation.Server.Api.Models; using Tgstation.Server.Api.Models.Internal; using Tgstation.Server.Host.Components.Deployment; +using Tgstation.Server.Host.IO; namespace Tgstation.Server.Host.Components.Engine { @@ -75,6 +76,7 @@ namespace Tgstation.Server.Host.Components.Engine /// /// Initializes a new instance of the class. /// + /// The for the . /// The value of . /// The value of . /// The value of . @@ -82,12 +84,14 @@ namespace Tgstation.Server.Host.Components.Engine /// If a CLI application is being used. /// The value of . public ByondInstallation( + IIOManager installationIOManager, Task installationTask, EngineVersion version, string dreamDaemonPath, string dreamMakerPath, bool supportsCli, bool supportsMapThreads) + : base(installationIOManager) { InstallationTask = installationTask ?? throw new ArgumentNullException(nameof(installationTask)); ArgumentNullException.ThrowIfNull(version); diff --git a/src/Tgstation.Server.Host/Components/Engine/ByondInstallerBase.cs b/src/Tgstation.Server.Host/Components/Engine/ByondInstallerBase.cs index a992073d10..81a68470e7 100644 --- a/src/Tgstation.Server.Host/Components/Engine/ByondInstallerBase.cs +++ b/src/Tgstation.Server.Host/Components/Engine/ByondInstallerBase.cs @@ -88,21 +88,22 @@ namespace Tgstation.Server.Host.Components.Engine { CheckVersionValidity(version); - var binPathForVersion = IOManager.ConcatPath(path, ByondBinPath); + var installationIOManager = new ResolvingIOManager(IOManager, path); var supportsMapThreads = version.Version >= MapThreadsVersion; return new ByondInstallation( + installationIOManager, installationTask, version, - IOManager.ResolvePath( - IOManager.ConcatPath( - binPathForVersion, + installationIOManager.ResolvePath( + installationIOManager.ConcatPath( + ByondBinPath, GetDreamDaemonName( version.Version!, out var supportsCli))), - IOManager.ResolvePath( - IOManager.ConcatPath( - binPathForVersion, + installationIOManager.ResolvePath( + installationIOManager.ConcatPath( + ByondBinPath, DreamMakerName)), supportsCli, supportsMapThreads); diff --git a/src/Tgstation.Server.Host/Components/Engine/EngineExecutableLock.cs b/src/Tgstation.Server.Host/Components/Engine/EngineExecutableLock.cs index 5d7a1d7fef..3e1a92be14 100644 --- a/src/Tgstation.Server.Host/Components/Engine/EngineExecutableLock.cs +++ b/src/Tgstation.Server.Host/Components/Engine/EngineExecutableLock.cs @@ -62,5 +62,9 @@ namespace Tgstation.Server.Host.Components.Engine accessIdentifier, port, cancellationToken); + + /// + public ValueTask?> LoadEnv(ILogger logger, bool forCompiler, CancellationToken cancellationToken) + => Instance.LoadEnv(logger, forCompiler, cancellationToken); } } diff --git a/src/Tgstation.Server.Host/Components/Engine/EngineInstallationBase.cs b/src/Tgstation.Server.Host/Components/Engine/EngineInstallationBase.cs index 5edf44609f..77666748c7 100644 --- a/src/Tgstation.Server.Host/Components/Engine/EngineInstallationBase.cs +++ b/src/Tgstation.Server.Host/Components/Engine/EngineInstallationBase.cs @@ -1,15 +1,19 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Text; using System.Threading; using System.Threading.Tasks; using System.Web; +using DotEnv.Core; + using Microsoft.Extensions.Logging; using Tgstation.Server.Api.Models; using Tgstation.Server.Api.Models.Internal; using Tgstation.Server.Host.Components.Deployment; +using Tgstation.Server.Host.IO; using Tgstation.Server.Host.System; namespace Tgstation.Server.Host.Components.Engine @@ -38,6 +42,11 @@ namespace Tgstation.Server.Host.Components.Engine /// public abstract Task InstallationTask { get; } + /// + /// The pointing to the installation directory. + /// + protected IIOManager InstallationIOManager { get; } + /// /// Encode given parameters for passing as world.params on the command line. /// @@ -56,6 +65,15 @@ namespace Tgstation.Server.Host.Components.Engine return parametersString; } + /// + /// Initializes a new instance of the class. + /// + /// The value of . + public EngineInstallationBase(IIOManager installationIOManager) + { + InstallationIOManager = installationIOManager ?? throw new ArgumentNullException(nameof(installationIOManager)); + } + /// public abstract string FormatCompilerArguments(string dmePath); @@ -69,10 +87,45 @@ namespace Tgstation.Server.Host.Components.Engine /// public virtual async ValueTask StopServerProcess(ILogger logger, IProcess process, string accessIdentifier, ushort port, CancellationToken cancellationToken) { + ArgumentNullException.ThrowIfNull(logger); cancellationToken.ThrowIfCancellationRequested(); logger.LogTrace("Terminating engine server process..."); process.Terminate(); await process.Lifetime; } + + /// + public async ValueTask?> LoadEnv(ILogger logger, bool forCompiler, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(logger); + + var envFile = forCompiler + ? "compiler.env" + : "server.env"; + + if (!await InstallationIOManager.FileExists(envFile, cancellationToken)) + { + logger.LogTrace("No {envFile} present in engine installation {version}", envFile, Version); + return null; + } + + logger.LogDebug("Loading {envFile} for engine installation {version}...", envFile, Version); + + var fileBytes = await InstallationIOManager.ReadAllBytes(envFile, cancellationToken); + var fileContents = Encoding.UTF8.GetString(fileBytes); + var parser = new EnvParser(); + + try + { + var variables = parser.Parse(fileContents); + + return variables.ToDictionary(); + } + catch (Exception ex) + { + logger.LogWarning(ex, "Unable to parse {envFile}!", envFile); + return null; + } + } } } diff --git a/src/Tgstation.Server.Host/Components/Engine/IEngineInstallation.cs b/src/Tgstation.Server.Host/Components/Engine/IEngineInstallation.cs index ff2e4155f3..402eac4a76 100644 --- a/src/Tgstation.Server.Host/Components/Engine/IEngineInstallation.cs +++ b/src/Tgstation.Server.Host/Components/Engine/IEngineInstallation.cs @@ -82,5 +82,14 @@ namespace Tgstation.Server.Host.Components.Engine /// The for the operation. /// A representing the running operation. ValueTask StopServerProcess(ILogger logger, IProcess process, string accessIdentifier, ushort port, CancellationToken cancellationToken); + + /// + /// Loads the environment settings for either the server or compiler. + /// + /// The to write to. + /// If server.env will be loaded. If compiler.env will be loaded. + /// The for the operation. + /// A resulting in the environment or if the target environment file doesn't exist. + ValueTask?> LoadEnv(ILogger logger, bool forCompiler, CancellationToken cancellationToken); } } diff --git a/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstallation.cs b/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstallation.cs index ab9eb923d8..ba400090e0 100644 --- a/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstallation.cs +++ b/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstallation.cs @@ -47,11 +47,6 @@ namespace Tgstation.Server.Host.Components.Engine /// public override Task InstallationTask { get; } - /// - /// The for the . - /// - readonly IIOManager ioManager; - /// /// The for the . /// @@ -65,7 +60,7 @@ namespace Tgstation.Server.Host.Components.Engine /// /// Initializes a new instance of the class. /// - /// The value of . + /// The for the . /// The value of . /// The value of . /// The value of . @@ -73,15 +68,15 @@ namespace Tgstation.Server.Host.Components.Engine /// The value of . /// The value of . public OpenDreamInstallation( - IIOManager ioManager, + IIOManager installationIOManager, IAsyncDelayer asyncDelayer, IAbstractHttpClientFactory httpClientFactory, string serverExePath, string compilerExePath, Task installationTask, EngineVersion version) + : base(installationIOManager) { - this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer)); this.httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); ServerExePath = serverExePath ?? throw new ArgumentNullException(nameof(serverExePath)); @@ -109,7 +104,7 @@ namespace Tgstation.Server.Host.Components.Engine var parametersString = EncodeParameters(parameters, launchParameters); - var arguments = $"--cvar {(logFilePath != null ? $"log.path=\"{ioManager.GetDirectoryName(logFilePath)}\" --cvar log.format=\"{ioManager.GetFileName(logFilePath)}\"" : "log.enabled=false")} --cvar watchdog.token={accessIdentifier} --cvar log.runtimelog=false --cvar net.port={launchParameters.Port!.Value} --cvar opendream.topic_port=0 --cvar opendream.world_params=\"{parametersString}\" --cvar opendream.json_path=\"./{dmbProvider.DmbName}\""; + var arguments = $"--cvar {(logFilePath != null ? $"log.path=\"{InstallationIOManager.GetDirectoryName(logFilePath)}\" --cvar log.format=\"{InstallationIOManager.GetFileName(logFilePath)}\"" : "log.enabled=false")} --cvar watchdog.token={accessIdentifier} --cvar log.runtimelog=false --cvar net.port={launchParameters.Port!.Value} --cvar opendream.topic_port=0 --cvar opendream.world_params=\"{parametersString}\" --cvar opendream.json_path=\"./{dmbProvider.DmbName}\""; return arguments; } @@ -125,6 +120,8 @@ namespace Tgstation.Server.Host.Components.Engine ushort port, CancellationToken cancellationToken) { + ArgumentNullException.ThrowIfNull(logger); + const int MaximumTerminationSeconds = 5; logger.LogTrace("Attempting Robust.Server graceful exit (Timeout: {seconds}s)...", MaximumTerminationSeconds); diff --git a/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs b/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs index f0d39c2b34..dbb5577bc8 100644 --- a/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs +++ b/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs @@ -124,7 +124,7 @@ namespace Tgstation.Server.Host.Components.Engine CheckVersionValidity(version); GetExecutablePaths(path, out var serverExePath, out var compilerExePath); return new OpenDreamInstallation( - IOManager, + new ResolvingIOManager(IOManager, path), asyncDelayer, httpClientFactory, serverExePath, diff --git a/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs b/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs index 316202a7ba..41c087b22f 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs @@ -490,6 +490,7 @@ namespace Tgstation.Server.Host.Components.Session CancellationToken cancellationToken) { // important to run on all ports to allow port changing + var environment = await engineLock.LoadEnv(logger, false, cancellationToken); var arguments = engineLock.FormatServerArguments( dmbProvider, new Dictionary @@ -507,7 +508,7 @@ namespace Tgstation.Server.Host.Components.Session engineLock.ServerExePath, dmbProvider.Directory, arguments, - null, + environment, logFilePath, engineLock.HasStandardOutput, true); diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index e6a0a19d97..af011cc872 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -69,6 +69,8 @@ + + diff --git a/tests/DMAPI/LongRunning/EngineActiveVersionChange-SetupEnv.bat b/tests/DMAPI/LongRunning/EngineActiveVersionChange-SetupEnv.bat new file mode 100644 index 0000000000..6f026682c0 --- /dev/null +++ b/tests/DMAPI/LongRunning/EngineActiveVersionChange-SetupEnv.bat @@ -0,0 +1,4 @@ +cd /D "%~dp0" +cd ../../Byond/%1 +echo # Comment > server.env +echo NOTA=Real Comment>> server.env diff --git a/tests/DMAPI/LongRunning/EngineActiveVersionChange-SetupEnv.sh b/tests/DMAPI/LongRunning/EngineActiveVersionChange-SetupEnv.sh new file mode 100644 index 0000000000..52b8090bd1 --- /dev/null +++ b/tests/DMAPI/LongRunning/EngineActiveVersionChange-SetupEnv.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -e + +cd "../../Byond/$1" + +echo -e '# This is a comment\nNOTA=Real Comment\n\n\n' > server.env diff --git a/tests/Tgstation.Server.Tests/Live/Instance/ConfigurationTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/ConfigurationTest.cs index 73c658487b..9ef929701b 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/ConfigurationTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/ConfigurationTest.cs @@ -118,18 +118,24 @@ namespace Tgstation.Server.Tests.Live.Instance await using var memoryStream2 = new MemoryStream(Encoding.UTF8.GetBytes("bbb")); await configurationClient.Write(staticFile2, memoryStream2, cancellationToken); - var shellScriptExtension = new PlatformIdentifier().IsWindows ? ".bat" : ".sh"; - var scriptName = $"PreCompile-GenerateRandomResource{shellScriptExtension}"; - var resourcingScript = new ConfigurationFileRequest + async ValueTask UploadScript(string scriptId) { - Path = $"/EventScripts/{scriptName}" - }; + var shellScriptExtension = new PlatformIdentifier().IsWindows ? ".bat" : ".sh"; + var scriptName = $"{scriptId}{shellScriptExtension}"; + var resourcingScript = new ConfigurationFileRequest + { + Path = $"/EventScripts/{scriptName}" + }; - await using var readStream = ioManager.GetFileStream($"../../../../DMAPI/LongRunning/{scriptName}", false); - await configurationClient.Write( - resourcingScript, - readStream, - cancellationToken); + await using var readStream = ioManager.GetFileStream($"../../../../DMAPI/LongRunning/{scriptName}", false); + await configurationClient.Write( + resourcingScript, + readStream, + cancellationToken); + } + + await UploadScript("PreCompile-GenerateRandomResource"); + await UploadScript("EngineActiveVersionChange-SetupEnv"); } return ValueTaskExtensions.WhenAll( diff --git a/tgstation-server.sln b/tgstation-server.sln index 581410a016..fead8a204d 100644 --- a/tgstation-server.sln +++ b/tgstation-server.sln @@ -155,6 +155,8 @@ EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "LongRunning", "LongRunning", "{EB1DDE8C-CA6F-4BE3-947B-597CA8EABEA5}" ProjectSection(SolutionItems) = preProject tests\DMAPI\LongRunning\Config.dm = tests\DMAPI\LongRunning\Config.dm + tests\DMAPI\LongRunning\EngineActiveVersionChange-SetupEnv.bat = tests\DMAPI\LongRunning\EngineActiveVersionChange-SetupEnv.bat + tests\DMAPI\LongRunning\EngineActiveVersionChange-SetupEnv.sh = tests\DMAPI\LongRunning\EngineActiveVersionChange-SetupEnv.sh tests\DMAPI\LongRunning\long_running_test.dme = tests\DMAPI\LongRunning\long_running_test.dme tests\DMAPI\LongRunning\long_running_test_copy.dme = tests\DMAPI\LongRunning\long_running_test_copy.dme tests\DMAPI\LongRunning\long_running_test_rooted.dme = tests\DMAPI\LongRunning\long_running_test_rooted.dme @@ -171,8 +173,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "ApiFree", "ApiFree", "{7B8F EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "BasicOperation", "BasicOperation", "{F32B9514-AAD9-429D-841A-ED810FC2598C}" ProjectSection(SolutionItems) = preProject - tests\DMAPI\BasicOperation\Config.dm = tests\DMAPI\BasicOperation\Config.dm tests\DMAPI\BasicOperation\basic operation_test.dme = tests\DMAPI\BasicOperation\basic operation_test.dme + tests\DMAPI\BasicOperation\Config.dm = tests\DMAPI\BasicOperation\Config.dm tests\DMAPI\BasicOperation\Test.dm = tests\DMAPI\BasicOperation\Test.dm EndProjectSection EndProject From 2468633ebff311ee7c7eb28bbde62999144bcf66 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Wed, 31 Jan 2024 17:27:05 -0500 Subject: [PATCH 7/9] Update `README.md` for env files and add a missing OD reference --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 333a62165f..2efb724ec3 100644 --- a/README.md +++ b/README.md @@ -539,7 +539,11 @@ Manual operations on the repository while an instance is running may lead to git #### Byond -The `Byond` folder contains installations of [BYOND](https://www.byond.com/) versions. The version which is used by your game code can be changed on a whim (Note that only versions >= 511.1385 have been thouroughly tested. Lower versions should work but if one doesn't function, please open an issue report) and the server will take care of installing it. +The `Byond` folder contains installations of [BYOND](https://www.byond.com/) or [OpenDream](https://github.com/OpenDreamProject/OpenDream) versions. The version which is used by your game code can be changed on a whim (Note that only versions >= 511.1385 have been thouroughly tested. Lower versions should work but if one doesn't function, please open an issue report) and the server will take care of installing it. + +##### Environment Variables + +You can specify additional environment variables to launch your server/compiler with by adding `server.env`/`compiler.env` to your engine installation directory (i.e. `/Byond/515.1530/server.env`). These are [.env](https://hexdocs.pm/dotenvy/dotenv-file-format.html) files. #### Compiler From 0497dd328ebd3c19ffa10e82a46e7537121d0f9a Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Wed, 31 Jan 2024 17:16:57 -0500 Subject: [PATCH 8/9] Fix duplicate artefact uploads/steps --- .github/workflows/ci-pipeline.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index e7cc858520..49d1a5bc69 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -618,7 +618,7 @@ jobs: path: C:/tgs_api.json - name: Package Server Service - if: ${{ matrix.configuration == 'Release' && matrix.watchdog-type == 'Basic' }} + if: ${{ matrix.configuration == 'Release' && matrix.watchdog-type == 'Basic' && matrix.database-type == 'PostgresSql' }} run: | cd src/Tgstation.Server.Host.Service dotnet publish -c ${{ matrix.configuration }} -o ../../artifacts/Service @@ -630,14 +630,14 @@ jobs: build/RemoveUnsupportedServiceRuntimes.ps1 artifacts/Service - name: Store Server Service - if: ${{ matrix.configuration == 'Release' && matrix.watchdog-type == 'Basic' }} + if: ${{ matrix.configuration == 'Release' && matrix.watchdog-type == 'Basic' && matrix.database-type == 'PostgresSql' }} uses: actions/upload-artifact@v4 with: name: ServerService path: artifacts/Service/ - name: Install Code Signing Certificate - if: ${{ matrix.configuration == 'Release' && matrix.watchdog-type == 'Basic' }} + if: ${{ matrix.configuration == 'Release' && matrix.watchdog-type == 'Basic' && matrix.database-type == 'PostgresSql' }} shell: powershell run: | $pfxBytes = [convert]::FromBase64String("${{ secrets.CODE_SIGNING_BASE64 }}") @@ -647,7 +647,7 @@ jobs: rm tg_codesigning.pfx - name: Test Sign Service .exe - if: ${{ matrix.configuration == 'Release' && matrix.watchdog-type == 'Basic' }} + if: ${{ matrix.configuration == 'Release' && matrix.watchdog-type == 'Basic' && matrix.database-type == 'PostgresSql' }} shell: powershell run: Set-AuthenticodeSignature artifacts/Service/Tgstation.Server.Host.Service.exe -Certificate (Get-ChildItem Cert:\CurrentUser\My | Where-Object { $_.Thumbprint -eq "${{ vars.CODE_SIGNING_THUMBPRINT }}" }) -TimestampServer "http://timestamp.digicert.com" From 033f263ec1cb250a64bd9363a9cea038fcb43cba Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Wed, 31 Jan 2024 20:06:59 -0500 Subject: [PATCH 9/9] Update Nuget packages --- build/TestCommon.props | 4 ++-- src/Tgstation.Server.Api/Tgstation.Server.Api.csproj | 2 +- src/Tgstation.Server.Host/Tgstation.Server.Host.csproj | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/build/TestCommon.props b/build/TestCommon.props index c533dded96..9811a480b8 100644 --- a/build/TestCommon.props +++ b/build/TestCommon.props @@ -18,9 +18,9 @@ - + - + diff --git a/src/Tgstation.Server.Api/Tgstation.Server.Api.csproj b/src/Tgstation.Server.Api/Tgstation.Server.Api.csproj index 1c5c6e8c65..4f64dba5b1 100644 --- a/src/Tgstation.Server.Api/Tgstation.Server.Api.csproj +++ b/src/Tgstation.Server.Api/Tgstation.Server.Api.csproj @@ -27,7 +27,7 @@ - + diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index af011cc872..300d9ffe1c 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -100,7 +100,7 @@ - + @@ -126,7 +126,7 @@ - +