From c4e7f0b04ac20fc68e186a77eb2640617b0d670f Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Tue, 30 Jan 2024 22:03:59 -0500 Subject: [PATCH] .env files for engine installations --- .../Components/Deployment/DreamMaker.cs | 2 + .../Components/Engine/ByondInstallation.cs | 4 ++ .../Components/Engine/ByondInstallerBase.cs | 15 +++--- .../Components/Engine/EngineExecutableLock.cs | 4 ++ .../Engine/EngineInstallationBase.cs | 53 +++++++++++++++++++ .../Components/Engine/IEngineInstallation.cs | 9 ++++ .../Engine/OpenDreamInstallation.cs | 15 +++--- .../Components/Engine/OpenDreamInstaller.cs | 2 +- .../Session/SessionControllerFactory.cs | 3 +- .../Tgstation.Server.Host.csproj | 2 + .../EngineActiveVersionChange-SetupEnv.bat | 4 ++ .../EngineActiveVersionChange-SetupEnv.sh | 7 +++ .../Live/Instance/ConfigurationTest.cs | 26 +++++---- tgstation-server.sln | 4 +- 14 files changed, 121 insertions(+), 29 deletions(-) create mode 100644 tests/DMAPI/LongRunning/EngineActiveVersionChange-SetupEnv.bat create mode 100644 tests/DMAPI/LongRunning/EngineActiveVersionChange-SetupEnv.sh diff --git a/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs b/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs index ac5c4576e3..408cc07bfb 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs @@ -852,6 +852,7 @@ namespace Tgstation.Server.Host.Components.Deployment /// A resulting in if compilation succeeded, otherwise. async ValueTask RunDreamMaker(IEngineExecutableLock engineLock, Models.CompileJob job, CancellationToken cancellationToken) { + var environment = await engineLock.LoadEnv(logger, true, cancellationToken); var arguments = engineLock.FormatCompilerArguments($"{job.DmeName}.{DmeExtension}"); await using var dm = processExecutor.LaunchProcess( @@ -859,6 +860,7 @@ namespace Tgstation.Server.Host.Components.Deployment ioManager.ResolvePath( job.DirectoryName!.Value.ToString()), arguments, + environment, readStandardHandles: true, noShellExecute: true); diff --git a/src/Tgstation.Server.Host/Components/Engine/ByondInstallation.cs b/src/Tgstation.Server.Host/Components/Engine/ByondInstallation.cs index ecfb965e58..781add748f 100644 --- a/src/Tgstation.Server.Host/Components/Engine/ByondInstallation.cs +++ b/src/Tgstation.Server.Host/Components/Engine/ByondInstallation.cs @@ -6,6 +6,7 @@ using System.Threading.Tasks; using Tgstation.Server.Api.Models; using Tgstation.Server.Api.Models.Internal; using Tgstation.Server.Host.Components.Deployment; +using Tgstation.Server.Host.IO; namespace Tgstation.Server.Host.Components.Engine { @@ -75,6 +76,7 @@ namespace Tgstation.Server.Host.Components.Engine /// /// Initializes a new instance of the class. /// + /// The for the . /// The value of . /// The value of . /// The value of . @@ -82,12 +84,14 @@ namespace Tgstation.Server.Host.Components.Engine /// If a CLI application is being used. /// The value of . public ByondInstallation( + IIOManager installationIOManager, Task installationTask, EngineVersion version, string dreamDaemonPath, string dreamMakerPath, bool supportsCli, bool supportsMapThreads) + : base(installationIOManager) { InstallationTask = installationTask ?? throw new ArgumentNullException(nameof(installationTask)); ArgumentNullException.ThrowIfNull(version); diff --git a/src/Tgstation.Server.Host/Components/Engine/ByondInstallerBase.cs b/src/Tgstation.Server.Host/Components/Engine/ByondInstallerBase.cs index a992073d10..81a68470e7 100644 --- a/src/Tgstation.Server.Host/Components/Engine/ByondInstallerBase.cs +++ b/src/Tgstation.Server.Host/Components/Engine/ByondInstallerBase.cs @@ -88,21 +88,22 @@ namespace Tgstation.Server.Host.Components.Engine { CheckVersionValidity(version); - var binPathForVersion = IOManager.ConcatPath(path, ByondBinPath); + var installationIOManager = new ResolvingIOManager(IOManager, path); var supportsMapThreads = version.Version >= MapThreadsVersion; return new ByondInstallation( + installationIOManager, installationTask, version, - IOManager.ResolvePath( - IOManager.ConcatPath( - binPathForVersion, + installationIOManager.ResolvePath( + installationIOManager.ConcatPath( + ByondBinPath, GetDreamDaemonName( version.Version!, out var supportsCli))), - IOManager.ResolvePath( - IOManager.ConcatPath( - binPathForVersion, + installationIOManager.ResolvePath( + installationIOManager.ConcatPath( + ByondBinPath, DreamMakerName)), supportsCli, supportsMapThreads); diff --git a/src/Tgstation.Server.Host/Components/Engine/EngineExecutableLock.cs b/src/Tgstation.Server.Host/Components/Engine/EngineExecutableLock.cs index 5d7a1d7fef..3e1a92be14 100644 --- a/src/Tgstation.Server.Host/Components/Engine/EngineExecutableLock.cs +++ b/src/Tgstation.Server.Host/Components/Engine/EngineExecutableLock.cs @@ -62,5 +62,9 @@ namespace Tgstation.Server.Host.Components.Engine accessIdentifier, port, cancellationToken); + + /// + public ValueTask?> LoadEnv(ILogger logger, bool forCompiler, CancellationToken cancellationToken) + => Instance.LoadEnv(logger, forCompiler, cancellationToken); } } diff --git a/src/Tgstation.Server.Host/Components/Engine/EngineInstallationBase.cs b/src/Tgstation.Server.Host/Components/Engine/EngineInstallationBase.cs index 5edf44609f..77666748c7 100644 --- a/src/Tgstation.Server.Host/Components/Engine/EngineInstallationBase.cs +++ b/src/Tgstation.Server.Host/Components/Engine/EngineInstallationBase.cs @@ -1,15 +1,19 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Text; using System.Threading; using System.Threading.Tasks; using System.Web; +using DotEnv.Core; + using Microsoft.Extensions.Logging; using Tgstation.Server.Api.Models; using Tgstation.Server.Api.Models.Internal; using Tgstation.Server.Host.Components.Deployment; +using Tgstation.Server.Host.IO; using Tgstation.Server.Host.System; namespace Tgstation.Server.Host.Components.Engine @@ -38,6 +42,11 @@ namespace Tgstation.Server.Host.Components.Engine /// public abstract Task InstallationTask { get; } + /// + /// The pointing to the installation directory. + /// + protected IIOManager InstallationIOManager { get; } + /// /// Encode given parameters for passing as world.params on the command line. /// @@ -56,6 +65,15 @@ namespace Tgstation.Server.Host.Components.Engine return parametersString; } + /// + /// Initializes a new instance of the class. + /// + /// The value of . + public EngineInstallationBase(IIOManager installationIOManager) + { + InstallationIOManager = installationIOManager ?? throw new ArgumentNullException(nameof(installationIOManager)); + } + /// public abstract string FormatCompilerArguments(string dmePath); @@ -69,10 +87,45 @@ namespace Tgstation.Server.Host.Components.Engine /// public virtual async ValueTask StopServerProcess(ILogger logger, IProcess process, string accessIdentifier, ushort port, CancellationToken cancellationToken) { + ArgumentNullException.ThrowIfNull(logger); cancellationToken.ThrowIfCancellationRequested(); logger.LogTrace("Terminating engine server process..."); process.Terminate(); await process.Lifetime; } + + /// + public async ValueTask?> LoadEnv(ILogger logger, bool forCompiler, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(logger); + + var envFile = forCompiler + ? "compiler.env" + : "server.env"; + + if (!await InstallationIOManager.FileExists(envFile, cancellationToken)) + { + logger.LogTrace("No {envFile} present in engine installation {version}", envFile, Version); + return null; + } + + logger.LogDebug("Loading {envFile} for engine installation {version}...", envFile, Version); + + var fileBytes = await InstallationIOManager.ReadAllBytes(envFile, cancellationToken); + var fileContents = Encoding.UTF8.GetString(fileBytes); + var parser = new EnvParser(); + + try + { + var variables = parser.Parse(fileContents); + + return variables.ToDictionary(); + } + catch (Exception ex) + { + logger.LogWarning(ex, "Unable to parse {envFile}!", envFile); + return null; + } + } } } diff --git a/src/Tgstation.Server.Host/Components/Engine/IEngineInstallation.cs b/src/Tgstation.Server.Host/Components/Engine/IEngineInstallation.cs index ff2e4155f3..402eac4a76 100644 --- a/src/Tgstation.Server.Host/Components/Engine/IEngineInstallation.cs +++ b/src/Tgstation.Server.Host/Components/Engine/IEngineInstallation.cs @@ -82,5 +82,14 @@ namespace Tgstation.Server.Host.Components.Engine /// The for the operation. /// A representing the running operation. ValueTask StopServerProcess(ILogger logger, IProcess process, string accessIdentifier, ushort port, CancellationToken cancellationToken); + + /// + /// Loads the environment settings for either the server or compiler. + /// + /// The to write to. + /// If server.env will be loaded. If compiler.env will be loaded. + /// The for the operation. + /// A resulting in the environment or if the target environment file doesn't exist. + ValueTask?> LoadEnv(ILogger logger, bool forCompiler, CancellationToken cancellationToken); } } diff --git a/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstallation.cs b/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstallation.cs index ab9eb923d8..ba400090e0 100644 --- a/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstallation.cs +++ b/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstallation.cs @@ -47,11 +47,6 @@ namespace Tgstation.Server.Host.Components.Engine /// public override Task InstallationTask { get; } - /// - /// The for the . - /// - readonly IIOManager ioManager; - /// /// The for the . /// @@ -65,7 +60,7 @@ namespace Tgstation.Server.Host.Components.Engine /// /// Initializes a new instance of the class. /// - /// The value of . + /// The for the . /// The value of . /// The value of . /// The value of . @@ -73,15 +68,15 @@ namespace Tgstation.Server.Host.Components.Engine /// The value of . /// The value of . public OpenDreamInstallation( - IIOManager ioManager, + IIOManager installationIOManager, IAsyncDelayer asyncDelayer, IAbstractHttpClientFactory httpClientFactory, string serverExePath, string compilerExePath, Task installationTask, EngineVersion version) + : base(installationIOManager) { - this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer)); this.httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); ServerExePath = serverExePath ?? throw new ArgumentNullException(nameof(serverExePath)); @@ -109,7 +104,7 @@ namespace Tgstation.Server.Host.Components.Engine var parametersString = EncodeParameters(parameters, launchParameters); - var arguments = $"--cvar {(logFilePath != null ? $"log.path=\"{ioManager.GetDirectoryName(logFilePath)}\" --cvar log.format=\"{ioManager.GetFileName(logFilePath)}\"" : "log.enabled=false")} --cvar watchdog.token={accessIdentifier} --cvar log.runtimelog=false --cvar net.port={launchParameters.Port!.Value} --cvar opendream.topic_port=0 --cvar opendream.world_params=\"{parametersString}\" --cvar opendream.json_path=\"./{dmbProvider.DmbName}\""; + var arguments = $"--cvar {(logFilePath != null ? $"log.path=\"{InstallationIOManager.GetDirectoryName(logFilePath)}\" --cvar log.format=\"{InstallationIOManager.GetFileName(logFilePath)}\"" : "log.enabled=false")} --cvar watchdog.token={accessIdentifier} --cvar log.runtimelog=false --cvar net.port={launchParameters.Port!.Value} --cvar opendream.topic_port=0 --cvar opendream.world_params=\"{parametersString}\" --cvar opendream.json_path=\"./{dmbProvider.DmbName}\""; return arguments; } @@ -125,6 +120,8 @@ namespace Tgstation.Server.Host.Components.Engine ushort port, CancellationToken cancellationToken) { + ArgumentNullException.ThrowIfNull(logger); + const int MaximumTerminationSeconds = 5; logger.LogTrace("Attempting Robust.Server graceful exit (Timeout: {seconds}s)...", MaximumTerminationSeconds); diff --git a/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs b/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs index f0d39c2b34..dbb5577bc8 100644 --- a/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs +++ b/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs @@ -124,7 +124,7 @@ namespace Tgstation.Server.Host.Components.Engine CheckVersionValidity(version); GetExecutablePaths(path, out var serverExePath, out var compilerExePath); return new OpenDreamInstallation( - IOManager, + new ResolvingIOManager(IOManager, path), asyncDelayer, httpClientFactory, serverExePath, diff --git a/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs b/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs index 316202a7ba..41c087b22f 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs @@ -490,6 +490,7 @@ namespace Tgstation.Server.Host.Components.Session CancellationToken cancellationToken) { // important to run on all ports to allow port changing + var environment = await engineLock.LoadEnv(logger, false, cancellationToken); var arguments = engineLock.FormatServerArguments( dmbProvider, new Dictionary @@ -507,7 +508,7 @@ namespace Tgstation.Server.Host.Components.Session engineLock.ServerExePath, dmbProvider.Directory, arguments, - null, + environment, logFilePath, engineLock.HasStandardOutput, true); diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index e6a0a19d97..af011cc872 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -69,6 +69,8 @@ + + diff --git a/tests/DMAPI/LongRunning/EngineActiveVersionChange-SetupEnv.bat b/tests/DMAPI/LongRunning/EngineActiveVersionChange-SetupEnv.bat new file mode 100644 index 0000000000..6f026682c0 --- /dev/null +++ b/tests/DMAPI/LongRunning/EngineActiveVersionChange-SetupEnv.bat @@ -0,0 +1,4 @@ +cd /D "%~dp0" +cd ../../Byond/%1 +echo # Comment > server.env +echo NOTA=Real Comment>> server.env diff --git a/tests/DMAPI/LongRunning/EngineActiveVersionChange-SetupEnv.sh b/tests/DMAPI/LongRunning/EngineActiveVersionChange-SetupEnv.sh new file mode 100644 index 0000000000..52b8090bd1 --- /dev/null +++ b/tests/DMAPI/LongRunning/EngineActiveVersionChange-SetupEnv.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -e + +cd "../../Byond/$1" + +echo -e '# This is a comment\nNOTA=Real Comment\n\n\n' > server.env diff --git a/tests/Tgstation.Server.Tests/Live/Instance/ConfigurationTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/ConfigurationTest.cs index 73c658487b..9ef929701b 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/ConfigurationTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/ConfigurationTest.cs @@ -118,18 +118,24 @@ namespace Tgstation.Server.Tests.Live.Instance await using var memoryStream2 = new MemoryStream(Encoding.UTF8.GetBytes("bbb")); await configurationClient.Write(staticFile2, memoryStream2, cancellationToken); - var shellScriptExtension = new PlatformIdentifier().IsWindows ? ".bat" : ".sh"; - var scriptName = $"PreCompile-GenerateRandomResource{shellScriptExtension}"; - var resourcingScript = new ConfigurationFileRequest + async ValueTask UploadScript(string scriptId) { - Path = $"/EventScripts/{scriptName}" - }; + var shellScriptExtension = new PlatformIdentifier().IsWindows ? ".bat" : ".sh"; + var scriptName = $"{scriptId}{shellScriptExtension}"; + var resourcingScript = new ConfigurationFileRequest + { + Path = $"/EventScripts/{scriptName}" + }; - await using var readStream = ioManager.GetFileStream($"../../../../DMAPI/LongRunning/{scriptName}", false); - await configurationClient.Write( - resourcingScript, - readStream, - cancellationToken); + await using var readStream = ioManager.GetFileStream($"../../../../DMAPI/LongRunning/{scriptName}", false); + await configurationClient.Write( + resourcingScript, + readStream, + cancellationToken); + } + + await UploadScript("PreCompile-GenerateRandomResource"); + await UploadScript("EngineActiveVersionChange-SetupEnv"); } return ValueTaskExtensions.WhenAll( diff --git a/tgstation-server.sln b/tgstation-server.sln index 581410a016..fead8a204d 100644 --- a/tgstation-server.sln +++ b/tgstation-server.sln @@ -155,6 +155,8 @@ EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "LongRunning", "LongRunning", "{EB1DDE8C-CA6F-4BE3-947B-597CA8EABEA5}" ProjectSection(SolutionItems) = preProject tests\DMAPI\LongRunning\Config.dm = tests\DMAPI\LongRunning\Config.dm + tests\DMAPI\LongRunning\EngineActiveVersionChange-SetupEnv.bat = tests\DMAPI\LongRunning\EngineActiveVersionChange-SetupEnv.bat + tests\DMAPI\LongRunning\EngineActiveVersionChange-SetupEnv.sh = tests\DMAPI\LongRunning\EngineActiveVersionChange-SetupEnv.sh tests\DMAPI\LongRunning\long_running_test.dme = tests\DMAPI\LongRunning\long_running_test.dme tests\DMAPI\LongRunning\long_running_test_copy.dme = tests\DMAPI\LongRunning\long_running_test_copy.dme tests\DMAPI\LongRunning\long_running_test_rooted.dme = tests\DMAPI\LongRunning\long_running_test_rooted.dme @@ -171,8 +173,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "ApiFree", "ApiFree", "{7B8F EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "BasicOperation", "BasicOperation", "{F32B9514-AAD9-429D-841A-ED810FC2598C}" ProjectSection(SolutionItems) = preProject - tests\DMAPI\BasicOperation\Config.dm = tests\DMAPI\BasicOperation\Config.dm tests\DMAPI\BasicOperation\basic operation_test.dme = tests\DMAPI\BasicOperation\basic operation_test.dme + tests\DMAPI\BasicOperation\Config.dm = tests\DMAPI\BasicOperation\Config.dm tests\DMAPI\BasicOperation\Test.dm = tests\DMAPI\BasicOperation\Test.dm EndProjectSection EndProject