diff --git a/src/Tgstation.Server.Host/Components/Engine/EngineExecutableLock.cs b/src/Tgstation.Server.Host/Components/Engine/EngineExecutableLock.cs index 02d9bfe493..36944a876f 100644 --- a/src/Tgstation.Server.Host/Components/Engine/EngineExecutableLock.cs +++ b/src/Tgstation.Server.Host/Components/Engine/EngineExecutableLock.cs @@ -1,9 +1,13 @@ using System.Collections.Generic; +using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + using Tgstation.Server.Api.Models; using Tgstation.Server.Api.Models.Internal; using Tgstation.Server.Host.Components.Deployment; +using Tgstation.Server.Host.System; using Tgstation.Server.Host.Utils; #nullable disable @@ -11,7 +15,7 @@ using Tgstation.Server.Host.Utils; namespace Tgstation.Server.Host.Components.Engine { /// - sealed class EngineExecutableLock : ReferenceCounter, IEngineExecutableLock + class EngineExecutableLock : ReferenceCounter, IEngineExecutableLock { /// public EngineVersion Version => Instance.Version; @@ -51,5 +55,14 @@ namespace Tgstation.Server.Host.Components.Engine /// public string FormatCompilerArguments(string dmePath) => Instance.FormatCompilerArguments(dmePath); + + /// + public ValueTask StopServerProcess(ILogger logger, IProcess process, string accessIdentifier, ushort port, CancellationToken cancellationToken) + => Instance.StopServerProcess( + logger, + process, + accessIdentifier, + port, + cancellationToken); } } diff --git a/src/Tgstation.Server.Host/Components/Engine/EngineInstallationBase.cs b/src/Tgstation.Server.Host/Components/Engine/EngineInstallationBase.cs index 2133bdade4..f2da09f345 100644 --- a/src/Tgstation.Server.Host/Components/Engine/EngineInstallationBase.cs +++ b/src/Tgstation.Server.Host/Components/Engine/EngineInstallationBase.cs @@ -1,12 +1,16 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Threading; using System.Threading.Tasks; using System.Web; +using Microsoft.Extensions.Logging; + using Tgstation.Server.Api.Models; using Tgstation.Server.Api.Models.Internal; using Tgstation.Server.Host.Components.Deployment; +using Tgstation.Server.Host.System; #nullable disable @@ -58,6 +62,18 @@ namespace Tgstation.Server.Host.Components.Engine public abstract string FormatCompilerArguments(string dmePath); /// - public abstract string FormatServerArguments(IDmbProvider dmbProvider, IReadOnlyDictionary parameters, DreamDaemonLaunchParameters launchParameters, string logFilePath); + public abstract string FormatServerArguments( + IDmbProvider dmbProvider, + IReadOnlyDictionary parameters, + DreamDaemonLaunchParameters launchParameters, + string logFilePath); + + /// + public virtual async ValueTask StopServerProcess(ILogger logger, IProcess process, string accessIdentifier, ushort port, CancellationToken cancellationToken) + { + logger.LogTrace("Terminating engine server process..."); + process.Terminate(); + await process.Lifetime; + } } } diff --git a/src/Tgstation.Server.Host/Components/Engine/IEngineInstallation.cs b/src/Tgstation.Server.Host/Components/Engine/IEngineInstallation.cs index 861d6ebe90..bd7fa9e180 100644 --- a/src/Tgstation.Server.Host/Components/Engine/IEngineInstallation.cs +++ b/src/Tgstation.Server.Host/Components/Engine/IEngineInstallation.cs @@ -1,9 +1,13 @@ using System.Collections.Generic; +using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + using Tgstation.Server.Api.Models; using Tgstation.Server.Api.Models.Internal; using Tgstation.Server.Host.Components.Deployment; +using Tgstation.Server.Host.System; #nullable disable @@ -53,7 +57,7 @@ namespace Tgstation.Server.Host.Components.Engine /// Return the command line arguments for launching with given . /// /// The . - /// The map of parameter s as a . Should NOT include the of . + /// The map of parameter s as a . MUST include . Should NOT include the of . /// The . /// The full path to the log file, if any. /// The formatted arguments . @@ -69,5 +73,16 @@ namespace Tgstation.Server.Host.Components.Engine /// The full path to the .dme to compile. /// The formatted arguments . string FormatCompilerArguments(string dmePath); + + /// + /// Kills a given engine server . + /// + /// The to write to. + /// The to be terminated. + /// The of the session. + /// The port the server is running on. + /// The for the operation. + /// A representing the running operation. + ValueTask StopServerProcess(ILogger logger, IProcess process, string accessIdentifier, ushort port, CancellationToken cancellationToken); } } diff --git a/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstallation.cs b/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstallation.cs index db3e7a7392..7e1fc27e47 100644 --- a/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstallation.cs +++ b/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstallation.cs @@ -1,11 +1,22 @@ using System; using System.Collections.Generic; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Net.Mime; +using System.Text; +using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + using Tgstation.Server.Api.Models; using Tgstation.Server.Api.Models.Internal; +using Tgstation.Server.Common.Http; using Tgstation.Server.Host.Components.Deployment; +using Tgstation.Server.Host.Components.Interop; using Tgstation.Server.Host.IO; +using Tgstation.Server.Host.System; +using Tgstation.Server.Host.Utils; #nullable disable @@ -42,22 +53,38 @@ namespace Tgstation.Server.Host.Components.Engine /// readonly IIOManager ioManager; + /// + /// The for the . + /// + readonly IAsyncDelayer asyncDelayer; + + /// + /// The for the . + /// + readonly IAbstractHttpClientFactory httpClientFactory; + /// /// Initializes a new instance of the class. /// /// The value of . + /// The value of . + /// The value of . /// The value of . /// The value of . /// The value of . /// The value of . public OpenDreamInstallation( IIOManager ioManager, + IAsyncDelayer asyncDelayer, + IAbstractHttpClientFactory httpClientFactory, string serverExePath, string compilerExePath, Task installationTask, EngineVersion version) { this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); + this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer)); + this.httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); ServerExePath = serverExePath ?? throw new ArgumentNullException(nameof(serverExePath)); CompilerExePath = compilerExePath ?? throw new ArgumentNullException(nameof(compilerExePath)); InstallationTask = installationTask ?? throw new ArgumentNullException(nameof(installationTask)); @@ -78,15 +105,65 @@ namespace Tgstation.Server.Host.Components.Engine ArgumentNullException.ThrowIfNull(parameters); ArgumentNullException.ThrowIfNull(launchParameters); + if (!parameters.TryGetValue(DMApiConstants.ParamAccessIdentifier, out var accessIdentifier)) + throw new ArgumentException($"parameters must have \"{DMApiConstants.ParamAccessIdentifier}\" set!", nameof(parameters)); + var parametersString = EncodeParameters(parameters, launchParameters); var loggingEnabled = logFilePath != null; - var arguments = $"--cvar {(loggingEnabled ? $"log.path=\"{ioManager.GetDirectoryName(logFilePath)}\" --cvar log.format=\"{ioManager.GetFileName(logFilePath)}\"" : "log.enabled=false")} --cvar log.runtimelog=false --cvar net.port={launchParameters.Port.Value} --cvar opendream.topic_port=0 --cvar opendream.world_params=\"{parametersString}\" --cvar opendream.json_path=\"./{dmbProvider.DmbName}\""; + var arguments = $"--cvar {(loggingEnabled ? $"log.path=\"{ioManager.GetDirectoryName(logFilePath)}\" --cvar log.format=\"{ioManager.GetFileName(logFilePath)}\"" : "log.enabled=false")} --cvar watchdog.token={accessIdentifier} --cvar log.runtimelog=false --cvar net.port={launchParameters.Port.Value} --cvar opendream.topic_port=0 --cvar opendream.world_params=\"{parametersString}\" --cvar opendream.json_path=\"./{dmbProvider.DmbName}\""; return arguments; } /// public override string FormatCompilerArguments(string dmePath) => $"--suppress-unimplemented --notices-enabled \"{dmePath ?? throw new ArgumentNullException(nameof(dmePath))}\""; + + /// + public override async ValueTask StopServerProcess( + ILogger logger, + IProcess process, + string accessIdentifier, + ushort port, + CancellationToken cancellationToken) + { + const int MaximumTerminationSeconds = 5; + + logger.LogTrace("Attempting Robust.Server graceful exit (Timeout: {seconds}s)...", MaximumTerminationSeconds); + var timeout = asyncDelayer.Delay(TimeSpan.FromSeconds(MaximumTerminationSeconds), cancellationToken); + var lifetime = process.Lifetime; + + using var httpClient = httpClientFactory.CreateClient(); + using var request = new HttpRequestMessage(); + request.Headers.Add("WatchdogToken", accessIdentifier); + request.RequestUri = new Uri($"http://localhost:{port}/shutdown"); + request.Content = new StringContent( + "{\"Reason\":\"TGS session termination\"}", + Encoding.UTF8, + new MediaTypeHeaderValue(MediaTypeNames.Application.Json)); + request.Method = HttpMethod.Post; + + var responseTask = httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken); + + await Task.WhenAny(timeout, lifetime, responseTask); + if (responseTask.IsCompleted) + { + using var response = await responseTask; + if (response.IsSuccessStatusCode) + { + logger.LogDebug("Robust.Server responded to the shutdown command successfully. Waiting for exit..."); + await Task.WhenAny(timeout, lifetime); + } + } + + if (lifetime.IsCompleted) + { + logger.LogTrace("Robust.Server gracefully exited"); + return; + } + + logger.LogWarning("Robust.Server graceful exit timed out!"); + await base.StopServerProcess(logger, process, accessIdentifier, port, cancellationToken); + } } } diff --git a/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs b/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs index cefdadd504..886b419a3e 100644 --- a/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs +++ b/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs @@ -8,12 +8,14 @@ using Microsoft.Extensions.Options; using Tgstation.Server.Api.Models; using Tgstation.Server.Common.Extensions; +using Tgstation.Server.Common.Http; using Tgstation.Server.Host.Common; using Tgstation.Server.Host.Components.Repository; using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.IO; using Tgstation.Server.Host.Jobs; using Tgstation.Server.Host.System; +using Tgstation.Server.Host.Utils; #nullable disable @@ -52,6 +54,11 @@ namespace Tgstation.Server.Host.Components.Engine /// protected IProcessExecutor ProcessExecutor { get; } + /// + /// The for the . + /// + protected GeneralConfiguration GeneralConfiguration { get; } + /// /// The for the . /// @@ -63,9 +70,14 @@ namespace Tgstation.Server.Host.Components.Engine readonly IRepositoryManager repositoryManager; /// - /// The for the . + /// The for the . /// - protected GeneralConfiguration GeneralConfiguration { get; } + readonly IAsyncDelayer asyncDelayer; + + /// + /// The for the . + /// + readonly IAbstractHttpClientFactory httpClientFactory; /// /// Initializes a new instance of the class. @@ -75,6 +87,8 @@ namespace Tgstation.Server.Host.Components.Engine /// The value of . /// The value of . /// The value of . + /// The value of . + /// The value of . /// The containing value of . public OpenDreamInstaller( IIOManager ioManager, @@ -82,12 +96,16 @@ namespace Tgstation.Server.Host.Components.Engine IPlatformIdentifier platformIdentifier, IProcessExecutor processExecutor, IRepositoryManager repositoryManager, + IAsyncDelayer asyncDelayer, + IAbstractHttpClientFactory httpClientFactory, IOptions generalConfigurationOptions) : base(ioManager, logger) { this.platformIdentifier = platformIdentifier ?? throw new ArgumentNullException(nameof(platformIdentifier)); ProcessExecutor = processExecutor ?? throw new ArgumentNullException(nameof(processExecutor)); this.repositoryManager = repositoryManager ?? throw new ArgumentNullException(nameof(repositoryManager)); + this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer)); + this.httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); GeneralConfiguration = generalConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(generalConfigurationOptions)); } @@ -101,6 +119,8 @@ namespace Tgstation.Server.Host.Components.Engine GetExecutablePaths(path, out var serverExePath, out var compilerExePath); return new OpenDreamInstallation( IOManager, + asyncDelayer, + httpClientFactory, serverExePath, compilerExePath, installationTask, diff --git a/src/Tgstation.Server.Host/Components/Engine/WindowsOpenDreamInstaller.cs b/src/Tgstation.Server.Host/Components/Engine/WindowsOpenDreamInstaller.cs index 3aa19f3e38..2fb12e80de 100644 --- a/src/Tgstation.Server.Host/Components/Engine/WindowsOpenDreamInstaller.cs +++ b/src/Tgstation.Server.Host/Components/Engine/WindowsOpenDreamInstaller.cs @@ -7,11 +7,13 @@ using Microsoft.Extensions.Options; using Tgstation.Server.Api.Models; using Tgstation.Server.Common.Extensions; +using Tgstation.Server.Common.Http; using Tgstation.Server.Host.Components.Repository; using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.IO; using Tgstation.Server.Host.Jobs; using Tgstation.Server.Host.System; +using Tgstation.Server.Host.Utils; #nullable disable @@ -35,6 +37,8 @@ namespace Tgstation.Server.Host.Components.Engine /// The for the . /// The for the . /// The for the . + /// The for the . + /// The for the . /// The of for the . /// The value of . public WindowsOpenDreamInstaller( @@ -43,6 +47,8 @@ namespace Tgstation.Server.Host.Components.Engine IPlatformIdentifier platformIdentifier, IProcessExecutor processExecutor, IRepositoryManager repositoryManager, + IAsyncDelayer asyncDelayer, + IAbstractHttpClientFactory httpClientFactory, IOptions generalConfigurationOptions, IFilesystemLinkFactory linkFactory) : base( @@ -51,6 +57,8 @@ namespace Tgstation.Server.Host.Components.Engine platformIdentifier, processExecutor, repositoryManager, + asyncDelayer, + httpClientFactory, generalConfigurationOptions) { this.linkFactory = linkFactory ?? throw new ArgumentNullException(nameof(linkFactory)); diff --git a/src/Tgstation.Server.Host/Components/Session/SessionController.cs b/src/Tgstation.Server.Host/Components/Session/SessionController.cs index 51f7cdc126..9fbd96cca9 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionController.cs @@ -133,7 +133,7 @@ namespace Tgstation.Server.Host.Components.Session /// /// The for the . /// - readonly IEngineExecutableLock byondLock; + readonly IEngineExecutableLock engineLock; /// /// The for the . @@ -226,7 +226,7 @@ namespace Tgstation.Server.Host.Components.Session /// The value of . /// The owning . /// The value of . - /// The value of . + /// The value of . /// The value of . /// The used to populate . /// The value of . @@ -242,7 +242,7 @@ namespace Tgstation.Server.Host.Components.Session ReattachInformation reattachInformation, Api.Models.Instance metadata, IProcess process, - IEngineExecutableLock byondLock, + IEngineExecutableLock engineLock, Byond.TopicSender.ITopicClient byondTopicSender, IChatTrackingContext chatTrackingContext, IBridgeRegistrar bridgeRegistrar, @@ -259,7 +259,7 @@ namespace Tgstation.Server.Host.Components.Session ReattachInformation = reattachInformation ?? throw new ArgumentNullException(nameof(reattachInformation)); this.metadata = metadata ?? throw new ArgumentNullException(nameof(metadata)); this.process = process ?? throw new ArgumentNullException(nameof(process)); - this.byondLock = byondLock ?? throw new ArgumentNullException(nameof(byondLock)); + this.engineLock = engineLock ?? throw new ArgumentNullException(nameof(engineLock)); this.byondTopicSender = byondTopicSender ?? throw new ArgumentNullException(nameof(byondTopicSender)); this.chatTrackingContext = chatTrackingContext ?? throw new ArgumentNullException(nameof(chatTrackingContext)); ArgumentNullException.ThrowIfNull(bridgeRegistrar); @@ -340,16 +340,21 @@ namespace Tgstation.Server.Host.Components.Session Logger.LogTrace("Disposing..."); reattachTopicCts.Cancel(); - var semaphoreLockTask = TopicSendSemaphore.Lock(CancellationToken.None); // DCT: None available + var cancellationToken = CancellationToken.None; // DCT: None available + var semaphoreLockTask = TopicSendSemaphore.Lock(cancellationToken); if (!released) { - process.Terminate(); - await process.Lifetime; + await engineLock.StopServerProcess( + Logger, + process, + ReattachInformation.AccessIdentifier, + ReattachInformation.Port, + cancellationToken); } await process.DisposeAsync(); - byondLock.Dispose(); + engineLock.Dispose(); bridgeRegistration?.Dispose(); var regularDmbDisposeTask = ReattachInformation.Dmb.DisposeAsync(); var initialDmb = ReattachInformation.InitialDmb; @@ -395,7 +400,7 @@ namespace Tgstation.Server.Host.Components.Session ReattachInformation.Dmb.KeepAlive(); ReattachInformation.InitialDmb?.KeepAlive(); - byondLock.DoNotDeleteThisSession(); + engineLock.DoNotDeleteThisSession(); released = true; return DisposeAsync(); } diff --git a/src/Tgstation.Server.Host/System/IProcess.cs b/src/Tgstation.Server.Host/System/IProcess.cs index af7409ee9d..b0783cbd7e 100644 --- a/src/Tgstation.Server.Host/System/IProcess.cs +++ b/src/Tgstation.Server.Host/System/IProcess.cs @@ -7,7 +7,7 @@ namespace Tgstation.Server.Host.System /// /// Abstraction over a . /// - interface IProcess : IProcessBase, IAsyncDisposable + public interface IProcess : IProcessBase, IAsyncDisposable { /// /// The ' ID. diff --git a/src/Tgstation.Server.Host/System/IProcessBase.cs b/src/Tgstation.Server.Host/System/IProcessBase.cs index 3d1030f645..d7a20f43b7 100644 --- a/src/Tgstation.Server.Host/System/IProcessBase.cs +++ b/src/Tgstation.Server.Host/System/IProcessBase.cs @@ -6,7 +6,7 @@ namespace Tgstation.Server.Host.System /// /// Represents process lifetime. /// - interface IProcessBase + public interface IProcessBase { /// /// The resulting in the exit code of the process or if the process was detached. diff --git a/tests/Tgstation.Server.Host.Tests/Components/Engine/TestOpenDreamInstaller.cs b/tests/Tgstation.Server.Host.Tests/Components/Engine/TestOpenDreamInstaller.cs index 6b8024ddac..30d995f99c 100644 --- a/tests/Tgstation.Server.Host.Tests/Components/Engine/TestOpenDreamInstaller.cs +++ b/tests/Tgstation.Server.Host.Tests/Components/Engine/TestOpenDreamInstaller.cs @@ -9,10 +9,12 @@ using Moq; using Tgstation.Server.Api.Models; using Tgstation.Server.Api.Models.Internal; +using Tgstation.Server.Common.Http; using Tgstation.Server.Host.Components.Repository; using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.IO; using Tgstation.Server.Host.System; +using Tgstation.Server.Host.Utils; namespace Tgstation.Server.Host.Components.Engine.Tests { @@ -69,6 +71,8 @@ namespace Tgstation.Server.Host.Components.Engine.Tests Mock.Of(), Mock.Of(), mockRepositoryManager.Object, + Mock.Of(), + Mock.Of(), mockGeneralConfigOptions.Object); var data = await installer.DownloadVersion( diff --git a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs index a50d4a6504..76bfa30b23 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs @@ -17,6 +17,7 @@ using Tgstation.Server.Api.Models.Request; using Tgstation.Server.Api.Models.Response; using Tgstation.Server.Client; using Tgstation.Server.Client.Components; +using Tgstation.Server.Common.Http; using Tgstation.Server.Host.Components; using Tgstation.Server.Host.Components.Engine; using Tgstation.Server.Host.Components.Events; @@ -25,6 +26,7 @@ using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.IO; using Tgstation.Server.Host.Jobs; using Tgstation.Server.Host.System; +using Tgstation.Server.Host.Utils; namespace Tgstation.Server.Tests.Live.Instance { @@ -116,6 +118,8 @@ namespace Tgstation.Server.Tests.Live.Instance Mock.Of>(), Mock.Of>(), genConfig), + Mock.Of(), + Mock.Of(), mockOptions.Object) : new PlatformIdentifier().IsWindows ? new WindowsByondInstaller( diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index c1778e63fa..4b08e3ae1f 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -1013,7 +1013,7 @@ namespace Tgstation.Server.Tests.Live.Instance Assert.IsNotNull(sessionObj); var session = (ISessionController)sessionObj; - return session.ReattachInformation.Port; + return session.ReattachInformation.TopicPort ?? session.ReattachInformation.Port; } // - Uses instance manager concrete