Support graceful OpenDream shutdowns

- SessionController now uses `IEngineExectuableLock` to stop servers.
  - Kills servers by default. Uses the RobustToolbox WatchdogApi for OpenDream.
This commit is contained in:
Jordan Dominion
2023-12-20 14:00:24 -05:00
parent 9711a787a8
commit 0883874186
12 changed files with 180 additions and 18 deletions
@@ -1,9 +1,13 @@
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Tgstation.Server.Api.Models;
using Tgstation.Server.Api.Models.Internal;
using Tgstation.Server.Host.Components.Deployment;
using Tgstation.Server.Host.System;
using Tgstation.Server.Host.Utils;
#nullable disable
@@ -11,7 +15,7 @@ using Tgstation.Server.Host.Utils;
namespace Tgstation.Server.Host.Components.Engine
{
/// <inheritdoc cref="IEngineExecutableLock" />
sealed class EngineExecutableLock : ReferenceCounter<IEngineInstallation>, IEngineExecutableLock
class EngineExecutableLock : ReferenceCounter<IEngineInstallation>, IEngineExecutableLock
{
/// <inheritdoc />
public EngineVersion Version => Instance.Version;
@@ -51,5 +55,14 @@ namespace Tgstation.Server.Host.Components.Engine
/// <inheritdoc />
public string FormatCompilerArguments(string dmePath) => Instance.FormatCompilerArguments(dmePath);
/// <inheritdoc />
public ValueTask StopServerProcess(ILogger logger, IProcess process, string accessIdentifier, ushort port, CancellationToken cancellationToken)
=> Instance.StopServerProcess(
logger,
process,
accessIdentifier,
port,
cancellationToken);
}
}
@@ -1,12 +1,16 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using System.Web;
using Microsoft.Extensions.Logging;
using Tgstation.Server.Api.Models;
using Tgstation.Server.Api.Models.Internal;
using Tgstation.Server.Host.Components.Deployment;
using Tgstation.Server.Host.System;
#nullable disable
@@ -58,6 +62,18 @@ namespace Tgstation.Server.Host.Components.Engine
public abstract string FormatCompilerArguments(string dmePath);
/// <inheritdoc />
public abstract string FormatServerArguments(IDmbProvider dmbProvider, IReadOnlyDictionary<string, string> parameters, DreamDaemonLaunchParameters launchParameters, string logFilePath);
public abstract string FormatServerArguments(
IDmbProvider dmbProvider,
IReadOnlyDictionary<string, string> parameters,
DreamDaemonLaunchParameters launchParameters,
string logFilePath);
/// <inheritdoc />
public virtual async ValueTask StopServerProcess(ILogger logger, IProcess process, string accessIdentifier, ushort port, CancellationToken cancellationToken)
{
logger.LogTrace("Terminating engine server process...");
process.Terminate();
await process.Lifetime;
}
}
}
@@ -1,9 +1,13 @@
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Tgstation.Server.Api.Models;
using Tgstation.Server.Api.Models.Internal;
using Tgstation.Server.Host.Components.Deployment;
using Tgstation.Server.Host.System;
#nullable disable
@@ -53,7 +57,7 @@ namespace Tgstation.Server.Host.Components.Engine
/// Return the command line arguments for launching with given <paramref name="launchParameters"/>.
/// </summary>
/// <param name="dmbProvider">The <see cref="IDmbProvider"/>.</param>
/// <param name="parameters">The map of parameter <see cref="string"/>s as a <see cref="IReadOnlyDictionary{TKey, TValue}"/>. Should NOT include the <see cref="DreamDaemonLaunchParameters.AdditionalParameters"/> of <paramref name="launchParameters"/>.</param>
/// <param name="parameters">The map of parameter <see cref="string"/>s as a <see cref="IReadOnlyDictionary{TKey, TValue}"/>. MUST include <see cref="Interop.DMApiConstants.ParamAccessIdentifier"/>. Should NOT include the <see cref="DreamDaemonLaunchParameters.AdditionalParameters"/> of <paramref name="launchParameters"/>.</param>
/// <param name="launchParameters">The <see cref="DreamDaemonLaunchParameters"/>.</param>
/// <param name="logFilePath">The full path to the log file, if any.</param>
/// <returns>The formatted arguments <see cref="string"/>.</returns>
@@ -69,5 +73,16 @@ namespace Tgstation.Server.Host.Components.Engine
/// <param name="dmePath">The full path to the .dme to compile.</param>
/// <returns>The formatted arguments <see cref="string"/>.</returns>
string FormatCompilerArguments(string dmePath);
/// <summary>
/// Kills a given engine server <paramref name="process"/>.
/// </summary>
/// <param name="logger">The <see cref="ILogger"/> to write to.</param>
/// <param name="process">The <see cref="IProcess"/> to be terminated.</param>
/// <param name="accessIdentifier">The <see cref="Interop.DMApiParameters.AccessIdentifier"/> of the session.</param>
/// <param name="port">The port the server is running on.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
/// <returns>A <see cref="ValueTask"/> representing the running operation.</returns>
ValueTask StopServerProcess(ILogger logger, IProcess process, string accessIdentifier, ushort port, CancellationToken cancellationToken);
}
}
@@ -1,11 +1,22 @@
using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Net.Mime;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Tgstation.Server.Api.Models;
using Tgstation.Server.Api.Models.Internal;
using Tgstation.Server.Common.Http;
using Tgstation.Server.Host.Components.Deployment;
using Tgstation.Server.Host.Components.Interop;
using Tgstation.Server.Host.IO;
using Tgstation.Server.Host.System;
using Tgstation.Server.Host.Utils;
#nullable disable
@@ -42,22 +53,38 @@ namespace Tgstation.Server.Host.Components.Engine
/// </summary>
readonly IIOManager ioManager;
/// <summary>
/// The <see cref="IAsyncDelayer"/> for the <see cref="OpenDreamInstallation"/>.
/// </summary>
readonly IAsyncDelayer asyncDelayer;
/// <summary>
/// The <see cref="IAbstractHttpClientFactory"/> for the <see cref="OpenDreamInstallation"/>.
/// </summary>
readonly IAbstractHttpClientFactory httpClientFactory;
/// <summary>
/// Initializes a new instance of the <see cref="OpenDreamInstallation"/> class.
/// </summary>
/// <param name="ioManager">The value of <see cref="ioManager"/>.</param>
/// <param name="asyncDelayer">The value of <see cref="asyncDelayer"/>.</param>
/// <param name="httpClientFactory">The value of <see cref="httpClientFactory"/>.</param>
/// <param name="serverExePath">The value of <see cref="ServerExePath"/>.</param>
/// <param name="compilerExePath">The value of <see cref="CompilerExePath"/>.</param>
/// <param name="installationTask">The value of <see cref="InstallationTask"/>.</param>
/// <param name="version">The value of <see cref="Version"/>.</param>
public OpenDreamInstallation(
IIOManager ioManager,
IAsyncDelayer asyncDelayer,
IAbstractHttpClientFactory httpClientFactory,
string serverExePath,
string compilerExePath,
Task installationTask,
EngineVersion version)
{
this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager));
this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer));
this.httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
ServerExePath = serverExePath ?? throw new ArgumentNullException(nameof(serverExePath));
CompilerExePath = compilerExePath ?? throw new ArgumentNullException(nameof(compilerExePath));
InstallationTask = installationTask ?? throw new ArgumentNullException(nameof(installationTask));
@@ -78,15 +105,65 @@ namespace Tgstation.Server.Host.Components.Engine
ArgumentNullException.ThrowIfNull(parameters);
ArgumentNullException.ThrowIfNull(launchParameters);
if (!parameters.TryGetValue(DMApiConstants.ParamAccessIdentifier, out var accessIdentifier))
throw new ArgumentException($"parameters must have \"{DMApiConstants.ParamAccessIdentifier}\" set!", nameof(parameters));
var parametersString = EncodeParameters(parameters, launchParameters);
var loggingEnabled = logFilePath != null;
var arguments = $"--cvar {(loggingEnabled ? $"log.path=\"{ioManager.GetDirectoryName(logFilePath)}\" --cvar log.format=\"{ioManager.GetFileName(logFilePath)}\"" : "log.enabled=false")} --cvar log.runtimelog=false --cvar net.port={launchParameters.Port.Value} --cvar opendream.topic_port=0 --cvar opendream.world_params=\"{parametersString}\" --cvar opendream.json_path=\"./{dmbProvider.DmbName}\"";
var arguments = $"--cvar {(loggingEnabled ? $"log.path=\"{ioManager.GetDirectoryName(logFilePath)}\" --cvar log.format=\"{ioManager.GetFileName(logFilePath)}\"" : "log.enabled=false")} --cvar watchdog.token={accessIdentifier} --cvar log.runtimelog=false --cvar net.port={launchParameters.Port.Value} --cvar opendream.topic_port=0 --cvar opendream.world_params=\"{parametersString}\" --cvar opendream.json_path=\"./{dmbProvider.DmbName}\"";
return arguments;
}
/// <inheritdoc />
public override string FormatCompilerArguments(string dmePath)
=> $"--suppress-unimplemented --notices-enabled \"{dmePath ?? throw new ArgumentNullException(nameof(dmePath))}\"";
/// <inheritdoc />
public override async ValueTask StopServerProcess(
ILogger logger,
IProcess process,
string accessIdentifier,
ushort port,
CancellationToken cancellationToken)
{
const int MaximumTerminationSeconds = 5;
logger.LogTrace("Attempting Robust.Server graceful exit (Timeout: {seconds}s)...", MaximumTerminationSeconds);
var timeout = asyncDelayer.Delay(TimeSpan.FromSeconds(MaximumTerminationSeconds), cancellationToken);
var lifetime = process.Lifetime;
using var httpClient = httpClientFactory.CreateClient();
using var request = new HttpRequestMessage();
request.Headers.Add("WatchdogToken", accessIdentifier);
request.RequestUri = new Uri($"http://localhost:{port}/shutdown");
request.Content = new StringContent(
"{\"Reason\":\"TGS session termination\"}",
Encoding.UTF8,
new MediaTypeHeaderValue(MediaTypeNames.Application.Json));
request.Method = HttpMethod.Post;
var responseTask = httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
await Task.WhenAny(timeout, lifetime, responseTask);
if (responseTask.IsCompleted)
{
using var response = await responseTask;
if (response.IsSuccessStatusCode)
{
logger.LogDebug("Robust.Server responded to the shutdown command successfully. Waiting for exit...");
await Task.WhenAny(timeout, lifetime);
}
}
if (lifetime.IsCompleted)
{
logger.LogTrace("Robust.Server gracefully exited");
return;
}
logger.LogWarning("Robust.Server graceful exit timed out!");
await base.StopServerProcess(logger, process, accessIdentifier, port, cancellationToken);
}
}
}
@@ -8,12 +8,14 @@ using Microsoft.Extensions.Options;
using Tgstation.Server.Api.Models;
using Tgstation.Server.Common.Extensions;
using Tgstation.Server.Common.Http;
using Tgstation.Server.Host.Common;
using Tgstation.Server.Host.Components.Repository;
using Tgstation.Server.Host.Configuration;
using Tgstation.Server.Host.IO;
using Tgstation.Server.Host.Jobs;
using Tgstation.Server.Host.System;
using Tgstation.Server.Host.Utils;
#nullable disable
@@ -52,6 +54,11 @@ namespace Tgstation.Server.Host.Components.Engine
/// </summary>
protected IProcessExecutor ProcessExecutor { get; }
/// <summary>
/// The <see cref="GeneralConfiguration"/> for the <see cref="OpenDreamInstaller"/>.
/// </summary>
protected GeneralConfiguration GeneralConfiguration { get; }
/// <summary>
/// The <see cref="IPlatformIdentifier"/> for the <see cref="OpenDreamInstaller"/>.
/// </summary>
@@ -63,9 +70,14 @@ namespace Tgstation.Server.Host.Components.Engine
readonly IRepositoryManager repositoryManager;
/// <summary>
/// The <see cref="GeneralConfiguration"/> for the <see cref="OpenDreamInstaller"/>.
/// The <see cref="IAsyncDelayer"/> for the <see cref="OpenDreamInstaller"/>.
/// </summary>
protected GeneralConfiguration GeneralConfiguration { get; }
readonly IAsyncDelayer asyncDelayer;
/// <summary>
/// The <see cref="IAbstractHttpClientFactory"/> for the <see cref="OpenDreamInstaller"/>.
/// </summary>
readonly IAbstractHttpClientFactory httpClientFactory;
/// <summary>
/// Initializes a new instance of the <see cref="OpenDreamInstaller"/> class.
@@ -75,6 +87,8 @@ namespace Tgstation.Server.Host.Components.Engine
/// <param name="platformIdentifier">The value of <see cref="platformIdentifier"/>.</param>
/// <param name="processExecutor">The value of <see cref="ProcessExecutor"/>.</param>
/// <param name="repositoryManager">The value of <see cref="repositoryManager"/>.</param>
/// <param name="asyncDelayer">The value of <see cref="asyncDelayer"/>.</param>
/// <param name="httpClientFactory">The value of <see cref="httpClientFactory"/>.</param>
/// <param name="generalConfigurationOptions">The <see cref="IOptions{TOptions}"/> containing value of <see cref="GeneralConfiguration"/>.</param>
public OpenDreamInstaller(
IIOManager ioManager,
@@ -82,12 +96,16 @@ namespace Tgstation.Server.Host.Components.Engine
IPlatformIdentifier platformIdentifier,
IProcessExecutor processExecutor,
IRepositoryManager repositoryManager,
IAsyncDelayer asyncDelayer,
IAbstractHttpClientFactory httpClientFactory,
IOptions<GeneralConfiguration> generalConfigurationOptions)
: base(ioManager, logger)
{
this.platformIdentifier = platformIdentifier ?? throw new ArgumentNullException(nameof(platformIdentifier));
ProcessExecutor = processExecutor ?? throw new ArgumentNullException(nameof(processExecutor));
this.repositoryManager = repositoryManager ?? throw new ArgumentNullException(nameof(repositoryManager));
this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer));
this.httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
GeneralConfiguration = generalConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(generalConfigurationOptions));
}
@@ -101,6 +119,8 @@ namespace Tgstation.Server.Host.Components.Engine
GetExecutablePaths(path, out var serverExePath, out var compilerExePath);
return new OpenDreamInstallation(
IOManager,
asyncDelayer,
httpClientFactory,
serverExePath,
compilerExePath,
installationTask,
@@ -7,11 +7,13 @@ using Microsoft.Extensions.Options;
using Tgstation.Server.Api.Models;
using Tgstation.Server.Common.Extensions;
using Tgstation.Server.Common.Http;
using Tgstation.Server.Host.Components.Repository;
using Tgstation.Server.Host.Configuration;
using Tgstation.Server.Host.IO;
using Tgstation.Server.Host.Jobs;
using Tgstation.Server.Host.System;
using Tgstation.Server.Host.Utils;
#nullable disable
@@ -35,6 +37,8 @@ namespace Tgstation.Server.Host.Components.Engine
/// <param name="platformIdentifier">The <see cref="IPlatformIdentifier"/> for the <see cref="OpenDreamInstaller"/>.</param>
/// <param name="processExecutor">The <see cref="IProcessExecutor"/> for the <see cref="OpenDreamInstaller"/>.</param>
/// <param name="repositoryManager">The <see cref="IRepositoryManager"/> for the <see cref="OpenDreamInstaller"/>.</param>
/// <param name="asyncDelayer">The <see cref="IAsyncDelayer"/> for the <see cref="OpenDreamInstaller"/>.</param>
/// <param name="httpClientFactory">The <see cref="IAbstractHttpClientFactory"/> for the <see cref="OpenDreamInstaller"/>.</param>
/// <param name="generalConfigurationOptions">The <see cref="IOptions{TOptions}"/> of <see cref="GeneralConfiguration"/> for the <see cref="OpenDreamInstaller"/>.</param>
/// <param name="linkFactory">The value of <see cref="linkFactory"/>.</param>
public WindowsOpenDreamInstaller(
@@ -43,6 +47,8 @@ namespace Tgstation.Server.Host.Components.Engine
IPlatformIdentifier platformIdentifier,
IProcessExecutor processExecutor,
IRepositoryManager repositoryManager,
IAsyncDelayer asyncDelayer,
IAbstractHttpClientFactory httpClientFactory,
IOptions<GeneralConfiguration> generalConfigurationOptions,
IFilesystemLinkFactory linkFactory)
: base(
@@ -51,6 +57,8 @@ namespace Tgstation.Server.Host.Components.Engine
platformIdentifier,
processExecutor,
repositoryManager,
asyncDelayer,
httpClientFactory,
generalConfigurationOptions)
{
this.linkFactory = linkFactory ?? throw new ArgumentNullException(nameof(linkFactory));
@@ -133,7 +133,7 @@ namespace Tgstation.Server.Host.Components.Session
/// <summary>
/// The <see cref="IEngineExecutableLock"/> for the <see cref="SessionController"/>.
/// </summary>
readonly IEngineExecutableLock byondLock;
readonly IEngineExecutableLock engineLock;
/// <summary>
/// The <see cref="IChatTrackingContext"/> for the <see cref="SessionController"/>.
@@ -226,7 +226,7 @@ namespace Tgstation.Server.Host.Components.Session
/// <param name="reattachInformation">The value of <see cref="ReattachInformation"/>.</param>
/// <param name="metadata">The owning <see cref="Instance"/>.</param>
/// <param name="process">The value of <see cref="process"/>.</param>
/// <param name="byondLock">The value of <see cref="byondLock"/>.</param>
/// <param name="engineLock">The value of <see cref="engineLock"/>.</param>
/// <param name="byondTopicSender">The value of <see cref="byondTopicSender"/>.</param>
/// <param name="bridgeRegistrar">The <see cref="IBridgeRegistrar"/> used to populate <see cref="bridgeRegistration"/>.</param>
/// <param name="chat">The value of <see cref="chat"/>.</param>
@@ -242,7 +242,7 @@ namespace Tgstation.Server.Host.Components.Session
ReattachInformation reattachInformation,
Api.Models.Instance metadata,
IProcess process,
IEngineExecutableLock byondLock,
IEngineExecutableLock engineLock,
Byond.TopicSender.ITopicClient byondTopicSender,
IChatTrackingContext chatTrackingContext,
IBridgeRegistrar bridgeRegistrar,
@@ -259,7 +259,7 @@ namespace Tgstation.Server.Host.Components.Session
ReattachInformation = reattachInformation ?? throw new ArgumentNullException(nameof(reattachInformation));
this.metadata = metadata ?? throw new ArgumentNullException(nameof(metadata));
this.process = process ?? throw new ArgumentNullException(nameof(process));
this.byondLock = byondLock ?? throw new ArgumentNullException(nameof(byondLock));
this.engineLock = engineLock ?? throw new ArgumentNullException(nameof(engineLock));
this.byondTopicSender = byondTopicSender ?? throw new ArgumentNullException(nameof(byondTopicSender));
this.chatTrackingContext = chatTrackingContext ?? throw new ArgumentNullException(nameof(chatTrackingContext));
ArgumentNullException.ThrowIfNull(bridgeRegistrar);
@@ -340,16 +340,21 @@ namespace Tgstation.Server.Host.Components.Session
Logger.LogTrace("Disposing...");
reattachTopicCts.Cancel();
var semaphoreLockTask = TopicSendSemaphore.Lock(CancellationToken.None); // DCT: None available
var cancellationToken = CancellationToken.None; // DCT: None available
var semaphoreLockTask = TopicSendSemaphore.Lock(cancellationToken);
if (!released)
{
process.Terminate();
await process.Lifetime;
await engineLock.StopServerProcess(
Logger,
process,
ReattachInformation.AccessIdentifier,
ReattachInformation.Port,
cancellationToken);
}
await process.DisposeAsync();
byondLock.Dispose();
engineLock.Dispose();
bridgeRegistration?.Dispose();
var regularDmbDisposeTask = ReattachInformation.Dmb.DisposeAsync();
var initialDmb = ReattachInformation.InitialDmb;
@@ -395,7 +400,7 @@ namespace Tgstation.Server.Host.Components.Session
ReattachInformation.Dmb.KeepAlive();
ReattachInformation.InitialDmb?.KeepAlive();
byondLock.DoNotDeleteThisSession();
engineLock.DoNotDeleteThisSession();
released = true;
return DisposeAsync();
}
+1 -1
View File
@@ -7,7 +7,7 @@ namespace Tgstation.Server.Host.System
/// <summary>
/// Abstraction over a <see cref="global::System.Diagnostics.Process"/>.
/// </summary>
interface IProcess : IProcessBase, IAsyncDisposable
public interface IProcess : IProcessBase, IAsyncDisposable
{
/// <summary>
/// The <see cref="IProcess"/>' ID.
@@ -6,7 +6,7 @@ namespace Tgstation.Server.Host.System
/// <summary>
/// Represents process lifetime.
/// </summary>
interface IProcessBase
public interface IProcessBase
{
/// <summary>
/// The <see cref="Task{TResult}"/> resulting in the exit code of the process or <see langword="null"/> if the process was detached.
@@ -9,10 +9,12 @@ using Moq;
using Tgstation.Server.Api.Models;
using Tgstation.Server.Api.Models.Internal;
using Tgstation.Server.Common.Http;
using Tgstation.Server.Host.Components.Repository;
using Tgstation.Server.Host.Configuration;
using Tgstation.Server.Host.IO;
using Tgstation.Server.Host.System;
using Tgstation.Server.Host.Utils;
namespace Tgstation.Server.Host.Components.Engine.Tests
{
@@ -69,6 +71,8 @@ namespace Tgstation.Server.Host.Components.Engine.Tests
Mock.Of<IPlatformIdentifier>(),
Mock.Of<IProcessExecutor>(),
mockRepositoryManager.Object,
Mock.Of<IAsyncDelayer>(),
Mock.Of<IAbstractHttpClientFactory>(),
mockGeneralConfigOptions.Object);
var data = await installer.DownloadVersion(
@@ -17,6 +17,7 @@ using Tgstation.Server.Api.Models.Request;
using Tgstation.Server.Api.Models.Response;
using Tgstation.Server.Client;
using Tgstation.Server.Client.Components;
using Tgstation.Server.Common.Http;
using Tgstation.Server.Host.Components;
using Tgstation.Server.Host.Components.Engine;
using Tgstation.Server.Host.Components.Events;
@@ -25,6 +26,7 @@ using Tgstation.Server.Host.Configuration;
using Tgstation.Server.Host.IO;
using Tgstation.Server.Host.Jobs;
using Tgstation.Server.Host.System;
using Tgstation.Server.Host.Utils;
namespace Tgstation.Server.Tests.Live.Instance
{
@@ -116,6 +118,8 @@ namespace Tgstation.Server.Tests.Live.Instance
Mock.Of<ILogger<Repository>>(),
Mock.Of<ILogger<RepositoryManager>>(),
genConfig),
Mock.Of<IAsyncDelayer>(),
Mock.Of<IAbstractHttpClientFactory>(),
mockOptions.Object)
: new PlatformIdentifier().IsWindows
? new WindowsByondInstaller(
@@ -1013,7 +1013,7 @@ namespace Tgstation.Server.Tests.Live.Instance
Assert.IsNotNull(sessionObj);
var session = (ISessionController)sessionObj;
return session.ReattachInformation.Port;
return session.ReattachInformation.TopicPort ?? session.ReattachInformation.Port;
}
// - Uses instance manager concrete