Merge pull request #1288 from tgstation/1286-GracefulKill

SIGUSR1 graceful shutdowns

build/Version.props

@@ -3,8 +3,8 @@
   <!-- Integration tests will ensure they match across the board -->
   <Import Project="ControlPanelVersion.props" />
   <PropertyGroup>
-    <TgsCoreVersion>4.12.1</TgsCoreVersion>
-    <TgsConfigVersion>3.1.0</TgsConfigVersion>
+    <TgsCoreVersion>4.13.0</TgsCoreVersion>
+    <TgsConfigVersion>4.0.0</TgsConfigVersion>
     <TgsApiVersion>9.0.1</TgsApiVersion>
     <TgsApiLibraryVersion>9.0.0</TgsApiLibraryVersion>
     <TgsClientVersion>10.0.0</TgsClientVersion>

src/Tgstation.Server.Host/.config/dotnet-tools.json

@@ -3,7 +3,7 @@
   "isRoot": true,
   "tools": {
     "dotnet-ef": {
-      "version": "3.1.13",
+      "version": "3.1.16",
       "commands": [
         "dotnet-ef"
       ]

src/Tgstation.Server.Host/Components/Chat/ChatManager.cs

@@ -493,9 +493,12 @@ namespace Tgstation.Server.Host.Components.Chat
 		}
 
 		/// <inheritdoc />
-		public Task HandleRestart(Version updateVersion, CancellationToken cancellationToken)
+		public Task HandleRestart(Version updateVersion, bool graceful, CancellationToken cancellationToken)
 		{
-			var message = updateVersion == null ? "TGS: Restart requested..." : String.Format(CultureInfo.InvariantCulture, "TGS: Updating to version {0}...", updateVersion);
+			var message =
+				updateVersion == null
+				? $"TGS: {(graceful ? "Graceful r" : "R")}estart requested..."
+				: $"TGS: Updating to version {updateVersion}...";
 			List<ulong> wdChannels;
 			lock (mappedChannels) // so it doesn't change while we're using it
 				wdChannels = mappedChannels.Select(x => x.Key).ToList();

src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs

@@ -404,8 +404,23 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		}
 
 		/// <inheritdoc />
-		public async Task HandleRestart(Version updateVersion, CancellationToken cancellationToken)
+		public async Task HandleRestart(Version updateVersion, bool graceful, CancellationToken cancellationToken)
 		{
+			if (graceful)
+			{
+				await Terminate(true, cancellationToken).ConfigureAwait(false);
+
+				if (Status != WatchdogStatus.Offline)
+				{
+					Logger.LogTrace("Waiting for server to gracefully shut down.");
+					await monitorTask.WithToken(cancellationToken).ConfigureAwait(false);
+				}
+				else
+					Logger.LogTrace("Graceful shutdown requested but server is already offline.");
+
+				return;
+			}
+
 			releaseServers = true;
 			if (Status == WatchdogStatus.Online)
 				await Chat.QueueWatchdogMessage("Detaching...", cancellationToken).ConfigureAwait(false);

src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs

@@ -105,7 +105,8 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			}
 			catch
 			{
-				_ = DisposeAsync();
+				// synchronous
+				DisposeAsync().AsTask().Wait();
 				throw;
 			}
 		}

src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs

@@ -51,9 +51,14 @@ namespace Tgstation.Server.Host.Configuration
 		const uint DefaultByondTopicTimeout = 5000;
 
 		/// <summary>
-		/// The default value for <see cref="RestartTimeout"/>.
+		/// The default value for <see cref="RestartTimeoutMinutes"/>.
 		/// </summary>
-		const uint DefaultRestartTimeout = 60000;
+		const uint DefaultRestartTimeoutMinutes = 1;
+
+		/// <summary>
+		/// The default value for <see cref="ShutdownTimeoutMinutes"/>.
+		/// </summary>
+		const uint DefaultShutdownTimeoutMinutes = 300;
 
 		/// <summary>
 		/// The current <see cref="ConfigVersion"/>.
@@ -87,9 +92,14 @@ namespace Tgstation.Server.Host.Configuration
 		public uint ByondTopicTimeout { get; set; } = DefaultByondTopicTimeout;
 
 		/// <summary>
-		/// The timeout milliseconds for restarting the server.
+		/// The timeout minutes for restarting the server.
 		/// </summary>
-		public uint RestartTimeout { get; set; } = DefaultRestartTimeout;
+		public uint RestartTimeoutMinutes { get; set; } = DefaultRestartTimeoutMinutes;
+
+		/// <summary>
+		/// The timeout minutes for gracefully stopping the server.
+		/// </summary>
+		public uint ShutdownTimeoutMinutes { get; set; } = DefaultShutdownTimeoutMinutes;
 
 		/// <summary>
 		/// If the <see cref="Components.Watchdog.BasicWatchdog"/> should be preferred.

src/Tgstation.Server.Host/Core/Application.cs

@@ -126,7 +126,7 @@ namespace Tgstation.Server.Host.Core
 
 			// Set the timeout for IHostedService.StopAsync
 			services.Configure<HostOptions>(
-				opts => opts.ShutdownTimeout = TimeSpan.FromMilliseconds(postSetupServices.GeneralConfiguration.RestartTimeout));
+				opts => opts.ShutdownTimeout = TimeSpan.FromMinutes(postSetupServices.GeneralConfiguration.RestartTimeoutMinutes));
 
 			static LogEventLevel? ConvertSeriLogLevel(LogLevel logLevel) =>
 				logLevel switch
@@ -321,6 +321,8 @@ namespace Tgstation.Server.Host.Core
 				// PosixProcessFeatures also needs a IProcessExecutor for gcore
 				services.AddSingleton(x => new Lazy<IProcessExecutor>(() => x.GetRequiredService<IProcessExecutor>(), true));
 				services.AddSingleton<INetworkPromptReaper, PosixNetworkPromptReaper>();
+
+				services.AddSingleton<IHostedService, PosixSignalHandler>();
 			}
 
 			// configure component/misc services

src/Tgstation.Server.Host/Core/IRestartHandler.cs

@@ -13,8 +13,9 @@ namespace Tgstation.Server.Host.Core
 		/// Handle a restart of the server.
 		/// </summary>
 		/// <param name="updateVersion">The <see cref="Version"/> being updated to, <see langword="null"/> if not being changed.</param>
+		/// <param name="graceful">If <see langword="true"/> the restart handler perform no destructive actions.</param>
 		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
 		/// <returns>A <see cref="Task"/> representing the running operation.</returns>
-		Task HandleRestart(Version updateVersion, CancellationToken cancellationToken);
+		Task HandleRestart(Version updateVersion, bool graceful, CancellationToken cancellationToken);
 	}
 }

src/Tgstation.Server.Host/Core/IServerControl.cs

@@ -43,6 +43,12 @@ namespace Tgstation.Server.Host.Core
 		/// <returns>A <see cref="Task"/> representing the running operation.</returns>
 		Task Restart();
 
+		/// <summary>
+		/// Gracefully shutsdown the <see cref="Host"/>.
+		/// </summary>
+		/// <returns>A <see cref="Task"/> representing the running operation.</returns>
+		Task GracefulShutdown();
+
 		/// <summary>
 		/// Kill the server with a fatal exception.
 		/// </summary>

src/Tgstation.Server.Host/Properties/AssemblyInfo.cs

@@ -1,5 +1,6 @@
 using System.Runtime.CompilerServices;
 
 [assembly: InternalsVisibleTo("Tgstation.Server.Host.Tests")]
+[assembly: InternalsVisibleTo("Tgstation.Server.Host.Tests.Signals")]
 [assembly: InternalsVisibleTo("Tgstation.Server.Tests")]
 [assembly: InternalsVisibleTo("DynamicProxyGenAssembly2")]

src/Tgstation.Server.Host/Server.cs

@@ -79,6 +79,11 @@ namespace Tgstation.Server.Host
 		/// </summary>
 		Exception propagatedException;
 
+		/// <summary>
+		/// If the server is being shut down or restarted.
+		/// </summary>
+		bool shutdownInProgress;
+
 		/// <summary>
 		/// Initializes a new instance of the <see cref="Server"/> class.
 		/// </summary>
@@ -153,9 +158,9 @@ namespace Tgstation.Server.Host
 
 			lock (restartLock)
 			{
-				if (UpdateInProgress || RestartRequested)
+				if (UpdateInProgress || shutdownInProgress)
 				{
-					logger.LogTrace("Aborted due to concurrency conflict!");
+					logger.LogDebug("Aborted update due to concurrency conflict!");
 					return false;
 				}
 
@@ -260,14 +265,14 @@ namespace Tgstation.Server.Host
 			CheckSanity(false);
 
 			lock (restartLock)
-				if (!RestartRequested)
+				if (!shutdownInProgress)
 				{
 					logger.LogTrace("Registering restart handler {0}...", handler);
 					restartHandlers.Add(handler);
 					return new RestartRegistration(() =>
 					{
 						lock (restartLock)
-							if (!RestartRequested)
+							if (!shutdownInProgress)
 								restartHandlers.Remove(handler);
 					});
 				}
@@ -278,6 +283,9 @@ namespace Tgstation.Server.Host
 		/// <inheritdoc />
 		public Task Restart() => Restart(null, null, true);
 
+		/// <inheritdoc />
+		public Task GracefulShutdown() => Restart(null, null, false);
+
 		/// <inheritdoc />
 		public Task Die(Exception exception) => Restart(null, exception, false);
 
@@ -320,28 +328,39 @@ namespace Tgstation.Server.Host
 		{
 			CheckSanity(requireWatchdog);
 
-			logger.LogTrace("Begin Restart...");
+			// if the watchdog isn't required and there's no issue, this is just a graceful shutdown
+			bool isGracefulShutdown = !requireWatchdog && exception == null;
+			logger.LogTrace(
+				"Begin {0}...",
+				isGracefulShutdown
+					? "graceful shutdown"
+					: "restart");
 
 			lock (restartLock)
 			{
-				if ((UpdateInProgress && newVersion == null) || RestartRequested)
+				if ((UpdateInProgress && newVersion == null) || shutdownInProgress)
 				{
-					logger.LogTrace("Aborted due to concurrency conflict!");
+					logger.LogTrace("Aborted restart due to concurrency conflict!");
 					return;
 				}
 
-				RestartRequested = true;
+				shutdownInProgress = true;
+				RestartRequested = !isGracefulShutdown;
 				propagatedException ??= exception;
 			}
 
 			if (exception == null)
 			{
-				logger.LogInformation("Restarting server...");
-				using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(generalConfiguration.RestartTimeout));
+				logger.LogInformation("Stopping server...");
+				using var cts = new CancellationTokenSource(
+					TimeSpan.FromMinutes(
+						isGracefulShutdown
+							? generalConfiguration.ShutdownTimeoutMinutes
+							: generalConfiguration.RestartTimeoutMinutes));
 				var cancellationToken = cts.Token;
 				var eventsTask = Task.WhenAll(
 					restartHandlers.Select(
-						x => x.HandleRestart(newVersion, cancellationToken))
+						x => x.HandleRestart(newVersion, isGracefulShutdown, cancellationToken))
 					.ToList());
 
 				logger.LogTrace("Joining restart handlers...");
@@ -351,9 +370,12 @@ namespace Tgstation.Server.Host
 				}
 				catch (OperationCanceledException ex)
 				{
-					logger.LogError(
-						ex,
-						"Restart timeout hit! Existing DreamDaemon processes will be lost and must be killed manually before being restarted with TGS!");
+					if (isGracefulShutdown)
+						logger.LogWarning(ex, "Graceful shutdown timeout hit! Existing DreamDaemon processes will be terminated!");
+					else
+						logger.LogError(
+							ex,
+							"Restart timeout hit! Existing DreamDaemon processes will be lost and must be killed manually before being restarted with TGS!");
 				}
 				catch (Exception e)
 				{

src/Tgstation.Server.Host/ServerFactory.cs

@@ -103,7 +103,7 @@ namespace Tgstation.Server.Host
 						.UseIISIntegration()
 						.UseApplication(postSetupServices)
 						.SuppressStatusMessages(true)
-						.UseShutdownTimeout(TimeSpan.FromMilliseconds(postSetupServices.GeneralConfiguration.RestartTimeout)));
+						.UseShutdownTimeout(TimeSpan.FromMinutes(postSetupServices.GeneralConfiguration.RestartTimeoutMinutes)));
 
 			if (updatePath != null)
 				hostBuilder.UseContentRoot(

src/Tgstation.Server.Host/Swarm/SwarmService.cs

@@ -177,7 +177,7 @@ namespace Tgstation.Server.Host.Swarm
 		DateTimeOffset? lastControllerHealthCheck;
 
 		/// <summary>
-		/// If <see cref="IRestartHandler.HandleRestart(Version, CancellationToken)"/> was called.
+		/// If <see cref="IRestartHandler.HandleRestart(Version, bool, CancellationToken)"/> was called.
 		/// </summary>
 		bool restarting;
 
@@ -661,7 +661,7 @@ namespace Tgstation.Server.Host.Swarm
 		}
 
 		/// <inheritdoc />
-		public Task HandleRestart(Version updateVersion, CancellationToken cancellationToken)
+		public Task HandleRestart(Version updateVersion, bool graceful, CancellationToken cancellationToken)
 		{
 			restarting = true;
 			return Task.CompletedTask;

src/Tgstation.Server.Host/System/PosixSignalHandler.cs

@@ -0,0 +1,131 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using Mono.Unix;
+using Mono.Unix.Native;
+
+using Tgstation.Server.Host.Core;
+
+namespace Tgstation.Server.Host.System
+{
+	/// <summary>
+	/// Handles POSIX signals.
+	/// </summary>
+	sealed class PosixSignalHandler : IHostedService, IDisposable
+	{
+		/// <summary>
+		/// Check for signals each time this amount of milliseconds pass.
+		/// </summary>
+		const int CheckDelayMs = 250;
+
+		/// <summary>
+		/// The <see cref="IServerControl"/> for the <see cref="PosixSignalHandler"/>.
+		/// </summary>
+		readonly IServerControl serverControl;
+
+		/// <summary>
+		/// The <see cref="IAsyncDelayer"/> for the <see cref="PosixSignalHandler"/>.
+		/// </summary>
+		readonly IAsyncDelayer asyncDelayer;
+
+		/// <summary>
+		/// The <see cref="ILogger"/> for the <see cref="PosixSignalHandler"/>.
+		/// </summary>
+		readonly ILogger<PosixSignalHandler> logger;
+
+		/// <summary>
+		/// The <see cref="CancellationTokenSource"/> used to stop the <see cref="signalCheckerTask"/>.
+		/// </summary>
+		readonly CancellationTokenSource cancellationTokenSource;
+
+		/// <summary>
+		/// The thread used to check the signal. See http://docs.go-mono.com/?link=T%3aMono.Unix.UnixSignal.
+		/// </summary>
+		Task signalCheckerTask;
+
+		/// <summary>
+		/// Initializes a new instance of the <see cref="PosixSignalHandler"/> class.
+		/// </summary>
+		/// <param name="serverControl">The value of <see cref="serverControl"/>.</param>
+		/// <param name="asyncDelayer">The value of <see cref="asyncDelayer"/>.</param>
+		/// <param name="logger">The value of <see cref="logger"/>.</param>
+		public PosixSignalHandler(IServerControl serverControl, IAsyncDelayer asyncDelayer, ILogger<PosixSignalHandler> logger)
+		{
+			this.serverControl = serverControl ?? throw new ArgumentNullException(nameof(serverControl));
+			this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer));
+			this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
+
+			cancellationTokenSource = new CancellationTokenSource();
+		}
+
+		/// <inheritdoc />
+		public void Dispose() => cancellationTokenSource.Dispose();
+
+		/// <inheritdoc />
+		public Task StartAsync(CancellationToken cancellationToken)
+		{
+			if (signalCheckerTask != null)
+				throw new InvalidOperationException("Attempted to start PosixSignalHandler twice!");
+
+			signalCheckerTask = SignalChecker();
+
+			return Task.CompletedTask;
+		}
+
+		/// <inheritdoc />
+		public async Task StopAsync(CancellationToken cancellationToken)
+		{
+			if (signalCheckerTask?.IsCompleted != false)
+				return;
+
+			logger.LogDebug("Stopping SignalCheckerThread...");
+			cancellationTokenSource.Cancel();
+
+			logger.LogTrace("Joining SignalCheckerThread...");
+			await signalCheckerTask.ConfigureAwait(false);
+		}
+
+		/// <summary>
+		/// Thread for listening to signal.
+		/// </summary>
+		/// <returns>A <see cref="Task"/> representing the running operation.</returns>
+		async Task SignalChecker()
+		{
+			try
+			{
+				logger.LogTrace("Started SignalChecker");
+
+				using var unixSignal = new UnixSignal(Signum.SIGUSR1);
+				if (!unixSignal.IsSet)
+				{
+					logger.LogTrace("Waiting for SIGUSR1...");
+					var cancellationToken = cancellationTokenSource.Token;
+					while (!unixSignal.IsSet)
+						await asyncDelayer.Delay(TimeSpan.FromMilliseconds(CheckDelayMs), cancellationToken);
+
+					logger.LogTrace("SIGUSR1 received!");
+				}
+				else
+					logger.LogDebug("SIGUSR1 has already been sent");
+
+				logger.LogTrace("Triggering graceful shutdown...");
+				await serverControl.GracefulShutdown().ConfigureAwait(false);
+			}
+			catch (OperationCanceledException ex)
+			{
+				logger.LogDebug(ex, "SignalChecker cancelled!");
+			}
+			catch (Exception ex)
+			{
+				logger.LogError(ex, "SignalChecker crashed!");
+			}
+			finally
+			{
+				logger.LogTrace("Exiting SignalChecker...");
+			}
+		}
+	}
+}

src/Tgstation.Server.Host/System/Process.cs

@@ -134,24 +134,31 @@ namespace Tgstation.Server.Host.System
 		{
 			if (combinedStringBuilder == null)
 				throw new InvalidOperationException("Output/Error stream reading was not enabled!");
-			await Task.WhenAll(standardOutputTask, standardErrorTask).WithToken(cancellationToken).ConfigureAwait(false);
+			await Task.WhenAll(
+				GetStandardOutput(cancellationToken),
+				GetErrorOutput(cancellationToken))
+				.ConfigureAwait(false);
 			return combinedStringBuilder.ToString().TrimStart(Environment.NewLine.ToCharArray());
 		}
 
 		/// <inheritdoc />
-		public Task<string> GetErrorOutput(CancellationToken cancellationToken)
+		public async Task<string> GetErrorOutput(CancellationToken cancellationToken)
 		{
 			if (standardErrorTask == null)
 				throw new InvalidOperationException("Error stream reading was not enabled!");
-			return standardErrorTask.WithToken(cancellationToken);
+			if (!standardErrorTask.IsCompleted)
+				logger.LogTrace("Waiting for PID {0} to close error stream...", Id);
+			return await standardErrorTask.WithToken(cancellationToken).ConfigureAwait(false);
 		}
 
 		/// <inheritdoc />
-		public Task<string> GetStandardOutput(CancellationToken cancellationToken)
+		public async Task<string> GetStandardOutput(CancellationToken cancellationToken)
 		{
 			if (standardOutputTask == null)
 				throw new InvalidOperationException("Output stream reading was not enabled!");
-			return standardOutputTask.WithToken(cancellationToken);
+			if (!standardOutputTask.IsCompleted)
+				logger.LogTrace("Waiting for PID {0} to close output stream...", Id);
+			return await standardOutputTask.WithToken(cancellationToken).ConfigureAwait(false);
 		}
 
 		/// <inheritdoc />

src/Tgstation.Server.Host/System/ProcessExecutor.cs

@@ -176,13 +176,17 @@ namespace Tgstation.Server.Host.System
 				{
 					combinedStringBuilder = new StringBuilder();
 
-					async Task<string> ConsumeReader(Func<TextReader> readerFunc)
+					async Task<string> ConsumeReader(Func<TextReader> readerFunc, bool isOutputStream)
 					{
 						var stringBuilder = new StringBuilder();
 						string text;
 
 						await processStartTcs.Task.ConfigureAwait(false);
 
+						var pid = handle.Id;
+						var streamType = isOutputStream ? "out" : "err";
+						logger.LogTrace("Starting std{0} read for PID {1}...", streamType, pid);
+
 						var reader = readerFunc();
 						while ((text = await reader.ReadLineAsync().ConfigureAwait(false)) != null)
 						{
@@ -192,18 +196,20 @@ namespace Tgstation.Server.Host.System
 							stringBuilder.Append(text);
 						}
 
+						logger.LogTrace("Finished std{0} read for PID {1}", streamType, pid);
+
 						return stringBuilder.ToString();
 					}
 
 					if (readOutput)
 					{
-						outputTask = ConsumeReader(() => handle.StandardOutput);
+						outputTask = ConsumeReader(() => handle.StandardOutput, true);
 						handle.StartInfo.RedirectStandardOutput = true;
 					}
 
 					if (readError)
 					{
-						errorTask = ConsumeReader(() => handle.StandardError);
+						errorTask = ConsumeReader(() => handle.StandardError, false);
 						handle.StartInfo.RedirectStandardError = true;
 					}
 				}

src/Tgstation.Server.Host/Tgstation.Server.Host.csproj

@@ -66,45 +66,45 @@
     <PackageReference Include="Byond.TopicSender" Version="5.0.0" />
     <PackageReference Include="Cyberboss.AspNetCore.AsyncInitializer" Version="1.2.0" />
     <PackageReference Include="Cyberboss.SmartIrc4net.Standard" Version="0.4.6" />
-    <PackageReference Include="Discord.Net.WebSocket" Version="2.3.1" />
+    <PackageReference Include="Discord.Net.WebSocket" Version="2.4.0" />
     <PackageReference Include="Elastic.CommonSchema.Serilog" Version="1.5.3" />
     <PackageReference Include="GitLabApiClient" Version="1.7.0" />
     <PackageReference Include="LibGit2Sharp" Version="0.27.0-preview-0034" />
-    <PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="3.1.13" />
-    <PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="3.1.13" />
+    <PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="3.1.16" />
+    <PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="3.1.16" />
     <PackageReference Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="5.0.3">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="Microsoft.EntityFrameworkCore" Version="3.1.13" />
-    <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="3.1.13">
+    <PackageReference Include="Microsoft.EntityFrameworkCore" Version="3.1.16" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="3.1.16">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="3.1.13" />
-    <PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer" Version="3.1.13" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="3.1.16" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer" Version="3.1.16" />
     <PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.10.8" />
     <PackageReference Include="Mono.Posix.NETStandard" Version="1.0.0" />
     <PackageReference Include="NetEscapades.Configuration.Yaml" Version="2.1.0" />
     <PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="3.1.11" />
     <PackageReference Include="Octokit" Version="0.50.0" />
-    <PackageReference Include="Pomelo.EntityFrameworkCore.MySql" Version="3.2.4" />
+    <PackageReference Include="Pomelo.EntityFrameworkCore.MySql" Version="3.2.6" />
     <PackageReference Include="Serilog.Extensions.Logging" Version="3.0.1" />
-    <PackageReference Include="Serilog.Sinks.Async" Version="1.4.0" />
+    <PackageReference Include="Serilog.Sinks.Async" Version="1.5.0" />
     <PackageReference Include="Serilog.Sinks.Console" Version="3.1.1" />
     <PackageReference Include="Serilog.Sinks.Elasticsearch" Version="8.4.1" />
-    <PackageReference Include="Serilog.Sinks.File" Version="4.1.0" />
+    <PackageReference Include="Serilog.Sinks.File" Version="5.0.0" />
     <PackageReference Include="StyleCop.Analyzers" Version="1.1.118">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="Swashbuckle.AspNetCore" Version="6.1.1" />
-    <PackageReference Include="Swashbuckle.AspNetCore.Newtonsoft" Version="6.1.1" />
+    <PackageReference Include="Swashbuckle.AspNetCore" Version="6.1.4" />
+    <PackageReference Include="Swashbuckle.AspNetCore.Newtonsoft" Version="6.1.4" />
     <PackageReference Include="System.Data.SqlClient" Version="4.8.2" />
     <PackageReference Include="System.DirectoryServices.AccountManagement" Version="5.0.0" />
-    <PackageReference Include="System.IdentityModel.Tokens.Jwt" Version="6.9.0" />
+    <PackageReference Include="System.IdentityModel.Tokens.Jwt" Version="6.11.1" />
     <PackageReference Include="System.Management" Version="5.0.0" />
-    <PackageReference Include="Z.EntityFramework.Plus.EFCore" Version="3.1.27" />
+    <PackageReference Include="Z.EntityFramework.Plus.EFCore" Version="3.2.2" />
   </ItemGroup>
 
   <ItemGroup>

src/Tgstation.Server.Host/appsettings.yml

@@ -3,7 +3,7 @@ General:
   GitHubAccessToken:
   SetupWizardMode: AutoDetect
   ByondTopicTimeout: 5000
-  RestartTimeout: 60000
+  RestartTimeoutMinutes: 1
   ApiPort: 5000
   UseBasicWatchdog: false
   UserLimit: 100

tests/Tgstation.Server.Api.Tests/Tgstation.Server.Api.Tests.csproj

@@ -8,14 +8,14 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="coverlet.collector" Version="3.0.3">
+    <PackageReference Include="coverlet.collector" Version="3.1.0">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
     <PackageReference Include="Microsoft.AspNetCore.Http" Version="2.2.2" />
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.9.1" />
-    <PackageReference Include="MSTest.TestAdapter" Version="2.2.3" />
-    <PackageReference Include="MSTest.TestFramework" Version="2.2.3" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.10.0" />
+    <PackageReference Include="MSTest.TestAdapter" Version="2.2.5" />
+    <PackageReference Include="MSTest.TestFramework" Version="2.2.5" />
   </ItemGroup>
 
   <ItemGroup>

tests/Tgstation.Server.Client.Tests/Tgstation.Server.Client.Tests.csproj

@@ -8,14 +8,14 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="coverlet.collector" Version="3.0.3">
+    <PackageReference Include="coverlet.collector" Version="3.1.0">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.9.1" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.10.0" />
     <PackageReference Include="Moq" Version="4.16.1" />
-    <PackageReference Include="MSTest.TestAdapter" Version="2.2.3" />
-    <PackageReference Include="MSTest.TestFramework" Version="2.2.3" />
+    <PackageReference Include="MSTest.TestAdapter" Version="2.2.5" />
+    <PackageReference Include="MSTest.TestFramework" Version="2.2.5" />
   </ItemGroup>
 
   <ItemGroup>

tests/Tgstation.Server.Host.Console.Tests/Tgstation.Server.Host.Console.Tests.csproj

@@ -8,14 +8,14 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="coverlet.collector" Version="3.0.3">
+    <PackageReference Include="coverlet.collector" Version="3.1.0">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.9.1" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.10.0" />
     <PackageReference Include="Moq" Version="4.16.1" />
-    <PackageReference Include="MSTest.TestAdapter" Version="2.2.3" />
-    <PackageReference Include="MSTest.TestFramework" Version="2.2.3" />
+    <PackageReference Include="MSTest.TestAdapter" Version="2.2.5" />
+    <PackageReference Include="MSTest.TestFramework" Version="2.2.5" />
   </ItemGroup>
 
   <ItemGroup>

tests/Tgstation.Server.Host.Service.Tests/Tgstation.Server.Host.Service.Tests.csproj

@@ -14,13 +14,13 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="coverlet.collector" Version="3.0.3">
+    <PackageReference Include="coverlet.collector" Version="3.1.0">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
     <PackageReference Include="Moq" Version="4.16.1" />
-    <PackageReference Include="MSTest.TestAdapter" Version="2.2.3" />
-    <PackageReference Include="MSTest.TestFramework" Version="2.2.3" />
+    <PackageReference Include="MSTest.TestAdapter" Version="2.2.5" />
+    <PackageReference Include="MSTest.TestFramework" Version="2.2.5" />
   </ItemGroup>
 
   <ItemGroup>

tests/Tgstation.Server.Host.Tests.Signals/Program.cs

@@ -0,0 +1,56 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+
+using Microsoft.Extensions.Logging;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using Mono.Unix.Native;
+using Moq;
+
+using Tgstation.Server.Host.Core;
+using Tgstation.Server.Host.Extensions;
+using Tgstation.Server.Host.System;
+
+namespace Tgstation.Server.Host.Tests.Signals
+{
+	static class Program
+	{
+		static async Task Main()
+		{
+			var mockServerControl = new Mock<IServerControl>();
+
+			var tcs = new TaskCompletionSource<object>();
+			mockServerControl
+				.Setup(x => x.GracefulShutdown())
+				.Callback(() => tcs.SetResult(null))
+				.Returns(Task.CompletedTask);
+
+			var mockAsyncDelayer = new Mock<IAsyncDelayer>();
+			mockAsyncDelayer.Setup(x => x.Delay(It.IsAny<TimeSpan>(), It.IsAny<CancellationToken>())).Returns(Task.CompletedTask);
+			using var signalHandler = new PosixSignalHandler(mockServerControl.Object, mockAsyncDelayer.Object, Mock.Of<ILogger<PosixSignalHandler>>());
+
+			Assert.IsFalse(tcs.Task.IsCompleted);
+
+			await signalHandler.StartAsync(default);
+
+			Assert.IsFalse(tcs.Task.IsCompleted);
+
+			await Assert.ThrowsExceptionAsync<InvalidOperationException>(() => signalHandler.StartAsync(default));
+
+			Assert.IsFalse(tcs.Task.IsCompleted);
+
+			using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
+			await signalHandler.StopAsync(default).WithToken(cts.Token).ConfigureAwait(false);
+			Assert.IsFalse(tcs.Task.IsCompleted);
+
+			using var signalHandler2 = new PosixSignalHandler(mockServerControl.Object, mockAsyncDelayer.Object, Mock.Of<ILogger<PosixSignalHandler>>());
+			await signalHandler2.StartAsync(default);
+
+			using var cts2 = new CancellationTokenSource(TimeSpan.FromSeconds(20));
+			await tcs.Task.WithToken(cts2.Token);
+
+			using var cts3 = new CancellationTokenSource(TimeSpan.FromSeconds(5));
+			await signalHandler2.StopAsync(default).WithToken(cts3.Token).ConfigureAwait(false);
+		}
+	}
+}

tests/Tgstation.Server.Host.Tests.Signals/Tgstation.Server.Host.Tests.Signals.csproj

@@ -0,0 +1,20 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
+    <OutputType>Exe</OutputType>
+
+    <IsPackable>false</IsPackable>
+    <LangVersion>latest</LangVersion>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Moq" Version="4.16.1" />
+    <PackageReference Include="MSTest.TestFramework" Version="2.2.5" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\src\Tgstation.Server.Host\Tgstation.Server.Host.csproj" />
+  </ItemGroup>
+
+</Project>

tests/Tgstation.Server.Host.Tests/System/TestPosixSignalHandler.cs

@@ -0,0 +1,93 @@
+using System;
+using System.IO;
+using System.Reflection;
+using System.Threading;
+using System.Threading.Tasks;
+
+using Microsoft.Extensions.Logging;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using Moq;
+
+using Tgstation.Server.Host.Core;
+using Tgstation.Server.Host.IO;
+
+namespace Tgstation.Server.Host.System.Tests
+{
+	[TestClass]
+	public sealed class TestPosixSignalHandler
+	{
+		[TestMethod]
+		public void TestConstruction()
+		{
+			Assert.ThrowsException<ArgumentNullException>(() => new PosixSignalHandler(null, null, null));
+
+			var mockServerControl = Mock.Of<IServerControl>();
+			Assert.ThrowsException<ArgumentNullException>(() => new PosixSignalHandler(mockServerControl, null, null));
+
+			var mockAsyncDelayer = Mock.Of<IAsyncDelayer>();
+			Assert.ThrowsException<ArgumentNullException>(() => new PosixSignalHandler(mockServerControl, mockAsyncDelayer, null));
+
+			new PosixSignalHandler(mockServerControl, mockAsyncDelayer, Mock.Of<ILogger<PosixSignalHandler>>()).Dispose();
+		}
+
+		[TestMethod]
+		public async Task TestSignalListening()
+		{
+			if (new PlatformIdentifier().IsWindows)
+				Assert.Inconclusive("POSIX only test.");
+
+			Assert.Inconclusive("This test fucking doesn't work (hangs on stdout/stderr processing). If this functionality breaks I will find you and devise a very temporarily traumatizing torture for you.");
+
+			// `kill`ing the test process results in it hanging, no idea why
+			// we need to run it as a standard dotnet process#if DEBUG
+#if DEBUG
+			const string CurrentConfig = "Debug";
+#else
+			const string CurrentConfig = "Release";
+#endif
+
+			var pathToSignalTestApp = $"{Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)}/../../../../Tgstation.Server.Host.Tests.Signals";
+			var loggerFactory = LoggerFactory.Create(builder =>
+			{
+				builder.AddConsole();
+				builder.SetMinimumLevel(LogLevel.Trace);
+			});
+
+			IProcessExecutor processExecutor = null;
+			processExecutor = new ProcessExecutor(
+				new PosixProcessFeatures(
+					new Lazy<IProcessExecutor>(() => processExecutor),
+					new DefaultIOManager(),
+					loggerFactory.CreateLogger<PosixProcessFeatures>()),
+				loggerFactory.CreateLogger<ProcessExecutor>(),
+				loggerFactory);
+			using var subProc = processExecutor
+				.LaunchProcess(
+					"dotnet",
+					pathToSignalTestApp,
+					$"run -c {CurrentConfig} --no-build",
+					true,
+					true,
+					true);
+
+			await Task.Delay(TimeSpan.FromSeconds(10));
+
+			using var killProc = global::System.Diagnostics.Process.Start("kill", $"-SIGUSR1 {subProc.Id}");
+			killProc.WaitForExit();
+
+			var exitTask = subProc.Lifetime;
+
+			await Task.WhenAny(exitTask, Task.Delay(TimeSpan.FromSeconds(10)));
+
+			if (!exitTask.IsCompleted)
+				subProc.Terminate();
+
+			var exitCode = await exitTask;
+
+			using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10));
+			global::System.Console.WriteLine(await subProc.GetCombinedOutput(cts.Token));
+
+			Assert.AreEqual(0, exitCode);
+		}
+	}
+}

tests/Tgstation.Server.Host.Tests/Tgstation.Server.Host.Tests.csproj

@@ -8,14 +8,14 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="coverlet.collector" Version="3.0.3">
+    <PackageReference Include="coverlet.collector" Version="3.1.0">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.9.1" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.10.0" />
     <PackageReference Include="Moq" Version="4.16.1" />
-    <PackageReference Include="MSTest.TestAdapter" Version="2.2.3" />
-    <PackageReference Include="MSTest.TestFramework" Version="2.2.3" />
+    <PackageReference Include="MSTest.TestAdapter" Version="2.2.5" />
+    <PackageReference Include="MSTest.TestFramework" Version="2.2.5" />
   </ItemGroup>
 
   <ItemGroup>

tests/Tgstation.Server.Host.Watchdog.Tests/Tgstation.Server.Host.Watchdog.Tests.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <TargetFramework>netcoreapp3.1</TargetFramework>
@@ -14,14 +14,14 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="coverlet.collector" Version="3.0.3">
+    <PackageReference Include="coverlet.collector" Version="3.1.0">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.9.1" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.10.0" />
     <PackageReference Include="Moq" Version="4.16.1" />
-    <PackageReference Include="MSTest.TestAdapter" Version="2.2.3" />
-    <PackageReference Include="MSTest.TestFramework" Version="2.2.3" />
+    <PackageReference Include="MSTest.TestAdapter" Version="2.2.5" />
+    <PackageReference Include="MSTest.TestFramework" Version="2.2.5" />
   </ItemGroup>
 
   <ItemGroup>

tests/Tgstation.Server.Tests/IntegrationTest.cs

@@ -115,6 +115,7 @@ namespace Tgstation.Server.Tests
 						Assert.Inconclusive(notSupportedException.Message);
 				}
 			}
+
 			Assert.IsTrue(server.RestartRequested, "Server not requesting restart!");
 		}
 

tests/Tgstation.Server.Tests/Tgstation.Server.Tests.csproj

@@ -8,14 +8,14 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="coverlet.collector" Version="3.0.3">
+    <PackageReference Include="coverlet.collector" Version="3.1.0">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.9.1" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.10.0" />
     <PackageReference Include="Moq" Version="4.16.1" />
-    <PackageReference Include="MSTest.TestAdapter" Version="2.2.3" />
-    <PackageReference Include="MSTest.TestFramework" Version="2.2.3" />
+    <PackageReference Include="MSTest.TestAdapter" Version="2.2.5" />
+    <PackageReference Include="MSTest.TestFramework" Version="2.2.5" />
   </ItemGroup>
 
   <ItemGroup>

tgstation-server.sln

@@ -48,6 +48,9 @@ EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tgstation.Server.Client.Tests", "tests\Tgstation.Server.Client.Tests\Tgstation.Server.Client.Tests.csproj", "{E5301AF1-4F74-4982-BF24-95F23CC5D5B2}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tgstation.Server.Host.Tests", "tests\Tgstation.Server.Host.Tests\Tgstation.Server.Host.Tests.csproj", "{A3362FF6-550F-480F-859E-8EC1EB6EAB31}"
+	ProjectSection(ProjectDependencies) = postProject
+		{5813CC33-B16C-485D-A74D-20204DDF6542} = {5813CC33-B16C-485D-A74D-20204DDF6542}
+	EndProjectSection
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tgstation.Server.Host.Watchdog", "src\Tgstation.Server.Host.Watchdog\Tgstation.Server.Host.Watchdog.csproj", "{5D2D682C-6BF0-439C-850B-6AB945BBEAEA}"
 	ProjectSection(ProjectDependencies) = postProject
@@ -180,6 +183,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "ISSUE_TEMPLATE", "ISSUE_TEM
 		.github\ISSUE_TEMPLATE\feature_request.md = .github\ISSUE_TEMPLATE\feature_request.md
 	EndProjectSection
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tgstation.Server.Host.Tests.Signals", "tests\Tgstation.Server.Host.Tests.Signals\Tgstation.Server.Host.Tests.Signals.csproj", "{5813CC33-B16C-485D-A74D-20204DDF6542}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -296,6 +301,14 @@ Global
 		{5CB51532-55F0-4255-B6E5-69ED5CCD14CD}.Release|Any CPU.Build.0 = Release|Any CPU
 		{5CB51532-55F0-4255-B6E5-69ED5CCD14CD}.ReleaseNoService|Any CPU.ActiveCfg = Release|Any CPU
 		{5CB51532-55F0-4255-B6E5-69ED5CCD14CD}.ReleaseNoService|Any CPU.Build.0 = Release|Any CPU
+		{5813CC33-B16C-485D-A74D-20204DDF6542}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{5813CC33-B16C-485D-A74D-20204DDF6542}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{5813CC33-B16C-485D-A74D-20204DDF6542}.DebugNoService|Any CPU.ActiveCfg = Debug|Any CPU
+		{5813CC33-B16C-485D-A74D-20204DDF6542}.DebugNoService|Any CPU.Build.0 = Debug|Any CPU
+		{5813CC33-B16C-485D-A74D-20204DDF6542}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{5813CC33-B16C-485D-A74D-20204DDF6542}.Release|Any CPU.Build.0 = Release|Any CPU
+		{5813CC33-B16C-485D-A74D-20204DDF6542}.ReleaseNoService|Any CPU.ActiveCfg = Release|Any CPU
+		{5813CC33-B16C-485D-A74D-20204DDF6542}.ReleaseNoService|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -322,6 +335,7 @@ Global
 		{103C61AB-67D6-46FE-AA47-CC633B88EE0F} = {82066812-6C73-4360-943B-B23F2F491261}
 		{28CDEB8F-2B2A-47A2-985B-5E2487E8D096} = {E82104F4-F5C4-4786-ACD4-B635166CDB21}
 		{CFFD7992-E73A-4D1F-9D7A-C817C07B7BEB} = {E82104F4-F5C4-4786-ACD4-B635166CDB21}
+		{5813CC33-B16C-485D-A74D-20204DDF6542} = {316141B0-CD21-4769-A013-D53DA9B9EC09}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {DFD36C95-3E49-41C7-ACDB-86BAF5B18A79}

tools/ReleaseNotes/ReleaseNotes.csproj

@@ -7,7 +7,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="octokit" Version="0.48.0" />
+    <PackageReference Include="octokit" Version="0.50.0" />
   </ItemGroup>
 
 </Project>