diff --git a/build/package/deb/debian/control b/build/package/deb/debian/control index dac12b8594..727f18b433 100644 --- a/build/package/deb/debian/control +++ b/build/package/deb/debian/control @@ -21,6 +21,7 @@ Depends: libstdc++6:i386 [amd64], libstdc++6 [i386], gcc-multilib [amd64], + libsystemd, Recommends: gdb, Description: A production scale tool for BYOND server management diff --git a/build/tgstation-server.service b/build/tgstation-server.service index bcd6de155d..21c6809372 100644 --- a/build/tgstation-server.service +++ b/build/tgstation-server.service @@ -7,13 +7,17 @@ After=postgresql.service After=mssql-server.service [Service] +Type=notify +NotifyAccess=all ExecStart=/bin/bash /opt/tgstation-server/tgs.sh General:SetupWizardMode=Never +TimeoutStartSec=600 Restart=Always KillMode=process RestartKillSignal=SIGUSR2 AmbientCapabilities=CAP_SYS_NICE StandardOutput=null StandardError=null +WatchdogSec=60 [Install] WantedBy=multi-user.target diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index b872c4eed6..ef312928a0 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -336,6 +336,9 @@ namespace Tgstation.Server.Host.Core services.AddSingleton(); services.AddSingleton(); + + services.AddSingleton(); + services.AddSingleton(x => x.GetRequiredService()); } // configure file transfer services diff --git a/src/Tgstation.Server.Host/ServerFactory.cs b/src/Tgstation.Server.Host/ServerFactory.cs index 7d34282f77..9dddebff23 100644 --- a/src/Tgstation.Server.Host/ServerFactory.cs +++ b/src/Tgstation.Server.Host/ServerFactory.cs @@ -1,5 +1,4 @@ using System; -using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; diff --git a/src/Tgstation.Server.Host/System/NativeMethods.cs b/src/Tgstation.Server.Host/System/NativeMethods.cs index 60b97ca90c..dacb4351bc 100644 --- a/src/Tgstation.Server.Host/System/NativeMethods.cs +++ b/src/Tgstation.Server.Host/System/NativeMethods.cs @@ -127,5 +127,15 @@ namespace Tgstation.Server.Host.System IntPtr expParam, IntPtr userStreamParam, IntPtr callbackParam); + + /// + /// See https://www.freedesktop.org/software/systemd/man/sd_notify.html. + /// +#pragma warning disable IDE0079 +#pragma warning disable CA2101 // https://github.com/dotnet/roslyn-analyzers/issues/5479#issuecomment-1603665900 + [DllImport("libsystemd.so.0", CharSet = CharSet.Ansi, BestFitMapping = false, ThrowOnUnmappableChar = true)] + public static extern int sd_notify(int unset_environment, [MarshalAs(UnmanagedType.LPUTF8Str)] string state); +#pragma warning restore CA2101 +#pragma warning restore IDE0079 } } diff --git a/src/Tgstation.Server.Host/System/SystemDManager.cs b/src/Tgstation.Server.Host/System/SystemDManager.cs new file mode 100644 index 0000000000..5dd66e259e --- /dev/null +++ b/src/Tgstation.Server.Host/System/SystemDManager.cs @@ -0,0 +1,217 @@ +using System; +using System.Globalization; +using System.Threading; +using System.Threading.Tasks; + +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +using Mono.Unix; + +using Tgstation.Server.Host.Components; +using Tgstation.Server.Host.Core; +using Tgstation.Server.Host.Extensions; + +namespace Tgstation.Server.Host.System +{ + /// + /// Implements the SystemD notify service protocol. + /// + sealed class SystemDManager : IHostedService, IRestartHandler, IDisposable + { + /// + /// The for the . + /// + readonly IHostApplicationLifetime applicationLifetime; + + /// + /// The for the . + /// + readonly IInstanceManager instanceManager; + + /// + /// The for the . + /// + readonly IRestartRegistration restartRegistration; + + /// + /// The for the . + /// + readonly ILogger logger; + + /// + /// The for . + /// + readonly CancellationTokenSource watchdogCts; + + /// + /// The main task executing in the . + /// + Task runTask; + + /// + /// If TGS is going to restart. + /// + bool restartInProgress; + + /// + /// Initializes a new instance of the class. + /// + /// The value of . + /// The value of . + /// The used to create the . + /// The value of . + public SystemDManager( + IHostApplicationLifetime applicationLifetime, + IInstanceManager instanceManager, + IServerControl serverControl, + ILogger logger) + { + this.applicationLifetime = applicationLifetime ?? throw new ArgumentNullException(nameof(applicationLifetime)); + this.instanceManager = instanceManager ?? throw new ArgumentNullException(nameof(instanceManager)); + + ArgumentNullException.ThrowIfNull(serverControl); + + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + restartRegistration = serverControl.RegisterForRestart(this); + try + { + watchdogCts = new CancellationTokenSource(); + } + catch + { + restartRegistration.Dispose(); + throw; + } + } + + /// + public void Dispose() + { + restartRegistration.Dispose(); + watchdogCts.Dispose(); + } + + /// + public Task HandleRestart(Version updateVersion, bool handlerMayDelayShutdownWithExtremelyLongRunningTasks, CancellationToken cancellationToken) + { + // If this is set, we know a gracefule SHUTDOWN was requested + restartInProgress = !handlerMayDelayShutdownWithExtremelyLongRunningTasks; + return Task.CompletedTask; + } + + /// + public Task StartAsync(CancellationToken cancellationToken) + { + if (SendSDNotify("RELOADING=1")) + { + logger.LogDebug("SystemD detected"); + runTask = RunAsync(watchdogCts.Token); + } + else + { + logger.LogDebug("SystemD not detected"); + runTask = Task.CompletedTask; + } + + return Task.CompletedTask; + } + + /// + public async Task StopAsync(CancellationToken cancellationToken) + { + watchdogCts.Cancel(); + await runTask.WithToken(cancellationToken); + } + + /// + /// Runs the . + /// + /// The for the operation. + /// A representing the running operation. + async Task RunAsync(CancellationToken cancellationToken) + { + if (applicationLifetime.ApplicationStarted.IsCancellationRequested) + throw new InvalidOperationException("RunAsync called after application started!"); + + logger.LogTrace("Installing lifetime handlers..."); + + var readyCounts = 0; + void CheckReady() + { + if (Interlocked.Increment(ref readyCounts) < 2) + return; + + SendSDNotify("READY=1"); + } + + applicationLifetime.ApplicationStarted.Register(() => CheckReady()); + applicationLifetime.ApplicationStopping.Register( + () => SendSDNotify( + restartInProgress + ? "RELOADING=1" + : "STOPPING=1")); + + try + { + await instanceManager.Ready.WithToken(cancellationToken); + CheckReady(); + + var watchdogUsec = Environment.GetEnvironmentVariable("WATCHDOG_USEC"); + if (String.IsNullOrWhiteSpace(watchdogUsec)) + { + logger.LogDebug("WATCHDOG_USEC not present, not starting watchdog loop"); + return; + } + + logger.LogDebug("Starting watchdog loop with interval of {usec}us", watchdogUsec); + + var microseconds = UInt64.Parse(watchdogUsec, CultureInfo.InvariantCulture); + var milliseconds = (int)(microseconds / 1000); + if (milliseconds == 0) + milliseconds = 1; + + while (!cancellationToken.IsCancellationRequested) + { + await Task.Delay(milliseconds, cancellationToken); + + logger.LogTrace("Sending sd_notify WATCHDOG=1"); + var result = NativeMethods.sd_notify(0, "WATCHDOG=1"); + if (result <= 0) + logger.LogError(new UnixIOException(result), "sd_notify READY=1 failed!"); + } + } + catch (OperationCanceledException ex) + { + logger.LogTrace(ex, "Watchdog loop cancelled!"); + } + catch (Exception ex) + { + logger.LogError(ex, "Watchdog loop crashed!"); + } + + logger.LogDebug("Exited watchdog loop"); + } + + /// + /// Send a sd_notify . + /// + /// The to send via sd_notify. + /// if the command succeeded, otherwise. + bool SendSDNotify(string command) + { + logger.LogTrace("Sending sd_notify {message}...", command); + var result = NativeMethods.sd_notify(0, command); + if (result > 0) + return true; + + if (result < 0) + logger.LogError(new UnixIOException(result), "sd_notify READY=1 failed!"); + else + logger.LogTrace("Could not send sd_notify {message}. Socket closed!", command); + + return false; + } + } +}