diff --git a/build/tgstation-server.service b/build/tgstation-server.service index 1b25e39393..bcd6de155d 100644 --- a/build/tgstation-server.service +++ b/build/tgstation-server.service @@ -10,7 +10,7 @@ After=mssql-server.service ExecStart=/bin/bash /opt/tgstation-server/tgs.sh General:SetupWizardMode=Never Restart=Always KillMode=process -RestartKillSignal=SIGUSR1 +RestartKillSignal=SIGUSR2 AmbientCapabilities=CAP_SYS_NICE StandardOutput=null StandardError=null diff --git a/src/Tgstation.Server.Host.Watchdog/Watchdog.cs b/src/Tgstation.Server.Host.Watchdog/Watchdog.cs index 3b9ee3bc63..d8a1d580ed 100644 --- a/src/Tgstation.Server.Host.Watchdog/Watchdog.cs +++ b/src/Tgstation.Server.Host.Watchdog/Watchdog.cs @@ -181,33 +181,37 @@ namespace Tgstation.Server.Host.Watchdog var processTask = tcs.Task; while (!processTask.IsCompleted) { - var signalTcs = new TaskCompletionSource(); + var signalTcs = new TaskCompletionSource(); using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); - async ValueTask CheckSignal() + async Task CheckSignal(Signum signum) { if (isWindows) return; try { - using var unixSignal = new UnixSignal(Signum.SIGUSR1); + using var unixSignal = new UnixSignal(signum); if (!unixSignal.IsSet) { - logger.LogTrace("Waiting for SIGUSR1..."); + logger.LogTrace("Waiting for {signum}...", signum); while (!unixSignal.IsSet) await Task.Delay(TimeSpan.FromMilliseconds(250), cts.Token); - logger.LogTrace("SIGUSR1 received!"); + logger.LogTrace("{signum} received!", signum); } else - logger.LogDebug("SIGUSR1 has already been sent"); + logger.LogDebug("{signum} has already been sent", signum); + + signalTcs.TrySetResult(signum); } catch (OperationCanceledException) { } } - var checkerTask = CheckSignal(); + var checkerTask = Task.WhenAll( + CheckSignal(Signum.SIGUSR1), + CheckSignal(Signum.SIGUSR2)); try { var signalTask = signalTcs.Task; @@ -215,12 +219,14 @@ namespace Tgstation.Server.Host.Watchdog var completedTask = await Task.WhenAny(processTask, signalTask); if (completedTask == signalTask) { - logger.LogInformation("Received SIGUSR1, forwarding to main TGS process!"); - var result = Syscall.kill(childPid, Signum.SIGUSR1); + var signalReceived = await signalTask; + logger.LogInformation("Received {signalReceived}, forwarding to main TGS process!", signalReceived); + var result = Syscall.kill(childPid, signalReceived); if (result != 0) logger.LogWarning( new UnixIOException(Stdlib.GetLastError()), - "Failed to forward SIGUSR1!"); + "Failed to forward {signalReceived}!", + signalReceived); } } finally diff --git a/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs b/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs index 95246f2e48..91cfdcade9 100644 --- a/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs +++ b/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs @@ -534,11 +534,10 @@ namespace Tgstation.Server.Host.Components.Chat } /// - public Task HandleRestart(Version updateVersion, bool gracefulShutdown, CancellationToken cancellationToken) + public Task HandleRestart(Version updateVersion, bool handlerMayDelayShutdownWithExtremelyLongRunningTasks, CancellationToken cancellationToken) { - var message = - updateVersion == null - ? $"TGS: {(gracefulShutdown ? "Graceful shutdown" : "Restart")} requested..." + var message = updateVersion == null + ? $"TGS: {(handlerMayDelayShutdownWithExtremelyLongRunningTasks ? "Graceful shutdown" : "Going down")}..." : $"TGS: Updating to version {updateVersion}..."; List wdChannels; lock (mappedChannels) // so it doesn't change while we're using it diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs index 450e1b6293..636f9df42a 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs @@ -402,15 +402,15 @@ namespace Tgstation.Server.Host.Components.Watchdog } /// - public async Task HandleRestart(Version updateVersion, bool gracefulShutdown, CancellationToken cancellationToken) + public async Task HandleRestart(Version updateVersion, bool handlerMayDelayShutdownWithExtremelyLongRunningTasks, CancellationToken cancellationToken) { - if (gracefulShutdown) + if (handlerMayDelayShutdownWithExtremelyLongRunningTasks) { await Terminate(true, cancellationToken); if (Status != WatchdogStatus.Offline) { - Logger.LogTrace("Waiting for server to gracefully shut down."); + Logger.LogDebug("Waiting for server to gracefully shut down."); await monitorTask.WithToken(cancellationToken); } else diff --git a/src/Tgstation.Server.Host/Core/IRestartHandler.cs b/src/Tgstation.Server.Host/Core/IRestartHandler.cs index 06549ff14f..93670c6f73 100644 --- a/src/Tgstation.Server.Host/Core/IRestartHandler.cs +++ b/src/Tgstation.Server.Host/Core/IRestartHandler.cs @@ -13,9 +13,9 @@ namespace Tgstation.Server.Host.Core /// Handle a restart of the server. /// /// The being updated to, if not being changed. - /// If the server should not expect to restart. + /// If the should aim to complete the returned from this function ASAP. /// The for the operation. /// A representing the running operation. - Task HandleRestart(Version updateVersion, bool gracefulShutdown, CancellationToken cancellationToken); + Task HandleRestart(Version updateVersion, bool handlerMayDelayShutdownWithExtremelyLongRunningTasks, CancellationToken cancellationToken); } } diff --git a/src/Tgstation.Server.Host/Core/IServerControl.cs b/src/Tgstation.Server.Host/Core/IServerControl.cs index d0b0f04bf8..bc081b3201 100644 --- a/src/Tgstation.Server.Host/Core/IServerControl.cs +++ b/src/Tgstation.Server.Host/Core/IServerControl.cs @@ -42,8 +42,9 @@ namespace Tgstation.Server.Host.Core /// /// Gracefully shutsdown the . /// + /// If the graceful shutdown should detach any running watchdog. If the server will wait for the next TgsReboot() or world exit before shutting down. /// A representing the running operation. - Task GracefulShutdown(); + Task GracefulShutdown(bool detach); /// /// Kill the server with a fatal exception. diff --git a/src/Tgstation.Server.Host/Server.cs b/src/Tgstation.Server.Host/Server.cs index 6a26ed8440..1a42d25b40 100644 --- a/src/Tgstation.Server.Host/Server.cs +++ b/src/Tgstation.Server.Host/Server.cs @@ -192,7 +192,7 @@ namespace Tgstation.Server.Host if (await updateExecutor.ExecuteUpdate(updatePath, criticalCancellationToken, criticalCancellationToken)) { logger.LogTrace("Update complete!"); - await Restart(newVersion, null, true); + await RestartImpl(newVersion, null, true, true); } else if (terminateIfUpdateFails) { @@ -235,13 +235,13 @@ namespace Tgstation.Server.Host } /// - public Task Restart() => Restart(null, null, true); + public Task Restart() => RestartImpl(null, null, true, true); /// - public Task GracefulShutdown() => Restart(null, null, false); + public Task GracefulShutdown(bool detach) => RestartImpl(null, null, false, detach); /// - public Task Die(Exception exception) => Restart(null, exception, false); + public Task Die(Exception exception) => RestartImpl(null, exception, false, true); /// /// Throws an if the cannot be used. @@ -277,8 +277,9 @@ namespace Tgstation.Server.Host /// The of any potential updates being applied. /// The potential value of . /// If the host watchdog is required for this "restart". + /// If the restart should wait for extremely long running tasks to complete (Like the current DreamDaemon world). /// A representing the running operation. - async Task Restart(Version newVersion, Exception exception, bool requireWatchdog) + async Task RestartImpl(Version newVersion, Exception exception, bool requireWatchdog, bool completeAsap) { CheckSanity(requireWatchdog); @@ -287,7 +288,9 @@ namespace Tgstation.Server.Host logger.LogTrace( "Begin {restartType}...", isGracefulShutdown - ? "graceful shutdown" + ? completeAsap + ? "semi-graceful shutdown" + : "graceful shutdown" : "restart"); lock (restartLock) @@ -304,10 +307,11 @@ namespace Tgstation.Server.Host if (exception == null) { + var giveHandlersTimeToWaitAround = isGracefulShutdown && !completeAsap; logger.LogInformation("Stopping server..."); using var cts = new CancellationTokenSource( TimeSpan.FromMinutes( - isGracefulShutdown + giveHandlersTimeToWaitAround ? generalConfiguration.ShutdownTimeoutMinutes : generalConfiguration.RestartTimeoutMinutes)); var cancellationToken = cts.Token; @@ -315,7 +319,7 @@ namespace Tgstation.Server.Host { var eventsTask = Task.WhenAll( restartHandlers.Select( - x => x.HandleRestart(newVersion, isGracefulShutdown, cancellationToken)) + x => x.HandleRestart(newVersion, giveHandlersTimeToWaitAround, cancellationToken)) .ToList()); logger.LogTrace("Joining restart handlers..."); diff --git a/src/Tgstation.Server.Host/System/PosixSignalHandler.cs b/src/Tgstation.Server.Host/System/PosixSignalHandler.cs index 93df0ae593..693324d679 100644 --- a/src/Tgstation.Server.Host/System/PosixSignalHandler.cs +++ b/src/Tgstation.Server.Host/System/PosixSignalHandler.cs @@ -71,7 +71,9 @@ namespace Tgstation.Server.Host.System if (signalCheckerTask != null) throw new InvalidOperationException("Attempted to start PosixSignalHandler twice!"); - signalCheckerTask = SignalChecker(); + signalCheckerTask = Task.WhenAll( + SignalChecker(Signum.SIGUSR1, false), + SignalChecker(Signum.SIGUSR2, true)); return Task.CompletedTask; } @@ -92,28 +94,30 @@ namespace Tgstation.Server.Host.System /// /// Thread for listening to signal. /// + /// The to monitor. + /// If the graceful shutdown should detach the watchdog. /// A representing the running operation. - async Task SignalChecker() + async Task SignalChecker(Signum signum, bool detach) { try { logger.LogTrace("Started SignalChecker"); - using var unixSignal = new UnixSignal(Signum.SIGUSR1); + using var unixSignal = new UnixSignal(signum); if (!unixSignal.IsSet) { - logger.LogTrace("Waiting for SIGUSR1..."); + logger.LogTrace("Waiting for {signum}...", signum); var cancellationToken = cancellationTokenSource.Token; while (!unixSignal.IsSet) await asyncDelayer.Delay(TimeSpan.FromMilliseconds(CheckDelayMs), cancellationToken); - logger.LogTrace("SIGUSR1 received!"); + logger.LogTrace("{signum} received!", signum); } else - logger.LogDebug("SIGUSR1 has already been sent"); + logger.LogDebug("{signum} has already been sent", signum); logger.LogTrace("Triggering graceful shutdown..."); - await serverControl.GracefulShutdown(); + await serverControl.GracefulShutdown(detach); } catch (OperationCanceledException ex) { diff --git a/tests/Tgstation.Server.Host.Tests.Signals/Program.cs b/tests/Tgstation.Server.Host.Tests.Signals/Program.cs index c60a40d3a9..995031cc4e 100644 --- a/tests/Tgstation.Server.Host.Tests.Signals/Program.cs +++ b/tests/Tgstation.Server.Host.Tests.Signals/Program.cs @@ -21,7 +21,7 @@ namespace Tgstation.Server.Host.Tests.Signals var tcs = new TaskCompletionSource(); mockServerControl - .Setup(x => x.GracefulShutdown()) + .Setup(x => x.GracefulShutdown(It.IsAny())) .Callback(() => tcs.SetResult()) .Returns(Task.CompletedTask);