cyberpresidentvanellope/tgstation-server
1
0
Fork 0
mirror of https://github.com/tgstation/tgstation-server.git synced 2026-08-22 12:37:24 +01:00
Code Issues Packages Projects Releases Wiki Activity

SIGUSR2 does not let DreamDaemon gracefully shutdown

Browse Source
This commit is contained in:
Jordan Dominion
2023-06-21 02:39:10 -04:00
parent 1bf2c0a47e
commit bedbb5ab8b
9 changed files with 51 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
build/tgstation-server.service
+1 -1
View File
@@ -10,7 +10,7 @@ After=mssql-server.service
ExecStart=/bin/bash /opt/tgstation-server/tgs.sh General:SetupWizardMode=Never
Restart=Always
KillMode=process
RestartKillSignal=SIGUSR1
RestartKillSignal=SIGUSR2
AmbientCapabilities=CAP_SYS_NICE
StandardOutput=null
StandardError=null
src/Tgstation.Server.Host.Watchdog/Watchdog.cs
+16 -10
View File
@@ -181,33 +181,37 @@ namespace Tgstation.Server.Host.Watchdog
var processTask = tcs.Task;
while (!processTask.IsCompleted)
{
var signalTcs = new TaskCompletionSource<object>();
var signalTcs = new TaskCompletionSource<Signum>();
using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
async ValueTask CheckSignal()
async Task CheckSignal(Signum signum)
{
if (isWindows)
return;
try
{
using var unixSignal = new UnixSignal(Signum.SIGUSR1);
using var unixSignal = new UnixSignal(signum);
if (!unixSignal.IsSet)
{
logger.LogTrace("Waiting for SIGUSR1...");
logger.LogTrace("Waiting for {signum}...", signum);
while (!unixSignal.IsSet)
await Task.Delay(TimeSpan.FromMilliseconds(250), cts.Token);
logger.LogTrace("SIGUSR1 received!");
logger.LogTrace("{signum} received!", signum);
}
else
logger.LogDebug("SIGUSR1 has already been sent");
logger.LogDebug("{signum} has already been sent", signum);
signalTcs.TrySetResult(signum);
}
catch (OperationCanceledException)
{
}
}
var checkerTask = CheckSignal();
var checkerTask = Task.WhenAll(
CheckSignal(Signum.SIGUSR1),
CheckSignal(Signum.SIGUSR2));
try
{
var signalTask = signalTcs.Task;
@@ -215,12 +219,14 @@ namespace Tgstation.Server.Host.Watchdog
var completedTask = await Task.WhenAny(processTask, signalTask);
if (completedTask == signalTask)
{
logger.LogInformation("Received SIGUSR1, forwarding to main TGS process!");
var result = Syscall.kill(childPid, Signum.SIGUSR1);
var signalReceived = await signalTask;
logger.LogInformation("Received {signalReceived}, forwarding to main TGS process!", signalReceived);
var result = Syscall.kill(childPid, signalReceived);
if (result != 0)
logger.LogWarning(
new UnixIOException(Stdlib.GetLastError()),
"Failed to forward SIGUSR1!");
"Failed to forward {signalReceived}!",
signalReceived);
}
}
finally
src/Tgstation.Server.Host/Components/Chat/ChatManager.cs
+3 -4
View File
@@ -534,11 +534,10 @@ namespace Tgstation.Server.Host.Components.Chat
}
/// <inheritdoc />
public Task HandleRestart(Version updateVersion, bool gracefulShutdown, CancellationToken cancellationToken)
public Task HandleRestart(Version updateVersion, bool handlerMayDelayShutdownWithExtremelyLongRunningTasks, CancellationToken cancellationToken)
{
var message =
updateVersion == null
? $"TGS: {(gracefulShutdown ? "Graceful shutdown" : "Restart")} requested..."
var message = updateVersion == null
? $"TGS: {(handlerMayDelayShutdownWithExtremelyLongRunningTasks ? "Graceful shutdown" : "Going down")}..."
: $"TGS: Updating to version {updateVersion}...";
List<ulong> wdChannels;
lock (mappedChannels) // so it doesn't change while we're using it
src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs
+3 -3
View File
@@ -402,15 +402,15 @@ namespace Tgstation.Server.Host.Components.Watchdog
}
/// <inheritdoc />
public async Task HandleRestart(Version updateVersion, bool gracefulShutdown, CancellationToken cancellationToken)
public async Task HandleRestart(Version updateVersion, bool handlerMayDelayShutdownWithExtremelyLongRunningTasks, CancellationToken cancellationToken)
{
if (gracefulShutdown)
if (handlerMayDelayShutdownWithExtremelyLongRunningTasks)
{
await Terminate(true, cancellationToken);
if (Status != WatchdogStatus.Offline)
{
Logger.LogTrace("Waiting for server to gracefully shut down.");
Logger.LogDebug("Waiting for server to gracefully shut down.");
await monitorTask.WithToken(cancellationToken);
}
else
src/Tgstation.Server.Host/Core/IRestartHandler.cs
+2 -2
View File
@@ -13,9 +13,9 @@ namespace Tgstation.Server.Host.Core
/// Handle a restart of the server.
/// </summary>
/// <param name="updateVersion">The <see cref="Version"/> being updated to, <see langword="null"/> if not being changed.</param>
/// <param name="gracefulShutdown">If <see langword="true"/> the server should not expect to restart.</param>
/// <param name="handlerMayDelayShutdownWithExtremelyLongRunningTasks">If <see langword="false"/> the <see cref="IRestartHandler"/> should aim to complete the <see cref="Task"/> returned from this function ASAP.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
/// <returns>A <see cref="Task"/> representing the running operation.</returns>
Task HandleRestart(Version updateVersion, bool gracefulShutdown, CancellationToken cancellationToken);
Task HandleRestart(Version updateVersion, bool handlerMayDelayShutdownWithExtremelyLongRunningTasks, CancellationToken cancellationToken);
}
}
src/Tgstation.Server.Host/Core/IServerControl.cs
+2 -1
View File
@@ -42,8 +42,9 @@ namespace Tgstation.Server.Host.Core
/// <summary>
/// Gracefully shutsdown the <see cref="Host"/>.
/// </summary>
/// <param name="detach">If the graceful shutdown should detach any running watchdog. If <see langword="false"/> the server will wait for the next TgsReboot() or world exit before shutting down.</param>
/// <returns>A <see cref="Task"/> representing the running operation.</returns>
Task GracefulShutdown();
Task GracefulShutdown(bool detach);
/// <summary>
/// Kill the server with a fatal exception.
src/Tgstation.Server.Host/Server.cs
+12 -8
View File
@@ -192,7 +192,7 @@ namespace Tgstation.Server.Host
if (await updateExecutor.ExecuteUpdate(updatePath, criticalCancellationToken, criticalCancellationToken))
{
logger.LogTrace("Update complete!");
await Restart(newVersion, null, true);
await RestartImpl(newVersion, null, true, true);
}
else if (terminateIfUpdateFails)
{
@@ -235,13 +235,13 @@ namespace Tgstation.Server.Host
}
/// <inheritdoc />
public Task Restart() => Restart(null, null, true);
public Task Restart() => RestartImpl(null, null, true, true);
/// <inheritdoc />
public Task GracefulShutdown() => Restart(null, null, false);
public Task GracefulShutdown(bool detach) => RestartImpl(null, null, false, detach);
/// <inheritdoc />
public Task Die(Exception exception) => Restart(null, exception, false);
public Task Die(Exception exception) => RestartImpl(null, exception, false, true);
/// <summary>
/// Throws an <see cref="InvalidOperationException"/> if the <see cref="IServerControl"/> cannot be used.
@@ -277,8 +277,9 @@ namespace Tgstation.Server.Host
/// <param name="newVersion">The <see cref="Version"/> of any potential updates being applied.</param>
/// <param name="exception">The potential value of <see cref="propagatedException"/>.</param>
/// <param name="requireWatchdog">If the host watchdog is required for this "restart".</param>
/// <param name="completeAsap">If the restart should wait for extremely long running tasks to complete (Like the current DreamDaemon world).</param>
/// <returns>A <see cref="Task"/> representing the running operation.</returns>
async Task Restart(Version newVersion, Exception exception, bool requireWatchdog)
async Task RestartImpl(Version newVersion, Exception exception, bool requireWatchdog, bool completeAsap)
{
CheckSanity(requireWatchdog);
@@ -287,7 +288,9 @@ namespace Tgstation.Server.Host
logger.LogTrace(
"Begin {restartType}...",
isGracefulShutdown
? "graceful shutdown"
? completeAsap
? "semi-graceful shutdown"
: "graceful shutdown"
: "restart");
lock (restartLock)
@@ -304,10 +307,11 @@ namespace Tgstation.Server.Host
if (exception == null)
{
var giveHandlersTimeToWaitAround = isGracefulShutdown && !completeAsap;
logger.LogInformation("Stopping server...");
using var cts = new CancellationTokenSource(
TimeSpan.FromMinutes(
isGracefulShutdown
giveHandlersTimeToWaitAround
? generalConfiguration.ShutdownTimeoutMinutes
: generalConfiguration.RestartTimeoutMinutes));
var cancellationToken = cts.Token;
@@ -315,7 +319,7 @@ namespace Tgstation.Server.Host
{
var eventsTask = Task.WhenAll(
restartHandlers.Select(
x => x.HandleRestart(newVersion, isGracefulShutdown, cancellationToken))
x => x.HandleRestart(newVersion, giveHandlersTimeToWaitAround, cancellationToken))
.ToList());
logger.LogTrace("Joining restart handlers...");
src/Tgstation.Server.Host/System/PosixSignalHandler.cs
+11 -7
View File
@@ -71,7 +71,9 @@ namespace Tgstation.Server.Host.System
if (signalCheckerTask != null)
throw new InvalidOperationException("Attempted to start PosixSignalHandler twice!");
signalCheckerTask = SignalChecker();
signalCheckerTask = Task.WhenAll(
SignalChecker(Signum.SIGUSR1, false),
SignalChecker(Signum.SIGUSR2, true));
return Task.CompletedTask;
}
@@ -92,28 +94,30 @@ namespace Tgstation.Server.Host.System
/// <summary>
/// Thread for listening to signal.
/// </summary>
/// <param name="signum">The <see cref="Signum"/> to monitor.</param>
/// <param name="detach">If the graceful shutdown should detach the watchdog.</param>
/// <returns>A <see cref="Task"/> representing the running operation.</returns>
async Task SignalChecker()
async Task SignalChecker(Signum signum, bool detach)
{
try
{
logger.LogTrace("Started SignalChecker");
using var unixSignal = new UnixSignal(Signum.SIGUSR1);
using var unixSignal = new UnixSignal(signum);
if (!unixSignal.IsSet)
{
logger.LogTrace("Waiting for SIGUSR1...");
logger.LogTrace("Waiting for {signum}...", signum);
var cancellationToken = cancellationTokenSource.Token;
while (!unixSignal.IsSet)
await asyncDelayer.Delay(TimeSpan.FromMilliseconds(CheckDelayMs), cancellationToken);
logger.LogTrace("SIGUSR1 received!");
logger.LogTrace("{signum} received!", signum);
}
else
logger.LogDebug("SIGUSR1 has already been sent");
logger.LogDebug("{signum} has already been sent", signum);
logger.LogTrace("Triggering graceful shutdown...");
await serverControl.GracefulShutdown();
await serverControl.GracefulShutdown(detach);
}
catch (OperationCanceledException ex)
{
tests/Tgstation.Server.Host.Tests.Signals/Program.cs
+1 -1
View File
@@ -21,7 +21,7 @@ namespace Tgstation.Server.Host.Tests.Signals
var tcs = new TaskCompletionSource();
mockServerControl
.Setup(x => x.GracefulShutdown())
.Setup(x => x.GracefulShutdown(It.IsAny<bool>()))
.Callback(() => tcs.SetResult())
.Returns(Task.CompletedTask);