diff --git a/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs
index 39ab878f09..5d5fa96f9a 100644
--- a/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs
+++ b/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs
@@ -33,9 +33,9 @@ namespace Tgstation.Server.Host.Components.Watchdog
protected ISessionController Server { get; private set; }
///
- /// If the server is set to gracefully reboot due to a pending dmb change.
+ /// If the server is set to gracefully reboot due to a pending dmb or settings change.
///
- bool gracefulRebootSetDueToNewDmb;
+ bool gracefulRebootRequired;
///
/// Initializes a new instance of the .
@@ -110,12 +110,11 @@ namespace Tgstation.Server.Host.Components.Watchdog
await Chat.SendWatchdogMessage(
String.Format(
CultureInfo.InvariantCulture,
- "Server {0}! Stopping due to graceful termination request...",
+ "Server {0}! Shutting down due to graceful termination request...",
exitWord),
false,
cancellationToken)
.ConfigureAwait(false);
- DisposeAndNullControllers();
monitorState.NextAction = MonitorAction.Exit;
}
else
@@ -134,7 +133,13 @@ namespace Tgstation.Server.Host.Components.Watchdog
break;
case MonitorActivationReason.ActiveServerRebooted:
var rebootState = Server.RebootState;
- gracefulRebootSetDueToNewDmb = false;
+ if (gracefulRebootRequired && rebootState == Watchdog.RebootState.Normal)
+ {
+ Logger.LogError("Watchdog reached normal reboot state with gracefulRebootRequired set!");
+ rebootState = Watchdog.RebootState.Restart;
+ }
+
+ gracefulRebootRequired = false;
Server.ResetRebootState();
switch (rebootState)
@@ -148,11 +153,10 @@ namespace Tgstation.Server.Host.Components.Watchdog
case Watchdog.RebootState.Shutdown:
// graceful shutdown time
await Chat.SendWatchdogMessage(
- "Active server rebooted! Stopping due to graceful termination request...",
+ "Active server rebooted! Shutting down due to graceful termination request...",
false,
cancellationToken)
.ConfigureAwait(false);
- DisposeAndNullControllers();
monitorState.NextAction = MonitorAction.Exit;
break;
default:
@@ -162,11 +166,10 @@ namespace Tgstation.Server.Host.Components.Watchdog
break;
case MonitorActivationReason.ActiveLaunchParametersUpdated:
await Server.SetRebootState(Watchdog.RebootState.Restart, cancellationToken).ConfigureAwait(false);
- monitorState.NextAction = MonitorAction.Continue;
+ gracefulRebootRequired = true;
break;
case MonitorActivationReason.NewDmbAvailable:
await HandleNewDmbAvailable(cancellationToken).ConfigureAwait(false);
- monitorState.NextAction = MonitorAction.Continue;
break;
case MonitorActivationReason.InactiveServerCrashed:
case MonitorActivationReason.InactiveServerRebooted:
@@ -192,7 +195,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
Server?.Dispose();
Server = null;
Running = false;
- gracefulRebootSetDueToNewDmb = false;
+ gracefulRebootRequired = false;
}
///
@@ -303,7 +306,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
/// A representing the running operation.
protected virtual Task HandleNewDmbAvailable(CancellationToken cancellationToken)
{
- gracefulRebootSetDueToNewDmb = true;
+ gracefulRebootRequired = true;
return Server.SetRebootState(Watchdog.RebootState.Restart, cancellationToken);
}
@@ -318,9 +321,10 @@ namespace Tgstation.Server.Host.Components.Watchdog
///
public override Task ResetRebootState(CancellationToken cancellationToken)
{
- if (gracefulRebootSetDueToNewDmb)
- return Task.CompletedTask;
- return base.ResetRebootState(cancellationToken);
+ if (!gracefulRebootRequired)
+ return base.ResetRebootState(cancellationToken);
+
+ return Restart(true, cancellationToken);
}
}
}
diff --git a/src/Tgstation.Server.Host/Components/Watchdog/ExperimentalWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/ExperimentalWatchdog.cs
index 90ede2c656..8f8383cc8e 100644
--- a/src/Tgstation.Server.Host/Components/Watchdog/ExperimentalWatchdog.cs
+++ b/src/Tgstation.Server.Host/Components/Watchdog/ExperimentalWatchdog.cs
@@ -235,11 +235,10 @@ namespace Tgstation.Server.Host.Components.Watchdog
await Chat.SendWatchdogMessage(
String.Format(
CultureInfo.InvariantCulture,
- "Active server {0}! Exiting due to graceful termination request...",
+ "Active server {0}! Shutting down due to graceful termination request...",
ExitWord(monitorState.ActiveServer)),
false,
cancellationToken).ConfigureAwait(false);
- DisposeAndNullControllers();
monitorState.NextAction = MonitorAction.Exit;
break;
}
@@ -312,11 +311,10 @@ namespace Tgstation.Server.Host.Components.Watchdog
case Watchdog.RebootState.Shutdown:
// graceful shutdown time
await Chat.SendWatchdogMessage(
- "Active server rebooted! Stopping due to graceful termination request...",
+ "Active server rebooted! Shutting down due to graceful termination request...",
false,
cancellationToken)
.ConfigureAwait(false);
- DisposeAndNullControllers();
monitorState.NextAction = MonitorAction.Exit;
return;
default:
diff --git a/src/Tgstation.Server.Host/Components/Watchdog/MonitorAction.cs b/src/Tgstation.Server.Host/Components/Watchdog/MonitorAction.cs
index 306bdccc40..3e5e26958e 100644
--- a/src/Tgstation.Server.Host/Components/Watchdog/MonitorAction.cs
+++ b/src/Tgstation.Server.Host/Components/Watchdog/MonitorAction.cs
@@ -26,7 +26,7 @@
Break,
///
- /// The monitor should exit. Does not kill servers
+ /// The monitor should end all sessions and exit.
///
Exit
}
diff --git a/src/Tgstation.Server.Host/Components/Watchdog/SessionController.cs b/src/Tgstation.Server.Host/Components/Watchdog/SessionController.cs
index c74bd16565..e91e5bb03d 100644
--- a/src/Tgstation.Server.Host/Components/Watchdog/SessionController.cs
+++ b/src/Tgstation.Server.Host/Components/Watchdog/SessionController.cs
@@ -612,6 +612,9 @@ namespace Tgstation.Server.Host.Components.Watchdog
{
if (RebootState == newRebootState)
return true;
+
+ logger.LogTrace("Changing reboot state to {0}", newRebootState);
+
reattachInformation.RebootState = newRebootState;
var result = await SendCommand(
new TopicParameters(newRebootState),
@@ -625,6 +628,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
public void ResetRebootState()
{
CheckDisposed();
+ logger.LogTrace("Resetting reboot state...");
reattachInformation.RebootState = RebootState.Normal;
}
diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs
index 88c6d1cf89..e6b05d2fc2 100644
--- a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs
+++ b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs
@@ -212,7 +212,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
return;
if (!graceful)
{
- var chatTask = announce ? Chat.SendWatchdogMessage("Terminating...", false, cancellationToken) : Task.CompletedTask;
+ var chatTask = announce ? Chat.SendWatchdogMessage("Shutting down...", false, cancellationToken) : Task.CompletedTask;
await StopMonitor().ConfigureAwait(false);
DisposeAndNullControllers();
@@ -226,7 +226,12 @@ namespace Tgstation.Server.Host.Components.Watchdog
// merely set the reboot state
var toKill = GetActiveController();
if (toKill != null)
+ {
await toKill.SetRebootState(Watchdog.RebootState.Shutdown, cancellationToken).ConfigureAwait(false);
+ Logger.LogTrace("Graceful termination requested");
+ }
+ else
+ Logger.LogTrace("Could not gracefully terminate as there is no active controller!");
}
///
@@ -237,10 +242,10 @@ namespace Tgstation.Server.Host.Components.Watchdog
/// A resulting in the next to take.
async Task HandleHeartbeat(ISessionController activeServer, CancellationToken cancellationToken)
{
- Logger.LogTrace("Sending heartbeat to session controller on :{0}", activeServer.Port);
-
+ Logger.LogTrace("Sending heartbeat to active server...");
var response = await activeServer.SendCommand(new TopicParameters(), cancellationToken).ConfigureAwait(false);
+ var shouldShutdown = activeServer.RebootState == Watchdog.RebootState.Shutdown;
if (response == null)
{
switch (++heartbeatsMissed)
@@ -254,16 +259,18 @@ namespace Tgstation.Server.Host.Components.Watchdog
await Chat.SendWatchdogMessage(message2, true, cancellationToken).ConfigureAwait(false);
break;
case 3:
- var message3 = "DEFCON 2: Watchdog has missed 3 heartbeats! If DreamDaemon does not respond to the next one, the server will be restarted!";
+ var actionToTake = shouldShutdown
+ ? "shutdown"
+ : "be restarted";
+ var message3 = $"DEFCON 2: Watchdog has missed 3 heartbeats! If DreamDaemon does not respond to the next one, the watchdog will {actionToTake}!";
Logger.LogWarning(message3);
await Chat.SendWatchdogMessage(message3, false, cancellationToken).ConfigureAwait(false);
break;
case 4:
- var shouldShutdown = activeServer.RebootState == Watchdog.RebootState.Shutdown;
- var actionToTake = shouldShutdown
+ var actionTaken = shouldShutdown
? "Shutting down due to graceful termination request"
: "Restarting";
- var message4 = $"DEFCON 1: Four heartbeats have been missed! {actionToTake}...";
+ var message4 = $"DEFCON 1: Four heartbeats have been missed! {actionTaken}...";
Logger.LogWarning(message4);
DisposeAndNullControllers();
await Chat.SendWatchdogMessage(message4, false, cancellationToken).ConfigureAwait(false);
@@ -302,7 +309,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
if (startMonitor && await StopMonitor().ConfigureAwait(false))
chatTask = Chat.SendWatchdogMessage("Automatic retry sequence cancelled by manual launch. Restarting...", false, cancellationToken);
else if (announce)
- chatTask = Chat.SendWatchdogMessage(reattachInfo == null ? "Starting..." : "Reattaching...", false, cancellationToken); // simple announce
+ chatTask = Chat.SendWatchdogMessage(reattachInfo == null ? "Launching..." : "Reattaching...", false, cancellationToken); // simple announce
else
chatTask = Task.CompletedTask; // no announce
@@ -448,7 +455,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
///
/// Handles the actions to take when the monitor has to "wake up"
///
- /// The that caused the invocation.
+ /// The that caused the invocation. Will never be .
/// The current .
/// The for the operation.
/// A representing the running operation.
@@ -457,6 +464,60 @@ namespace Tgstation.Server.Host.Components.Watchdog
MonitorState monitorState,
CancellationToken cancellationToken);
+ private async Task MonitorRestart(CancellationToken cancellationToken)
+ {
+ Logger.LogTrace("Monitor restart!");
+ DisposeAndNullControllers();
+
+ var chatTask = Task.CompletedTask;
+ for (var retryAttempts = 1; ; ++retryAttempts)
+ {
+ Exception launchException = null;
+ using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false))
+ try
+ {
+ // use LaunchImplNoLock without announcements or restarting the monitor
+ await LaunchImplNoLock(false, false, null, cancellationToken).ConfigureAwait(false);
+ if (Running)
+ {
+ Logger.LogDebug("Relaunch successful, resetting monitor state...");
+ return new MonitorState();
+ }
+ }
+ catch (OperationCanceledException)
+ {
+ throw;
+ }
+ catch (Exception e)
+ {
+ launchException = e;
+ }
+
+ await chatTask.ConfigureAwait(false);
+ if (!Running)
+ {
+ if (launchException == null)
+ Logger.LogWarning("Failed to automatically restart the watchdog! Attempt: {0}", retryAttempts);
+ else
+ Logger.LogWarning("Failed to automatically restart the watchdog! Attempt: {0}, Exception: {1}", retryAttempts, launchException);
+ var retryDelay = Math.Min(
+ Convert.ToInt32(
+ Math.Pow(2, retryAttempts)),
+ TimeSpan.FromHours(1).Seconds); // max of one hour, increasing by a power of 2 each time
+ chatTask = Chat.SendWatchdogMessage(
+ $"Failed to restart (Attempt: {retryAttempts}), retrying in {retryDelay}",
+ false,
+ cancellationToken);
+ await Task.WhenAll(
+ AsyncDelayer.Delay(
+ TimeSpan.FromSeconds(retryDelay),
+ cancellationToken),
+ chatTask)
+ .ConfigureAwait(false);
+ }
+ }
+ }
+
///
/// The loop that watches the watchdog.
///
@@ -468,169 +529,135 @@ namespace Tgstation.Server.Host.Components.Watchdog
// this function is responsible for calling HandlerMonitorWakeup when necessary and manitaining the MonitorState
var iteration = 1;
- for (var monitorState = new MonitorState(); monitorState.NextAction != MonitorAction.Exit; ++iteration)
+ try
{
- Logger.LogDebug("Iteration {0} of monitor loop", iteration);
- try
- {
- // load the activation tasks into local variables
- cancellationToken.ThrowIfCancellationRequested();
- var serverTasks = GetMonitoredServerTasks(monitorState);
- if (serverTasks.Count != 5)
- throw new InvalidOperationException("Expected 5 monitored server tasks!");
-
- var activeServerLifetime = serverTasks[MonitorActivationReason.ActiveServerCrashed];
- var activeServerReboot = serverTasks[MonitorActivationReason.ActiveServerRebooted];
- var inactiveServerLifetime = serverTasks[MonitorActivationReason.InactiveServerCrashed];
- var inactiveServerReboot = serverTasks[MonitorActivationReason.InactiveServerRebooted];
- var inactiveStartupComplete = serverTasks[MonitorActivationReason.InactiveServerStartupComplete];
- Task activeLaunchParametersChanged = ActiveParametersUpdated.Task;
- var newDmbAvailable = DmbFactory.OnNewerDmb;
- var heartbeatSeconds = ActiveLaunchParameters.HeartbeatSeconds.Value;
- var heartbeat = heartbeatSeconds == 0
- ? Extensions.TaskExtensions.InfiniteTask()
- : Task.Delay(TimeSpan.FromSeconds(heartbeatSeconds));
-
- // cancel waiting if requested
- var cancelTcs = new TaskCompletionSource