Merge pull request #1057 from tgstation/976-ThisTimeForSure

Refactor watchdog running into status enum

src/Tgstation.Server.Api/Models/DreamDaemon.cs

@@ -19,9 +19,10 @@ namespace Tgstation.Server.Api.Models
 		public CompileJob? StagedCompileJob { get; set; }
 
 		/// <summary>
-		/// The current status of <see cref="DreamDaemon"/>
+		/// The current <see cref="WatchdogStatus"/>.
 		/// </summary>
-		public bool? Running { get; set; }
+		[EnumDataType(typeof(WatchdogStatus))]
+		public WatchdogStatus? Status { get; set; }
 
 		/// <summary>
 		/// The current <see cref="DreamDaemonSecurity"/> of <see cref="DreamDaemon"/>. May be downgraded due to requirements of <see cref="ActiveCompileJob"/>

src/Tgstation.Server.Api/Models/WatchdogStatus.cs

@@ -0,0 +1,28 @@
+namespace Tgstation.Server.Api.Models
+{
+	/// <summary>
+	/// The current status of the watchdog.
+	/// </summary>
+	public enum WatchdogStatus
+	{
+		/// <summary>
+		/// The watchdog is not running.
+		/// </summary>
+		Offline,
+
+		/// <summary>
+		/// The watchdog is online and attempting to bring DreamDaemon back to operational status.
+		/// </summary>
+		Restoring,
+
+		/// <summary>
+		/// The watchdog is online and DreamDaemon is running.
+		/// </summary>
+		Online,
+
+		/// <summary>
+		/// The watchdog is online and in a delayed sleep to bring DreamDaemon back.
+		/// </summary>
+		DelayedRestart,
+	}
+}

src/Tgstation.Server.Api/Rights/DreamDaemonRights.cs

@@ -34,7 +34,7 @@ namespace Tgstation.Server.Api.Rights
 		SetSecurity = 8,
 
 		/// <summary>
-		/// User can read all ports, <see cref="Models.DreamDaemon.SoftRestart"/>, <see cref="Models.DreamDaemon.SoftShutdown"/>, <see cref="Models.DreamDaemon.Running"/>, <see cref="Models.Internal.DreamDaemonLaunchParameters.AllowWebClient"/>, and <see cref="Models.Internal.DreamDaemonSettings.AutoStart"/>
+		/// User can read all ports, <see cref="Models.DreamDaemon.SoftRestart"/>, <see cref="Models.DreamDaemon.SoftShutdown"/>, <see cref="Models.DreamDaemon.Status"/>, <see cref="Models.Internal.DreamDaemonLaunchParameters.AllowWebClient"/>, and <see cref="Models.Internal.DreamDaemonSettings.AutoStart"/>
 		/// </summary>
 		ReadMetadata = 16,
 

src/Tgstation.Server.Host/Components/Chat/Commands/ByondCommand.cs

@@ -3,6 +3,7 @@ using System.Globalization;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
+using Tgstation.Server.Api.Models;
 using Tgstation.Server.Host.Components.Byond;
 using Tgstation.Server.Host.Components.Watchdog;
 
@@ -48,7 +49,7 @@ namespace Tgstation.Server.Host.Components.Chat.Commands
 		{
 			if (arguments.Split(' ').Any(x => x.ToUpperInvariant() == "--ACTIVE"))
 				return Task.FromResult(byondManager.ActiveVersion == null ? "None!" : String.Format(CultureInfo.InvariantCulture, "{0}.{1}", byondManager.ActiveVersion.Major, byondManager.ActiveVersion.Minor));
-			if (!watchdog.Running)
+			if (watchdog.Status == WatchdogStatus.Offline)
 				return Task.FromResult("Server offline!");
 			return Task.FromResult(watchdog.ActiveCompileJob.ByondVersion);
 		}

src/Tgstation.Server.Host/Components/Chat/Commands/PullRequestsCommand.cs

@@ -5,6 +5,7 @@ using System.Globalization;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
+using Tgstation.Server.Api.Models;
 using Tgstation.Server.Host.Components.Repository;
 using Tgstation.Server.Host.Components.Watchdog;
 using Tgstation.Server.Host.Database;
@@ -12,7 +13,7 @@ using Tgstation.Server.Host.Database;
 namespace Tgstation.Server.Host.Components.Chat.Commands
 {
 	/// <summary>
-	/// Command for reading the active <see cref="Api.Models.TestMerge"/>s
+	/// Command for reading the active <see cref="TestMerge"/>s
 	/// </summary>
 	sealed class PullRequestsCommand : ICommand
 	{
@@ -94,7 +95,7 @@ namespace Tgstation.Server.Host.Components.Chat.Commands
 			}
 			else
 			{
-				if (!watchdog.Running)
+				if (watchdog.Status == WatchdogStatus.Offline)
 					return "Server offline!";
 				results = watchdog.ActiveCompileJob?.RevisionInformation.ActiveTestMerges.Select(x => x.TestMerge).ToList() ?? new List<Models.TestMerge>();
 			}

src/Tgstation.Server.Host/Components/Chat/Commands/RevisionCommand.cs

@@ -3,6 +3,7 @@ using System.Globalization;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
+using Tgstation.Server.Api.Models;
 using Tgstation.Server.Host.Components.Repository;
 using Tgstation.Server.Host.Components.Watchdog;
 
@@ -56,7 +57,7 @@ namespace Tgstation.Server.Host.Components.Chat.Commands
 				}
 			else
 			{
-				if (!watchdog.Running)
+				if (watchdog.Status == WatchdogStatus.Offline)
 					return "Server offline!";
 				result = watchdog.ActiveCompileJob?.RevisionInformation.CommitSha;
 			}

src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs

@@ -204,7 +204,6 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		{
 			Server?.Dispose();
 			Server = null;
-			Running = false;
 			gracefulRebootRequired = false;
 		}
 
@@ -212,7 +211,10 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		protected sealed override ISessionController GetActiveController() => Server;
 
 		/// <inheritdoc />
-		protected sealed override async Task InitControllers(Action callBeforeRecurse, Task chatTask, DualReattachInformation reattachInfo, CancellationToken cancellationToken)
+		protected sealed override async Task InitControllers(
+			Task chatTask,
+			DualReattachInformation reattachInfo,
+			CancellationToken cancellationToken)
 		{
 			var serverToReattach = reattachInfo?.Alpha ?? reattachInfo?.Bravo;
 			var serverToKill = reattachInfo?.Bravo ?? reattachInfo?.Alpha;
@@ -274,8 +276,11 @@ namespace Tgstation.Server.Host.Components.Watchdog
 				// possiblity of null servers due to failed reattaches
 				if (Server == null)
 				{
-					callBeforeRecurse();
-					await NotifyOfFailedReattach(thereIsAnInactiveServerToKill && !inactiveServerWasKilled, cancellationToken).ConfigureAwait(false);
+					await ReattachFailure(
+						chatTask,
+						thereIsAnInactiveServerToKill && !inactiveServerWasKilled,
+						cancellationToken)
+						.ConfigureAwait(false);
 					return;
 				}
 
@@ -337,24 +342,6 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			return Restart(true, cancellationToken);
 		}
 
-		/// <summary>
-		/// Send a chat message and log about a failed reattach operation and attempts another call to <see cref="WatchdogBase.LaunchNoLock(bool, bool, DualReattachInformation, CancellationToken)"/>.
-		/// </summary>
-		/// <param name="inactiveReattachSuccess">If the inactive server was reattached successfully.</param>
-		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation/</param>
-		/// <returns>A <see cref="Task"/> representing the running operation.</returns>
-		async Task NotifyOfFailedReattach(bool inactiveReattachSuccess, CancellationToken cancellationToken)
-		{
-			// we lost the server, just restart entirely
-			DisposeAndNullControllers();
-			const string FailReattachMessage = "Unable to properly reattach to server! Restarting...";
-			Logger.LogWarning(FailReattachMessage);
-			Logger.LogDebug(inactiveReattachSuccess ? "Also could not reattach to inactive server!" : "Inactive server was reattached successfully!");
-			Task chatTask = Chat.SendWatchdogMessage(FailReattachMessage, false, cancellationToken);
-			await LaunchNoLock(true, false, null, cancellationToken).ConfigureAwait(false);
-			await chatTask.ConfigureAwait(false);
-		}
-
 		/// <inheritdoc />
 		public sealed override Task InstanceRenamed(string newInstanceName, CancellationToken cancellationToken)
 			=> Server?.InstanceRenamed(newInstanceName, cancellationToken) ?? Task.CompletedTask;

src/Tgstation.Server.Host/Components/Watchdog/ExperimentalWatchdog.cs

@@ -5,6 +5,7 @@ using System.Diagnostics;
 using System.Globalization;
 using System.Threading;
 using System.Threading.Tasks;
+using Tgstation.Server.Api.Models;
 using Tgstation.Server.Api.Models.Internal;
 using Tgstation.Server.Host.Components.Chat;
 using Tgstation.Server.Host.Components.Deployment;
@@ -34,7 +35,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		public override Models.CompileJob ActiveCompileJob => (AlphaIsActive ? alphaServer : bravoServer)?.Dmb.CompileJob;
 
 		/// <inheritdoc />
-		public override RebootState? RebootState => Running ? (AlphaIsActive ? alphaServer?.RebootState : bravoServer?.RebootState) : null;
+		public override RebootState? RebootState => Status != WatchdogStatus.Offline ? (AlphaIsActive ? alphaServer?.RebootState : bravoServer?.RebootState) : null;
 
 		/// <summary>
 		/// Server designation alpha
@@ -407,7 +408,6 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			alphaServer = null;
 			bravoServer?.Dispose();
 			bravoServer = null;
-			Running = false;
 		}
 
 		/// <inheritdoc />
@@ -445,7 +445,10 @@ namespace Tgstation.Server.Host.Components.Watchdog
 
 		/// <inheritdoc />
 		#pragma warning disable CA1502 // TODO: Decomplexify
-		protected override async Task InitControllers(Action callBeforeRecurse, Task chatTask, DualReattachInformation reattachInfo, CancellationToken cancellationToken)
+		protected override async Task InitControllers(
+			Task chatTask,
+			DualReattachInformation reattachInfo,
+			CancellationToken cancellationToken)
 		{
 			Debug.Assert(alphaServer == null && bravoServer == null, "Entered LaunchNoLock with one or more of the servers not being null!");
 
@@ -517,14 +520,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 						|| (bravoServer == null && !reattachInfo.AlphaIsActive))
 					{
 						// we lost the active server, just restart entirely
-						DisposeAndNullControllers();
-						const string FailReattachMessage = "Unable to properly reattach to active server! Restarting...";
-						Logger.LogWarning(FailReattachMessage);
-						Logger.LogDebug(bothServersDead ? "Also could not reattach to inactive server!" : "Inactive server was reattached successfully!");
-						chatTask = Chat.SendWatchdogMessage(FailReattachMessage, false, cancellationToken);
-						callBeforeRecurse();
-						await LaunchNoLock(true, false, null, cancellationToken).ConfigureAwait(false);
-						await chatTask.ConfigureAwait(false);
+						await ReattachFailure(chatTask, !bothServersDead, cancellationToken).ConfigureAwait(false);
 						return;
 					}
 

src/Tgstation.Server.Host/Components/Watchdog/IWatchdog.cs

@@ -2,6 +2,7 @@
 using System;
 using System.Threading;
 using System.Threading.Tasks;
+using Tgstation.Server.Api.Models;
 using Tgstation.Server.Api.Models.Internal;
 using Tgstation.Server.Host.Components.Events;
 using Tgstation.Server.Host.Components.Session;
@@ -14,9 +15,9 @@ namespace Tgstation.Server.Host.Components.Watchdog
 	public interface IWatchdog : IHostedService, IDisposable, IEventConsumer, IRenameNotifyee
 	{
 		/// <summary>
-		/// If the watchdog is running
+		/// The current <see cref="WatchdogStatus"/>.
 		/// </summary>
-		bool Running { get; }
+		WatchdogStatus Status { get; }
 
 		/// <summary>
 		/// If the alpha server is the active server
@@ -24,7 +25,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		bool AlphaIsActive { get; }
 
 		/// <summary>
-		/// The <see cref="CompileJob"/> currently running on the server
+		/// The <see cref="Models.CompileJob"/> currently running on the server
 		/// </summary>
 		Models.CompileJob ActiveCompileJob { get; }
 
@@ -51,7 +52,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		Task Launch(CancellationToken cancellationToken);
 
 		/// <summary>
-		/// Changes the <see cref="ActiveLaunchParameters"/>. If currently <see cref="Running"/> triggers a graceful restart
+		/// Changes the <see cref="ActiveLaunchParameters"/>. If currently running, may trigger a graceful restart.
 		/// </summary>
 		/// <param name="launchParameters">The new <see cref="DreamDaemonLaunchParameters"/>. May be modified</param>
 		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>

src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs

@@ -30,13 +30,13 @@ namespace Tgstation.Server.Host.Components.Watchdog
 	abstract class WatchdogBase : IWatchdog, ICustomCommandHandler, IRestartHandler
 	{
 		/// <inheritdoc />
-		public bool Running
+		public WatchdogStatus Status
 		{
-			get => running;
+			get => status;
 			set
 			{
-				running = value;
-				Logger.LogTrace("Running set to {0}", running);
+				status = value;
+				Logger.LogTrace("Status set to {0}", status);
 			}
 		}
 
@@ -56,7 +56,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		public abstract RebootState? RebootState { get; }
 
 		/// <summary>
-		/// <see cref="TaskCompletionSource{TResult}"/> that completes when <see cref="ActiveLaunchParameters"/> are changed and we are <see cref="Running"/>.
+		/// <see cref="TaskCompletionSource{TResult}"/> that completes when <see cref="ActiveLaunchParameters"/> are changed and we are running
 		/// </summary>
 		protected TaskCompletionSource<object> ActiveParametersUpdated { get; set; }
 
@@ -150,6 +150,11 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// </summary>
 		Task monitorTask;
 
+		/// <summary>
+		/// Backing field for <see cref="Status"/>.
+		/// </summary>
+		WatchdogStatus status;
+
 		/// <summary>
 		/// The number of hearbeats missed.
 		/// </summary>
@@ -160,11 +165,6 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// </summary>
 		bool releaseServers;
 
-		/// <summary>
-		/// Backing field for <see cref="Running"/>.
-		/// </summary>
-		bool running;
-
 		/// <summary>
 		/// If the <see cref="WatchdogBase"/> has been <see cref="Dispose"/>d.
 		/// </summary>
@@ -259,7 +259,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// <returns>A <see cref="Task"/> representing the running operation.</returns>
 		async Task TerminateNoLock(bool graceful, bool announce, CancellationToken cancellationToken)
 		{
-			if (!Running)
+			if (Status == WatchdogStatus.Offline)
 				return;
 			if (!graceful)
 			{
@@ -328,8 +328,8 @@ namespace Tgstation.Server.Host.Components.Watchdog
 							: "Restarting";
 						var message4 = $"DEFCON 1: Four heartbeats have been missed! {actionTaken}...";
 						Logger.LogWarning(message4);
-						DisposeAndNullControllers();
 						await Chat.SendWatchdogMessage(message4, false, cancellationToken).ConfigureAwait(false);
+						DisposeAndNullControllers();
 						return shouldShutdown ? MonitorAction.Exit : MonitorAction.Restart;
 					default:
 						Logger.LogError("Invalid heartbeats missed count: {0}", heartbeatsMissed);
@@ -354,17 +354,12 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		{
 			Logger.LogTrace("Begin LaunchImplNoLock");
 
-			if (Running)
-				throw new JobException(ErrorCode.WatchdogRunning);
-
 			if (reattachInfo == null && !DmbFactory.DmbAvailable)
 				throw new JobException(ErrorCode.WatchdogCompileJobCorrupted);
 
 			// this is necessary, the monitor could be in it's sleep loop trying to restart, if so cancel THAT monitor and start our own with blackjack and hookers
 			Task announceTask;
-			if (startMonitor && await StopMonitor().ConfigureAwait(false))
-				announceTask = Chat.SendWatchdogMessage("Automatic retry sequence cancelled by manual launch. Restarting...", false, cancellationToken);
-			else if (announce)
+			if (announce)
 			{
 				announceTask = Chat.SendWatchdogMessage(reattachInfo == null ? "Launching..." : "Reattaching...", false, cancellationToken); // simple announce
 				if (reattachInfo == null)
@@ -379,41 +374,27 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			LastLaunchParameters = ActiveLaunchParameters;
 			heartbeatsMissed = 0;
 
-			// for when we call ourself and want to not catch thrown exceptions
-			var recursiveCallToHappen = false;
 			try
 			{
-				await InitControllers(() => recursiveCallToHappen = true, announceTask, reattachInfo, cancellationToken).ConfigureAwait(false);
-				if (recursiveCallToHappen)
-					return;
-
-				await announceTask.ConfigureAwait(false);
-
-				Logger.LogInformation("Launched servers successfully");
-				Running = true;
-
-				if (startMonitor)
-				{
-					monitorCts = new CancellationTokenSource();
-					monitorTask = MonitorLifetimes(monitorCts.Token);
-				}
+				await InitControllers(announceTask, reattachInfo, cancellationToken).ConfigureAwait(false);
+			}
+			catch (OperationCanceledException)
+			{
+				Logger.LogTrace("Controller initialization canceled!");
+				throw;
 			}
 			catch (Exception e)
 			{
 				// don't try to send chat tasks or warning logs if were suppressing exceptions or cancelled
-				if (!recursiveCallToHappen && !cancellationToken.IsCancellationRequested)
+				var originalChatTask = announceTask;
+				async Task ChainChatTaskWithErrorMessage()
 				{
-					var originalChatTask = announceTask;
-					async Task ChainChatTaskWithErrorMessage()
-					{
-						await originalChatTask.ConfigureAwait(false);
-						await Chat.SendWatchdogMessage("Startup failed!", false, cancellationToken).ConfigureAwait(false);
-					}
-
-					announceTask = ChainChatTaskWithErrorMessage();
-					Logger.LogWarning("Failed to start watchdog: {0}", e.ToString());
+					await originalChatTask.ConfigureAwait(false);
+					await Chat.SendWatchdogMessage("Startup failed!", false, cancellationToken).ConfigureAwait(false);
 				}
 
+				announceTask = ChainChatTaskWithErrorMessage();
+				Logger.LogWarning("Failed to start watchdog: {0}", e.ToString());
 				throw;
 			}
 			finally
@@ -428,6 +409,14 @@ namespace Tgstation.Server.Host.Components.Watchdog
 					Logger.LogTrace("Announcement task canceled!");
 				}
 			}
+
+			Logger.LogInformation("Controller(s) initialized successfully");
+
+			if (startMonitor)
+			{
+				monitorCts = new CancellationTokenSource();
+				monitorTask = MonitorLifetimes(monitorCts.Token);
+			}
 		}
 
 		/// <summary>
@@ -472,7 +461,34 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		}
 
 		/// <summary>
-		/// Call <see cref="IDisposable.Dispose"/> and null the fields for all <see cref="ISessionController"/>s and set <see cref="Running"/> to <see langword="false"/>.
+		/// Call from <see cref="InitControllers(Task, DualReattachInformation, CancellationToken)"/> when a reattach operation fails to attempt a fresh start.
+		/// </summary>
+		/// <param name="chatTask">A, possibly active, <see cref="Task"/> for an outgoing chat message.</param>
+		/// <param name="reattachedInactive">If the inactive server was successfully reattached.</param>
+		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
+		/// <returns>A <see cref="Task"/> representing the running operation.</returns>
+		protected async Task ReattachFailure(Task chatTask, bool reattachedInactive, CancellationToken cancellationToken)
+		{
+			// we lost the server, just restart entirely
+			DisposeAndNullControllers();
+			const string FailReattachMessage = "Unable to properly reattach to server! Restarting watchdog...";
+			Logger.LogWarning(FailReattachMessage);
+			Logger.LogDebug(
+				reattachedInactive
+				? "Also could not reattach to inactive server!"
+				: "Inactive server was reattached successfully!");
+
+			async Task ChainChatTask()
+			{
+				await chatTask.ConfigureAwait(false);
+				await Chat.SendWatchdogMessage(FailReattachMessage, false, cancellationToken).ConfigureAwait(false);
+			}
+
+			await InitControllers(ChainChatTask(), null, cancellationToken).ConfigureAwait(false);
+		}
+
+		/// <summary>
+		/// Call <see cref="IDisposable.Dispose"/> and null the fields for all <see cref="ISessionController"/>s.
 		/// </summary>
 		protected abstract void DisposeAndNullControllersImpl();
 
@@ -531,17 +547,15 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			var chatTask = Task.CompletedTask;
 			for (var retryAttempts = 1; ; ++retryAttempts)
 			{
-				Exception launchException = null;
+				Status = WatchdogStatus.Restoring;
+				Exception launchException;
 				using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false))
 					try
 					{
 						// use LaunchImplNoLock without announcements or restarting the monitor
 						await LaunchNoLock(false, false, null, cancellationToken).ConfigureAwait(false);
-						if (Running)
-						{
-							Logger.LogDebug("Relaunch successful, resetting monitor state...");
-							return new MonitorState();
-						}
+						Logger.LogDebug("Relaunch successful, resetting monitor state...");
+						return new MonitorState();
 					}
 					catch (OperationCanceledException)
 					{
@@ -551,29 +565,29 @@ namespace Tgstation.Server.Host.Components.Watchdog
 					{
 						launchException = e;
 					}
+					finally
+					{
+						await chatTask.ConfigureAwait(false);
+					}
 
-				await chatTask.ConfigureAwait(false);
-				if (!Running)
-				{
-					if (launchException == null)
-						Logger.LogWarning("Failed to automatically restart the watchdog! Attempt: {0}", retryAttempts);
-					else
-						Logger.LogWarning("Failed to automatically restart the watchdog! Attempt: {0}, Exception: {1}", retryAttempts, launchException);
-					var retryDelay = Math.Min(
-						Convert.ToInt32(
-							Math.Pow(2, retryAttempts)),
-						TimeSpan.FromHours(1).Seconds); // max of one hour, increasing by a power of 2 each time
-					chatTask = Chat.SendWatchdogMessage(
-						$"Failed to restart (Attempt: {retryAttempts}), retrying in {retryDelay}",
-						false,
-						cancellationToken);
-					await Task.WhenAll(
-						AsyncDelayer.Delay(
-							TimeSpan.FromSeconds(retryDelay),
-							cancellationToken),
-						chatTask)
-						.ConfigureAwait(false);
-				}
+				Logger.LogWarning("Failed to automatically restart the watchdog! Attempt: {0}, Exception: {1}", retryAttempts, launchException);
+
+				var retryDelay = Math.Min(
+					Convert.ToInt32(
+						Math.Pow(2, retryAttempts)),
+					TimeSpan.FromHours(1).Seconds); // max of one hour, increasing by a power of 2 each time
+
+				chatTask = Chat.SendWatchdogMessage(
+					$"Failed to restart (Attempt: {retryAttempts}), retrying in {retryDelay}",
+					false,
+					cancellationToken);
+
+				await Task.WhenAll(
+					AsyncDelayer.Delay(
+						TimeSpan.FromSeconds(retryDelay),
+						cancellationToken),
+					chatTask)
+					.ConfigureAwait(false);
 			}
 		}
 
@@ -585,6 +599,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		private async Task MonitorLifetimes(CancellationToken cancellationToken)
 		{
 			Logger.LogTrace("Entered MonitorLifetimes");
+			Status = WatchdogStatus.Online;
 			using var _ = cancellationToken.Register(() => Logger.LogTrace("Monitor cancellationToken triggered"));
 
 			// this function is responsible for calling HandlerMonitorWakeup when necessary and manitaining the MonitorState
@@ -738,6 +753,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			}
 
 			DisposeAndNullControllers();
+			Status = WatchdogStatus.Offline;
 
 			Logger.LogTrace("Monitor exiting...");
 		}
@@ -745,12 +761,11 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// <summary>
 		/// Starts all <see cref="ISessionController"/>s.
 		/// </summary>
-		/// <param name="callBeforeRecurse">An <see cref="Action"/> that must be run before making a recursive call to <see cref="LaunchNoLock(bool, bool, DualReattachInformation, CancellationToken)"/>.</param>
 		/// <param name="chatTask">A, possibly active, <see cref="Task"/> for an outgoing chat message.</param>
 		/// <param name="reattachInfo"><see cref="DualReattachInformation"/> to use, if any</param>
 		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
 		/// <returns>A <see cref="Task"/> representing the running operation</returns>
-		protected abstract Task InitControllers(Action callBeforeRecurse, Task chatTask, DualReattachInformation reattachInfo, CancellationToken cancellationToken);
+		protected abstract Task InitControllers(Task chatTask, DualReattachInformation reattachInfo, CancellationToken cancellationToken);
 
 		/// <inheritdoc />
 		public async Task ChangeSettings(DreamDaemonLaunchParameters launchParameters, CancellationToken cancellationToken)
@@ -759,7 +774,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			{
 				bool match = launchParameters.CanApplyWithoutReboot(ActiveLaunchParameters);
 				ActiveLaunchParameters = launchParameters;
-				if (match || !Running)
+				if (match || Status == WatchdogStatus.Offline)
 					return;
 
 				ActiveParametersUpdated.TrySetResult(null); // queue an update
@@ -807,8 +822,8 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		{
 			using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false))
 			{
-				if (!Running)
-					return "ERROR: Server offline!";
+				if (Status == WatchdogStatus.Offline)
+					return "TGS: Server offline!";
 
 				var commandObject = new ChatCommand(sender, commandName, arguments);
 
@@ -831,6 +846,8 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// <inheritdoc />
 		public async Task Launch(CancellationToken cancellationToken)
 		{
+			if (Status != WatchdogStatus.Offline)
+				throw new JobException(ErrorCode.WatchdogRunning);
 			using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false))
 				await LaunchNoLock(true, true, null, cancellationToken).ConfigureAwait(false);
 		}
@@ -840,7 +857,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		{
 			using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false))
 			{
-				if (!Running)
+				if (Status == WatchdogStatus.Offline)
 					return;
 				var toClear = GetActiveController();
 				if (toClear != null)
@@ -851,25 +868,19 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// <inheritdoc />
 		public async Task Restart(bool graceful, CancellationToken cancellationToken)
 		{
-			if (!Running)
-				throw new JobException(ErrorCode.WatchdogRunning);
+			if (Status == WatchdogStatus.Offline)
+				throw new JobException(ErrorCode.WatchdogNotRunning);
 
 			Logger.LogTrace("Begin Restart. Graceful: {0}", graceful);
 			using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false))
 			{
 				if (!graceful)
 				{
-					Task chatTask;
-					bool hard = Running;
-					if (hard)
-					{
-						chatTask = Chat.SendWatchdogMessage("Manual restart triggered...", false, cancellationToken);
-						await TerminateNoLock(false, false, cancellationToken).ConfigureAwait(false);
-					}
-					else
-						chatTask = Task.CompletedTask;
-					await LaunchNoLock(true, !hard, null, cancellationToken).ConfigureAwait(false);
+					var chatTask = Chat.SendWatchdogMessage("Manual restart triggered...", false, cancellationToken);
+					await TerminateNoLock(false, false, cancellationToken).ConfigureAwait(false);
+					await LaunchNoLock(true, false, null, cancellationToken).ConfigureAwait(false);
 					await chatTask.ConfigureAwait(false);
+					return;
 				}
 
 				var toReboot = GetActiveController();
@@ -941,7 +952,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		public async Task HandleRestart(Version updateVersion, CancellationToken cancellationToken)
 		{
 			releaseServers = true;
-			if (Running)
+			if (Status == WatchdogStatus.Online)
 				await Chat.SendWatchdogMessage("Detaching...", false, cancellationToken).ConfigureAwait(false);
 		}
 

src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs

@@ -63,7 +63,7 @@ namespace Tgstation.Server.Host.Controllers
 			// alias for launching DD
 			var instance = instanceManager.GetInstance(Instance);
 
-			if (instance.Watchdog.Running)
+			if (instance.Watchdog.Status != WatchdogStatus.Offline)
 				return Conflict(new ErrorMessage(ErrorCode.WatchdogRunning));
 
 			var job = new Models.Job
@@ -130,7 +130,7 @@ namespace Tgstation.Server.Host.Controllers
 				result.CurrentAllowWebclient = llp?.AllowWebClient.Value;
 				result.PrimaryPort = settings.PrimaryPort.Value;
 				result.AllowWebClient = settings.AllowWebClient.Value;
-				result.Running = dd.Running;
+				result.Status = dd.Status;
 				result.SecondaryPort = settings.SecondaryPort.Value;
 				result.SecurityLevel = settings.SecurityLevel.Value;
 				result.SoftRestart = rstate == RebootState.Restart;
@@ -143,7 +143,10 @@ namespace Tgstation.Server.Host.Controllers
 			if (revision)
 			{
 				var latestCompileJob = instance.LatestCompileJob();
-				result.ActiveCompileJob = ((dd.Running ? dd.ActiveCompileJob : latestCompileJob) ?? latestCompileJob)?.ToApi();
+				result.ActiveCompileJob = ((instance.Watchdog.Status == WatchdogStatus.Offline
+					? dd.ActiveCompileJob
+					: latestCompileJob) ?? latestCompileJob)
+					?.ToApi();
 				if (latestCompileJob?.Id != result.ActiveCompileJob?.Id)
 					result.StagedCompileJob = latestCompileJob?.ToApi();
 			}
@@ -296,7 +299,7 @@ namespace Tgstation.Server.Host.Controllers
 
 			var watchdog = instanceManager.GetInstance(Instance).Watchdog;
 
-			if (!watchdog.Running)
+			if (watchdog.Status == WatchdogStatus.Offline)
 				return Conflict(new ErrorMessage(ErrorCode.WatchdogNotRunning));
 
 			await jobManager.RegisterOperation(job, (paramJob, databaseContextFactory, progressReporter, ct) => watchdog.Restart(false, ct), cancellationToken).ConfigureAwait(false);
@@ -325,7 +328,7 @@ namespace Tgstation.Server.Host.Controllers
 
 			var watchdog = instanceManager.GetInstance(Instance).Watchdog;
 
-			if (!watchdog.Running)
+			if (watchdog.Status == WatchdogStatus.Offline)
 				return Conflict(new ErrorMessage(ErrorCode.WatchdogNotRunning));
 
 			await jobManager.RegisterOperation(

tests/Tgstation.Server.Tests/Instance/WatchdogTest.cs

@@ -73,7 +73,7 @@ namespace Tgstation.Server.Tests.Instance
 			global::System.Console.WriteLine("TEST: WATCHDOG BASIC TEST");
 			var daemonStatus = await DeployTestDme("BasicOperation/basic_operation_test", DreamDaemonSecurity.Ultrasafe, cancellationToken);
 
-			Assert.IsFalse(daemonStatus.Running.Value);
+			Assert.AreEqual(WatchdogStatus.Offline, daemonStatus.Status.Value);
 			Assert.IsNotNull(daemonStatus.ActiveCompileJob);
 			Assert.IsNull(daemonStatus.StagedCompileJob);
 			Assert.AreEqual(DMApiConstants.Version, daemonStatus.ActiveCompileJob.DMApiVersion);
@@ -84,14 +84,14 @@ namespace Tgstation.Server.Tests.Instance
 			await WaitForJob(startJob, 10, false, cancellationToken);
 
 			daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
-			Assert.IsTrue(daemonStatus.Running.Value);
+			Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value);
 			Assert.AreEqual(false, daemonStatus.SoftRestart);
 			Assert.AreEqual(false, daemonStatus.SoftShutdown);
 
 			await GracefulWatchdogShutdown(30, cancellationToken);
 
 			daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
-			Assert.IsFalse(daemonStatus.Running.Value);
+			Assert.AreEqual(WatchdogStatus.Offline, daemonStatus.Status.Value);
 
 			await CheckDMApiFail(daemonStatus.ActiveCompileJob, cancellationToken);
 		}
@@ -112,7 +112,7 @@ namespace Tgstation.Server.Tests.Instance
 			// lock on to DD and pause it so it can't heartbeat
 			var ddProcs = System.Diagnostics.Process.GetProcessesByName("DreamDaemon").ToList();
 			if (ddProcs.Count != 1)
-				Assert.Inconclusive($"Incorrect number of DD processes: {ddProcs.Count}");
+				Assert.Fail($"Incorrect number of DD processes: {ddProcs.Count}");
 
 			using var ddProc = ddProcs.Single();
 			IProcessExecutor executor = null;
@@ -144,7 +144,7 @@ namespace Tgstation.Server.Tests.Instance
 				await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken).ConfigureAwait(false);
 				var ddStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
 				Assert.AreEqual(1U, ddStatus.HeartbeatSeconds.Value);
-				if (!ddStatus.Running.Value)
+				if (ddStatus.Status.Value == WatchdogStatus.Offline)
 					break;
 
 				if (--timeout == 0)
@@ -167,7 +167,7 @@ namespace Tgstation.Server.Tests.Instance
 			var daemonStatus = await DeployTestDme(DmeName, DreamDaemonSecurity.Trusted, cancellationToken);
 
 			var initialCompileJob = daemonStatus.ActiveCompileJob;
-			Assert.IsFalse(daemonStatus.Running.Value);
+			Assert.AreEqual(WatchdogStatus.Offline, daemonStatus.Status.Value);
 			Assert.IsNotNull(daemonStatus.ActiveCompileJob);
 			Assert.IsNull(daemonStatus.StagedCompileJob);
 			Assert.AreEqual(DMApiConstants.Version, daemonStatus.ActiveCompileJob.DMApiVersion);
@@ -179,7 +179,7 @@ namespace Tgstation.Server.Tests.Instance
 
 			daemonStatus = await DeployTestDme(DmeName, DreamDaemonSecurity.Safe, cancellationToken);
 
-			Assert.IsTrue(daemonStatus.Running.Value);
+			Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value);
 
 			Assert.AreEqual(initialCompileJob.Id, daemonStatus.ActiveCompileJob.Id);
 			var newerCompileJob = daemonStatus.StagedCompileJob;
@@ -197,7 +197,7 @@ namespace Tgstation.Server.Tests.Instance
 			await instanceClient.DreamDaemon.Shutdown(cancellationToken);
 
 			daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
-			Assert.IsFalse(daemonStatus.Running.Value);
+			Assert.AreEqual(WatchdogStatus.Offline, daemonStatus.Status.Value);
 		}
 
 		async Task RunLongRunningTestThenUpdateWithByondVersionSwitch(CancellationToken cancellationToken)
@@ -226,7 +226,7 @@ namespace Tgstation.Server.Tests.Instance
 			await DeployTestDme(DmeName, DreamDaemonSecurity.Safe, cancellationToken);
 
 			var daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
-			Assert.IsTrue(daemonStatus.Running.Value);
+			Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value);
 			Assert.IsNotNull(daemonStatus.ActiveCompileJob);
 
 
@@ -246,7 +246,7 @@ namespace Tgstation.Server.Tests.Instance
 			await instanceClient.DreamDaemon.Shutdown(cancellationToken);
 
 			daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
-			Assert.IsFalse(daemonStatus.Running.Value);
+			Assert.AreEqual(WatchdogStatus.Offline, daemonStatus.Status.Value);
 		}
 
 		public async Task StartAndLeaveRunning(CancellationToken cancellationToken)
@@ -259,6 +259,22 @@ namespace Tgstation.Server.Tests.Instance
 			var startJob = await instanceClient.DreamDaemon.Start(cancellationToken).ConfigureAwait(false);
 
 			await WaitForJob(startJob, 40, false, cancellationToken);
+
+			var daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
+			Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value);
+
+			// Try killing the DD process to ensure it gets set to the restoring state
+			var ddProcs = System.Diagnostics.Process.GetProcessesByName("DreamDaemon").ToList();
+			if (ddProcs.Count != 1)
+				Assert.Fail($"Incorrect number of DD processes: {ddProcs.Count}");
+
+			using var ddProc = ddProcs.Single();
+			ddProc.Kill();
+			ddProc.WaitForExit();
+
+			await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken);
+			daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
+			Assert.AreEqual(WatchdogStatus.Restoring, daemonStatus.Status.Value);
 		}
 
 		async Task TellWorldToReboot(CancellationToken cancellationToken)
@@ -308,13 +324,13 @@ namespace Tgstation.Server.Tests.Instance
 			}, cancellationToken);
 
 			var newStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
-			Assert.IsTrue(newStatus.SoftShutdown.Value || !newStatus.Running.Value);
+			Assert.IsTrue(newStatus.SoftShutdown.Value || (newStatus.Status.Value == WatchdogStatus.Offline));
 
 			do
 			{
 				await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken).ConfigureAwait(false);
 				var ddStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
-				if (!ddStatus.Running.Value)
+				if (ddStatus.Status.Value == WatchdogStatus.Offline)
 					break;
 
 				if (--timeout == 0)

tests/Tgstation.Server.Tests/IntegrationTest.cs

@@ -247,7 +247,7 @@ namespace Tgstation.Server.Tests
 					await new JobsRequiredTest(instanceClient.Jobs).WaitForJob(reattachJob, 40, false, cancellationToken);
 
 					var dd = await instanceClient.DreamDaemon.Read(cancellationToken);
-					Assert.IsTrue(dd.Running.Value);
+					Assert.AreEqual(WatchdogStatus.Online, dd.Status.Value);
 
 					await instanceClient.DreamDaemon.Shutdown(cancellationToken);
 					await instanceClient.DreamDaemon.Update(new DreamDaemon
@@ -291,7 +291,7 @@ namespace Tgstation.Server.Tests
 
 					var dd = await instanceClient.DreamDaemon.Read(cancellationToken);
 
-					Assert.IsTrue(dd.Running.Value);
+					Assert.AreEqual(WatchdogStatus.Online, dd.Status.Value);
 
 					var repoTest = new RepositoryTest(instanceClient.Repository, instanceClient.Jobs).RunPostTest(cancellationToken);
 					await new ChatTest(instanceClient.ChatBots, adminClient.Instances, instance).RunPostTest(cancellationToken);