diff --git a/src/Tgstation.Server.Api/Models/DreamDaemon.cs b/src/Tgstation.Server.Api/Models/DreamDaemon.cs index 8f2545af4f..aab484014c 100644 --- a/src/Tgstation.Server.Api/Models/DreamDaemon.cs +++ b/src/Tgstation.Server.Api/Models/DreamDaemon.cs @@ -19,9 +19,10 @@ namespace Tgstation.Server.Api.Models public CompileJob? StagedCompileJob { get; set; } /// - /// The current status of + /// The current . /// - public bool? Running { get; set; } + [EnumDataType(typeof(WatchdogStatus))] + public WatchdogStatus? Status { get; set; } /// /// The current of . May be downgraded due to requirements of diff --git a/src/Tgstation.Server.Api/Models/WatchdogStatus.cs b/src/Tgstation.Server.Api/Models/WatchdogStatus.cs new file mode 100644 index 0000000000..5250f3d740 --- /dev/null +++ b/src/Tgstation.Server.Api/Models/WatchdogStatus.cs @@ -0,0 +1,28 @@ +namespace Tgstation.Server.Api.Models +{ + /// + /// The current status of the watchdog. + /// + public enum WatchdogStatus + { + /// + /// The watchdog is not running. + /// + Offline, + + /// + /// The watchdog is online and attempting to bring DreamDaemon back to operational status. + /// + Restoring, + + /// + /// The watchdog is online and DreamDaemon is running. + /// + Online, + + /// + /// The watchdog is online and in a delayed sleep to bring DreamDaemon back. + /// + DelayedRestart, + } +} \ No newline at end of file diff --git a/src/Tgstation.Server.Api/Rights/DreamDaemonRights.cs b/src/Tgstation.Server.Api/Rights/DreamDaemonRights.cs index 083c7d4229..089cd810c8 100644 --- a/src/Tgstation.Server.Api/Rights/DreamDaemonRights.cs +++ b/src/Tgstation.Server.Api/Rights/DreamDaemonRights.cs @@ -34,7 +34,7 @@ namespace Tgstation.Server.Api.Rights SetSecurity = 8, /// - /// User can read all ports, , , , , and + /// User can read all ports, , , , , and /// ReadMetadata = 16, diff --git a/src/Tgstation.Server.Host/Components/Chat/Commands/ByondCommand.cs b/src/Tgstation.Server.Host/Components/Chat/Commands/ByondCommand.cs index 7226c55ba6..2d8eeaabd8 100644 --- a/src/Tgstation.Server.Host/Components/Chat/Commands/ByondCommand.cs +++ b/src/Tgstation.Server.Host/Components/Chat/Commands/ByondCommand.cs @@ -3,6 +3,7 @@ using System.Globalization; using System.Linq; using System.Threading; using System.Threading.Tasks; +using Tgstation.Server.Api.Models; using Tgstation.Server.Host.Components.Byond; using Tgstation.Server.Host.Components.Watchdog; @@ -48,7 +49,7 @@ namespace Tgstation.Server.Host.Components.Chat.Commands { if (arguments.Split(' ').Any(x => x.ToUpperInvariant() == "--ACTIVE")) return Task.FromResult(byondManager.ActiveVersion == null ? "None!" : String.Format(CultureInfo.InvariantCulture, "{0}.{1}", byondManager.ActiveVersion.Major, byondManager.ActiveVersion.Minor)); - if (!watchdog.Running) + if (watchdog.Status == WatchdogStatus.Offline) return Task.FromResult("Server offline!"); return Task.FromResult(watchdog.ActiveCompileJob.ByondVersion); } diff --git a/src/Tgstation.Server.Host/Components/Chat/Commands/PullRequestsCommand.cs b/src/Tgstation.Server.Host/Components/Chat/Commands/PullRequestsCommand.cs index 5e63fa62a3..2c645130b4 100644 --- a/src/Tgstation.Server.Host/Components/Chat/Commands/PullRequestsCommand.cs +++ b/src/Tgstation.Server.Host/Components/Chat/Commands/PullRequestsCommand.cs @@ -5,6 +5,7 @@ using System.Globalization; using System.Linq; using System.Threading; using System.Threading.Tasks; +using Tgstation.Server.Api.Models; using Tgstation.Server.Host.Components.Repository; using Tgstation.Server.Host.Components.Watchdog; using Tgstation.Server.Host.Database; @@ -12,7 +13,7 @@ using Tgstation.Server.Host.Database; namespace Tgstation.Server.Host.Components.Chat.Commands { /// - /// Command for reading the active s + /// Command for reading the active s /// sealed class PullRequestsCommand : ICommand { @@ -94,7 +95,7 @@ namespace Tgstation.Server.Host.Components.Chat.Commands } else { - if (!watchdog.Running) + if (watchdog.Status == WatchdogStatus.Offline) return "Server offline!"; results = watchdog.ActiveCompileJob?.RevisionInformation.ActiveTestMerges.Select(x => x.TestMerge).ToList() ?? new List(); } diff --git a/src/Tgstation.Server.Host/Components/Chat/Commands/RevisionCommand.cs b/src/Tgstation.Server.Host/Components/Chat/Commands/RevisionCommand.cs index 107c3c2d54..014322a25f 100644 --- a/src/Tgstation.Server.Host/Components/Chat/Commands/RevisionCommand.cs +++ b/src/Tgstation.Server.Host/Components/Chat/Commands/RevisionCommand.cs @@ -3,6 +3,7 @@ using System.Globalization; using System.Linq; using System.Threading; using System.Threading.Tasks; +using Tgstation.Server.Api.Models; using Tgstation.Server.Host.Components.Repository; using Tgstation.Server.Host.Components.Watchdog; @@ -56,7 +57,7 @@ namespace Tgstation.Server.Host.Components.Chat.Commands } else { - if (!watchdog.Running) + if (watchdog.Status == WatchdogStatus.Offline) return "Server offline!"; result = watchdog.ActiveCompileJob?.RevisionInformation.CommitSha; } diff --git a/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs index e9e5083e3f..4fc0000179 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs @@ -204,7 +204,6 @@ namespace Tgstation.Server.Host.Components.Watchdog { Server?.Dispose(); Server = null; - Running = false; gracefulRebootRequired = false; } @@ -212,7 +211,10 @@ namespace Tgstation.Server.Host.Components.Watchdog protected sealed override ISessionController GetActiveController() => Server; /// - protected sealed override async Task InitControllers(Action callBeforeRecurse, Task chatTask, DualReattachInformation reattachInfo, CancellationToken cancellationToken) + protected sealed override async Task InitControllers( + Task chatTask, + DualReattachInformation reattachInfo, + CancellationToken cancellationToken) { var serverToReattach = reattachInfo?.Alpha ?? reattachInfo?.Bravo; var serverToKill = reattachInfo?.Bravo ?? reattachInfo?.Alpha; @@ -274,8 +276,11 @@ namespace Tgstation.Server.Host.Components.Watchdog // possiblity of null servers due to failed reattaches if (Server == null) { - callBeforeRecurse(); - await NotifyOfFailedReattach(thereIsAnInactiveServerToKill && !inactiveServerWasKilled, cancellationToken).ConfigureAwait(false); + await ReattachFailure( + chatTask, + thereIsAnInactiveServerToKill && !inactiveServerWasKilled, + cancellationToken) + .ConfigureAwait(false); return; } @@ -337,24 +342,6 @@ namespace Tgstation.Server.Host.Components.Watchdog return Restart(true, cancellationToken); } - /// - /// Send a chat message and log about a failed reattach operation and attempts another call to . - /// - /// If the inactive server was reattached successfully. - /// The for the operation/ - /// A representing the running operation. - async Task NotifyOfFailedReattach(bool inactiveReattachSuccess, CancellationToken cancellationToken) - { - // we lost the server, just restart entirely - DisposeAndNullControllers(); - const string FailReattachMessage = "Unable to properly reattach to server! Restarting..."; - Logger.LogWarning(FailReattachMessage); - Logger.LogDebug(inactiveReattachSuccess ? "Also could not reattach to inactive server!" : "Inactive server was reattached successfully!"); - Task chatTask = Chat.SendWatchdogMessage(FailReattachMessage, false, cancellationToken); - await LaunchNoLock(true, false, null, cancellationToken).ConfigureAwait(false); - await chatTask.ConfigureAwait(false); - } - /// public sealed override Task InstanceRenamed(string newInstanceName, CancellationToken cancellationToken) => Server?.InstanceRenamed(newInstanceName, cancellationToken) ?? Task.CompletedTask; diff --git a/src/Tgstation.Server.Host/Components/Watchdog/ExperimentalWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/ExperimentalWatchdog.cs index 320ca1a96c..e547e00568 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/ExperimentalWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/ExperimentalWatchdog.cs @@ -5,6 +5,7 @@ using System.Diagnostics; using System.Globalization; using System.Threading; using System.Threading.Tasks; +using Tgstation.Server.Api.Models; using Tgstation.Server.Api.Models.Internal; using Tgstation.Server.Host.Components.Chat; using Tgstation.Server.Host.Components.Deployment; @@ -34,7 +35,7 @@ namespace Tgstation.Server.Host.Components.Watchdog public override Models.CompileJob ActiveCompileJob => (AlphaIsActive ? alphaServer : bravoServer)?.Dmb.CompileJob; /// - public override RebootState? RebootState => Running ? (AlphaIsActive ? alphaServer?.RebootState : bravoServer?.RebootState) : null; + public override RebootState? RebootState => Status != WatchdogStatus.Offline ? (AlphaIsActive ? alphaServer?.RebootState : bravoServer?.RebootState) : null; /// /// Server designation alpha @@ -407,7 +408,6 @@ namespace Tgstation.Server.Host.Components.Watchdog alphaServer = null; bravoServer?.Dispose(); bravoServer = null; - Running = false; } /// @@ -445,7 +445,10 @@ namespace Tgstation.Server.Host.Components.Watchdog /// #pragma warning disable CA1502 // TODO: Decomplexify - protected override async Task InitControllers(Action callBeforeRecurse, Task chatTask, DualReattachInformation reattachInfo, CancellationToken cancellationToken) + protected override async Task InitControllers( + Task chatTask, + DualReattachInformation reattachInfo, + CancellationToken cancellationToken) { Debug.Assert(alphaServer == null && bravoServer == null, "Entered LaunchNoLock with one or more of the servers not being null!"); @@ -517,14 +520,7 @@ namespace Tgstation.Server.Host.Components.Watchdog || (bravoServer == null && !reattachInfo.AlphaIsActive)) { // we lost the active server, just restart entirely - DisposeAndNullControllers(); - const string FailReattachMessage = "Unable to properly reattach to active server! Restarting..."; - Logger.LogWarning(FailReattachMessage); - Logger.LogDebug(bothServersDead ? "Also could not reattach to inactive server!" : "Inactive server was reattached successfully!"); - chatTask = Chat.SendWatchdogMessage(FailReattachMessage, false, cancellationToken); - callBeforeRecurse(); - await LaunchNoLock(true, false, null, cancellationToken).ConfigureAwait(false); - await chatTask.ConfigureAwait(false); + await ReattachFailure(chatTask, !bothServersDead, cancellationToken).ConfigureAwait(false); return; } diff --git a/src/Tgstation.Server.Host/Components/Watchdog/IWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/IWatchdog.cs index bc2fdbd8a7..b593667a5b 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/IWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/IWatchdog.cs @@ -2,6 +2,7 @@ using System; using System.Threading; using System.Threading.Tasks; +using Tgstation.Server.Api.Models; using Tgstation.Server.Api.Models.Internal; using Tgstation.Server.Host.Components.Events; using Tgstation.Server.Host.Components.Session; @@ -14,9 +15,9 @@ namespace Tgstation.Server.Host.Components.Watchdog public interface IWatchdog : IHostedService, IDisposable, IEventConsumer, IRenameNotifyee { /// - /// If the watchdog is running + /// The current . /// - bool Running { get; } + WatchdogStatus Status { get; } /// /// If the alpha server is the active server @@ -24,7 +25,7 @@ namespace Tgstation.Server.Host.Components.Watchdog bool AlphaIsActive { get; } /// - /// The currently running on the server + /// The currently running on the server /// Models.CompileJob ActiveCompileJob { get; } @@ -51,7 +52,7 @@ namespace Tgstation.Server.Host.Components.Watchdog Task Launch(CancellationToken cancellationToken); /// - /// Changes the . If currently triggers a graceful restart + /// Changes the . If currently running, may trigger a graceful restart. /// /// The new . May be modified /// The for the operation diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs index 6c41a43985..01e34a45af 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs @@ -30,13 +30,13 @@ namespace Tgstation.Server.Host.Components.Watchdog abstract class WatchdogBase : IWatchdog, ICustomCommandHandler, IRestartHandler { /// - public bool Running + public WatchdogStatus Status { - get => running; + get => status; set { - running = value; - Logger.LogTrace("Running set to {0}", running); + status = value; + Logger.LogTrace("Status set to {0}", status); } } @@ -56,7 +56,7 @@ namespace Tgstation.Server.Host.Components.Watchdog public abstract RebootState? RebootState { get; } /// - /// that completes when are changed and we are . + /// that completes when are changed and we are running /// protected TaskCompletionSource ActiveParametersUpdated { get; set; } @@ -150,6 +150,11 @@ namespace Tgstation.Server.Host.Components.Watchdog /// Task monitorTask; + /// + /// Backing field for . + /// + WatchdogStatus status; + /// /// The number of hearbeats missed. /// @@ -160,11 +165,6 @@ namespace Tgstation.Server.Host.Components.Watchdog /// bool releaseServers; - /// - /// Backing field for . - /// - bool running; - /// /// If the has been d. /// @@ -259,7 +259,7 @@ namespace Tgstation.Server.Host.Components.Watchdog /// A representing the running operation. async Task TerminateNoLock(bool graceful, bool announce, CancellationToken cancellationToken) { - if (!Running) + if (Status == WatchdogStatus.Offline) return; if (!graceful) { @@ -328,8 +328,8 @@ namespace Tgstation.Server.Host.Components.Watchdog : "Restarting"; var message4 = $"DEFCON 1: Four heartbeats have been missed! {actionTaken}..."; Logger.LogWarning(message4); - DisposeAndNullControllers(); await Chat.SendWatchdogMessage(message4, false, cancellationToken).ConfigureAwait(false); + DisposeAndNullControllers(); return shouldShutdown ? MonitorAction.Exit : MonitorAction.Restart; default: Logger.LogError("Invalid heartbeats missed count: {0}", heartbeatsMissed); @@ -354,17 +354,12 @@ namespace Tgstation.Server.Host.Components.Watchdog { Logger.LogTrace("Begin LaunchImplNoLock"); - if (Running) - throw new JobException(ErrorCode.WatchdogRunning); - if (reattachInfo == null && !DmbFactory.DmbAvailable) throw new JobException(ErrorCode.WatchdogCompileJobCorrupted); // this is necessary, the monitor could be in it's sleep loop trying to restart, if so cancel THAT monitor and start our own with blackjack and hookers Task announceTask; - if (startMonitor && await StopMonitor().ConfigureAwait(false)) - announceTask = Chat.SendWatchdogMessage("Automatic retry sequence cancelled by manual launch. Restarting...", false, cancellationToken); - else if (announce) + if (announce) { announceTask = Chat.SendWatchdogMessage(reattachInfo == null ? "Launching..." : "Reattaching...", false, cancellationToken); // simple announce if (reattachInfo == null) @@ -379,41 +374,27 @@ namespace Tgstation.Server.Host.Components.Watchdog LastLaunchParameters = ActiveLaunchParameters; heartbeatsMissed = 0; - // for when we call ourself and want to not catch thrown exceptions - var recursiveCallToHappen = false; try { - await InitControllers(() => recursiveCallToHappen = true, announceTask, reattachInfo, cancellationToken).ConfigureAwait(false); - if (recursiveCallToHappen) - return; - - await announceTask.ConfigureAwait(false); - - Logger.LogInformation("Launched servers successfully"); - Running = true; - - if (startMonitor) - { - monitorCts = new CancellationTokenSource(); - monitorTask = MonitorLifetimes(monitorCts.Token); - } + await InitControllers(announceTask, reattachInfo, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + Logger.LogTrace("Controller initialization canceled!"); + throw; } catch (Exception e) { // don't try to send chat tasks or warning logs if were suppressing exceptions or cancelled - if (!recursiveCallToHappen && !cancellationToken.IsCancellationRequested) + var originalChatTask = announceTask; + async Task ChainChatTaskWithErrorMessage() { - var originalChatTask = announceTask; - async Task ChainChatTaskWithErrorMessage() - { - await originalChatTask.ConfigureAwait(false); - await Chat.SendWatchdogMessage("Startup failed!", false, cancellationToken).ConfigureAwait(false); - } - - announceTask = ChainChatTaskWithErrorMessage(); - Logger.LogWarning("Failed to start watchdog: {0}", e.ToString()); + await originalChatTask.ConfigureAwait(false); + await Chat.SendWatchdogMessage("Startup failed!", false, cancellationToken).ConfigureAwait(false); } + announceTask = ChainChatTaskWithErrorMessage(); + Logger.LogWarning("Failed to start watchdog: {0}", e.ToString()); throw; } finally @@ -428,6 +409,14 @@ namespace Tgstation.Server.Host.Components.Watchdog Logger.LogTrace("Announcement task canceled!"); } } + + Logger.LogInformation("Controller(s) initialized successfully"); + + if (startMonitor) + { + monitorCts = new CancellationTokenSource(); + monitorTask = MonitorLifetimes(monitorCts.Token); + } } /// @@ -472,7 +461,34 @@ namespace Tgstation.Server.Host.Components.Watchdog } /// - /// Call and null the fields for all s and set to . + /// Call from when a reattach operation fails to attempt a fresh start. + /// + /// A, possibly active, for an outgoing chat message. + /// If the inactive server was successfully reattached. + /// The for the operation. + /// A representing the running operation. + protected async Task ReattachFailure(Task chatTask, bool reattachedInactive, CancellationToken cancellationToken) + { + // we lost the server, just restart entirely + DisposeAndNullControllers(); + const string FailReattachMessage = "Unable to properly reattach to server! Restarting watchdog..."; + Logger.LogWarning(FailReattachMessage); + Logger.LogDebug( + reattachedInactive + ? "Also could not reattach to inactive server!" + : "Inactive server was reattached successfully!"); + + async Task ChainChatTask() + { + await chatTask.ConfigureAwait(false); + await Chat.SendWatchdogMessage(FailReattachMessage, false, cancellationToken).ConfigureAwait(false); + } + + await InitControllers(ChainChatTask(), null, cancellationToken).ConfigureAwait(false); + } + + /// + /// Call and null the fields for all s. /// protected abstract void DisposeAndNullControllersImpl(); @@ -531,17 +547,15 @@ namespace Tgstation.Server.Host.Components.Watchdog var chatTask = Task.CompletedTask; for (var retryAttempts = 1; ; ++retryAttempts) { - Exception launchException = null; + Status = WatchdogStatus.Restoring; + Exception launchException; using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false)) try { // use LaunchImplNoLock without announcements or restarting the monitor await LaunchNoLock(false, false, null, cancellationToken).ConfigureAwait(false); - if (Running) - { - Logger.LogDebug("Relaunch successful, resetting monitor state..."); - return new MonitorState(); - } + Logger.LogDebug("Relaunch successful, resetting monitor state..."); + return new MonitorState(); } catch (OperationCanceledException) { @@ -551,29 +565,29 @@ namespace Tgstation.Server.Host.Components.Watchdog { launchException = e; } + finally + { + await chatTask.ConfigureAwait(false); + } - await chatTask.ConfigureAwait(false); - if (!Running) - { - if (launchException == null) - Logger.LogWarning("Failed to automatically restart the watchdog! Attempt: {0}", retryAttempts); - else - Logger.LogWarning("Failed to automatically restart the watchdog! Attempt: {0}, Exception: {1}", retryAttempts, launchException); - var retryDelay = Math.Min( - Convert.ToInt32( - Math.Pow(2, retryAttempts)), - TimeSpan.FromHours(1).Seconds); // max of one hour, increasing by a power of 2 each time - chatTask = Chat.SendWatchdogMessage( - $"Failed to restart (Attempt: {retryAttempts}), retrying in {retryDelay}", - false, - cancellationToken); - await Task.WhenAll( - AsyncDelayer.Delay( - TimeSpan.FromSeconds(retryDelay), - cancellationToken), - chatTask) - .ConfigureAwait(false); - } + Logger.LogWarning("Failed to automatically restart the watchdog! Attempt: {0}, Exception: {1}", retryAttempts, launchException); + + var retryDelay = Math.Min( + Convert.ToInt32( + Math.Pow(2, retryAttempts)), + TimeSpan.FromHours(1).Seconds); // max of one hour, increasing by a power of 2 each time + + chatTask = Chat.SendWatchdogMessage( + $"Failed to restart (Attempt: {retryAttempts}), retrying in {retryDelay}", + false, + cancellationToken); + + await Task.WhenAll( + AsyncDelayer.Delay( + TimeSpan.FromSeconds(retryDelay), + cancellationToken), + chatTask) + .ConfigureAwait(false); } } @@ -585,6 +599,7 @@ namespace Tgstation.Server.Host.Components.Watchdog private async Task MonitorLifetimes(CancellationToken cancellationToken) { Logger.LogTrace("Entered MonitorLifetimes"); + Status = WatchdogStatus.Online; using var _ = cancellationToken.Register(() => Logger.LogTrace("Monitor cancellationToken triggered")); // this function is responsible for calling HandlerMonitorWakeup when necessary and manitaining the MonitorState @@ -738,6 +753,7 @@ namespace Tgstation.Server.Host.Components.Watchdog } DisposeAndNullControllers(); + Status = WatchdogStatus.Offline; Logger.LogTrace("Monitor exiting..."); } @@ -745,12 +761,11 @@ namespace Tgstation.Server.Host.Components.Watchdog /// /// Starts all s. /// - /// An that must be run before making a recursive call to . /// A, possibly active, for an outgoing chat message. /// to use, if any /// The for the operation /// A representing the running operation - protected abstract Task InitControllers(Action callBeforeRecurse, Task chatTask, DualReattachInformation reattachInfo, CancellationToken cancellationToken); + protected abstract Task InitControllers(Task chatTask, DualReattachInformation reattachInfo, CancellationToken cancellationToken); /// public async Task ChangeSettings(DreamDaemonLaunchParameters launchParameters, CancellationToken cancellationToken) @@ -759,7 +774,7 @@ namespace Tgstation.Server.Host.Components.Watchdog { bool match = launchParameters.CanApplyWithoutReboot(ActiveLaunchParameters); ActiveLaunchParameters = launchParameters; - if (match || !Running) + if (match || Status == WatchdogStatus.Offline) return; ActiveParametersUpdated.TrySetResult(null); // queue an update @@ -807,8 +822,8 @@ namespace Tgstation.Server.Host.Components.Watchdog { using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false)) { - if (!Running) - return "ERROR: Server offline!"; + if (Status == WatchdogStatus.Offline) + return "TGS: Server offline!"; var commandObject = new ChatCommand(sender, commandName, arguments); @@ -831,6 +846,8 @@ namespace Tgstation.Server.Host.Components.Watchdog /// public async Task Launch(CancellationToken cancellationToken) { + if (Status != WatchdogStatus.Offline) + throw new JobException(ErrorCode.WatchdogRunning); using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false)) await LaunchNoLock(true, true, null, cancellationToken).ConfigureAwait(false); } @@ -840,7 +857,7 @@ namespace Tgstation.Server.Host.Components.Watchdog { using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false)) { - if (!Running) + if (Status == WatchdogStatus.Offline) return; var toClear = GetActiveController(); if (toClear != null) @@ -851,25 +868,19 @@ namespace Tgstation.Server.Host.Components.Watchdog /// public async Task Restart(bool graceful, CancellationToken cancellationToken) { - if (!Running) - throw new JobException(ErrorCode.WatchdogRunning); + if (Status == WatchdogStatus.Offline) + throw new JobException(ErrorCode.WatchdogNotRunning); Logger.LogTrace("Begin Restart. Graceful: {0}", graceful); using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false)) { if (!graceful) { - Task chatTask; - bool hard = Running; - if (hard) - { - chatTask = Chat.SendWatchdogMessage("Manual restart triggered...", false, cancellationToken); - await TerminateNoLock(false, false, cancellationToken).ConfigureAwait(false); - } - else - chatTask = Task.CompletedTask; - await LaunchNoLock(true, !hard, null, cancellationToken).ConfigureAwait(false); + var chatTask = Chat.SendWatchdogMessage("Manual restart triggered...", false, cancellationToken); + await TerminateNoLock(false, false, cancellationToken).ConfigureAwait(false); + await LaunchNoLock(true, false, null, cancellationToken).ConfigureAwait(false); await chatTask.ConfigureAwait(false); + return; } var toReboot = GetActiveController(); @@ -941,7 +952,7 @@ namespace Tgstation.Server.Host.Components.Watchdog public async Task HandleRestart(Version updateVersion, CancellationToken cancellationToken) { releaseServers = true; - if (Running) + if (Status == WatchdogStatus.Online) await Chat.SendWatchdogMessage("Detaching...", false, cancellationToken).ConfigureAwait(false); } diff --git a/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs b/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs index a2daa40649..a9d6ec4d32 100644 --- a/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs +++ b/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs @@ -63,7 +63,7 @@ namespace Tgstation.Server.Host.Controllers // alias for launching DD var instance = instanceManager.GetInstance(Instance); - if (instance.Watchdog.Running) + if (instance.Watchdog.Status != WatchdogStatus.Offline) return Conflict(new ErrorMessage(ErrorCode.WatchdogRunning)); var job = new Models.Job @@ -130,7 +130,7 @@ namespace Tgstation.Server.Host.Controllers result.CurrentAllowWebclient = llp?.AllowWebClient.Value; result.PrimaryPort = settings.PrimaryPort.Value; result.AllowWebClient = settings.AllowWebClient.Value; - result.Running = dd.Running; + result.Status = dd.Status; result.SecondaryPort = settings.SecondaryPort.Value; result.SecurityLevel = settings.SecurityLevel.Value; result.SoftRestart = rstate == RebootState.Restart; @@ -143,7 +143,10 @@ namespace Tgstation.Server.Host.Controllers if (revision) { var latestCompileJob = instance.LatestCompileJob(); - result.ActiveCompileJob = ((dd.Running ? dd.ActiveCompileJob : latestCompileJob) ?? latestCompileJob)?.ToApi(); + result.ActiveCompileJob = ((instance.Watchdog.Status == WatchdogStatus.Offline + ? dd.ActiveCompileJob + : latestCompileJob) ?? latestCompileJob) + ?.ToApi(); if (latestCompileJob?.Id != result.ActiveCompileJob?.Id) result.StagedCompileJob = latestCompileJob?.ToApi(); } @@ -296,7 +299,7 @@ namespace Tgstation.Server.Host.Controllers var watchdog = instanceManager.GetInstance(Instance).Watchdog; - if (!watchdog.Running) + if (watchdog.Status == WatchdogStatus.Offline) return Conflict(new ErrorMessage(ErrorCode.WatchdogNotRunning)); await jobManager.RegisterOperation(job, (paramJob, databaseContextFactory, progressReporter, ct) => watchdog.Restart(false, ct), cancellationToken).ConfigureAwait(false); @@ -325,7 +328,7 @@ namespace Tgstation.Server.Host.Controllers var watchdog = instanceManager.GetInstance(Instance).Watchdog; - if (!watchdog.Running) + if (watchdog.Status == WatchdogStatus.Offline) return Conflict(new ErrorMessage(ErrorCode.WatchdogNotRunning)); await jobManager.RegisterOperation( diff --git a/tests/Tgstation.Server.Tests/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Instance/WatchdogTest.cs index 92d06ca4d7..6cfd528c12 100644 --- a/tests/Tgstation.Server.Tests/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Instance/WatchdogTest.cs @@ -73,7 +73,7 @@ namespace Tgstation.Server.Tests.Instance global::System.Console.WriteLine("TEST: WATCHDOG BASIC TEST"); var daemonStatus = await DeployTestDme("BasicOperation/basic_operation_test", DreamDaemonSecurity.Ultrasafe, cancellationToken); - Assert.IsFalse(daemonStatus.Running.Value); + Assert.AreEqual(WatchdogStatus.Offline, daemonStatus.Status.Value); Assert.IsNotNull(daemonStatus.ActiveCompileJob); Assert.IsNull(daemonStatus.StagedCompileJob); Assert.AreEqual(DMApiConstants.Version, daemonStatus.ActiveCompileJob.DMApiVersion); @@ -84,14 +84,14 @@ namespace Tgstation.Server.Tests.Instance await WaitForJob(startJob, 10, false, cancellationToken); daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); - Assert.IsTrue(daemonStatus.Running.Value); + Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value); Assert.AreEqual(false, daemonStatus.SoftRestart); Assert.AreEqual(false, daemonStatus.SoftShutdown); await GracefulWatchdogShutdown(30, cancellationToken); daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); - Assert.IsFalse(daemonStatus.Running.Value); + Assert.AreEqual(WatchdogStatus.Offline, daemonStatus.Status.Value); await CheckDMApiFail(daemonStatus.ActiveCompileJob, cancellationToken); } @@ -112,7 +112,7 @@ namespace Tgstation.Server.Tests.Instance // lock on to DD and pause it so it can't heartbeat var ddProcs = System.Diagnostics.Process.GetProcessesByName("DreamDaemon").ToList(); if (ddProcs.Count != 1) - Assert.Inconclusive($"Incorrect number of DD processes: {ddProcs.Count}"); + Assert.Fail($"Incorrect number of DD processes: {ddProcs.Count}"); using var ddProc = ddProcs.Single(); IProcessExecutor executor = null; @@ -144,7 +144,7 @@ namespace Tgstation.Server.Tests.Instance await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken).ConfigureAwait(false); var ddStatus = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.AreEqual(1U, ddStatus.HeartbeatSeconds.Value); - if (!ddStatus.Running.Value) + if (ddStatus.Status.Value == WatchdogStatus.Offline) break; if (--timeout == 0) @@ -167,7 +167,7 @@ namespace Tgstation.Server.Tests.Instance var daemonStatus = await DeployTestDme(DmeName, DreamDaemonSecurity.Trusted, cancellationToken); var initialCompileJob = daemonStatus.ActiveCompileJob; - Assert.IsFalse(daemonStatus.Running.Value); + Assert.AreEqual(WatchdogStatus.Offline, daemonStatus.Status.Value); Assert.IsNotNull(daemonStatus.ActiveCompileJob); Assert.IsNull(daemonStatus.StagedCompileJob); Assert.AreEqual(DMApiConstants.Version, daemonStatus.ActiveCompileJob.DMApiVersion); @@ -179,7 +179,7 @@ namespace Tgstation.Server.Tests.Instance daemonStatus = await DeployTestDme(DmeName, DreamDaemonSecurity.Safe, cancellationToken); - Assert.IsTrue(daemonStatus.Running.Value); + Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value); Assert.AreEqual(initialCompileJob.Id, daemonStatus.ActiveCompileJob.Id); var newerCompileJob = daemonStatus.StagedCompileJob; @@ -197,7 +197,7 @@ namespace Tgstation.Server.Tests.Instance await instanceClient.DreamDaemon.Shutdown(cancellationToken); daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); - Assert.IsFalse(daemonStatus.Running.Value); + Assert.AreEqual(WatchdogStatus.Offline, daemonStatus.Status.Value); } async Task RunLongRunningTestThenUpdateWithByondVersionSwitch(CancellationToken cancellationToken) @@ -226,7 +226,7 @@ namespace Tgstation.Server.Tests.Instance await DeployTestDme(DmeName, DreamDaemonSecurity.Safe, cancellationToken); var daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); - Assert.IsTrue(daemonStatus.Running.Value); + Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value); Assert.IsNotNull(daemonStatus.ActiveCompileJob); @@ -246,7 +246,7 @@ namespace Tgstation.Server.Tests.Instance await instanceClient.DreamDaemon.Shutdown(cancellationToken); daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); - Assert.IsFalse(daemonStatus.Running.Value); + Assert.AreEqual(WatchdogStatus.Offline, daemonStatus.Status.Value); } public async Task StartAndLeaveRunning(CancellationToken cancellationToken) @@ -259,6 +259,22 @@ namespace Tgstation.Server.Tests.Instance var startJob = await instanceClient.DreamDaemon.Start(cancellationToken).ConfigureAwait(false); await WaitForJob(startJob, 40, false, cancellationToken); + + var daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); + Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value); + + // Try killing the DD process to ensure it gets set to the restoring state + var ddProcs = System.Diagnostics.Process.GetProcessesByName("DreamDaemon").ToList(); + if (ddProcs.Count != 1) + Assert.Fail($"Incorrect number of DD processes: {ddProcs.Count}"); + + using var ddProc = ddProcs.Single(); + ddProc.Kill(); + ddProc.WaitForExit(); + + await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken); + daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); + Assert.AreEqual(WatchdogStatus.Restoring, daemonStatus.Status.Value); } async Task TellWorldToReboot(CancellationToken cancellationToken) @@ -308,13 +324,13 @@ namespace Tgstation.Server.Tests.Instance }, cancellationToken); var newStatus = await instanceClient.DreamDaemon.Read(cancellationToken); - Assert.IsTrue(newStatus.SoftShutdown.Value || !newStatus.Running.Value); + Assert.IsTrue(newStatus.SoftShutdown.Value || (newStatus.Status.Value == WatchdogStatus.Offline)); do { await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken).ConfigureAwait(false); var ddStatus = await instanceClient.DreamDaemon.Read(cancellationToken); - if (!ddStatus.Running.Value) + if (ddStatus.Status.Value == WatchdogStatus.Offline) break; if (--timeout == 0) diff --git a/tests/Tgstation.Server.Tests/IntegrationTest.cs b/tests/Tgstation.Server.Tests/IntegrationTest.cs index eeb4562a50..6584ec587d 100644 --- a/tests/Tgstation.Server.Tests/IntegrationTest.cs +++ b/tests/Tgstation.Server.Tests/IntegrationTest.cs @@ -247,7 +247,7 @@ namespace Tgstation.Server.Tests await new JobsRequiredTest(instanceClient.Jobs).WaitForJob(reattachJob, 40, false, cancellationToken); var dd = await instanceClient.DreamDaemon.Read(cancellationToken); - Assert.IsTrue(dd.Running.Value); + Assert.AreEqual(WatchdogStatus.Online, dd.Status.Value); await instanceClient.DreamDaemon.Shutdown(cancellationToken); await instanceClient.DreamDaemon.Update(new DreamDaemon @@ -291,7 +291,7 @@ namespace Tgstation.Server.Tests var dd = await instanceClient.DreamDaemon.Read(cancellationToken); - Assert.IsTrue(dd.Running.Value); + Assert.AreEqual(WatchdogStatus.Online, dd.Status.Value); var repoTest = new RepositoryTest(instanceClient.Repository, instanceClient.Jobs).RunPostTest(cancellationToken); await new ChatTest(instanceClient.ChatBots, adminClient.Instances, instance).RunPostTest(cancellationToken);