From 3932f0047eee8cf20763ce69e774b9a8e642c6fb Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Tue, 2 Jun 2020 13:05:00 -0400 Subject: [PATCH] Watchdog docs - Cleanup code names/docs - Error code 80 returned when attempting to restart a stopped watchdog --- build/Version.props | 6 +- src/Tgstation.Server.Api/Models/ErrorCode.cs | 6 + .../Components/Session/README.md | 2 +- .../Components/Watchdog/BasicWatchdog.cs | 18 +++ .../Watchdog/ExperimentalWatchdog.cs | 2 +- .../Components/Watchdog/README.md | 52 ++++++++ .../Components/Watchdog/WatchdogBase.cs | 114 ++++++++---------- 7 files changed, 133 insertions(+), 67 deletions(-) create mode 100644 src/Tgstation.Server.Host/Components/Watchdog/README.md diff --git a/build/Version.props b/build/Version.props index 264b6860ac..2a24e56613 100644 --- a/build/Version.props +++ b/build/Version.props @@ -2,9 +2,9 @@ - 4.3.1 - 6.5.1 - 7.1.1 + 4.3.2 + 6.6.0 + 7.2.0 5.2.2 0.4.0 1.1.0 diff --git a/src/Tgstation.Server.Api/Models/ErrorCode.cs b/src/Tgstation.Server.Api/Models/ErrorCode.cs index 1f8141301d..f54147b304 100644 --- a/src/Tgstation.Server.Api/Models/ErrorCode.cs +++ b/src/Tgstation.Server.Api/Models/ErrorCode.cs @@ -489,5 +489,11 @@ namespace Tgstation.Server.Api.Models /// [Description("The deployment succeeded but one or more notification events failed!")] PostDeployFailure, + + /// + /// Attempted to restart a stopped watchdog. + /// + [Description("Cannot restart the watchdog as it is not running!")] + WatchdogNotRunning, } } \ No newline at end of file diff --git a/src/Tgstation.Server.Host/Components/Session/README.md b/src/Tgstation.Server.Host/Components/Session/README.md index c8d6756420..cf6b8f3516 100644 --- a/src/Tgstation.Server.Host/Components/Session/README.md +++ b/src/Tgstation.Server.Host/Components/Session/README.md @@ -2,7 +2,7 @@ "Session" refers to a single invocation of DreamDaemon. The code in here is meant for creating and managing sessions. This is different from the watchdog as it is more low level. -[ISessionController](./ISessionController.cs) ([implementation](SessionController.cs)) is the representation of DreamDaemon. It handles most of the DMAPI interop, such as DMAPI validation, and raising events for things such as the world rebooting, or the process ending. These are created by the [ISessionControllerFactory](./ISessionControllerFactory.cs) ([implementation](./SessionControllerFactory.cs)) which handles actually launching DreamDaemon and linking together all the components it needs. Sessions give out a [LaunchResult](./LaunchResult.cs) which indicates how long it took for DreamDaemon to become responsive or if it exited before it did. +[ISessionController](./ISessionController.cs) ([implementation](SessionController.cs)) is the representation of DreamDaemon. It handles most of the DMAPI interop, such as DMAPI validation, and raising events for things such as the world rebooting, or the process ending. These are created by the [ISessionControllerFactory](./ISessionControllerFactory.cs) ([implementation](./SessionControllerFactory.cs)) which handles actually launching DreamDaemon and linking together all the components it needs. It also runs some sanity checks, such as if the port requested is avaiable or if the BYOND pager is running (which can cause issues). Sessions give out a [LaunchResult](./LaunchResult.cs) which indicates how long it took for DreamDaemon to become responsive or if it exited before it did. Sessions can be detached and reattached. When a session is detached, it generates [ReattachInformation](./ReattachInformation.cs). This is stored in and the database as [DualReattachInformation](./DualReattachInformation.cs) by passing it into and out of [IReattachInfoHandler](./IReattachInfoHandler.cs) ([implementation](./ReattachInfoHandler.cs)). This data can be passed back into the `ISessionControllerFactory` to reattach the session. diff --git a/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs index e3e8968fd8..1722376728 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs @@ -328,5 +328,23 @@ namespace Tgstation.Server.Host.Components.Watchdog return Restart(true, cancellationToken); } + + /// + /// Send a chat message and log about a failed reattach operation and attempts another call to . + /// + /// If the inactive server was reattached successfully. + /// The for the operation/ + /// A representing the running operation. + async Task NotifyOfFailedReattach(bool inactiveReattachSuccess, CancellationToken cancellationToken) + { + // we lost the server, just restart entirely + DisposeAndNullControllers(); + const string FailReattachMessage = "Unable to properly reattach to server! Restarting..."; + Logger.LogWarning(FailReattachMessage); + Logger.LogDebug(inactiveReattachSuccess ? "Also could not reattach to inactive server!" : "Inactive server was reattached successfully!"); + Task chatTask = Chat.SendWatchdogMessage(FailReattachMessage, false, cancellationToken); + await LaunchNoLock(true, false, null, cancellationToken).ConfigureAwait(false); + await chatTask.ConfigureAwait(false); + } } } diff --git a/src/Tgstation.Server.Host/Components/Watchdog/ExperimentalWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/ExperimentalWatchdog.cs index 6547ef6d62..07c73f7376 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/ExperimentalWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/ExperimentalWatchdog.cs @@ -512,7 +512,7 @@ namespace Tgstation.Server.Host.Components.Watchdog Logger.LogDebug(bothServersDead ? "Also could not reattach to inactive server!" : "Inactive server was reattached successfully!"); chatTask = Chat.SendWatchdogMessage(FailReattachMessage, false, cancellationToken); callBeforeRecurse(); - await LaunchImplNoLock(true, false, null, cancellationToken).ConfigureAwait(false); + await LaunchNoLock(true, false, null, cancellationToken).ConfigureAwait(false); await chatTask.ConfigureAwait(false); return; } diff --git a/src/Tgstation.Server.Host/Components/Watchdog/README.md b/src/Tgstation.Server.Host/Components/Watchdog/README.md new file mode 100644 index 0000000000..8927a46a83 --- /dev/null +++ b/src/Tgstation.Server.Host/Components/Watchdog/README.md @@ -0,0 +1,52 @@ +# The Watchdog + +The watchdog is what keeps DreamDaemon alive and well. It's responsible for starting, stopping, rebooting it, sending TGS events to the DMAPI, running heartbeats, and applying patches. + +There are three main implementations of the interface [IWatchdog](./IWatchdog.cs) that deal with how it handles the last of those jobs. + +- The [BasicWatchdog](./BasicWatchdog.cs) kills DreamDaemon when the world reboots and immediately restarts it with the newly compiled .dmb. +- The [WindowsWatchdog](./WindowsWatchdog.cs) uses a quirk of the interaction of DD with the Windows file system. A symlink to the game folder is created. From here, we launch the game. When a new .dmb is available, this symlink is deleted and then immediately recreated pointing to the new game directory. When DreamDaemon reboots it will load the new game. +- The [ExperimentalWatchdog](./ExperimentalWatchdog) starts new servers when the new .dmb comes in. But it uses some, rather invasive, port trickery to reroute traffic from the old game server to the new one. The idea for this is to always have two servers running so one can act as a fallback for if the main one fails. This is currently a big work-in-progress as it has issues with BYOND bugs and static file sharing. + +[WatchdogBase](./WatchdogBase.cs) is the parent of all these implementations and contains the bulk of the monitoring code. More on that later. + +`IWatchdog`s are created via the [IWatchdogFactory](./IWatchdogFactory.cs) interface. The two implementations, [WatchdogFactory](./WatchdogFactory.cs) and [WindowsWatchdogFactory](./WindowsWatchdogFactory.cs), differ in that the latter creates `WindowsWatchdog`s as opposed to `BasicWatchdog`s. + +## Startup + +From the perspective of `WatchdogBase` + +- When an instance is onlined, `StartAsync()` is called on `WatchdogBase`. If the instance is configured to autostart the server, or if reattach information is available from the `IReattachInfoHandler` the watchdog will be launched here. Otherwise, it waits to be activated from the controller. +- We first do some sanity checks such as if the watchdog is already running and if the `IDmbFactory` has a IDmbProvider for us. +- We send out the annoucement of launch/reattach to the chat system. +- The `ActiveLaunchParameters` are made the `LastLaunchParameters` for reference. +- `InitControllers()` is called, which abstractly creates the `ISessionController`(s) for the watchdog. + - For the `BasicWatchdog`, the controller is created normally. + - For the `WindowsWatchdog`, we replace the `IDmbProvider` with a `WindowsSwappableDmbProvider` and setup the symlinking before deferring to the `BasicWatchdog` + - For the `ExperimentalWatchdog` we launch two servers in tandem. + - For reattaching, all three work similarly in that they use `ISessionControllerFactory` to reattach their sessions. The `ExperimentalWatchdog` will insert a `DeadSessionController` in place of one if only one server could not reattach. +- `MonitorLifetimes()` is called which is the "main loop" of the watchdog. + +## Monitoring + +Watchdog monitoring takes place in the `MonitorLifetimes()` functiona and is entirely event based. It responds to a set of [MonitorActivationReason](./MonitorActivationReason.cs)s and takes the apporprate action. This includes: +- When a server crashes. +- When a server calls `/world/prod/Reboot()`. +- When a new .dmb is deployed. +- When DreamDaemon settings are changed via the API. +- When it is time to make a heartbeat. + +Here's how this works. + +- A [MonitorState](./MonitorState.cs) object is created +- `Task`s are created for all possible activation reasons and an asynchrounous wait is made on them. +- Each activation reason is processed in a particular order via a call to `HandleMonitorWakeup()`. This function is allowed to mutate `MonitorState`. + - How each activation reason is handled depends on the watchdog type. For example, the `BasicWatchdog` will queue a graceful reboot when a new .dmb becomes available, wheras the `WindowsWatchdog` will immediately swap the symlink to it. + - Some actions are universal. i.e. if the server crashes, a reboot will occur. Or the server will shutdown on reboot if a graceful stop was queued. + - If the state's [MonitorAction](./MonitorAction.cs) changes, what happens after each activation may change + +This continues until `StopMonitor` is called, which triggers the `CancellationToken` in `MonitorLifetimes()` and terminates all `ISessionController`s. + +## Detaching + +`WatchdogBase` maintains a `IRestartRegistration` with TGS, meaning that, when TGS is rebooted, `HandleRestart()` will be called before `StopAsync()`. This has the effect of adjusting what happens when the monitor exits. Instead of terminating the server(s) it will instead call `ISessionController.Release()` on them. This generates the `ReattachInformation` which will be saved to the database in `StopAsync()`. diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs index 9874ce67fa..d2400a8170 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs @@ -119,7 +119,7 @@ namespace Tgstation.Server.Host.Components.Watchdog readonly object controllerDisposeLock; /// - /// If the should in + /// If the should in /// readonly bool autoStart; @@ -332,14 +332,14 @@ namespace Tgstation.Server.Host.Components.Watchdog /// to use, if any /// The for the operation /// A representing the running operation - protected async Task LaunchImplNoLock(bool startMonitor, bool announce, DualReattachInformation reattachInfo, CancellationToken cancellationToken) + protected async Task LaunchNoLock(bool startMonitor, bool announce, DualReattachInformation reattachInfo, CancellationToken cancellationToken) { Logger.LogTrace("Begin LaunchImplNoLock"); if (Running) throw new JobException(ErrorCode.WatchdogRunning); - if (!DmbFactory.DmbAvailable) + if (reattachInfo == null && !DmbFactory.DmbAvailable) throw new JobException(ErrorCode.WatchdogCompileJobCorrupted); // this is necessary, the monitor could be in it's sleep loop trying to restart, if so cancel THAT monitor and start our own with blackjack and hookers @@ -356,10 +356,13 @@ namespace Tgstation.Server.Host.Components.Watchdog heartbeatsMissed = 0; // for when we call ourself and want to not catch thrown exceptions - var ignoreNestedException = false; + var recursiveCallToHappen = false; try { - await InitControllers(() => ignoreNestedException = true, chatTask, reattachInfo, cancellationToken).ConfigureAwait(false); + await InitControllers(() => recursiveCallToHappen = true, chatTask, reattachInfo, cancellationToken).ConfigureAwait(false); + if (recursiveCallToHappen) + return; + await chatTask.ConfigureAwait(false); Logger.LogInformation("Launched servers successfully"); @@ -367,13 +370,14 @@ namespace Tgstation.Server.Host.Components.Watchdog if (startMonitor) { - StartMonitor(); + monitorCts = new CancellationTokenSource(); + monitorTask = MonitorLifetimes(monitorCts.Token); } } catch (Exception e) { // don't try to send chat tasks or warning logs if were suppressing exceptions or cancelled - if (!ignoreNestedException && !cancellationToken.IsCancellationRequested) + if (!recursiveCallToHappen && !cancellationToken.IsCancellationRequested) { var originalChatTask = chatTask; async Task ChainChatTaskWithErrorMessage() @@ -399,15 +403,6 @@ namespace Tgstation.Server.Host.Components.Watchdog } } - /// - /// Call and setup and . - /// - protected void StartMonitor() - { - monitorCts = new CancellationTokenSource(); - monitorTask = MonitorLifetimes(monitorCts.Token); - } - /// /// Stops . Doesn't kill the servers /// @@ -427,24 +422,6 @@ namespace Tgstation.Server.Host.Components.Watchdog return wasRunning; } - /// - /// Send a chat message and log about a failed reattach operation and attempts another call to . - /// - /// If the inactive server was reattached successfully. - /// The for the operation/ - /// A representing the running operation. - protected async Task NotifyOfFailedReattach(bool inactiveReattachSuccess, CancellationToken cancellationToken) - { - // we lost the server, just restart entirely - DisposeAndNullControllers(); - const string FailReattachMessage = "Unable to properly reattach to server! Restarting..."; - Logger.LogWarning(FailReattachMessage); - Logger.LogDebug(inactiveReattachSuccess ? "Also could not reattach to inactive server!" : "Inactive server was reattached successfully!"); - Task chatTask = Chat.SendWatchdogMessage(FailReattachMessage, false, cancellationToken); - await LaunchImplNoLock(true, false, null, cancellationToken).ConfigureAwait(false); - await chatTask.ConfigureAwait(false); - } - /// /// Check the of a given for errors and throw a if any are detected. /// @@ -477,6 +454,7 @@ namespace Tgstation.Server.Host.Components.Watchdog /// protected void DisposeAndNullControllers() { + Logger.LogTrace("DisposeAndNullControllers"); lock (controllerDisposeLock) DisposeAndNullControllersImpl(); } @@ -513,6 +491,11 @@ namespace Tgstation.Server.Host.Components.Watchdog MonitorState monitorState, CancellationToken cancellationToken); + /// + /// Attempt to restart the monitor from scratch. + /// + /// The for the operation. + /// A resulting in the new . private async Task MonitorRestart(CancellationToken cancellationToken) { Logger.LogTrace("Monitor restart!"); @@ -526,7 +509,7 @@ namespace Tgstation.Server.Host.Components.Watchdog try { // use LaunchImplNoLock without announcements or restarting the monitor - await LaunchImplNoLock(false, false, null, cancellationToken).ConfigureAwait(false); + await LaunchNoLock(false, false, null, cancellationToken).ConfigureAwait(false); if (Running) { Logger.LogDebug("Relaunch successful, resetting monitor state..."); @@ -568,7 +551,7 @@ namespace Tgstation.Server.Host.Components.Watchdog } /// - /// The loop that watches the watchdog. + /// The main loop of the watchdog. Ayschronously waits for events to occur and then responds to them. /// /// The for the operation. /// A representing the running operation. @@ -607,35 +590,32 @@ namespace Tgstation.Server.Host.Components.Watchdog // cancel waiting if requested var cancelTcs = new TaskCompletionSource(); - using (cancellationToken.Register(() => cancelTcs.SetCanceled())) - { - var toWaitOn = Task.WhenAny( - activeServerLifetime, - activeServerReboot, - inactiveServerLifetime, - inactiveServerReboot, - inactiveStartupComplete, - heartbeat, - newDmbAvailable, - cancelTcs.Task, - activeLaunchParametersChanged); + var toWaitOn = Task.WhenAny( + activeServerLifetime, + activeServerReboot, + inactiveServerLifetime, + inactiveServerReboot, + inactiveStartupComplete, + heartbeat, + newDmbAvailable, + cancelTcs.Task, + activeLaunchParametersChanged); - // wait for something to happen + // wait for something to happen + using (cancellationToken.Register(() => cancelTcs.SetCanceled())) await toWaitOn.ConfigureAwait(false); - } cancellationToken.ThrowIfCancellationRequested(); Logger.LogTrace("Monitor activated"); + // always run HandleMonitorWakeup from the context of the semaphore lock using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false)) { - // always run HandleMonitorWakeup from the context of the semaphore lock // multiple things may have happened, handle them one at a time for (var moreActivationsToProcess = true; moreActivationsToProcess && (monitorState.NextAction == MonitorAction.Continue || monitorState.NextAction == MonitorAction.Skip);) { MonitorActivationReason activationReason = default; // this will always be assigned before being used - // process the tasks in this order and call HandlerMonitorWakup for each bool CheckActivationReason(ref Task task, MonitorActivationReason testActivationReason) { var taskCompleted = task?.IsCompleted == true; @@ -651,6 +631,7 @@ namespace Tgstation.Server.Host.Components.Watchdog return false; } + // process the tasks in this order and call HandlerMonitorWakup for each depending on the new monitorState var anyActivation = CheckActivationReason(ref activeServerLifetime, MonitorActivationReason.ActiveServerCrashed) || CheckActivationReason(ref activeServerReboot, MonitorActivationReason.ActiveServerRebooted) || CheckActivationReason(ref newDmbAvailable, MonitorActivationReason.NewDmbAvailable) @@ -681,11 +662,14 @@ namespace Tgstation.Server.Host.Components.Watchdog } Logger.LogTrace("Next monitor action is to {0}", monitorState.NextAction); + + // Restart if requested if (monitorState.NextAction == MonitorAction.Restart) monitorState = await MonitorRestart(cancellationToken).ConfigureAwait(false); } catch (OperationCanceledException) { + // let this bubble, other exceptions caught below throw; } catch (Exception e) @@ -715,6 +699,7 @@ namespace Tgstation.Server.Host.Components.Watchdog } catch (OperationCanceledException) { + // stop signal Logger.LogDebug("Monitor cancelled"); if (releaseServers) @@ -732,7 +717,7 @@ namespace Tgstation.Server.Host.Components.Watchdog /// /// Starts all s. /// - /// An that must be run before making a recursive call to . + /// An that must be run before making a recursive call to . /// A, possibly active, for an outgoing chat message. /// to use, if any /// The for the operation @@ -818,7 +803,7 @@ namespace Tgstation.Server.Host.Components.Watchdog public async Task Launch(CancellationToken cancellationToken) { using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false)) - await LaunchImplNoLock(true, true, null, cancellationToken).ConfigureAwait(false); + await LaunchNoLock(true, true, null, cancellationToken).ConfigureAwait(false); } /// @@ -837,10 +822,13 @@ namespace Tgstation.Server.Host.Components.Watchdog /// public async Task Restart(bool graceful, CancellationToken cancellationToken) { + if (!Running) + throw new JobException(ErrorCode.WatchdogRunning); + Logger.LogTrace("Begin Restart. Graceful: {0}", graceful); using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false)) { - if (!graceful || !Running) + if (!graceful) { Task chatTask; if (Running) @@ -850,16 +838,14 @@ namespace Tgstation.Server.Host.Components.Watchdog } else chatTask = Task.CompletedTask; - await LaunchImplNoLock(true, !Running, null, cancellationToken).ConfigureAwait(false); + await LaunchNoLock(true, !Running, null, cancellationToken).ConfigureAwait(false); await chatTask.ConfigureAwait(false); } var toReboot = GetActiveController(); - if (toReboot != null) - { - if (!await toReboot.SetRebootState(Session.RebootState.Restart, cancellationToken).ConfigureAwait(false)) - Logger.LogWarning("Unable to send reboot state change event!"); - } + if (toReboot != null + && !await toReboot.SetRebootState(Session.RebootState.Restart, cancellationToken).ConfigureAwait(false)) + Logger.LogWarning("Unable to send reboot state change event!"); } } @@ -898,7 +884,7 @@ namespace Tgstation.Server.Host.Components.Watchdog await jobManager.RegisterOperation(job, async (j, databaseContextFactory, progressFunction, ct) => { using (await SemaphoreSlimContext.Lock(Semaphore, ct).ConfigureAwait(false)) - await LaunchImplNoLock(true, true, reattachInfo, ct).ConfigureAwait(false); + await LaunchNoLock(true, true, reattachInfo, ct).ConfigureAwait(false); }, cancellationToken).ConfigureAwait(false); } @@ -907,7 +893,11 @@ namespace Tgstation.Server.Host.Components.Watchdog { await TerminateNoLock(false, !releaseServers, cancellationToken).ConfigureAwait(false); if (releasedReattachInformation != null) + { await reattachInfoHandler.Save(releasedReattachInformation, cancellationToken).ConfigureAwait(false); + releasedReattachInformation = null; + releaseServers = false; + } } ///