diff --git a/src/Tgstation.Server.Host/Components/Instance.cs b/src/Tgstation.Server.Host/Components/Instance.cs index 899310152a..5353b9d6f8 100644 --- a/src/Tgstation.Server.Host/Components/Instance.cs +++ b/src/Tgstation.Server.Host/Components/Instance.cs @@ -372,7 +372,7 @@ namespace Tgstation.Server.Host.Components var startSha = repo.Head; if (!repo.Tracking) { - logger.LogTrace("Aborting repo update, not tracking origin!"); + logger.LogTrace("Aborting repo update, active ref not tracking any remote branch!"); deploySha = startSha; return; } diff --git a/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs new file mode 100644 index 0000000000..2f9f9f5b8f --- /dev/null +++ b/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs @@ -0,0 +1,359 @@ +using Byond.TopicSender; +using Microsoft.Extensions.Logging; +using System; +using System.Globalization; +using System.Threading; +using System.Threading.Tasks; +using Tgstation.Server.Api.Models.Internal; +using Tgstation.Server.Host.Components.Chat; +using Tgstation.Server.Host.Components.Compiler; +using Tgstation.Server.Host.Core; + +namespace Tgstation.Server.Host.Components.Watchdog +{ + /// + /// A that manages one server. + /// + sealed class BasicWatchdog : WatchdogBase + { + /// + public override bool AlphaIsActive => true; + + /// + public override Models.CompileJob ActiveCompileJob => server?.Dmb.CompileJob; + + /// + public override RebootState? RebootState => server?.RebootState; + + /// + /// The single . + /// + ISessionController server; + + /// + /// Initializes a new instance of the . + /// + /// The for the . + /// The for the . + /// The for the . + /// The for the . + /// The for the . + /// The for the . + /// The for the . + /// The for the . + /// The for the . + /// The for the . + /// The for the . + /// The for the . + /// The for the . + /// The autostart value for the . + public BasicWatchdog( + IChat chat, + ISessionControllerFactory sessionControllerFactory, + IDmbFactory dmbFactory, + IReattachInfoHandler reattachInfoHandler, + IDatabaseContextFactory databaseContextFactory, + IByondTopicSender byondTopicSender, + IEventConsumer eventConsumer, + IJobManager jobManager, + IServerControl serverControl, + IAsyncDelayer asyncDelayer, + ILogger logger, + DreamDaemonLaunchParameters initialLaunchParameters, + Api.Models.Instance instance, + bool autoStart) + : base( + chat, + sessionControllerFactory, + dmbFactory, + reattachInfoHandler, + databaseContextFactory, + byondTopicSender, + eventConsumer, + jobManager, + serverControl, + asyncDelayer, + logger, + initialLaunchParameters, + instance, + autoStart) + { } + + async Task HandleMonitorWakeup(MonitorActivationReason reason, CancellationToken cancellationToken) + { + switch (reason) + { + case MonitorActivationReason.ActiveServerCrashed: + string exitWord = server.TerminationWasRequested ? "exited" : "crashed"; + if (server.RebootState == Watchdog.RebootState.Shutdown) + { + // the time for graceful shutdown is now + await Chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Server {0}! Exiting due to graceful termination request...", exitWord), cancellationToken).ConfigureAwait(false); + DisposeAndNullControllers(); + return MonitorAction.Exit; + } + + await Chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Server {0}! Rebooting...", exitWord), cancellationToken).ConfigureAwait(false); + return MonitorAction.Restart; + case MonitorActivationReason.ActiveServerRebooted: + var rebootState = server.RebootState; + server.ResetRebootState(); + + switch (rebootState) + { + case Watchdog.RebootState.Normal: + bool dmbUpdatePending = ActiveLaunchParameters != LastLaunchParameters; + return dmbUpdatePending ? MonitorAction.Restart : MonitorAction.Continue; + case Watchdog.RebootState.Restart: + return MonitorAction.Restart; + case Watchdog.RebootState.Shutdown: + // graceful shutdown time + await Chat.SendWatchdogMessage("Active server rebooted! Exiting due to graceful termination request...", cancellationToken).ConfigureAwait(false); + DisposeAndNullControllers(); + return MonitorAction.Exit; + default: + throw new InvalidOperationException($"Invalid reboot state: {rebootState}"); + } + + case MonitorActivationReason.ActiveLaunchParametersUpdated: + case MonitorActivationReason.NewDmbAvailable: + await server.SetRebootState(Watchdog.RebootState.Restart, cancellationToken).ConfigureAwait(false); + return MonitorAction.Continue; + case MonitorActivationReason.InactiveServerCrashed: + case MonitorActivationReason.InactiveServerRebooted: + case MonitorActivationReason.InactiveServerStartupComplete: + throw new NotSupportedException($"Unsupported activation reason: {reason}"); + default: + throw new InvalidOperationException($"Invalid activation reason: {reason}"); + } + } + + /// + protected override WatchdogReattachInformation CreateReattachInformation() + => new WatchdogReattachInformation + { + AlphaIsActive = true, + Alpha = server?.Release() + }; + + /// + protected override void DisposeAndNullControllers() + { + server?.Dispose(); + server = null; + Running = false; + } + + /// + protected override ISessionController GetActiveController() => server; + + /// + protected override async Task InitControllers(Action callBeforeRecurse, Task chatTask, WatchdogReattachInformation reattachInfo, CancellationToken cancellationToken) + { + // good ole sanity, should never fucking trigger but i don't trust myself even though I should + // TODO: Unit test this instead? + if (server != null) + throw new InvalidOperationException("Entered LaunchNoLock with server not being null!"); + + // don't need a new dmb if reattaching + var doesntNeedNewDmb = reattachInfo?.Alpha != null && reattachInfo?.Bravo != null; + var dmbToUse = doesntNeedNewDmb ? null : DmbFactory.LockNextDmb(1); + + var serverToReattach = reattachInfo?.Alpha ?? reattachInfo?.Bravo; + var serverToKill = reattachInfo?.Bravo ?? reattachInfo?.Alpha; + + // vice versa + if (reattachInfo?.AlphaIsActive == false) + { + var temp = serverToReattach; + serverToReattach = serverToKill; + serverToKill = temp; + } + + // if this try catches something, both servers are killed + bool inactiveServerWasKilled = false; + try + { + // start the alpha server task, either by launch a new process or attaching to an existing one + // The tasks returned are mainly for writing interop files to the directories among other things and should generally never fail + // The tasks pertaining to server startup times are in the ISessionControllers + Task serverLaunchTask, inactiveReattachTask; + if (!doesntNeedNewDmb) + serverLaunchTask = SessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbToUse, null, true, true, false, cancellationToken); + else + serverLaunchTask = SessionControllerFactory.Reattach(serverToReattach, cancellationToken); + + bool thereIsAnInactiveServerToKill = serverToKill != null; + if (thereIsAnInactiveServerToKill) + inactiveReattachTask = SessionControllerFactory.Reattach(serverToKill, cancellationToken); + else + inactiveReattachTask = Task.FromResult(null); + + // retrieve the session controller + server = await serverLaunchTask.ConfigureAwait(false); + + // failed reattaches will return null + server?.SetHighPriority(); + + var inactiveServerController = await inactiveReattachTask.ConfigureAwait(false); + inactiveServerController?.Dispose(); + inactiveServerWasKilled = inactiveServerController != null; + + // possiblity of null servers due to failed reattaches + if (server == null) + { + callBeforeRecurse(); + await NotifyOfFailedReattach(thereIsAnInactiveServerToKill && !inactiveServerWasKilled, cancellationToken).ConfigureAwait(false); + return; + } + + await CheckLaunchResult(server, "Server", cancellationToken).ConfigureAwait(false); + + server.EnableCustomChatCommands(); + } + catch + { + // kill the controllers + bool serverWasActive = server != null; + DisposeAndNullControllers(); + + // server didn't get control of this dmb + if (dmbToUse != null && !serverWasActive) + dmbToUse.Dispose(); + + if (serverToKill != null && !inactiveServerWasKilled) + serverToKill.Dmb.Dispose(); + throw; + } + } + + /// + protected override async Task MonitorLifetimes(CancellationToken cancellationToken) + { + Logger.LogTrace("Entered MonitorLifetimes"); + + // this function is responsible for calling HandlerMonitorWakeup when necessary and manitaining the MonitorState + var iteration = 1; + for (MonitorAction nextAction = MonitorAction.Continue; nextAction != MonitorAction.Exit; ++iteration) + { + // always start out with continue + nextAction = MonitorAction.Continue; + + // dump some info to the logs + Logger.LogDebug("Iteration {0} of monitor loop", iteration); + try + { + Logger.LogDebug("Server Compile Job ID: {0}", server.Dmb.CompileJob.Id); + + // load the activation tasks into local variables + Task activeServerLifetime = server.Lifetime; + var activeServerReboot = server.OnReboot; + Task activeLaunchParametersChanged = ActiveParametersUpdated.Task; + var newDmbAvailable = DmbFactory.OnNewerDmb; + + // cancel waiting if requested + var cancelTcs = new TaskCompletionSource(); + using (cancellationToken.Register(() => cancelTcs.SetCanceled())) + { + var toWaitOn = Task.WhenAny(activeServerLifetime, activeServerReboot, newDmbAvailable, cancelTcs.Task, activeLaunchParametersChanged); + + // wait for something to happen + await toWaitOn.ConfigureAwait(false); + cancellationToken.ThrowIfCancellationRequested(); + } + + var chatTask = Task.CompletedTask; + using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false)) + { + // always run HandleMonitorWakeup from the context of the semaphore lock + // multiple things may have happened, handle them one at a time + for (var moreActivationsToProcess = true; moreActivationsToProcess && (nextAction == MonitorAction.Continue || nextAction == MonitorAction.Skip);) + { + MonitorActivationReason activationReason = default; // this will always be assigned before being used + + // process the tasks in this order and call HandlerMonitorWakup for each + bool CheckActivationReason(ref Task task, MonitorActivationReason testActivationReason) + { + var taskCompleted = task?.IsCompleted == true; + task = null; + if (nextAction == MonitorAction.Skip) + nextAction = MonitorAction.Continue; + else if (taskCompleted) + { + activationReason = testActivationReason; + return true; + } + + return false; + } + + if (CheckActivationReason(ref activeServerLifetime, MonitorActivationReason.ActiveServerCrashed) + || CheckActivationReason(ref activeServerReboot, MonitorActivationReason.ActiveServerRebooted) + || CheckActivationReason(ref newDmbAvailable, MonitorActivationReason.NewDmbAvailable) + || CheckActivationReason(ref activeLaunchParametersChanged, MonitorActivationReason.ActiveLaunchParametersUpdated)) + nextAction = await HandleMonitorWakeup(activationReason, cancellationToken).ConfigureAwait(false); + else + moreActivationsToProcess = false; + } + } + + // full reboot required + if (nextAction == MonitorAction.Restart) + { + Logger.LogDebug("Next state action is to restart"); + DisposeAndNullControllers(); + + for (var retryAttempts = 1; nextAction == MonitorAction.Restart; ++retryAttempts) + { + Exception launchException = null; + using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false)) + try + { + // use LaunchImplNoLock without announcements or restarting the monitor + await LaunchImplNoLock(false, false, null, cancellationToken).ConfigureAwait(false); + if (Running) + { + Logger.LogDebug("Relaunch successful, resetting monitor state..."); + break; // continue on main loop + } + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception e) + { + launchException = e; + } + + await chatTask.ConfigureAwait(false); + if (!Running) + { + if (launchException == null) + Logger.LogWarning("Failed to automatically restart the watchdog! Attempt: {0}", retryAttempts); + else + Logger.LogWarning("Failed to automatically restart the watchdog! Attempt: {0}, Exception: {1}", retryAttempts, launchException); + var retryDelay = Math.Min(Math.Pow(2, retryAttempts), 3600); // max of one hour, increasing by a power of 2 each time + chatTask = Chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Failed to restart watchdog (Attempt: {0}), retrying in {1} seconds...", retryAttempts, retryDelay), cancellationToken); + await Task.WhenAll(AsyncDelayer.Delay(TimeSpan.FromSeconds(retryDelay), cancellationToken), chatTask).ConfigureAwait(false); + } + } + } + } + catch (OperationCanceledException) + { + Logger.LogDebug("Monitor cancelled"); + break; + } + catch (Exception e) + { + // really, this should NEVER happen + Logger.LogError("Monitor crashed! Iteration: {0}, NextAction: {1}, Exception: {2}", iteration, nextAction, e); + await Chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Monitor crashed, this should NEVER happen! Please report this, full details in logs! Restarting monitor... Error: {0}", e.Message), cancellationToken).ConfigureAwait(false); + } + } + + Logger.LogTrace("Monitor exiting..."); + } + } +} diff --git a/src/Tgstation.Server.Host/Components/Watchdog/ExperimentalWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/ExperimentalWatchdog.cs new file mode 100644 index 0000000000..ed2284241e --- /dev/null +++ b/src/Tgstation.Server.Host/Components/Watchdog/ExperimentalWatchdog.cs @@ -0,0 +1,649 @@ +using Byond.TopicSender; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; +using System; +using System.Diagnostics; +using System.Globalization; +using System.Threading; +using System.Threading.Tasks; +using Tgstation.Server.Api.Models.Internal; +using Tgstation.Server.Host.Components.Chat; +using Tgstation.Server.Host.Components.Compiler; +using Tgstation.Server.Host.Core; + +namespace Tgstation.Server.Host.Components.Watchdog +{ + /// + /// A that tries to manage 2 servers at once for maximum uptime. + /// + sealed class ExperimentalWatchdog : WatchdogBase + { + /// + /// The time in seconds to wait from starting to start . Does not take responsiveness into account + /// + const int AlphaBravoStartupSeperationInterval = 10; // TODO: Make this configurable + + /// + public override bool AlphaIsActive => alphaIsActive; + + /// + public override Models.CompileJob ActiveCompileJob => (AlphaIsActive ? alphaServer : bravoServer)?.Dmb.CompileJob; + + /// + public override RebootState? RebootState => Running ? (AlphaIsActive ? alphaServer?.RebootState : bravoServer?.RebootState) : null; + + /// + /// Server designation alpha + /// + ISessionController alphaServer; + + /// + /// Server designation bravo + /// + ISessionController bravoServer; + + /// + /// Backing field for . + /// + bool alphaIsActive; + + /// + /// Initializes a new instance of the . + /// + /// The for the . + /// The for the . + /// The for the . + /// The for the . + /// The for the . + /// The for the . + /// The for the . + /// The for the . + /// The for the . + /// The for the . + /// The for the . + /// The for the . + /// The for the . + /// The autostart value for the . + public ExperimentalWatchdog(IChat chat, ISessionControllerFactory sessionControllerFactory, IDmbFactory dmbFactory, IReattachInfoHandler reattachInfoHandler, IDatabaseContextFactory databaseContextFactory, IByondTopicSender byondTopicSender, IEventConsumer eventConsumer, IJobManager jobManager, IServerControl serverControl, IAsyncDelayer asyncDelayer, ILogger logger, DreamDaemonLaunchParameters initialLaunchParameters, Api.Models.Instance instance, bool autoStart) + : base( + chat, + sessionControllerFactory, + dmbFactory, + reattachInfoHandler, + databaseContextFactory, + byondTopicSender, + eventConsumer, + jobManager, + serverControl, + asyncDelayer, + logger, + initialLaunchParameters, + instance, + autoStart) + { + alphaIsActive = true; + } + + /// + /// Handles the actions to take when the monitor has to "wake up" + /// + /// The that caused the invocation + /// The current . Will be modified upon retrn + /// The for the operation + /// A representing the running operation + #pragma warning disable CA1502 // TODO: Decomplexify + async Task HandlerMonitorWakeup(MonitorActivationReason activationReason, MonitorState monitorState, CancellationToken cancellationToken) + { + Logger.LogDebug("Monitor activation. Reason: {0}", activationReason); + + // this is where the bulk of the watchdog handling code lives and is fraught with lambdas, sorry not sorry + // I'll do my best to walk you through it + + // returns true if the inactive server can't be used immediately + // also sets monitor to restart if the above holds + bool FullRestartDeadInactive() + { + if (monitorState.RebootingInactiveServer || monitorState.InactiveServerCritFail) + { + Logger.LogInformation("Inactive server is {0}! Restarting monitor...", monitorState.InactiveServerCritFail ? "critically failed" : "still rebooting"); + monitorState.NextAction = MonitorAction.Restart; // will dispose server + return true; + } + + return false; + } + + // trys to set inactive server's port to the public game port + // doesn't handle closing active server's port + // returns true on success and swaps inactiveserver and activeserver also sets LastLaunchParameters to ActiveLaunchParameters + // on failure, sets monitor to restart + async Task MakeInactiveActive() + { + Logger.LogDebug("Setting inactive server to port {0}...", ActiveLaunchParameters.PrimaryPort.Value); + var result = await monitorState.InactiveServer.SetPort(ActiveLaunchParameters.PrimaryPort.Value, cancellationToken).ConfigureAwait(false); + + if (!result) + { + Logger.LogWarning("Failed to activate inactive server! Restarting monitor..."); + monitorState.NextAction = MonitorAction.Restart; // will dispose server + return false; + } + + // inactive server should always be using active launch parameters + LastLaunchParameters = ActiveLaunchParameters; + + var tmp = monitorState.ActiveServer; + monitorState.ActiveServer = monitorState.InactiveServer; + monitorState.InactiveServer = tmp; + alphaIsActive = !AlphaIsActive; + monitorState.ActiveServer.EnableCustomChatCommands(); + return true; + } + + // Kills and tries to launch inactive server with the latest dmb + // falls back to current dmb on failure + // Sets critfail on inactive server failing that + // returns false if the backup dmb was used successfully, true otherwise + async Task UpdateAndRestartInactiveServer(bool breakAfter) + { + ActiveParametersUpdated = new TaskCompletionSource(); + monitorState.InactiveServer.Dispose(); // kill or recycle it + var desiredNextAction = breakAfter ? MonitorAction.Break : MonitorAction.Continue; + monitorState.NextAction = desiredNextAction; + + Logger.LogInformation("Rebooting inactive server..."); + var newDmb = DmbFactory.LockNextDmb(1); + try + { + monitorState.InactiveServer = await SessionControllerFactory.LaunchNew(ActiveLaunchParameters, newDmb, null, false, !monitorState.ActiveServer.IsPrimary, false, cancellationToken).ConfigureAwait(false); + monitorState.InactiveServer.SetHighPriority(); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception e) + { + Logger.LogError("Error occurred while recreating server! Attempting backup strategy of running DMB of running server! Exception: {0}", e.ToString()); + + // ahh jeez, what do we do here? + // this is our fault, so it should never happen but + // idk maybe a database error while handling the newest dmb? + // either way try to start it using the active server's dmb as a backup + try + { + var dmbBackup = await DmbFactory.FromCompileJob(monitorState.ActiveServer.Dmb.CompileJob, cancellationToken).ConfigureAwait(false); + + if (dmbBackup == null) // NANI!? + throw new JobException("Creating backup DMB provider failed!"); // just give up, if THAT compile job is failing then the ActiveServer is gonna crash soon too or already has + + monitorState.InactiveServer = await SessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbBackup, null, false, !monitorState.ActiveServer.IsPrimary, false, cancellationToken).ConfigureAwait(false); + monitorState.InactiveServer.SetHighPriority(); + await Chat.SendWatchdogMessage("Staging newest DMB on inactive server failed: {0} Falling back to previous dmb...", cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception e2) + { + // fuuuuucckkk + Logger.LogError("Backup strategy failed! Monitor will restart when active server reboots! Exception: {0}", e2.ToString()); + monitorState.InactiveServerCritFail = true; + await Chat.SendWatchdogMessage("Attempted reboot of inactive server failed. Watchdog will reset when active server fails or exits", cancellationToken).ConfigureAwait(false); + return; + } + } + + Logger.LogInformation("Successfully relaunched inactive server!"); + monitorState.RebootingInactiveServer = true; + } + + string ExitWord(ISessionController controller) => controller.TerminationWasRequested ? "exited" : "crashed"; + + // reason handling + switch (activationReason) + { + case MonitorActivationReason.ActiveServerCrashed: + if (monitorState.ActiveServer.RebootState == Watchdog.RebootState.Shutdown) + { + // the time for graceful shutdown is now + await Chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Active server {0}! Exiting due to graceful termination request...", ExitWord(monitorState.ActiveServer)), cancellationToken).ConfigureAwait(false); + DisposeAndNullControllers(); + monitorState.NextAction = MonitorAction.Exit; + break; + } + + if (FullRestartDeadInactive()) + { + // tell chat about it and go ahead + await Chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Active server {0}! Inactive server unable to online!", ExitWord(monitorState.ActiveServer)), cancellationToken).ConfigureAwait(false); + + // we've already been set to restart + break; + } + + // tell chat about it + await Chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Active server {0}! Onlining inactive server...", ExitWord(monitorState.ActiveServer)), cancellationToken).ConfigureAwait(false); + + // try to activate the inactive server + if (!await MakeInactiveActive().ConfigureAwait(false)) + break; // failing that, we've already been set to restart + + // bring up another inactive server + await UpdateAndRestartInactiveServer(true).ConfigureAwait(false); + break; + case MonitorActivationReason.InactiveServerCrashed: + // just announce and try to bring it back + await Chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Inactive server {0}! Rebooting...", ExitWord(monitorState.InactiveServer)), cancellationToken).ConfigureAwait(false); + await UpdateAndRestartInactiveServer(false).ConfigureAwait(false); + break; + case MonitorActivationReason.ActiveServerRebooted: + // ideal goal: active server just closed its port + // tell inactive server to open it's port and that's now the active server + var rebootState = monitorState.ActiveServer.RebootState; + monitorState.ActiveServer.ResetRebootState(); // the DMAPI has already done this internally + + if (FullRestartDeadInactive() && rebootState != Watchdog.RebootState.Shutdown) + break; // full restart if the inactive server is being fucky + + // what matters here is the RebootState + var restartOnceSwapped = false; + + switch (rebootState) + { + case Watchdog.RebootState.Normal: + // life as normal + break; + case Watchdog.RebootState.Restart: + // reboot the current active server once the inactive one activates + restartOnceSwapped = true; + break; + case Watchdog.RebootState.Shutdown: + // graceful shutdown time + await Chat.SendWatchdogMessage("Active server rebooted! Exiting due to graceful termination request...", cancellationToken).ConfigureAwait(false); + DisposeAndNullControllers(); + monitorState.NextAction = MonitorAction.Exit; + return; + default: + throw new InvalidOperationException($"Invalid reboot state: {rebootState}"); + } + + // are both servers now running the same CompileJob? + var sameCompileJob = monitorState.InactiveServer.Dmb.CompileJob.Id == monitorState.ActiveServer.Dmb.CompileJob.Id; + + if (!sameCompileJob || ActiveLaunchParameters != LastLaunchParameters) + restartOnceSwapped = true; // need a new launch to update either settings or compile job + + if (restartOnceSwapped) + /* + * we need to manually restart active server + * just kill it here, easier that way + */ + monitorState.ActiveServer.Dispose(); + + var activeServerStillHasPortOpen = !restartOnceSwapped && !monitorState.ActiveServer.ClosePortOnReboot; + + if (activeServerStillHasPortOpen) + /* we didn't want active server to swap for some reason and it still has it's port open + * just continue as normal + */ + break; + + if (!await MakeInactiveActive().ConfigureAwait(false)) + break; // monitor will restart + + // servers now swapped + // enable this now if inactive server is not still valid + monitorState.ActiveServer.ClosePortOnReboot = restartOnceSwapped; + + if (!restartOnceSwapped) + /* + * now try to reopen it on the private port + * failing that, just reboot it + */ + restartOnceSwapped = !await monitorState.InactiveServer.SetPort(ActiveLaunchParameters.SecondaryPort.Value, cancellationToken).ConfigureAwait(false); + + // break either way because any issues past this point would be solved by the reboot + if (restartOnceSwapped) // for one reason or another + await UpdateAndRestartInactiveServer(true).ConfigureAwait(false); // update and reboot + else + monitorState.NextAction = MonitorAction.Skip; // only skip checking inactive server rebooted, it's guaranteed InactiveServerStartup complete wouldn't fire this iteration + break; + case MonitorActivationReason.InactiveServerRebooted: + // just don't let the active server close it's port if the inactive server isn't ready + monitorState.RebootingInactiveServer = true; + monitorState.InactiveServer.ResetRebootState(); + monitorState.ActiveServer.ClosePortOnReboot = false; + monitorState.NextAction = MonitorAction.Continue; + break; + case MonitorActivationReason.InactiveServerStartupComplete: + // opposite of above case + monitorState.RebootingInactiveServer = false; + monitorState.ActiveServer.ClosePortOnReboot = true; + monitorState.NextAction = MonitorAction.Continue; + break; + case MonitorActivationReason.NewDmbAvailable: + case MonitorActivationReason.ActiveLaunchParametersUpdated: + // just reload the inactive server and wait for a swap to apply the changes + await UpdateAndRestartInactiveServer(true).ConfigureAwait(false); + break; + default: + Trace.Assert(false, String.Format(CultureInfo.InvariantCulture, "Invalid monitor activation reason: {0}!", activationReason)); + break; + } + } + #pragma warning restore CA1502 + + /// + /// Call on and and set them to + /// + protected override void DisposeAndNullControllers() + { + alphaServer?.Dispose(); + alphaServer = null; + bravoServer?.Dispose(); + bravoServer = null; + Running = false; + } + + /// + #pragma warning disable CA1502 // TODO: Decomplexify + protected override async Task MonitorLifetimes(CancellationToken cancellationToken) + { + Logger.LogTrace("Entered MonitorLifetimes"); + + // this function is responsible for calling HandlerMonitorWakeup when necessary and manitaining the MonitorState + var iteration = 1; + for (var monitorState = new MonitorState(); monitorState.NextAction != MonitorAction.Exit; ++iteration) + { + // always start out with continue + monitorState.NextAction = MonitorAction.Continue; + + // dump some info to the logs + Logger.LogDebug("Iteration {0} of monitor loop", iteration); + try + { + if (AlphaIsActive) + Logger.LogDebug("Alpha is the active server"); + else + Logger.LogDebug("Bravo is the active server"); + + if (monitorState.RebootingInactiveServer) + Logger.LogDebug("Inactive server is rebooting"); + + // update the monitor state with the inactive/active servers + monitorState.ActiveServer = AlphaIsActive ? alphaServer : bravoServer; + monitorState.InactiveServer = AlphaIsActive ? bravoServer : alphaServer; + + if (monitorState.ActiveServer.ClosePortOnReboot) + Logger.LogDebug("Active server will close port on reboot"); + if (monitorState.InactiveServer.ClosePortOnReboot) + Logger.LogDebug("Inactive server will close port on reboot"); + + Logger.LogDebug("Active server Compile Job ID: {0}", monitorState.ActiveServer.Dmb.CompileJob.Id); + Logger.LogDebug("Inactive server Compile Job ID: {0}", monitorState.InactiveServer.Dmb.CompileJob.Id); + + // load the activation tasks into local variables + Task activeServerLifetime = monitorState.ActiveServer.Lifetime; + Task inactiveServerLifetime = monitorState.InactiveServer.Lifetime; + var activeServerReboot = monitorState.ActiveServer.OnReboot; + var inactiveServerReboot = monitorState.InactiveServer.OnReboot; + Task inactiveServerStartup = monitorState.RebootingInactiveServer ? monitorState.InactiveServer.LaunchResult : null; + Task activeLaunchParametersChanged = ActiveParametersUpdated.Task; + var newDmbAvailable = DmbFactory.OnNewerDmb; + + // cancel waiting if requested + var cancelTcs = new TaskCompletionSource(); + using (cancellationToken.Register(() => cancelTcs.SetCanceled())) + { + var toWaitOn = Task.WhenAny(activeServerLifetime, inactiveServerLifetime, activeServerReboot, inactiveServerReboot, newDmbAvailable, cancelTcs.Task, activeLaunchParametersChanged); + if (monitorState.RebootingInactiveServer) + toWaitOn = Task.WhenAny(toWaitOn, inactiveServerStartup); + + // wait for something to happen + await toWaitOn.ConfigureAwait(false); + cancellationToken.ThrowIfCancellationRequested(); + } + + var chatTask = Task.CompletedTask; + using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false)) + { + // always run HandleMonitorWakeup from the context of the semaphore lock + // multiple things may have happened, handle them one at a time + for (var moreActivationsToProcess = true; moreActivationsToProcess && (monitorState.NextAction == MonitorAction.Continue || monitorState.NextAction == MonitorAction.Skip);) + { + MonitorActivationReason activationReason = default; // this will always be assigned before being used + + // process the tasks in this order and call HandlerMonitorWakup for each + bool CheckActivationReason(ref Task task, MonitorActivationReason testActivationReason) + { + var taskCompleted = task?.IsCompleted == true; + task = null; + if (monitorState.NextAction == MonitorAction.Skip) + monitorState.NextAction = MonitorAction.Continue; + else if (taskCompleted) + { + activationReason = testActivationReason; + return true; + } + + return false; + } + + if (CheckActivationReason(ref activeServerLifetime, MonitorActivationReason.ActiveServerCrashed) + || CheckActivationReason(ref inactiveServerLifetime, MonitorActivationReason.InactiveServerCrashed) + || CheckActivationReason(ref activeServerReboot, MonitorActivationReason.ActiveServerRebooted) + || CheckActivationReason(ref inactiveServerReboot, MonitorActivationReason.InactiveServerRebooted) + || CheckActivationReason(ref inactiveServerStartup, MonitorActivationReason.InactiveServerStartupComplete) + || CheckActivationReason(ref newDmbAvailable, MonitorActivationReason.NewDmbAvailable) + || CheckActivationReason(ref activeLaunchParametersChanged, MonitorActivationReason.ActiveLaunchParametersUpdated)) + await HandlerMonitorWakeup(activationReason, monitorState, cancellationToken).ConfigureAwait(false); + else + moreActivationsToProcess = false; + } + + // writeback alphaServer and bravoServer from monitor state in case they changesd + alphaServer = AlphaIsActive ? monitorState.ActiveServer : monitorState.InactiveServer; + bravoServer = !AlphaIsActive ? monitorState.ActiveServer : monitorState.InactiveServer; + } + + // full reboot required + if (monitorState.NextAction == MonitorAction.Restart) + { + Logger.LogDebug("Next state action is to restart"); + DisposeAndNullControllers(); + chatTask = Chat.SendWatchdogMessage("Restarting entirely due to complications...", cancellationToken); + + for (var retryAttempts = 1; monitorState.NextAction == MonitorAction.Restart; ++retryAttempts) + { + Exception launchException = null; + using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false)) + try + { + // use LaunchImplNoLock without announcements or restarting the monitor + await LaunchImplNoLock(false, false, null, cancellationToken).ConfigureAwait(false); + if (Running) + { + Logger.LogDebug("Relaunch successful, resetting monitor state..."); + monitorState = new MonitorState(); // clean the slate and continue + } + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception e) + { + launchException = e; + } + + await chatTask.ConfigureAwait(false); + if (!Running) + { + if (launchException == null) + Logger.LogWarning("Failed to automatically restart the watchdog! Attempt: {0}", retryAttempts); + else + Logger.LogWarning("Failed to automatically restart the watchdog! Attempt: {0}, Exception: {1}", retryAttempts, launchException); + var retryDelay = Math.Min(Math.Pow(2, retryAttempts), 3600); // max of one hour, increasing by a power of 2 each time + chatTask = Chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Failed to restart watchdog (Attempt: {0}), retrying in {1} seconds...", retryAttempts, retryDelay), cancellationToken); + await Task.WhenAll(AsyncDelayer.Delay(TimeSpan.FromSeconds(retryDelay), cancellationToken), chatTask).ConfigureAwait(false); + } + } + } + } + catch (OperationCanceledException) + { + Logger.LogDebug("Monitor cancelled"); + break; + } + catch (Exception e) + { + // really, this should NEVER happen + Logger.LogError("Monitor crashed! Iteration: {0}, State: {1}, Exception: {2}", iteration, JsonConvert.SerializeObject(monitorState), e); + await Chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Monitor crashed, this should NEVER happen! Please report this, full details in logs! Restarting monitor... Error: {0}", e.Message), cancellationToken).ConfigureAwait(false); + } + } + + Logger.LogTrace("Monitor exiting..."); + } + #pragma warning restore CA1502 + + /// + #pragma warning disable CA1502 // TODO: Decomplexify + protected override async Task InitControllers(Action callBeforeRecurse, Task chatTask, WatchdogReattachInformation reattachInfo, CancellationToken cancellationToken) + { + // good ole sanity, should never fucking trigger but i don't trust myself even though I should + // TODO: Unit test this instead? + if (alphaServer != null || bravoServer != null) + throw new InvalidOperationException("Entered LaunchNoLock with one or more of the servers not being null!"); + + // don't need a new dmb if reattaching + var doesntNeedNewDmb = reattachInfo?.Alpha != null && reattachInfo?.Bravo != null; + var dmbToUse = doesntNeedNewDmb ? null : DmbFactory.LockNextDmb(2); + + // if this try catches something, both servers are killed + try + { + // start the alpha server task, either by launch a new process or attaching to an existing one + // The tasks returned are mainly for writing interop files to the directories among other things and should generally never fail + // The tasks pertaining to server startup times are in the ISessionControllers + Task alphaServerTask; + if (!doesntNeedNewDmb) + alphaServerTask = SessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbToUse, null, true, true, false, cancellationToken); + else + alphaServerTask = SessionControllerFactory.Reattach(reattachInfo.Alpha, cancellationToken); + + // retrieve the session controller + var startTime = DateTimeOffset.Now; + alphaServer = await alphaServerTask.ConfigureAwait(false); + + // failed reattaches will return null + alphaServer?.SetHighPriority(); + + // extra delay for total ordering + var now = DateTimeOffset.Now; + var delay = now - startTime; + + // definitely not if reattaching though + if (reattachInfo == null && delay.TotalSeconds < AlphaBravoStartupSeperationInterval) + await AsyncDelayer.Delay(startTime.AddSeconds(AlphaBravoStartupSeperationInterval) - now, cancellationToken).ConfigureAwait(false); + + // now bring bravo up + if (!doesntNeedNewDmb) + bravoServer = await SessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbToUse, null, false, false, false, cancellationToken).ConfigureAwait(false); + else + bravoServer = await SessionControllerFactory.Reattach(reattachInfo.Bravo, cancellationToken).ConfigureAwait(false); + + // failed reattaches will return null + bravoServer?.SetHighPriority(); + + // possiblity of null servers due to failed reattaches + if (alphaServer == null || bravoServer == null) + { + await chatTask.ConfigureAwait(false); + var bothServersDead = alphaServer == null && bravoServer == null; + if (bothServersDead + || (alphaServer == null && reattachInfo.AlphaIsActive) + || (bravoServer == null && !reattachInfo.AlphaIsActive)) + { + // we lost the active server, just restart entirely + DisposeAndNullControllers(); + const string FailReattachMessage = "Unable to properly reattach to active server! Restarting..."; + Logger.LogWarning(FailReattachMessage); + Logger.LogDebug(bothServersDead ? "Also could not reattach to inactive server!" : "Inactive server was reattached successfully!"); + chatTask = Chat.SendWatchdogMessage(FailReattachMessage, cancellationToken); + callBeforeRecurse(); + await LaunchImplNoLock(true, false, null, cancellationToken).ConfigureAwait(false); + await chatTask.ConfigureAwait(false); + return; + } + + // we still have the active server but the other one is dead to us, hand it off to the monitor to restart + const string InactiveReattachFailureMessage = "Unable to reattach to inactive server. Leaving for monitor to reboot..."; + chatTask = Chat.SendWatchdogMessage(InactiveReattachFailureMessage, cancellationToken); + Logger.LogWarning(InactiveReattachFailureMessage); + + if (reattachInfo.AlphaIsActive) + bravoServer = SessionControllerFactory.CreateDeadSession(reattachInfo.Bravo.Dmb); + else + alphaServer = SessionControllerFactory.CreateDeadSession(reattachInfo.Alpha.Dmb); + } + + var alphaLrt = CheckLaunchResult(alphaServer, "Alpha", cancellationToken); + var bravoLrt = CheckLaunchResult(bravoServer, "Bravo", cancellationToken); + + // this task completes when both serers have finished booting + var allTask = Task.WhenAll(alphaLrt, bravoLrt); + + await allTask.ConfigureAwait(false); + + // both servers are now running, alpha is the active server(unless reattach), huzzah + alphaIsActive = reattachInfo?.AlphaIsActive ?? true; + + var activeServer = AlphaIsActive ? alphaServer : bravoServer; + activeServer.EnableCustomChatCommands(); + activeServer.ClosePortOnReboot = true; + } + catch + { + if (dmbToUse != null) + { + // we locked 2 dmbs + if (bravoServer == null) + { + // bravo didn't get control of his + dmbToUse.Dispose(); + if (alphaServer == null) + dmbToUse.Dispose(); // alpha didn't get control of his + } + } + else if (doesntNeedNewDmb) // we have reattachInfo + if (bravoServer == null) + { + // bravo didn't get control of his + reattachInfo.Bravo?.Dmb.Dispose(); + if (alphaServer == null) + reattachInfo.Alpha?.Dmb.Dispose(); // alpha didn't get control of his + } + + // kill the controllers + DisposeAndNullControllers(); + throw; + } + } + #pragma warning restore CA1502 + + /// + protected override ISessionController GetActiveController() => AlphaIsActive ? alphaServer : bravoServer; + + /// + protected override WatchdogReattachInformation CreateReattachInformation() + => new WatchdogReattachInformation + { + AlphaIsActive = AlphaIsActive, + Alpha = alphaServer?.Release(), + Bravo = bravoServer?.Release() + }; + } +} diff --git a/src/Tgstation.Server.Host/Components/Watchdog/MonitorState.cs b/src/Tgstation.Server.Host/Components/Watchdog/MonitorState.cs index 571874a370..e6eb14e0b5 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/MonitorState.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/MonitorState.cs @@ -3,7 +3,7 @@ namespace Tgstation.Server.Host.Components.Watchdog { /// - /// The (absolute) state of the + /// The (absolute) state of the /// sealed class MonitorState { @@ -18,7 +18,7 @@ namespace Tgstation.Server.Host.Components.Watchdog public bool InactiveServerCritFail { get; set; } /// - /// The next to take in + /// The next to take in /// public MonitorAction NextAction { get; set; } diff --git a/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs deleted file mode 100644 index 84ea3fb90d..0000000000 --- a/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs +++ /dev/null @@ -1,1107 +0,0 @@ -using Byond.TopicSender; -using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.Logging; -using Newtonsoft.Json; -using Newtonsoft.Json.Serialization; -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Globalization; -using System.Linq; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using Tgstation.Server.Api.Models.Internal; -using Tgstation.Server.Api.Rights; -using Tgstation.Server.Host.Components.Chat; -using Tgstation.Server.Host.Components.Compiler; -using Tgstation.Server.Host.Components.Interop; -using Tgstation.Server.Host.Core; - -namespace Tgstation.Server.Host.Components.Watchdog -{ - /// - #pragma warning disable CA1506 // TODO: Decomplexify - sealed class Watchdog : IWatchdog, ICustomCommandHandler, IRestartHandler - { - /// - /// The time in seconds to wait from starting to start . Does not take responsiveness into account - /// - const int AlphaBravoStartupSeperationInterval = 10; - - /// - public bool Running { get; private set; } - - /// - public bool AlphaIsActive { get; private set; } - - /// - public Models.CompileJob ActiveCompileJob => (AlphaIsActive ? alphaServer : bravoServer)?.Dmb.CompileJob; - - /// - public DreamDaemonLaunchParameters ActiveLaunchParameters { get; private set; } - - /// - public DreamDaemonLaunchParameters LastLaunchParameters { get; private set; } - - /// - public RebootState? RebootState => Running ? (AlphaIsActive ? alphaServer?.RebootState : bravoServer?.RebootState) : null; - - /// - /// The for the - /// - readonly IChat chat; - - /// - /// The for the - /// - readonly ISessionControllerFactory sessionControllerFactory; - - /// - /// The for the - /// - readonly IDmbFactory dmbFactory; - - /// - /// The for the - /// - readonly IReattachInfoHandler reattachInfoHandler; - - /// - /// The for the - /// - readonly IDatabaseContextFactory databaseContextFactory; - - /// - /// The for the - /// - readonly IByondTopicSender byondTopicSender; - - /// - /// The for the - /// - readonly IEventConsumer eventConsumer; - - /// - /// The for the - /// - readonly IJobManager jobManager; - - /// - /// The for the - /// - readonly IRestartRegistration restartRegistration; - - /// - /// The for the - /// - readonly IAsyncDelayer asyncDelayer; - - /// - /// The for the - /// - readonly ILogger logger; - - /// - /// The for the - /// - readonly SemaphoreSlim semaphore; - - /// - /// The for the - /// - readonly Api.Models.Instance instance; - - /// - /// If the should in - /// - readonly bool autoStart; - - /// - /// The for the monitor loop - /// - CancellationTokenSource monitorCts; - - /// - /// The running the monitor loop - /// - Task monitorTask; - - /// - /// that completes when are changed and we are - /// - TaskCompletionSource activeParametersUpdated; - - /// - /// Server designation alpha - /// - ISessionController alphaServer; - - /// - /// Server designation bravo - /// - ISessionController bravoServer; - - /// - /// If the servers should be released instead of shutdown - /// - bool releaseServers; - - /// - /// Construct a - /// - /// The value of - /// The value of - /// The value of - /// The value of - /// The value of - /// The value of - /// The value of - /// The value of - /// The to populate with - /// The value of - /// The value of - /// The initial value of . May be modified - /// The value of - /// The value of - public Watchdog(IChat chat, ISessionControllerFactory sessionControllerFactory, IDmbFactory dmbFactory, IReattachInfoHandler reattachInfoHandler, IDatabaseContextFactory databaseContextFactory, IByondTopicSender byondTopicSender, IEventConsumer eventConsumer, IJobManager jobManager, IServerControl serverControl, IAsyncDelayer asyncDelayer, ILogger logger, DreamDaemonLaunchParameters initialLaunchParameters, Api.Models.Instance instance, bool autoStart) - { - this.chat = chat ?? throw new ArgumentNullException(nameof(chat)); - this.sessionControllerFactory = sessionControllerFactory ?? throw new ArgumentNullException(nameof(sessionControllerFactory)); - this.dmbFactory = dmbFactory ?? throw new ArgumentNullException(nameof(dmbFactory)); - this.reattachInfoHandler = reattachInfoHandler ?? throw new ArgumentNullException(nameof(reattachInfoHandler)); - this.databaseContextFactory = databaseContextFactory ?? throw new ArgumentNullException(nameof(databaseContextFactory)); - this.byondTopicSender = byondTopicSender ?? throw new ArgumentNullException(nameof(byondTopicSender)); - this.eventConsumer = eventConsumer ?? throw new ArgumentNullException(nameof(eventConsumer)); - this.jobManager = jobManager ?? throw new ArgumentNullException(nameof(jobManager)); - this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer)); - this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); - ActiveLaunchParameters = initialLaunchParameters ?? throw new ArgumentNullException(nameof(initialLaunchParameters)); - this.instance = instance ?? throw new ArgumentNullException(nameof(instance)); - this.autoStart = autoStart; - - if (serverControl == null) - throw new ArgumentNullException(nameof(serverControl)); - - chat.RegisterCommandHandler(this); - - AlphaIsActive = true; - ActiveLaunchParameters = initialLaunchParameters; - releaseServers = false; - activeParametersUpdated = new TaskCompletionSource(); - - restartRegistration = serverControl.RegisterForRestart(this); - try - { - semaphore = new SemaphoreSlim(1); - } - catch - { - restartRegistration.Dispose(); - throw; - } - } - - /// - public void Dispose() - { - DisposeAndNullControllers(); - semaphore.Dispose(); - restartRegistration.Dispose(); - - // mostly here to please fxcop, but it definitely should be disposed already - Debug.Assert(monitorCts == null, "We reached Disposes() an monitorCts is not null!"); - monitorCts?.Dispose(); - } - - /// - /// Call on and and set them to - /// - void DisposeAndNullControllers() - { - alphaServer?.Dispose(); - alphaServer = null; - bravoServer?.Dispose(); - bravoServer = null; - Running = false; - } - - /// - /// Implementation of . Does not lock - /// - /// If the termination will be delayed until a reboot is detected in the active server's DMAPI and this function will return immediately - /// If the termination will be announced using - /// The for the operation - async Task TerminateNoLock(bool graceful, bool announce, CancellationToken cancellationToken) - { - if (!Running) - return; - if (!graceful) - { - var chatTask = announce ? chat.SendWatchdogMessage("Terminating...", cancellationToken) : Task.CompletedTask; - await StopMonitor().ConfigureAwait(false); - DisposeAndNullControllers(); - LastLaunchParameters = null; - await chatTask.ConfigureAwait(false); - return; - } - - // merely set the reboot state - var toKill = AlphaIsActive ? alphaServer : bravoServer; - var other = AlphaIsActive ? bravoServer : alphaServer; - if (toKill != null) - await toKill.SetRebootState(Components.Watchdog.RebootState.Shutdown, cancellationToken).ConfigureAwait(false); - } - - /// - /// Handles the actions to take when the monitor has to "wake up" - /// - /// The that caused the invocation - /// The current . Will be modified upon retrn - /// The for the operation - /// A representing the running operation - #pragma warning disable CA1502 // TODO: Decomplexify - async Task HandlerMonitorWakeup(MonitorActivationReason activationReason, MonitorState monitorState, CancellationToken cancellationToken) - { - logger.LogDebug("Monitor activation. Reason: {0}", activationReason); - - // this is where the bulk of the watchdog handling code lives and is fraught with lambdas, sorry not sorry - // I'll do my best to walk you through it - - // returns true if the inactive server can't be used immediately - // also sets monitor to restart if the above holds - bool FullRestartDeadInactive() - { - if (monitorState.RebootingInactiveServer || monitorState.InactiveServerCritFail) - { - logger.LogInformation("Inactive server is {0}! Restarting monitor...", monitorState.InactiveServerCritFail ? "critically failed" : "still rebooting"); - monitorState.NextAction = MonitorAction.Restart; // will dispose server - return true; - } - - return false; - } - - // trys to set inactive server's port to the public game port - // doesn't handle closing active server's port - // returns true on success and swaps inactiveserver and activeserver also sets LastLaunchParameters to ActiveLaunchParameters - // on failure, sets monitor to restart - async Task MakeInactiveActive() - { - logger.LogDebug("Setting inactive server to port {0}...", ActiveLaunchParameters.PrimaryPort.Value); - var result = await monitorState.InactiveServer.SetPort(ActiveLaunchParameters.PrimaryPort.Value, cancellationToken).ConfigureAwait(false); - - if (!result) - { - logger.LogWarning("Failed to activate inactive server! Restarting monitor..."); - monitorState.NextAction = MonitorAction.Restart; // will dispose server - return false; - } - - // inactive server should always be using active launch parameters - LastLaunchParameters = ActiveLaunchParameters; - - var tmp = monitorState.ActiveServer; - monitorState.ActiveServer = monitorState.InactiveServer; - monitorState.InactiveServer = tmp; - AlphaIsActive = !AlphaIsActive; - monitorState.ActiveServer.EnableCustomChatCommands(); - return true; - } - - // Kills and tries to launch inactive server with the latest dmb - // falls back to current dmb on failure - // Sets critfail on inactive server failing that - // returns false if the backup dmb was used successfully, true otherwise - async Task UpdateAndRestartInactiveServer(bool breakAfter) - { - activeParametersUpdated = new TaskCompletionSource(); - monitorState.InactiveServer.Dispose(); // kill or recycle it - var desiredNextAction = breakAfter ? MonitorAction.Break : MonitorAction.Continue; - monitorState.NextAction = desiredNextAction; - - logger.LogInformation("Rebooting inactive server..."); - var newDmb = dmbFactory.LockNextDmb(1); - try - { - monitorState.InactiveServer = await sessionControllerFactory.LaunchNew(ActiveLaunchParameters, newDmb, null, false, !monitorState.ActiveServer.IsPrimary, false, cancellationToken).ConfigureAwait(false); - monitorState.InactiveServer.SetHighPriority(); - } - catch (OperationCanceledException) - { - throw; - } - catch (Exception e) - { - logger.LogError("Error occurred while recreating server! Attempting backup strategy of running DMB of running server! Exception: {0}", e.ToString()); - - // ahh jeez, what do we do here? - // this is our fault, so it should never happen but - // idk maybe a database error while handling the newest dmb? - // either way try to start it using the active server's dmb as a backup - try - { - var dmbBackup = await dmbFactory.FromCompileJob(monitorState.ActiveServer.Dmb.CompileJob, cancellationToken).ConfigureAwait(false); - - if (dmbBackup == null) // NANI!? - throw new JobException("Creating backup DMB provider failed!"); // just give up, if THAT compile job is failing then the ActiveServer is gonna crash soon too or already has - - monitorState.InactiveServer = await sessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbBackup, null, false, !monitorState.ActiveServer.IsPrimary, false, cancellationToken).ConfigureAwait(false); - monitorState.InactiveServer.SetHighPriority(); - await chat.SendWatchdogMessage("Staging newest DMB on inactive server failed: {0} Falling back to previous dmb...", cancellationToken).ConfigureAwait(false); - } - catch (OperationCanceledException) - { - throw; - } - catch (Exception e2) - { - // fuuuuucckkk - logger.LogError("Backup strategy failed! Monitor will restart when active server reboots! Exception: {0}", e2.ToString()); - monitorState.InactiveServerCritFail = true; - await chat.SendWatchdogMessage("Attempted reboot of inactive server failed. Watchdog will reset when active server fails or exits", cancellationToken).ConfigureAwait(false); - return; - } - } - - logger.LogInformation("Successfully relaunched inactive server!"); - monitorState.RebootingInactiveServer = true; - } - - string ExitWord(ISessionController controller) => controller.TerminationWasRequested ? "exited" : "crashed"; - - // reason handling - switch (activationReason) - { - case MonitorActivationReason.ActiveServerCrashed: - if (monitorState.ActiveServer.RebootState == Components.Watchdog.RebootState.Shutdown) - { - // the time for graceful shutdown is now - await chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Active server {0}! Exiting due to graceful termination request...", ExitWord(monitorState.ActiveServer)), cancellationToken).ConfigureAwait(false); - DisposeAndNullControllers(); - monitorState.NextAction = MonitorAction.Exit; - break; - } - - if (FullRestartDeadInactive()) - { - // tell chat about it and go ahead - await chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Active server {0}! Inactive server unable to online!", ExitWord(monitorState.ActiveServer)), cancellationToken).ConfigureAwait(false); - - // we've already been set to restart - break; - } - - // tell chat about it - await chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Active server {0}! Onlining inactive server...", ExitWord(monitorState.ActiveServer)), cancellationToken).ConfigureAwait(false); - - // try to activate the inactive server - if (!await MakeInactiveActive().ConfigureAwait(false)) - break; // failing that, we've already been set to restart - - // bring up another inactive server - await UpdateAndRestartInactiveServer(true).ConfigureAwait(false); - break; - case MonitorActivationReason.InactiveServerCrashed: - // just announce and try to bring it back - await chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Inactive server {0}! Rebooting...", ExitWord(monitorState.InactiveServer)), cancellationToken).ConfigureAwait(false); - await UpdateAndRestartInactiveServer(false).ConfigureAwait(false); - break; - case MonitorActivationReason.ActiveServerRebooted: - // ideal goal: active server just closed its port - // tell inactive server to open it's port and that's now the active server - var rebootState = monitorState.ActiveServer.RebootState; - monitorState.ActiveServer.ResetRebootState(); // the DMAPI has already done this internally - - if (FullRestartDeadInactive() && rebootState != Components.Watchdog.RebootState.Shutdown) - break; // full restart if the inactive server is being fucky - - // what matters here is the RebootState - var restartOnceSwapped = false; - - switch (rebootState) - { - case Components.Watchdog.RebootState.Normal: - // life as normal - break; - case Components.Watchdog.RebootState.Restart: - // reboot the current active server once the inactive one activates - restartOnceSwapped = true; - break; - case Components.Watchdog.RebootState.Shutdown: - // graceful shutdown time - await chat.SendWatchdogMessage("Active server rebooted! Exiting due to graceful termination request...", cancellationToken).ConfigureAwait(false); - DisposeAndNullControllers(); - monitorState.NextAction = MonitorAction.Exit; - return; - default: - Trace.Assert(false, String.Format(CultureInfo.InvariantCulture, "Invalid RebootState: {0}!", rebootState)); - break; - } - - // are both servers now running the same CompileJob? - var sameCompileJob = monitorState.InactiveServer.Dmb.CompileJob.Id == monitorState.ActiveServer.Dmb.CompileJob.Id; - - if (!sameCompileJob || ActiveLaunchParameters != LastLaunchParameters) - restartOnceSwapped = true; // need a new launch to update either settings or compile job - - if (restartOnceSwapped) - /* - * we need to manually restart active server - * just kill it here, easier that way - */ - monitorState.ActiveServer.Dispose(); - - var activeServerStillHasPortOpen = !restartOnceSwapped && !monitorState.ActiveServer.ClosePortOnReboot; - - if (activeServerStillHasPortOpen) - /* we didn't want active server to swap for some reason and it still has it's port open - * just continue as normal - */ - break; - - if (!await MakeInactiveActive().ConfigureAwait(false)) - break; // monitor will restart - - // servers now swapped - // enable this now if inactive server is not still valid - monitorState.ActiveServer.ClosePortOnReboot = restartOnceSwapped; - - if (!restartOnceSwapped) - /* - * now try to reopen it on the private port - * failing that, just reboot it - */ - restartOnceSwapped = !await monitorState.InactiveServer.SetPort(ActiveLaunchParameters.SecondaryPort.Value, cancellationToken).ConfigureAwait(false); - - // break either way because any issues past this point would be solved by the reboot - if (restartOnceSwapped) // for one reason or another - await UpdateAndRestartInactiveServer(true).ConfigureAwait(false); // update and reboot - else - monitorState.NextAction = MonitorAction.Skip; // only skip checking inactive server rebooted, it's guaranteed InactiveServerStartup complete wouldn't fire this iteration - break; - case MonitorActivationReason.InactiveServerRebooted: - // just don't let the active server close it's port if the inactive server isn't ready - monitorState.RebootingInactiveServer = true; - monitorState.InactiveServer.ResetRebootState(); - monitorState.ActiveServer.ClosePortOnReboot = false; - monitorState.NextAction = MonitorAction.Continue; - break; - case MonitorActivationReason.InactiveServerStartupComplete: - // opposite of above case - monitorState.RebootingInactiveServer = false; - monitorState.ActiveServer.ClosePortOnReboot = true; - monitorState.NextAction = MonitorAction.Continue; - break; - case MonitorActivationReason.NewDmbAvailable: - case MonitorActivationReason.ActiveLaunchParametersUpdated: - // just reload the inactive server and wait for a swap to apply the changes - await UpdateAndRestartInactiveServer(true).ConfigureAwait(false); - break; - default: - Trace.Assert(false, String.Format(CultureInfo.InvariantCulture, "Invalid monitor activation reason: {0}!", activationReason)); - break; - } - } - #pragma warning restore CA1502 - - /// - /// The loop that watches the watchdog - /// - /// The for the operation - /// A representing the running operation -#pragma warning disable CA1502 // TODO: Decomplexify - async Task MonitorLifetimes(CancellationToken cancellationToken) - { - logger.LogTrace("Entered MonitorLifetimes"); - - // this function is responsible for calling HandlerMonitorWakeup when necessary and manitaining the MonitorState - var iteration = 1; - for (var monitorState = new MonitorState(); monitorState.NextAction != MonitorAction.Exit; ++iteration) - { - // always start out with continue - monitorState.NextAction = MonitorAction.Continue; - - // dump some info to the logs - logger.LogDebug("Iteration {0} of monitor loop", iteration); - try - { - if (AlphaIsActive) - logger.LogDebug("Alpha is the active server"); - else - logger.LogDebug("Bravo is the active server"); - - if (monitorState.RebootingInactiveServer) - logger.LogDebug("Inactive server is rebooting"); - - // update the monitor state with the inactive/active servers - monitorState.ActiveServer = AlphaIsActive ? alphaServer : bravoServer; - monitorState.InactiveServer = AlphaIsActive ? bravoServer : alphaServer; - - if (monitorState.ActiveServer.ClosePortOnReboot) - logger.LogDebug("Active server will close port on reboot"); - if (monitorState.InactiveServer.ClosePortOnReboot) - logger.LogDebug("Inactive server will close port on reboot"); - - logger.LogDebug("Active server Compile Job ID: {0}", monitorState.ActiveServer.Dmb.CompileJob.Id); - logger.LogDebug("Inactive server Compile Job ID: {0}", monitorState.InactiveServer.Dmb.CompileJob.Id); - - // load the activation tasks into local variables - Task activeServerLifetime = monitorState.ActiveServer.Lifetime; - Task inactiveServerLifetime = monitorState.InactiveServer.Lifetime; - var activeServerReboot = monitorState.ActiveServer.OnReboot; - var inactiveServerReboot = monitorState.InactiveServer.OnReboot; - Task inactiveServerStartup = monitorState.RebootingInactiveServer ? monitorState.InactiveServer.LaunchResult : null; - Task activeLaunchParametersChanged = activeParametersUpdated.Task; - var newDmbAvailable = dmbFactory.OnNewerDmb; - - // cancel waiting if requested - var cancelTcs = new TaskCompletionSource(); - using (cancellationToken.Register(() => cancelTcs.SetCanceled())) - { - var toWaitOn = Task.WhenAny(activeServerLifetime, inactiveServerLifetime, activeServerReboot, inactiveServerReboot, newDmbAvailable, cancelTcs.Task, activeLaunchParametersChanged); - if (monitorState.RebootingInactiveServer) - toWaitOn = Task.WhenAny(toWaitOn, inactiveServerStartup); - - // wait for something to happen - await toWaitOn.ConfigureAwait(false); - cancellationToken.ThrowIfCancellationRequested(); - } - - var chatTask = Task.CompletedTask; - using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken).ConfigureAwait(false)) - { - // always run HandleMonitorWakeup from the context of the semaphore lock - // multiple things may have happened, handle them one at a time - for (var moreActivationsToProcess = true; moreActivationsToProcess && (monitorState.NextAction == MonitorAction.Continue || monitorState.NextAction == MonitorAction.Skip);) - { - MonitorActivationReason activationReason = default; // this will always be assigned before being used - - // process the tasks in this order and call HandlerMonitorWakup for each - bool CheckActivationReason(ref Task task, MonitorActivationReason testActivationReason) - { - var taskCompleted = task?.IsCompleted == true; - task = null; - if (monitorState.NextAction == MonitorAction.Skip) - monitorState.NextAction = MonitorAction.Continue; - else if (taskCompleted) - { - activationReason = testActivationReason; - return true; - } - - return false; - } - - if (CheckActivationReason(ref activeServerLifetime, MonitorActivationReason.ActiveServerCrashed) - || CheckActivationReason(ref inactiveServerLifetime, MonitorActivationReason.InactiveServerCrashed) - || CheckActivationReason(ref activeServerReboot, MonitorActivationReason.ActiveServerRebooted) - || CheckActivationReason(ref inactiveServerReboot, MonitorActivationReason.InactiveServerRebooted) - || CheckActivationReason(ref inactiveServerStartup, MonitorActivationReason.InactiveServerStartupComplete) - || CheckActivationReason(ref newDmbAvailable, MonitorActivationReason.NewDmbAvailable) - || CheckActivationReason(ref activeLaunchParametersChanged, MonitorActivationReason.ActiveLaunchParametersUpdated)) - await HandlerMonitorWakeup(activationReason, monitorState, cancellationToken).ConfigureAwait(false); - else - moreActivationsToProcess = false; - } - - // writeback alphaServer and bravoServer from monitor state in case they changesd - alphaServer = AlphaIsActive ? monitorState.ActiveServer : monitorState.InactiveServer; - bravoServer = !AlphaIsActive ? monitorState.ActiveServer : monitorState.InactiveServer; - } - - // full reboot required - if (monitorState.NextAction == MonitorAction.Restart) - { - logger.LogDebug("Next state action is to restart"); - DisposeAndNullControllers(); - chatTask = chat.SendWatchdogMessage("Restarting entirely due to complications...", cancellationToken); - - for (var retryAttempts = 1; monitorState.NextAction == MonitorAction.Restart; ++retryAttempts) - { - Exception launchException = null; - using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken).ConfigureAwait(false)) - try - { - // use LaunchImplNoLock without announcements or restarting the monitor - await LaunchImplNoLock(false, false, null, cancellationToken).ConfigureAwait(false); - if (Running) - { - logger.LogDebug("Relaunch successful, resetting monitor state..."); - monitorState = new MonitorState(); // clean the slate and continue - } - } - catch (OperationCanceledException) - { - throw; - } - catch (Exception e) - { - launchException = e; - } - - await chatTask.ConfigureAwait(false); - if (!Running) - { - if (launchException == null) - logger.LogWarning("Failed to automatically restart the watchdog! Attempt: {0}", retryAttempts); - else - logger.LogWarning("Failed to automatically restart the watchdog! Attempt: {0}, Exception: {1}", retryAttempts, launchException); - var retryDelay = Math.Min(Math.Pow(2, retryAttempts), 3600); // max of one hour, increasing by a power of 2 each time - chatTask = chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Failed to restart watchdog (Attempt: {0}), retrying in {1} seconds...", retryAttempts, retryDelay), cancellationToken); - await Task.WhenAll(asyncDelayer.Delay(TimeSpan.FromSeconds(retryDelay), cancellationToken), chatTask).ConfigureAwait(false); - } - } - } - } - catch (OperationCanceledException) - { - logger.LogDebug("Monitor cancelled"); - break; - } - catch (Exception e) - { - // really, this should NEVER happen - logger.LogError("Monitor crashed! Iteration: {0}, State: {1}, Exception: {2}", iteration, JsonConvert.SerializeObject(monitorState), e); - await chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Monitor crashed, this should NEVER happen! Please report this, full details in logs! Restarting monitor... Error: {0}", e.Message), cancellationToken).ConfigureAwait(false); - } - } - - logger.LogTrace("Monitor exiting..."); - } - #pragma warning restore CA1502 - - /// - /// Stops . Doesn't kill the servers - /// - /// if the monitor was running, otherwise - async Task StopMonitor() - { - logger.LogTrace("StopMonitor"); - if (monitorTask == null) - return false; - monitorCts.Cancel(); - await monitorTask.ConfigureAwait(false); - monitorCts.Dispose(); - monitorCts = null; - monitorTask = null; - return true; - } - - /// - public async Task ChangeSettings(DreamDaemonLaunchParameters launchParameters, CancellationToken cancellationToken) - { - using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken).ConfigureAwait(false)) - { - if (launchParameters.Match(ActiveLaunchParameters)) - return; - ActiveLaunchParameters = launchParameters; - if (Running) - activeParametersUpdated.TrySetResult(null); // queue an update - } - } - - /// - /// Launches the - /// - /// If should be started by this function - /// If the launch should be announced to chat by this function - /// to use, if any - /// The for the operation - /// A representing the running operation - #pragma warning disable CA1502 // TODO: Decomplexify - async Task LaunchImplNoLock(bool startMonitor, bool announce, WatchdogReattachInformation reattachInfo, CancellationToken cancellationToken) - { - logger.LogTrace("Begin LaunchNoLock"); - - if (Running) - throw new JobException("Watchdog already running!"); - - // this is necessary, the monitor could be in it's sleep loop trying to restart, if so cancel THAT monitor and start our own with blackjack and hookers - Task chatTask; - if (startMonitor && await StopMonitor().ConfigureAwait(false)) - chatTask = chat.SendWatchdogMessage("Automatic retry sequence cancelled by manual launch. Restarting...", cancellationToken); - else if (announce) - chatTask = chat.SendWatchdogMessage(reattachInfo == null ? "Starting..." : "Reattaching...", cancellationToken); // simple announce - else - chatTask = Task.CompletedTask; // no announce - - // since neither server is running, this is safe to do - LastLaunchParameters = ActiveLaunchParameters; - - // for when we call ourself and want to not catch thrown exceptions - var ignoreNestedException = false; - try - { - // good ole sanity, should never fucking trigger but i don't trust myself even though I should - // TODO: Unit test this instead? - if (alphaServer != null || bravoServer != null) - throw new InvalidOperationException("Entered LaunchNoLock with one or more of the servers not being null!"); - - // don't need a new dmb if reattaching - var doesntNeedNewDmb = reattachInfo?.Alpha != null && reattachInfo?.Bravo != null; - var dmbToUse = doesntNeedNewDmb ? null : dmbFactory.LockNextDmb(2); - - // if this try catches something, both servers are killed - try - { - // start the alpha server task, either by launch a new process or attaching to an existing one - // The tasks returned are mainly for writing interop files to the directories among other things and should generally never fail - // The tasks pertaining to server startup times are in the ISessionControllers - Task alphaServerTask; - if (!doesntNeedNewDmb) - alphaServerTask = sessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbToUse, null, true, true, false, cancellationToken); - else - alphaServerTask = sessionControllerFactory.Reattach(reattachInfo.Alpha, cancellationToken); - - // retrieve the session controller - var startTime = DateTimeOffset.Now; - alphaServer = await alphaServerTask.ConfigureAwait(false); - - // failed reattaches will return null - alphaServer?.SetHighPriority(); - - // extra delay for total ordering - var now = DateTimeOffset.Now; - var delay = now - startTime; - - // definitely not if reattaching though - if (reattachInfo == null && delay.TotalSeconds < AlphaBravoStartupSeperationInterval) - await asyncDelayer.Delay(startTime.AddSeconds(AlphaBravoStartupSeperationInterval) - now, cancellationToken).ConfigureAwait(false); - - // now bring bravo up - if (!doesntNeedNewDmb) - bravoServer = await sessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbToUse, null, false, false, false, cancellationToken).ConfigureAwait(false); - else - bravoServer = await sessionControllerFactory.Reattach(reattachInfo.Bravo, cancellationToken).ConfigureAwait(false); - - // failed reattaches will return null - bravoServer?.SetHighPriority(); - - // possiblity of null servers due to failed reattaches - if (alphaServer == null || bravoServer == null) - { - await chatTask.ConfigureAwait(false); - var bothServersDead = alphaServer == null && bravoServer == null; - if (bothServersDead - || (alphaServer == null && reattachInfo.AlphaIsActive) - || (bravoServer == null && !reattachInfo.AlphaIsActive)) - { - // we lost the active server, just restart entirely - DisposeAndNullControllers(); - const string FailReattachMessage = "Unable to properly reattach to active server! Restarting..."; - logger.LogWarning(FailReattachMessage); - logger.LogDebug(bothServersDead ? "Also could not reattach to inactive server!" : "Inactive server was reattached successfully!"); - chatTask = chat.SendWatchdogMessage(FailReattachMessage, cancellationToken); - ignoreNestedException = true; - await LaunchImplNoLock(true, false, null, cancellationToken).ConfigureAwait(false); - return; - } - - // we still have the active server but the other one is dead to us, hand it off to the monitor to restart - const string InactiveReattachFailureMessage = "Unable to reattach to inactive server. Leaving for monitor to reboot..."; - chatTask = chat.SendWatchdogMessage(InactiveReattachFailureMessage, cancellationToken); - logger.LogWarning(InactiveReattachFailureMessage); - - if (reattachInfo.AlphaIsActive) - bravoServer = sessionControllerFactory.CreateDeadSession(reattachInfo.Bravo.Dmb); - else - alphaServer = sessionControllerFactory.CreateDeadSession(reattachInfo.Alpha.Dmb); - } - - // throws a JobException if something went wrong with a launch - // Dead sessions won't trigger this - async Task CheckLaunch(ISessionController controller, string serverName) - { - var launch = await controller.LaunchResult.ConfigureAwait(false); - if (launch.ExitCode.HasValue) // you killed us ray... - throw new JobException(String.Format(CultureInfo.InvariantCulture, "{1} server failed to start: {0}", launch.ToString(), serverName)); - if (!launch.StartupTime.HasValue) - throw new JobException(String.Format(CultureInfo.InvariantCulture, "{1} server timed out on startup: {0}s", launch.ToString(), ActiveLaunchParameters.StartupTimeout.Value)); - return launch; - } - - var alphaLrt = CheckLaunch(alphaServer, "Alpha"); - var bravoLrt = CheckLaunch(bravoServer, "Bravo"); - - // this task completes when both serers have finished booting - var allTask = Task.WhenAll(alphaLrt, bravoLrt); - - // don't forget about the cancellationToken - var cancelTcs = new TaskCompletionSource(); - using (cancellationToken.Register(() => cancelTcs.SetCanceled())) - await Task.WhenAny(allTask, cancelTcs.Task).ConfigureAwait(false); - cancellationToken.ThrowIfCancellationRequested(); - - await allTask.ConfigureAwait(false); - - // both servers are now running, alpha is the active server(unless reattach), huzzah - AlphaIsActive = reattachInfo?.AlphaIsActive ?? true; - - var activeServer = AlphaIsActive ? alphaServer : bravoServer; - activeServer.EnableCustomChatCommands(); - activeServer.ClosePortOnReboot = true; - - logger.LogInformation("Launched servers successfully"); - Running = true; - - if (startMonitor) - { - monitorCts = new CancellationTokenSource(); - monitorTask = MonitorLifetimes(monitorCts.Token); - } - } - catch - { - if (dmbToUse != null) - { - // we locked 2 dmbs - if (bravoServer == null) - { - // bravo didn't get control of his - dmbToUse.Dispose(); - if (alphaServer == null) - dmbToUse.Dispose(); // alpha didn't get control of his - } - } - else if (doesntNeedNewDmb) // we have reattachInfo - if (bravoServer == null) - { - // bravo didn't get control of his - reattachInfo.Bravo?.Dmb.Dispose(); - if (alphaServer == null) - reattachInfo.Alpha?.Dmb.Dispose(); // alpha didn't get control of his - } - - // kill the controllers - DisposeAndNullControllers(); - throw; - } - } - catch (Exception e) - { - // don't try to send chat tasks or warning logs if were suppressing exceptions or cancelled - if (!ignoreNestedException && !cancellationToken.IsCancellationRequested) - { - var originalChatTask = chatTask; - async Task ChainChatTaskWithErrorMessage() - { - await originalChatTask.ConfigureAwait(false); - await chat.SendWatchdogMessage("Startup failed!", cancellationToken).ConfigureAwait(false); - } - - chatTask = ChainChatTaskWithErrorMessage(); - logger.LogWarning("Failed to start watchdog: {0}", e.ToString()); - } - - throw; - } - finally - { - // finish the chat task that's in flight - try - { - await chatTask.ConfigureAwait(false); - } - catch (OperationCanceledException) { } - } - } - #pragma warning restore CA1502 - - /// - public async Task Launch(CancellationToken cancellationToken) - { - using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken).ConfigureAwait(false)) - await LaunchImplNoLock(true, true, null, cancellationToken).ConfigureAwait(false); - } - - /// - public async Task ResetRebootState(CancellationToken cancellationToken) - { - using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken).ConfigureAwait(false)) - { - if (!Running) - return; - var toClear = AlphaIsActive ? alphaServer : bravoServer; - if (toClear != null) - toClear.ResetRebootState(); - } - } - - /// - public async Task Restart(bool graceful, CancellationToken cancellationToken) - { - logger.LogTrace("Begin Restart. Graceful: {0}", graceful); - using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken).ConfigureAwait(false)) - { - if (!graceful || !Running) - { - Task chatTask; - if (Running) - { - chatTask = chat.SendWatchdogMessage("Manual restart triggered...", cancellationToken); - await TerminateNoLock(false, false, cancellationToken).ConfigureAwait(false); - } - else - chatTask = Task.CompletedTask; - await LaunchImplNoLock(true, !Running, null, cancellationToken).ConfigureAwait(false); - await chatTask.ConfigureAwait(false); - } - - var toReboot = AlphaIsActive ? alphaServer : bravoServer; - if (toReboot != null) - { - if (!await toReboot.SetRebootState(Components.Watchdog.RebootState.Restart, cancellationToken).ConfigureAwait(false)) - logger.LogWarning("Unable to send reboot state change event!"); - } - } - } - - /// - public async Task Terminate(bool graceful, CancellationToken cancellationToken) - { - using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken).ConfigureAwait(false)) - await TerminateNoLock(graceful, !releaseServers, cancellationToken).ConfigureAwait(false); - } - - /// - public async Task StartAsync(CancellationToken cancellationToken) - { - var reattachInfo = await reattachInfoHandler.Load(cancellationToken).ConfigureAwait(false); - if (!autoStart && reattachInfo == null) - return; - - long? adminUserId = null; - - await databaseContextFactory.UseContext(async db => adminUserId = await db.Users - .Where(x => x.CanonicalName == Api.Models.User.AdminName.ToUpperInvariant()) - .Select(x => x.Id) - .FirstAsync(cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); - var job = new Models.Job - { - StartedBy = new Models.User - { - Id = adminUserId.Value - }, - Instance = new Models.Instance - { - Id = instance.Id - }, - Description = "Instance startup watchdog launch", - CancelRight = (ulong)DreamDaemonRights.Shutdown, - CancelRightsType = RightsType.DreamDaemon - }; - await jobManager.RegisterOperation(job, async (j, databaseContext, progressFunction, ct) => - { - using (await SemaphoreSlimContext.Lock(semaphore, ct).ConfigureAwait(false)) - await LaunchImplNoLock(true, true, reattachInfo, ct).ConfigureAwait(false); - }, cancellationToken).ConfigureAwait(false); - } - - /// - public async Task StopAsync(CancellationToken cancellationToken) - { - try - { - if (releaseServers && Running) - { - await StopMonitor().ConfigureAwait(false); - - var reattachInformation = new WatchdogReattachInformation - { - AlphaIsActive = AlphaIsActive - }; - reattachInformation.Alpha = alphaServer?.Release(); - reattachInformation.Bravo = bravoServer?.Release(); - await reattachInfoHandler.Save(reattachInformation, cancellationToken).ConfigureAwait(false); - } - - await Terminate(false, cancellationToken).ConfigureAwait(false); - } - catch - { - releaseServers = false; - throw; - } - } - - /// - public async Task HandleEvent(EventType eventType, IEnumerable parameters, CancellationToken cancellationToken) - { - string results; - using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken).ConfigureAwait(false)) - { - if (!Running) - return true; - - var builder = new StringBuilder(Constants.DMTopicEvent); - builder.Append('&'); - var notification = new EventNotification - { - Type = eventType, - Parameters = parameters - }; - var json = JsonConvert.SerializeObject(notification); - builder.Append(byondTopicSender.SanitizeString(Constants.DMParameterData)); - builder.Append('='); - builder.Append(byondTopicSender.SanitizeString(json)); - - var activeServer = AlphaIsActive ? alphaServer : bravoServer; - results = await activeServer.SendCommand(builder.ToString(), cancellationToken).ConfigureAwait(false); - } - - if (results == Constants.DMResponseSuccess) - return true; - - List responses; - try - { - responses = JsonConvert.DeserializeObject>(results); - } - catch - { - logger.LogInformation("Recieved invalid response from DD when parsing event {0}:{1}{2}", eventType, Environment.NewLine, results); - return true; - } - - await Task.WhenAll(responses.Select(x => chat.SendMessage(x.Message, x.ChannelIds, cancellationToken))).ConfigureAwait(false); - - return true; - } - - /// - public async Task HandleChatCommand(string commandName, string arguments, Chat.User sender, CancellationToken cancellationToken) - { - using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken).ConfigureAwait(false)) - { - if (!Running) - return "ERROR: Server offline!"; - - var commandObject = new ChatCommand - { - Command = commandName, - Params = arguments, - User = sender - }; - - var json = JsonConvert.SerializeObject(commandObject, new JsonSerializerSettings - { - ContractResolver = new CamelCasePropertyNamesContractResolver() - }); - - var command = String.Format(CultureInfo.InvariantCulture, "{0}&{1}={2}", byondTopicSender.SanitizeString(Constants.DMTopicChatCommand), byondTopicSender.SanitizeString(Constants.DMParameterData), byondTopicSender.SanitizeString(json)); - - var activeServer = AlphaIsActive ? alphaServer : bravoServer; - return await activeServer.SendCommand(command, cancellationToken).ConfigureAwait(false) ?? "ERROR: Bad topic exchange!"; - } - } - - /// - public async Task HandleRestart(Version updateVersion, CancellationToken cancellationToken) - { - releaseServers = true; - if (Running) - await chat.SendWatchdogMessage("Detaching...", cancellationToken).ConfigureAwait(false); - } - } -} diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs new file mode 100644 index 0000000000..2a6c6ac54a --- /dev/null +++ b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs @@ -0,0 +1,614 @@ +using Byond.TopicSender; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; +using Newtonsoft.Json.Serialization; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Tgstation.Server.Api.Models.Internal; +using Tgstation.Server.Api.Rights; +using Tgstation.Server.Host.Components.Chat; +using Tgstation.Server.Host.Components.Compiler; +using Tgstation.Server.Host.Components.Interop; +using Tgstation.Server.Host.Core; + +namespace Tgstation.Server.Host.Components.Watchdog +{ + /// + /// Base class for s. + /// + #pragma warning disable CA1506 // TODO: Decomplexify + abstract class WatchdogBase : IWatchdog, ICustomCommandHandler, IRestartHandler + { + /// + public bool Running { get; protected set; } + + /// + public abstract bool AlphaIsActive { get; } + + /// + public abstract Models.CompileJob ActiveCompileJob { get; } + + /// + public DreamDaemonLaunchParameters ActiveLaunchParameters { get; protected set; } + + /// + public DreamDaemonLaunchParameters LastLaunchParameters { get; protected set; } + + /// + public abstract RebootState? RebootState { get; } + + /// + /// that completes when are changed and we are . + /// + protected TaskCompletionSource ActiveParametersUpdated { get; set; } + + /// + /// The for the . + /// + protected SemaphoreSlim Semaphore { get; } + + /// + /// The for the . + /// + protected ILogger Logger { get; } + + /// + /// The for the + /// + protected IChat Chat { get; } + + /// + /// The for the + /// + protected ISessionControllerFactory SessionControllerFactory { get; } + + /// + /// The for the + /// + protected IDmbFactory DmbFactory { get; } + + /// + /// The for the . + /// + protected IAsyncDelayer AsyncDelayer { get; } + + /// + /// The for the . + /// + readonly Api.Models.Instance instance; + + /// + /// The for the + /// + readonly IReattachInfoHandler reattachInfoHandler; + + /// + /// The for the + /// + readonly IDatabaseContextFactory databaseContextFactory; + + /// + /// The for the + /// + readonly IByondTopicSender byondTopicSender; + + /// + /// The for the + /// + readonly IEventConsumer eventConsumer; + + /// + /// The for the + /// + readonly IJobManager jobManager; + + /// + /// The for the + /// + readonly IRestartRegistration restartRegistration; + + /// + /// If the should in + /// + readonly bool autoStart; + + /// + /// The for the monitor loop + /// + CancellationTokenSource monitorCts; + + /// + /// The running the monitor loop + /// + Task monitorTask; + + /// + /// If the servers should be released instead of shutdown + /// + bool releaseServers; + + /// + /// Initializes a new instance of the . + /// + /// The value of + /// The value of + /// The value of + /// The value of + /// The value of + /// The value of + /// The value of + /// The value of + /// The to populate with + /// The value of . + /// The value of + /// The initial value of . May be modified + /// The value of + /// The value of + protected WatchdogBase( + IChat chat, + ISessionControllerFactory sessionControllerFactory, + IDmbFactory dmbFactory, + IReattachInfoHandler reattachInfoHandler, + IDatabaseContextFactory databaseContextFactory, + IByondTopicSender byondTopicSender, + IEventConsumer eventConsumer, + IJobManager jobManager, + IServerControl serverControl, + IAsyncDelayer asyncDelayer, + ILogger logger, + DreamDaemonLaunchParameters initialLaunchParameters, + Api.Models.Instance instance, + bool autoStart) + { + Chat = chat ?? throw new ArgumentNullException(nameof(chat)); + SessionControllerFactory = sessionControllerFactory ?? throw new ArgumentNullException(nameof(sessionControllerFactory)); + DmbFactory = dmbFactory ?? throw new ArgumentNullException(nameof(dmbFactory)); + AsyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer)); + this.reattachInfoHandler = reattachInfoHandler ?? throw new ArgumentNullException(nameof(reattachInfoHandler)); + this.databaseContextFactory = databaseContextFactory ?? throw new ArgumentNullException(nameof(databaseContextFactory)); + this.byondTopicSender = byondTopicSender ?? throw new ArgumentNullException(nameof(byondTopicSender)); + this.eventConsumer = eventConsumer ?? throw new ArgumentNullException(nameof(eventConsumer)); + this.jobManager = jobManager ?? throw new ArgumentNullException(nameof(jobManager)); + Logger = logger ?? throw new ArgumentNullException(nameof(logger)); + ActiveLaunchParameters = initialLaunchParameters ?? throw new ArgumentNullException(nameof(initialLaunchParameters)); + this.instance = instance ?? throw new ArgumentNullException(nameof(instance)); + this.autoStart = autoStart; + + if (serverControl == null) + throw new ArgumentNullException(nameof(serverControl)); + + chat.RegisterCommandHandler(this); + + ActiveLaunchParameters = initialLaunchParameters; + releaseServers = false; + ActiveParametersUpdated = new TaskCompletionSource(); + + restartRegistration = serverControl.RegisterForRestart(this); + try + { + Semaphore = new SemaphoreSlim(1); + } + catch + { + restartRegistration.Dispose(); + throw; + } + } + + /// + public void Dispose() + { + Semaphore.Dispose(); + restartRegistration.Dispose(); + DisposeAndNullControllers(); + + Debug.Assert(monitorCts == null, "Expected monitorCts to be null!"); + monitorCts?.Dispose(); + } + + /// + /// Implementation of . Does not lock + /// + /// If the termination will be delayed until a reboot is detected in the active server's DMAPI and this function will return immediately + /// If the termination will be announced using + /// The for the operation + /// A representing the running operation. + async Task TerminateNoLock(bool graceful, bool announce, CancellationToken cancellationToken) + { + if (!Running) + return; + if (!graceful) + { + var chatTask = announce ? Chat.SendWatchdogMessage("Terminating...", cancellationToken) : Task.CompletedTask; + await StopMonitor().ConfigureAwait(false); + DisposeAndNullControllers(); + LastLaunchParameters = null; + await chatTask.ConfigureAwait(false); + return; + } + + // merely set the reboot state + var toKill = GetActiveController(); + if (toKill != null) + await toKill.SetRebootState(Watchdog.RebootState.Shutdown, cancellationToken).ConfigureAwait(false); + } + + /// + /// Launches the watchdog. + /// + /// If should be started by this function + /// If the launch should be announced to chat by this function + /// to use, if any + /// The for the operation + /// A representing the running operation + protected async Task LaunchImplNoLock(bool startMonitor, bool announce, WatchdogReattachInformation reattachInfo, CancellationToken cancellationToken) + { + Logger.LogTrace("Begin LaunchNoLock"); + + if (Running) + throw new JobException("Watchdog already running!"); + + // this is necessary, the monitor could be in it's sleep loop trying to restart, if so cancel THAT monitor and start our own with blackjack and hookers + Task chatTask; + if (startMonitor && await StopMonitor().ConfigureAwait(false)) + chatTask = Chat.SendWatchdogMessage("Automatic retry sequence cancelled by manual launch. Restarting...", cancellationToken); + else if (announce) + chatTask = Chat.SendWatchdogMessage(reattachInfo == null ? "Starting..." : "Reattaching...", cancellationToken); // simple announce + else + chatTask = Task.CompletedTask; // no announce + + // since neither server is running, this is safe to do + LastLaunchParameters = ActiveLaunchParameters; + + // for when we call ourself and want to not catch thrown exceptions + var ignoreNestedException = false; + try + { + await InitControllers(() => ignoreNestedException = true, chatTask, reattachInfo, cancellationToken).ConfigureAwait(false); + await chatTask.ConfigureAwait(false); + + Logger.LogInformation("Launched servers successfully"); + Running = true; + + if (startMonitor) + { + StartMonitor(); + } + } + catch (Exception e) + { + // don't try to send chat tasks or warning logs if were suppressing exceptions or cancelled + if (!ignoreNestedException && !cancellationToken.IsCancellationRequested) + { + var originalChatTask = chatTask; + async Task ChainChatTaskWithErrorMessage() + { + await originalChatTask.ConfigureAwait(false); + await Chat.SendWatchdogMessage("Startup failed!", cancellationToken).ConfigureAwait(false); + } + + chatTask = ChainChatTaskWithErrorMessage(); + Logger.LogWarning("Failed to start watchdog: {0}", e.ToString()); + } + + throw; + } + finally + { + // finish the chat task that's in flight + try + { + await chatTask.ConfigureAwait(false); + } + catch (OperationCanceledException) { } + } + } + + /// + /// Call and setup and . + /// + protected void StartMonitor() + { + monitorCts = new CancellationTokenSource(); + monitorTask = MonitorLifetimes(monitorCts.Token); + } + + /// + /// Stops . Doesn't kill the servers + /// + /// if the monitor was running, otherwise + protected async Task StopMonitor() + { + Logger.LogTrace("StopMonitor"); + if (monitorTask == null) + return false; + monitorCts.Cancel(); + await monitorTask.ConfigureAwait(false); + monitorCts.Dispose(); + monitorTask = null; + monitorCts = null; + return true; + } + + /// + /// Send a chat message and log about a failed reattach operation and attempts another call to . + /// + /// If the inactive server was reattached successfully. + /// The for the operation/ + /// A representing the running operation. + protected async Task NotifyOfFailedReattach(bool inactiveReattachSuccess, CancellationToken cancellationToken) + { + // we lost the server, just restart entirely + DisposeAndNullControllers(); + const string FailReattachMessage = "Unable to properly reattach to server! Restarting..."; + Logger.LogWarning(FailReattachMessage); + Logger.LogDebug(inactiveReattachSuccess ? "Also could not reattach to inactive server!" : "Inactive server was reattached successfully!"); + Task chatTask = Chat.SendWatchdogMessage(FailReattachMessage, cancellationToken); + await LaunchImplNoLock(true, false, null, cancellationToken).ConfigureAwait(false); + await chatTask.ConfigureAwait(false); + } + + /// + /// Check the of a given for errors and throw a if any are detected. + /// + /// The to checkou. + /// The name of the server being checked. + /// The for the operation. + /// A representing the running operation. + protected async Task CheckLaunchResult(ISessionController controller, string serverName, CancellationToken cancellationToken) + { + var launchResult = await controller.LaunchResult.WithToken(cancellationToken).ConfigureAwait(false); + + // Dead sessions won't trigger this + if (launchResult.ExitCode.HasValue) // you killed us ray... + throw new JobException(String.Format(CultureInfo.InvariantCulture, "{0} failed to start: {1}", serverName, launchResult)); + if (!launchResult.StartupTime.HasValue) + throw new JobException(String.Format(CultureInfo.InvariantCulture, "{0} timed out on startup: {1}s", serverName, ActiveLaunchParameters.StartupTimeout.Value)); + } + + /// + /// Call and null the fields for all s and set to . + /// + protected abstract void DisposeAndNullControllers(); + + /// + /// Get the active . + /// + /// The active . + protected abstract ISessionController GetActiveController(); + + /// + /// Create the for the s. + /// + /// A new . + protected abstract WatchdogReattachInformation CreateReattachInformation(); + + /// + /// The loop that watches the watchdog + /// + /// The for the operation + /// A representing the running operation + protected abstract Task MonitorLifetimes(CancellationToken cancellationToken); + + /// + /// Starts all s. + /// + /// An that must be run before making a recursive call to . + /// A, possibly active, for an outgoing chat message. + /// to use, if any + /// The for the operation + /// A representing the running operation + protected abstract Task InitControllers(Action callBeforeRecurse, Task chatTask, WatchdogReattachInformation reattachInfo, CancellationToken cancellationToken); + + /// + public async Task ChangeSettings(DreamDaemonLaunchParameters launchParameters, CancellationToken cancellationToken) + { + using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false)) + { + if (launchParameters.Match(ActiveLaunchParameters)) + return; + ActiveLaunchParameters = launchParameters; + if (Running) + ActiveParametersUpdated.TrySetResult(null); // queue an update + } + } + + /// + public async Task HandleEvent(EventType eventType, IEnumerable parameters, CancellationToken cancellationToken) + { + string results; + using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false)) + { + if (!Running) + return true; + + var builder = new StringBuilder(Constants.DMTopicEvent); + builder.Append('&'); + var notification = new EventNotification + { + Type = eventType, + Parameters = parameters + }; + var json = JsonConvert.SerializeObject(notification); + builder.Append(byondTopicSender.SanitizeString(Constants.DMParameterData)); + builder.Append('='); + builder.Append(byondTopicSender.SanitizeString(json)); + + var activeServer = GetActiveController(); + results = await activeServer.SendCommand(builder.ToString(), cancellationToken).ConfigureAwait(false); + } + + if (results == Constants.DMResponseSuccess) + return true; + + List responses; + try + { + responses = JsonConvert.DeserializeObject>(results); + } + catch + { + Logger.LogInformation("Recieved invalid response from DD when parsing event {0}:{1}{2}", eventType, Environment.NewLine, results); + return true; + } + + await Task.WhenAll(responses.Select(x => Chat.SendMessage(x.Message, x.ChannelIds, cancellationToken))).ConfigureAwait(false); + + return true; + } + + /// + public async Task HandleChatCommand(string commandName, string arguments, Chat.User sender, CancellationToken cancellationToken) + { + using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false)) + { + if (!Running) + return "ERROR: Server offline!"; + + var commandObject = new ChatCommand + { + Command = commandName, + Params = arguments, + User = sender + }; + + var json = JsonConvert.SerializeObject(commandObject, new JsonSerializerSettings + { + ContractResolver = new CamelCasePropertyNamesContractResolver() + }); + + var command = String.Format(CultureInfo.InvariantCulture, "{0}&{1}={2}", byondTopicSender.SanitizeString(Constants.DMTopicChatCommand), byondTopicSender.SanitizeString(Constants.DMParameterData), byondTopicSender.SanitizeString(json)); + + var activeServer = GetActiveController(); + return await activeServer.SendCommand(command, cancellationToken).ConfigureAwait(false) ?? "ERROR: Bad topic exchange!"; + } + } + + /// + public async Task Launch(CancellationToken cancellationToken) + { + using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false)) + await LaunchImplNoLock(true, true, null, cancellationToken).ConfigureAwait(false); + } + + /// + public async Task ResetRebootState(CancellationToken cancellationToken) + { + using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false)) + { + if (!Running) + return; + var toClear = GetActiveController(); + if (toClear != null) + toClear.ResetRebootState(); + } + } + + /// + public async Task Restart(bool graceful, CancellationToken cancellationToken) + { + Logger.LogTrace("Begin Restart. Graceful: {0}", graceful); + using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false)) + { + if (!graceful || !Running) + { + Task chatTask; + if (Running) + { + chatTask = Chat.SendWatchdogMessage("Manual restart triggered...", cancellationToken); + await TerminateNoLock(false, false, cancellationToken).ConfigureAwait(false); + } + else + chatTask = Task.CompletedTask; + await LaunchImplNoLock(true, !Running, null, cancellationToken).ConfigureAwait(false); + await chatTask.ConfigureAwait(false); + } + + var toReboot = GetActiveController(); + if (toReboot != null) + { + if (!await toReboot.SetRebootState(Watchdog.RebootState.Restart, cancellationToken).ConfigureAwait(false)) + Logger.LogWarning("Unable to send reboot state change event!"); + } + } + } + + /// + public async Task StartAsync(CancellationToken cancellationToken) + { + var reattachInfo = await reattachInfoHandler.Load(cancellationToken).ConfigureAwait(false); + if (!autoStart && reattachInfo == null) + return; + + long? adminUserId = null; + + await databaseContextFactory.UseContext(async db => adminUserId = await db.Users + .Where(x => x.CanonicalName == Api.Models.User.AdminName.ToUpperInvariant()) + .Select(x => x.Id) + .FirstAsync(cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); + var job = new Models.Job + { + StartedBy = new Models.User + { + Id = adminUserId.Value + }, + Instance = new Models.Instance + { + Id = instance.Id + }, + Description = "Instance startup watchdog launch", + CancelRight = (ulong)DreamDaemonRights.Shutdown, + CancelRightsType = RightsType.DreamDaemon + }; + await jobManager.RegisterOperation(job, async (j, databaseContext, progressFunction, ct) => + { + using (await SemaphoreSlimContext.Lock(Semaphore, ct).ConfigureAwait(false)) + await LaunchImplNoLock(true, true, reattachInfo, ct).ConfigureAwait(false); + }, cancellationToken).ConfigureAwait(false); + } + + /// + public async Task StopAsync(CancellationToken cancellationToken) + { + try + { + if (releaseServers && Running) + { + await StopMonitor().ConfigureAwait(false); + + var reattachInformation = CreateReattachInformation(); + await reattachInfoHandler.Save(reattachInformation, cancellationToken).ConfigureAwait(false); + } + + await Terminate(false, cancellationToken).ConfigureAwait(false); + } + catch + { + releaseServers = false; + throw; + } + } + + /// + public async Task Terminate(bool graceful, CancellationToken cancellationToken) + { + using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false)) + await TerminateNoLock(graceful, !releaseServers, cancellationToken).ConfigureAwait(false); + } + + /// + public async Task HandleRestart(Version updateVersion, CancellationToken cancellationToken) + { + releaseServers = true; + if (Running) + await Chat.SendWatchdogMessage("Detaching...", cancellationToken).ConfigureAwait(false); + } + } +} diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogFactory.cs b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogFactory.cs index 1c03523ee7..46a565bdb7 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogFactory.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogFactory.cs @@ -1,9 +1,11 @@ using Byond.TopicSender; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using System; using Tgstation.Server.Api.Models.Internal; using Tgstation.Server.Host.Components.Chat; using Tgstation.Server.Host.Components.Compiler; +using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.Core; namespace Tgstation.Server.Host.Components.Watchdog @@ -41,6 +43,11 @@ namespace Tgstation.Server.Host.Components.Watchdog /// readonly IAsyncDelayer asyncDelayer; + /// + /// The for the + /// + readonly GeneralConfiguration generalConfiguration; + /// /// Construct a /// @@ -50,7 +57,8 @@ namespace Tgstation.Server.Host.Components.Watchdog /// The value of /// The value of /// The value of - public WatchdogFactory(IServerControl serverControl, ILoggerFactory loggerFactory, IDatabaseContextFactory databaseContextFactory, IByondTopicSender byondTopicSender, IJobManager jobManager, IAsyncDelayer asyncDelayer) + /// The containing the value of + public WatchdogFactory(IServerControl serverControl, ILoggerFactory loggerFactory, IDatabaseContextFactory databaseContextFactory, IByondTopicSender byondTopicSender, IJobManager jobManager, IAsyncDelayer asyncDelayer, IOptions generalConfigurationOptions) { this.serverControl = serverControl ?? throw new ArgumentNullException(nameof(serverControl)); this.loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory)); @@ -58,9 +66,16 @@ namespace Tgstation.Server.Host.Components.Watchdog this.byondTopicSender = byondTopicSender ?? throw new ArgumentNullException(nameof(byondTopicSender)); this.jobManager = jobManager ?? throw new ArgumentNullException(nameof(jobManager)); this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer)); + generalConfiguration = generalConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(generalConfigurationOptions)); } /// - public IWatchdog CreateWatchdog(IChat chat, IDmbFactory dmbFactory, IReattachInfoHandler reattachInfoHandler, IEventConsumer eventConsumer, ISessionControllerFactory sessionControllerFactory, Api.Models.Instance instance, DreamDaemonSettings settings) => new Watchdog(chat, sessionControllerFactory, dmbFactory, reattachInfoHandler, databaseContextFactory, byondTopicSender, eventConsumer, jobManager, serverControl, asyncDelayer, loggerFactory.CreateLogger(), settings, instance, settings.AutoStart.Value); + public IWatchdog CreateWatchdog(IChat chat, IDmbFactory dmbFactory, IReattachInfoHandler reattachInfoHandler, IEventConsumer eventConsumer, ISessionControllerFactory sessionControllerFactory, Api.Models.Instance instance, DreamDaemonSettings settings) + { + if (generalConfiguration.UseExperimentalWatchdog) + return new ExperimentalWatchdog(chat, sessionControllerFactory, dmbFactory, reattachInfoHandler, databaseContextFactory, byondTopicSender, eventConsumer, jobManager, serverControl, asyncDelayer, loggerFactory.CreateLogger(), settings, instance, settings.AutoStart.Value); + + return new BasicWatchdog(chat, sessionControllerFactory, dmbFactory, reattachInfoHandler, databaseContextFactory, byondTopicSender, eventConsumer, jobManager, serverControl, asyncDelayer, loggerFactory.CreateLogger(), settings, instance, settings.AutoStart.Value); + } } } diff --git a/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs b/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs index 8cc77dcde6..78ac49b33f 100644 --- a/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs +++ b/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs @@ -53,5 +53,10 @@ namespace Tgstation.Server.Host.Configuration /// The timeout milliseconds for restarting the server /// public int RestartTimeout { get; set; } = DefaultRestartTimeout; + + /// + /// If the should be used. + /// + public bool UseExperimentalWatchdog { get; set; } } } diff --git a/src/Tgstation.Server.Host/appsettings.json b/src/Tgstation.Server.Host/appsettings.json index 08135f778f..5c06b38672 100644 --- a/src/Tgstation.Server.Host/appsettings.json +++ b/src/Tgstation.Server.Host/appsettings.json @@ -4,7 +4,8 @@ "GitHubAccessToken": null, "SetupWizardMode": "AutoDetect", "ByondTopicTimeout": 5000, - "RestartTimeout": 10000 + "RestartTimeout": 10000, + "UseExperimentalWatchdog" : false }, "FileLogging": { "Directory": null, //use the default path diff --git a/tests/Tgstation.Server.Host.Tests/Components/Watchdog/TestWatchdog.cs b/tests/Tgstation.Server.Host.Tests/Components/Watchdog/TestExperimentalWatchdog.cs similarity index 53% rename from tests/Tgstation.Server.Host.Tests/Components/Watchdog/TestWatchdog.cs rename to tests/Tgstation.Server.Host.Tests/Components/Watchdog/TestExperimentalWatchdog.cs index ff95faf9f4..28ad98600b 100644 --- a/tests/Tgstation.Server.Host.Tests/Components/Watchdog/TestWatchdog.cs +++ b/tests/Tgstation.Server.Host.Tests/Components/Watchdog/TestExperimentalWatchdog.cs @@ -14,55 +14,55 @@ using Tgstation.Server.Host.Core; namespace Tgstation.Server.Host.Components.Watchdog.Tests { [TestClass] - public sealed class TestWatchdog + public sealed class TestExperimentalWatchdog { [TestMethod] public void TestConstruction() { - Assert.ThrowsException(() => new Watchdog(null, null, null, null, null, null, null, null, null, null, null, null, null, default)); + Assert.ThrowsException(() => new ExperimentalWatchdog(null, null, null, null, null, null, null, null, null, null, null, null, null, default)); var mockChat = new Mock(); mockChat.Setup(x => x.RegisterCommandHandler(It.IsNotNull())).Verifiable(); - Assert.ThrowsException(() => new Watchdog(mockChat.Object, null, null, null, null, null, null, null, null, null, null, null, null, default)); + Assert.ThrowsException(() => new ExperimentalWatchdog(mockChat.Object, null, null, null, null, null, null, null, null, null, null, null, null, default)); var mockSessionControllerFactory = new Mock(); - Assert.ThrowsException(() => new Watchdog(mockChat.Object, mockSessionControllerFactory.Object, null, null, null, null, null, null, null, null, null, null, null, default)); + Assert.ThrowsException(() => new ExperimentalWatchdog(mockChat.Object, mockSessionControllerFactory.Object, null, null, null, null, null, null, null, null, null, null, null, default)); var mockDmbFactory = new Mock(); - Assert.ThrowsException(() => new Watchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, null, null, null, null, null, null, null, null, null, null, default)); + Assert.ThrowsException(() => new ExperimentalWatchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, null, null, null, null, null, null, null, null, null, null, default)); var mockReattachInfoHandler = new Mock(); - Assert.ThrowsException(() => new Watchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, mockReattachInfoHandler.Object, null, null, null, null, null, null, null, null, null, default)); + Assert.ThrowsException(() => new ExperimentalWatchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, mockReattachInfoHandler.Object, null, null, null, null, null, null, null, null, null, default)); var mockDatabaseContextFactory = new Mock(); - Assert.ThrowsException(() => new Watchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, mockReattachInfoHandler.Object, mockDatabaseContextFactory.Object, null, null, null, null, null, null, null, null, default)); + Assert.ThrowsException(() => new ExperimentalWatchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, mockReattachInfoHandler.Object, mockDatabaseContextFactory.Object, null, null, null, null, null, null, null, null, default)); var mockByondTopicSender = new Mock(); - Assert.ThrowsException(() => new Watchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, mockReattachInfoHandler.Object, mockDatabaseContextFactory.Object, mockByondTopicSender.Object, null, null, null, null, null, null, null, default)); + Assert.ThrowsException(() => new ExperimentalWatchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, mockReattachInfoHandler.Object, mockDatabaseContextFactory.Object, mockByondTopicSender.Object, null, null, null, null, null, null, null, default)); var mockEventConsumer = new Mock(); - Assert.ThrowsException(() => new Watchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, mockReattachInfoHandler.Object, mockDatabaseContextFactory.Object, mockByondTopicSender.Object, mockEventConsumer.Object, null, null, null, null, null, null, default)); + Assert.ThrowsException(() => new ExperimentalWatchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, mockReattachInfoHandler.Object, mockDatabaseContextFactory.Object, mockByondTopicSender.Object, mockEventConsumer.Object, null, null, null, null, null, null, default)); var mockJobManager = new Mock(); - Assert.ThrowsException(() => new Watchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, mockReattachInfoHandler.Object, mockDatabaseContextFactory.Object, mockByondTopicSender.Object, mockEventConsumer.Object, mockJobManager.Object, null, null, null, null, null, default)); + Assert.ThrowsException(() => new ExperimentalWatchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, mockReattachInfoHandler.Object, mockDatabaseContextFactory.Object, mockByondTopicSender.Object, mockEventConsumer.Object, mockJobManager.Object, null, null, null, null, null, default)); var mockRestartRegistration = new Mock(); mockRestartRegistration.Setup(x => x.Dispose()).Verifiable(); var mockServerControl = new Mock(); mockServerControl.Setup(x => x.RegisterForRestart(It.IsNotNull())).Returns(mockRestartRegistration.Object).Verifiable(); - Assert.ThrowsException(() => new Watchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, mockReattachInfoHandler.Object, mockDatabaseContextFactory.Object, mockByondTopicSender.Object, mockEventConsumer.Object, mockJobManager.Object, mockServerControl.Object, null, null, null, null, default)); + Assert.ThrowsException(() => new ExperimentalWatchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, mockReattachInfoHandler.Object, mockDatabaseContextFactory.Object, mockByondTopicSender.Object, mockEventConsumer.Object, mockJobManager.Object, mockServerControl.Object, null, null, null, null, default)); var mockAsyncDelayer = new Mock(); - Assert.ThrowsException(() => new Watchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, mockReattachInfoHandler.Object, mockDatabaseContextFactory.Object, mockByondTopicSender.Object, mockEventConsumer.Object, mockJobManager.Object, mockServerControl.Object, mockAsyncDelayer.Object, null, null, null, default)); + Assert.ThrowsException(() => new ExperimentalWatchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, mockReattachInfoHandler.Object, mockDatabaseContextFactory.Object, mockByondTopicSender.Object, mockEventConsumer.Object, mockJobManager.Object, mockServerControl.Object, mockAsyncDelayer.Object, null, null, null, default)); - var mockLogger = new Mock>(); - Assert.ThrowsException(() => new Watchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, mockReattachInfoHandler.Object, mockDatabaseContextFactory.Object, mockByondTopicSender.Object, mockEventConsumer.Object, mockJobManager.Object, mockServerControl.Object, mockAsyncDelayer.Object, mockLogger.Object, null, null, default)); + var mockLogger = new Mock>(); + Assert.ThrowsException(() => new ExperimentalWatchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, mockReattachInfoHandler.Object, mockDatabaseContextFactory.Object, mockByondTopicSender.Object, mockEventConsumer.Object, mockJobManager.Object, mockServerControl.Object, mockAsyncDelayer.Object, mockLogger.Object, null, null, default)); var mockLaunchParameters = new DreamDaemonLaunchParameters(); - Assert.ThrowsException(() => new Watchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, mockReattachInfoHandler.Object, mockDatabaseContextFactory.Object, mockByondTopicSender.Object, mockEventConsumer.Object, mockJobManager.Object, mockServerControl.Object, mockAsyncDelayer.Object, mockLogger.Object, mockLaunchParameters, null, default)); + Assert.ThrowsException(() => new ExperimentalWatchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, mockReattachInfoHandler.Object, mockDatabaseContextFactory.Object, mockByondTopicSender.Object, mockEventConsumer.Object, mockJobManager.Object, mockServerControl.Object, mockAsyncDelayer.Object, mockLogger.Object, mockLaunchParameters, null, default)); var mockInstance = new Models.Instance(); - new Watchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, mockReattachInfoHandler.Object, mockDatabaseContextFactory.Object, mockByondTopicSender.Object, mockEventConsumer.Object, mockJobManager.Object, mockServerControl.Object, mockAsyncDelayer.Object, mockLogger.Object, mockLaunchParameters, mockInstance, default).Dispose(); + new ExperimentalWatchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, mockReattachInfoHandler.Object, mockDatabaseContextFactory.Object, mockByondTopicSender.Object, mockEventConsumer.Object, mockJobManager.Object, mockServerControl.Object, mockAsyncDelayer.Object, mockLogger.Object, mockLaunchParameters, mockInstance, default).Dispose(); mockRestartRegistration.VerifyAll(); mockServerControl.VerifyAll(); @@ -76,7 +76,7 @@ namespace Tgstation.Server.Host.Components.Watchdog.Tests mockChat.Setup(x => x.RegisterCommandHandler(It.IsNotNull())).Verifiable(); var mockSessionControllerFactory = new Mock(); var mockDmbFactory = new Mock(); - var mockLogger = new Mock>(); + var mockLogger = new Mock>(); var mockReattachInfoHandler = new Mock(); var mockDatabaseContextFactory = new Mock(); var mockByondTopicSender = new Mock(); @@ -90,7 +90,7 @@ namespace Tgstation.Server.Host.Components.Watchdog.Tests var mockInstance = new Models.Instance(); var mockAsyncDelayer = new Mock(); - using (var wd = new Watchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, mockReattachInfoHandler.Object, mockDatabaseContextFactory.Object, mockByondTopicSender.Object, mockEventConsumer.Object, mockJobManager.Object, mockServerControl.Object, mockAsyncDelayer.Object, mockLogger.Object, mockLaunchParameters, mockInstance, default)) + using (var wd = new ExperimentalWatchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, mockReattachInfoHandler.Object, mockDatabaseContextFactory.Object, mockByondTopicSender.Object, mockEventConsumer.Object, mockJobManager.Object, mockServerControl.Object, mockAsyncDelayer.Object, mockLogger.Object, mockLaunchParameters, mockInstance, default)) using (var cts = new CancellationTokenSource()) { var mockCompileJob = new Models.CompileJob();