From beda35034242263d1919633b1d64ba9c22595658 Mon Sep 17 00:00:00 2001 From: Cyberboss Date: Tue, 17 Jul 2018 16:52:45 -0400 Subject: [PATCH 1/7] Work on the watchdog event handling actions --- .../Components/Watchdog/MonitorState.cs | 2 + .../Components/Watchdog/Watchdog.cs | 100 +++++++++++++++--- 2 files changed, 89 insertions(+), 13 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Watchdog/MonitorState.cs b/src/Tgstation.Server.Host/Components/Watchdog/MonitorState.cs index 09f24d30c5..5914cb2f50 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/MonitorState.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/MonitorState.cs @@ -5,6 +5,8 @@ public bool RebootingInactiveServer { get; set; } public bool InactiveServerHasStagedDmb { get; set; } + public bool InactiveServerCritFail { get; set; } + public MonitorAction NextAction { get; set; } public ISessionController ActiveServer { get; set; } diff --git a/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs index c64d6212c7..20523ca6d2 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs @@ -199,11 +199,82 @@ namespace Tgstation.Server.Host.Components.Watchdog await toKill.SetRebootState(Components.Watchdog.RebootState.Shutdown, cancellationToken).ConfigureAwait(false); } - async Task HandlerMonitorWakeup(MonitorActivationReason activationReason, MonitorState monitorState) + async Task HandlerMonitorWakeup(MonitorActivationReason activationReason, MonitorState monitorState, CancellationToken cancellationToken) { logger.LogInformation("Monitor activation. Reason: {0}", activationReason); - await Task.Yield(); - throw new NotImplementedException(nameof(monitorState)); + switch (activationReason) + { + case MonitorActivationReason.ActiveServerCrashed: + if (monitorState.RebootingInactiveServer || monitorState.InactiveServerCritFail) + { + logger.LogInformation("Inactive server is {0}! Restarting monitor...", monitorState.InactiveServerCritFail ? "critically failed" : "still rebooting"); + monitorState.NextAction = MonitorAction.Restart; + break; + } + + var dasDmbTask = dmbFactory.LockNextDmb(cancellationToken); + var result = await monitorState.InactiveServer.SetPort(ActiveLaunchParameters.PrimaryPort.Value, cancellationToken).ConfigureAwait(false); + + if (!result) + { + logger.LogWarning("Failed to activate inactive server! Restarting monitor..."); + monitorState.NextAction = MonitorAction.Restart; + break; + } + + monitorState.InactiveServerHasStagedDmb = false; + LastLaunchParameters = ActiveLaunchParameters; + + var deadServer = monitorState.ActiveServer; + monitorState.ActiveServer = monitorState.InactiveServer; + monitorState.NextAction = MonitorAction.Continue; + + try + { + monitorState.InactiveServer = await sessionControllerFactory.LaunchNew(ActiveLaunchParameters, await dasDmbTask.ConfigureAwait(false), null, false, !monitorState.ActiveServer.IsPrimary, false, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception e) + { + logger.LogError("Exception occurred while recreating crashed server! Attempting backup strategy of running DMB of running server! Exception: {0}", e.ToString()); + //ahh jeez, what do we do here? + //this is our fault, so it should never happen + //try to start it using the active server's dmb as a backup + try + { + var dmbBackup = dmbFactory.FromCompileJob(monitorState.ActiveServer.Dmb.CompileJob); + monitorState.InactiveServer = await sessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbBackup, null, false, !monitorState.ActiveServer.IsPrimary, false, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception e2) + { + //fuuuuucckkk + logger.LogError("Backup strategy failed! Monitor will restart when active server reboots! This Exception: {0}", e2.ToString()); + monitorState.InactiveServerCritFail = true; + break; + } + } + + logger.LogInformation("Successfully relaunched inactive server!"); + monitorState.RebootingInactiveServer = true; + break; + case MonitorActivationReason.InactiveServerCrashed: + throw new NotImplementedException(); + case MonitorActivationReason.ActiveServerRebooted: + throw new NotImplementedException(); + case MonitorActivationReason.InactiveServerRebooted: + throw new NotImplementedException(); + case MonitorActivationReason.InactiveServerStartupComplete: + throw new NotImplementedException(); + case MonitorActivationReason.NewDmbAvailable: + throw new NotImplementedException(); + } } /// @@ -254,7 +325,7 @@ namespace Tgstation.Server.Host.Components.Watchdog { MonitorActivationReason activationReason = default; //multiple things may have happened, handle them one at a time - for (var moreActivationsToProcess = true; moreActivationsToProcess && state.NextAction == MonitorAction.Continue; await HandlerMonitorWakeup(activationReason, state).ConfigureAwait(false)) + for (var moreActivationsToProcess = true; moreActivationsToProcess && state.NextAction == MonitorAction.Continue; ) { if (activeServerLifetime?.IsCompleted == true) { @@ -289,14 +360,17 @@ namespace Tgstation.Server.Host.Components.Watchdog else moreActivationsToProcess = false; } - //full reboot required - if (state.NextAction == MonitorAction.Restart) - { - logger.LogDebug("Next state action is to restart"); - DisposeAndNullControllers(); - Running = false; - chatTask = chat.SendWatchdogMessage("Restarting due to complications...", cancellationToken); - } + + await HandlerMonitorWakeup(activationReason, state, cancellationToken).ConfigureAwait(false); + } + + //full reboot required + if (state.NextAction == MonitorAction.Restart) + { + logger.LogDebug("Next state action is to restart"); + DisposeAndNullControllers(); + Running = false; + chatTask = chat.SendWatchdogMessage("Restarting due to complications...", cancellationToken); } for (var retryAttempts = 1; state.NextAction == MonitorAction.Restart; ++retryAttempts) @@ -307,7 +381,7 @@ namespace Tgstation.Server.Host.Components.Watchdog await chatTask.ConfigureAwait(false); if (Running) - state.NextAction = MonitorAction.Continue; + state = new MonitorState(); //clean the slate else { logger.LogWarning("Failed to automatically restart the watchdog! Alpha: {0}; Bravo: {1}", result.Alpha.ToString(), result.Bravo.ToString()); From 2ba4002dd78251a08e68263bdf35b105beeff963 Mon Sep 17 00:00:00 2001 From: Cyberboss Date: Tue, 17 Jul 2018 17:15:23 -0400 Subject: [PATCH 2/7] InactiveServerStartupComplete --- .../Components/Watchdog/Watchdog.cs | 91 ++++++++++--------- 1 file changed, 48 insertions(+), 43 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs index 20523ca6d2..dede7bdd79 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs @@ -205,64 +205,66 @@ namespace Tgstation.Server.Host.Components.Watchdog switch (activationReason) { case MonitorActivationReason.ActiveServerCrashed: - if (monitorState.RebootingInactiveServer || monitorState.InactiveServerCritFail) + using (monitorState.ActiveServer) //it's dead, dispose it when we're done { - logger.LogInformation("Inactive server is {0}! Restarting monitor...", monitorState.InactiveServerCritFail ? "critically failed" : "still rebooting"); - monitorState.NextAction = MonitorAction.Restart; - break; - } + if (monitorState.RebootingInactiveServer || monitorState.InactiveServerCritFail) + { + logger.LogInformation("Inactive server is {0}! Restarting monitor...", monitorState.InactiveServerCritFail ? "critically failed" : "still rebooting"); + monitorState.NextAction = MonitorAction.Restart; + break; + } - var dasDmbTask = dmbFactory.LockNextDmb(cancellationToken); - var result = await monitorState.InactiveServer.SetPort(ActiveLaunchParameters.PrimaryPort.Value, cancellationToken).ConfigureAwait(false); + var dasDmbTask = dmbFactory.LockNextDmb(cancellationToken); + var result = await monitorState.InactiveServer.SetPort(ActiveLaunchParameters.PrimaryPort.Value, cancellationToken).ConfigureAwait(false); - if (!result) - { - logger.LogWarning("Failed to activate inactive server! Restarting monitor..."); - monitorState.NextAction = MonitorAction.Restart; - break; - } + if (!result) + { + logger.LogWarning("Failed to activate inactive server! Restarting monitor..."); + monitorState.NextAction = MonitorAction.Restart; + break; + } - monitorState.InactiveServerHasStagedDmb = false; - LastLaunchParameters = ActiveLaunchParameters; + monitorState.InactiveServerHasStagedDmb = false; + LastLaunchParameters = ActiveLaunchParameters; - var deadServer = monitorState.ActiveServer; - monitorState.ActiveServer = monitorState.InactiveServer; - monitorState.NextAction = MonitorAction.Continue; + monitorState.ActiveServer = monitorState.InactiveServer; + monitorState.NextAction = MonitorAction.Continue; - try - { - monitorState.InactiveServer = await sessionControllerFactory.LaunchNew(ActiveLaunchParameters, await dasDmbTask.ConfigureAwait(false), null, false, !monitorState.ActiveServer.IsPrimary, false, cancellationToken).ConfigureAwait(false); - } - catch (OperationCanceledException) - { - throw; - } - catch (Exception e) - { - logger.LogError("Exception occurred while recreating crashed server! Attempting backup strategy of running DMB of running server! Exception: {0}", e.ToString()); - //ahh jeez, what do we do here? - //this is our fault, so it should never happen - //try to start it using the active server's dmb as a backup try { - var dmbBackup = dmbFactory.FromCompileJob(monitorState.ActiveServer.Dmb.CompileJob); - monitorState.InactiveServer = await sessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbBackup, null, false, !monitorState.ActiveServer.IsPrimary, false, cancellationToken).ConfigureAwait(false); + monitorState.InactiveServer = await sessionControllerFactory.LaunchNew(ActiveLaunchParameters, await dasDmbTask.ConfigureAwait(false), null, false, !monitorState.ActiveServer.IsPrimary, false, cancellationToken).ConfigureAwait(false); } catch (OperationCanceledException) { throw; } - catch (Exception e2) + catch (Exception e) { - //fuuuuucckkk - logger.LogError("Backup strategy failed! Monitor will restart when active server reboots! This Exception: {0}", e2.ToString()); - monitorState.InactiveServerCritFail = true; - break; + logger.LogError("Exception occurred while recreating crashed server! Attempting backup strategy of running DMB of running server! Exception: {0}", e.ToString()); + //ahh jeez, what do we do here? + //this is our fault, so it should never happen + //try to start it using the active server's dmb as a backup + try + { + var dmbBackup = dmbFactory.FromCompileJob(monitorState.ActiveServer.Dmb.CompileJob); + monitorState.InactiveServer = await sessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbBackup, null, false, !monitorState.ActiveServer.IsPrimary, false, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception e2) + { + //fuuuuucckkk + logger.LogError("Backup strategy failed! Monitor will restart when active server reboots! This Exception: {0}", e2.ToString()); + monitorState.InactiveServerCritFail = true; + break; + } } - } - logger.LogInformation("Successfully relaunched inactive server!"); - monitorState.RebootingInactiveServer = true; + logger.LogInformation("Successfully relaunched inactive server!"); + monitorState.RebootingInactiveServer = true; + } break; case MonitorActivationReason.InactiveServerCrashed: throw new NotImplementedException(); @@ -271,7 +273,10 @@ namespace Tgstation.Server.Host.Components.Watchdog case MonitorActivationReason.InactiveServerRebooted: throw new NotImplementedException(); case MonitorActivationReason.InactiveServerStartupComplete: - throw new NotImplementedException(); + //eziest case of my life + monitorState.RebootingInactiveServer = false; + monitorState.NextAction = MonitorAction.Continue; + break; case MonitorActivationReason.NewDmbAvailable: throw new NotImplementedException(); } From 94729b10bab6c959d0ff91816d8824f56c01e82e Mon Sep 17 00:00:00 2001 From: Cyberboss Date: Wed, 18 Jul 2018 10:56:17 -0400 Subject: [PATCH 3/7] Many more watch monitor cases --- .../Components/Watchdog/MonitorAction.cs | 1 - .../Components/Watchdog/MonitorState.cs | 1 + .../Components/Watchdog/Watchdog.cs | 266 ++++++++++++++---- 3 files changed, 209 insertions(+), 59 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Watchdog/MonitorAction.cs b/src/Tgstation.Server.Host/Components/Watchdog/MonitorAction.cs index 3140b9e854..b5178c7b55 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/MonitorAction.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/MonitorAction.cs @@ -6,7 +6,6 @@ enum MonitorAction { Continue, - Break, Restart, Exit } diff --git a/src/Tgstation.Server.Host/Components/Watchdog/MonitorState.cs b/src/Tgstation.Server.Host/Components/Watchdog/MonitorState.cs index 5914cb2f50..367c2c3e80 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/MonitorState.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/MonitorState.cs @@ -3,6 +3,7 @@ sealed class MonitorState { public bool RebootingInactiveServer { get; set; } + public bool InactiveServerHasStagedDmb { get; set; } public bool InactiveServerCritFail { get; set; } diff --git a/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs index dede7bdd79..ac949b65b2 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs @@ -90,10 +90,23 @@ namespace Tgstation.Server.Host.Components.Watchdog /// readonly bool autoStart; + /// + /// The for the monitor loop + /// CancellationTokenSource monitorCts; + + /// + /// The running the monitor loop + /// Task monitorTask; + /// + /// Server designation alpha + /// ISessionController alphaServer; + /// + /// Server designation bravo + /// ISessionController bravoServer; /// @@ -147,6 +160,9 @@ namespace Tgstation.Server.Host.Components.Watchdog semaphore.Dispose(); } + /// + /// Call on and and set them to + /// void DisposeAndNullControllers() { logger.LogTrace("DisposeAndNullControllers"); @@ -156,6 +172,9 @@ namespace Tgstation.Server.Host.Components.Watchdog bravoServer = null; } + /// + /// Implementation of . Does not lock + /// async Task RestartNoLock(bool graceful, CancellationToken cancellationToken) { var running = Running; @@ -181,6 +200,12 @@ namespace Tgstation.Server.Host.Components.Watchdog return null; } + /// + /// Implementation of . Does not lock + /// + /// If the termination will be delayed until a reboot is detected in the active server's DMAPI and this function will return immediately + /// If the termination will be announced using + /// The for the operation async Task TerminateNoLock(bool graceful, bool announce, CancellationToken cancellationToken) { if (!Running) @@ -199,82 +224,208 @@ namespace Tgstation.Server.Host.Components.Watchdog await toKill.SetRebootState(Components.Watchdog.RebootState.Shutdown, cancellationToken).ConfigureAwait(false); } + /// + /// Handles the actions to take when the monitor has to "wake up" + /// + /// The that caused the invocation + /// The current . Will be modified upon retrn + /// The for the operation + /// A representing the running operation async Task HandlerMonitorWakeup(MonitorActivationReason activationReason, MonitorState monitorState, CancellationToken cancellationToken) { logger.LogInformation("Monitor activation. Reason: {0}", activationReason); + + //returns true if the inactive server can't be used immediately + bool FullRestartDeadInactive() + { + if (monitorState.RebootingInactiveServer || monitorState.InactiveServerCritFail) + { + logger.LogInformation("Inactive server is {0}! Restarting monitor...", monitorState.InactiveServerCritFail ? "critically failed" : "still rebooting"); + monitorState.NextAction = MonitorAction.Restart; //will dispose server + return true; + } + return false; + }; + + //trys to set inactive server's port to the private port + async Task MakeInactiveActive() + { + logger.LogInformation("Setting inactive server to port {0}...", ActiveLaunchParameters.PrimaryPort.Value); + var result = await monitorState.InactiveServer.SetPort(ActiveLaunchParameters.PrimaryPort.Value, cancellationToken).ConfigureAwait(false); + + if (!result) + { + logger.LogWarning("Failed to activate inactive server! Restarting monitor..."); + monitorState.NextAction = MonitorAction.Restart; //will dispose server + return false; + } + + // should always be set for InactiveServer + monitorState.InactiveServer.ClosePortOnReboot = false; + monitorState.ActiveServer.ClosePortOnReboot = true; + + //inactive server should always be using active launch parameters + LastLaunchParameters = ActiveLaunchParameters; + + var tmp = monitorState.ActiveServer; + monitorState.ActiveServer = monitorState.InactiveServer; + monitorState.InactiveServer = tmp; + return true; + } + + //trys to load inactive server with latest dmb, falling back to current dmb on failure and returning false + async Task RestartInactiveServer() + { + logger.LogInformation("Rebooting inactive server..."); + var newDmb = dmbFactory.LockNextDmb(cancellationToken); + bool usedMostRecentDmb; + try + { + monitorState.InactiveServer = await sessionControllerFactory.LaunchNew(ActiveLaunchParameters, await newDmb.ConfigureAwait(false), null, false, !monitorState.ActiveServer.IsPrimary, false, cancellationToken).ConfigureAwait(false); + usedMostRecentDmb = true; + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception e) + { + logger.LogError("Exception occurred while recreating server! Attempting backup strategy of running DMB of running server! Exception: {0}", e.ToString()); + //ahh jeez, what do we do here? + //this is our fault, so it should never happen but + //idk maybe a database error while handling the newest dmb? + //either way try to start it using the active server's dmb as a backup + try + { + var dmbBackup = dmbFactory.FromCompileJob(monitorState.ActiveServer.Dmb.CompileJob); + monitorState.InactiveServer = await sessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbBackup, null, false, !monitorState.ActiveServer.IsPrimary, false, cancellationToken).ConfigureAwait(false); + usedMostRecentDmb = false; + await chat.SendWatchdogMessage("Staging newest DMB on inactive server failed: {0} Falling back to previous dmb...", cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception e2) + { + //fuuuuucckkk + logger.LogError("Backup strategy failed! Monitor will restart when active server reboots! This Exception: {0}", e2.ToString()); + monitorState.InactiveServerCritFail = true; + await chat.SendWatchdogMessage("Attempted reboot of inactive server failed. Watchdog will reset when active server fails or exits", cancellationToken).ConfigureAwait(false); + return true; //we didn't use the old dmb + } + } + + logger.LogInformation("Successfully relaunched inactive server!"); + monitorState.RebootingInactiveServer = true; + // should always be set for InactiveServer + monitorState.InactiveServer.ClosePortOnReboot = false; + return usedMostRecentDmb; + } + + //reason handling switch (activationReason) { case MonitorActivationReason.ActiveServerCrashed: - using (monitorState.ActiveServer) //it's dead, dispose it when we're done + if(monitorState.ActiveServer.RebootState == Components.Watchdog.RebootState.Shutdown) { - if (monitorState.RebootingInactiveServer || monitorState.InactiveServerCritFail) - { - logger.LogInformation("Inactive server is {0}! Restarting monitor...", monitorState.InactiveServerCritFail ? "critically failed" : "still rebooting"); - monitorState.NextAction = MonitorAction.Restart; - break; - } + await chat.SendWatchdogMessage("Active server crashed! Exiting due to graceful termination request...", cancellationToken).ConfigureAwait(false); + monitorState.NextAction = MonitorAction.Exit; + break; + } - var dasDmbTask = dmbFactory.LockNextDmb(cancellationToken); - var result = await monitorState.InactiveServer.SetPort(ActiveLaunchParameters.PrimaryPort.Value, cancellationToken).ConfigureAwait(false); + if (FullRestartDeadInactive()) + { + await chat.SendWatchdogMessage("Active server crashed! Inactive server unable to online!", cancellationToken).ConfigureAwait(false); + break; + } - if (!result) - { - logger.LogWarning("Failed to activate inactive server! Restarting monitor..."); - monitorState.NextAction = MonitorAction.Restart; - break; - } + await chat.SendWatchdogMessage("Active server crashed! Onlining inactive server...", cancellationToken).ConfigureAwait(false); + if (!await MakeInactiveActive().ConfigureAwait(false)) + break; - monitorState.InactiveServerHasStagedDmb = false; - LastLaunchParameters = ActiveLaunchParameters; - - monitorState.ActiveServer = monitorState.InactiveServer; + monitorState.NextAction = MonitorAction.Continue; + monitorState.ActiveServer.ClosePortOnReboot = false; + goto case MonitorActivationReason.InactiveServerCrashed; + case MonitorActivationReason.InactiveServerCrashed: + using (monitorState.InactiveServer) //it's dead, dispose it when we're done + { monitorState.NextAction = MonitorAction.Continue; - - try + var usedLatestDmb = await RestartInactiveServer().ConfigureAwait(false); + if (monitorState.NextAction == MonitorAction.Continue) { - monitorState.InactiveServer = await sessionControllerFactory.LaunchNew(ActiveLaunchParameters, await dasDmbTask.ConfigureAwait(false), null, false, !monitorState.ActiveServer.IsPrimary, false, cancellationToken).ConfigureAwait(false); + monitorState.ActiveServer.ClosePortOnReboot = false; + if (monitorState.InactiveServerHasStagedDmb && !usedLatestDmb) + monitorState.InactiveServerHasStagedDmb = false; //don't try to load it again though } - catch (OperationCanceledException) - { - throw; - } - catch (Exception e) - { - logger.LogError("Exception occurred while recreating crashed server! Attempting backup strategy of running DMB of running server! Exception: {0}", e.ToString()); - //ahh jeez, what do we do here? - //this is our fault, so it should never happen - //try to start it using the active server's dmb as a backup - try - { - var dmbBackup = dmbFactory.FromCompileJob(monitorState.ActiveServer.Dmb.CompileJob); - monitorState.InactiveServer = await sessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbBackup, null, false, !monitorState.ActiveServer.IsPrimary, false, cancellationToken).ConfigureAwait(false); - } - catch (OperationCanceledException) - { - throw; - } - catch (Exception e2) - { - //fuuuuucckkk - logger.LogError("Backup strategy failed! Monitor will restart when active server reboots! This Exception: {0}", e2.ToString()); - monitorState.InactiveServerCritFail = true; - break; - } - } - - logger.LogInformation("Successfully relaunched inactive server!"); - monitorState.RebootingInactiveServer = true; } break; - case MonitorActivationReason.InactiveServerCrashed: - throw new NotImplementedException(); case MonitorActivationReason.ActiveServerRebooted: - throw new NotImplementedException(); + if (FullRestartDeadInactive()) + break; + + //what matters here is the RebootState + bool restartOnceSwapped = false; + var rebootState = monitorState.ActiveServer.RebootState; + monitorState.ActiveServer.ResetRebootState(); //the DMAPI has already done this internally + + /* TODO: This should be handled when ActiveLaunchParameters is SET + + if(LastLaunchParameters != ActiveLaunchParameters && rebootState != Components.Watchdog.RebootState.Shutdown) + //they need a relaunch with active parameters + rebootState = Components.Watchdog.RebootState.Restart; + */ + + switch (rebootState) + { + case Components.Watchdog.RebootState.Normal: + break; + case Components.Watchdog.RebootState.Restart: + restartOnceSwapped = true; + break; + case Components.Watchdog.RebootState.Shutdown: + await chat.SendWatchdogMessage("Active server rebooted! Exiting due to graceful termination request...", cancellationToken).ConfigureAwait(false); + DisposeAndNullControllers(); + Running = false; + monitorState.NextAction = MonitorAction.Exit; + return; + } + + if (monitorState.InactiveServerHasStagedDmb) + { + if (monitorState.InactiveServer.Dmb.CompileJob.Id == monitorState.ActiveServer.Dmb.CompileJob.Id) + //both servers up to date + monitorState.InactiveServerHasStagedDmb = false; + else + //need to load a new dmb in ActiveServer + restartOnceSwapped = true; + } + + if (!await MakeInactiveActive().ConfigureAwait(false)) + break; + + if(!restartOnceSwapped) + //try to reopen inactive server on the private port so it's not pinging all the time + //failing that, just reboot it + restartOnceSwapped = !await monitorState.InactiveServer.SetPort(ActiveLaunchParameters.SecondaryPort.Value, cancellationToken).ConfigureAwait(false); + + if (restartOnceSwapped) //for one reason or another, + { + monitorState.InactiveServer.Dispose(); + monitorState.InactiveServerHasStagedDmb = await RestartInactiveServer().ConfigureAwait(false); + } + break; case MonitorActivationReason.InactiveServerRebooted: - throw new NotImplementedException(); + //should never happen but okay + logger.LogWarning("Inactive server rebooted, this is a bug in DM code!"); + monitorState.RebootingInactiveServer = true; + monitorState.ActiveServer.ClosePortOnReboot = false; + monitorState.NextAction = MonitorAction.Continue; + break; case MonitorActivationReason.InactiveServerStartupComplete: //eziest case of my life monitorState.RebootingInactiveServer = false; + monitorState.ActiveServer.ClosePortOnReboot = true; monitorState.NextAction = MonitorAction.Continue; break; case MonitorActivationReason.NewDmbAvailable: @@ -374,8 +525,7 @@ namespace Tgstation.Server.Host.Components.Watchdog { logger.LogDebug("Next state action is to restart"); DisposeAndNullControllers(); - Running = false; - chatTask = chat.SendWatchdogMessage("Restarting due to complications...", cancellationToken); + chatTask = chat.SendWatchdogMessage("Restarting entirely due to complications...", cancellationToken); } for (var retryAttempts = 1; state.NextAction == MonitorAction.Restart; ++retryAttempts) From 4af3bd811ee87b60302a70e44e6cf84e78e511cf Mon Sep 17 00:00:00 2001 From: Cyberboss Date: Wed, 18 Jul 2018 10:58:02 -0400 Subject: [PATCH 4/7] Implement ISessionController.ResetRebootState() --- .../Components/Watchdog/ISessionController.cs | 7 ++++++- .../Components/Watchdog/SessionController.cs | 7 +++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Components/Watchdog/ISessionController.cs b/src/Tgstation.Server.Host/Components/Watchdog/ISessionController.cs index faa997cbf1..6624d03145 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/ISessionController.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/ISessionController.cs @@ -79,5 +79,10 @@ namespace Tgstation.Server.Host.Components.Watchdog /// The for the operation /// A resulting in if the operation succeeded, otherwise Task SetRebootState(RebootState newRebootState, CancellationToken cancellationToken); - } + + /// + /// Changes to without telling the DMAPI + /// + void ResetRebootState(); + } } diff --git a/src/Tgstation.Server.Host/Components/Watchdog/SessionController.cs b/src/Tgstation.Server.Host/Components/Watchdog/SessionController.cs index 4937f7e170..757734d4f1 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/SessionController.cs @@ -333,5 +333,12 @@ namespace Tgstation.Server.Host.Components.Watchdog return await SendCommand(String.Format(CultureInfo.InvariantCulture, "{0}&{1}={2}", InteropConstants.DMTopicChangeReboot, InteropConstants.DMParameterNewRebootMode, (int)newRebootState), cancellationToken).ConfigureAwait(false) == InteropConstants.DMResponseSuccess; } + + /// + public void ResetRebootState() + { + CheckDisposed(); + reattachInformation.RebootState = RebootState.Normal; + } } } From e467ac457f4c82285e6d3b5dbfb64450d35c955f Mon Sep 17 00:00:00 2001 From: Cyberboss Date: Wed, 18 Jul 2018 11:18:03 -0400 Subject: [PATCH 5/7] Good shit --- .../Watchdog/MonitorActivationReason.cs | 3 +- .../Components/Watchdog/Watchdog.cs | 83 ++++++++++++------- 2 files changed, 56 insertions(+), 30 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Watchdog/MonitorActivationReason.cs b/src/Tgstation.Server.Host/Components/Watchdog/MonitorActivationReason.cs index 1c36c6f0d6..45349a4be0 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/MonitorActivationReason.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/MonitorActivationReason.cs @@ -7,6 +7,7 @@ ActiveServerRebooted, InactiveServerRebooted, NewDmbAvailable, - InactiveServerStartupComplete + InactiveServerStartupComplete, + ActiveLaunchParametersUpdated } } diff --git a/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs index ac949b65b2..9d060f4063 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs @@ -100,6 +100,11 @@ namespace Tgstation.Server.Host.Components.Watchdog /// Task monitorTask; + /// + /// that completes when are changed and we are + /// + TaskCompletionSource activeParametersUpdated; + /// /// Server designation alpha /// @@ -151,6 +156,7 @@ namespace Tgstation.Server.Host.Components.Watchdog ActiveLaunchParameters = initialLaunchParameters; releaseServers = false; semaphore = new SemaphoreSlim(1); + activeParametersUpdated = new TaskCompletionSource(); } /// @@ -198,7 +204,7 @@ namespace Tgstation.Server.Host.Components.Watchdog //todo, log the result await toReboot.SetRebootState(Components.Watchdog.RebootState.Restart, cancellationToken).ConfigureAwait(false); return null; - } + } /// /// Implementation of . Does not lock @@ -270,10 +276,11 @@ namespace Tgstation.Server.Host.Components.Watchdog var tmp = monitorState.ActiveServer; monitorState.ActiveServer = monitorState.InactiveServer; monitorState.InactiveServer = tmp; + AlphaIsActive = !AlphaIsActive; return true; } - //trys to load inactive server with latest dmb, falling back to current dmb on failure and returning false + // Tries to load inactive server with latest dmb, falling back to current dmb on failure. Requires a lock on async Task RestartInactiveServer() { logger.LogInformation("Rebooting inactive server..."); @@ -312,7 +319,7 @@ namespace Tgstation.Server.Host.Components.Watchdog logger.LogError("Backup strategy failed! Monitor will restart when active server reboots! This Exception: {0}", e2.ToString()); monitorState.InactiveServerCritFail = true; await chat.SendWatchdogMessage("Attempted reboot of inactive server failed. Watchdog will reset when active server fails or exits", cancellationToken).ConfigureAwait(false); - return true; //we didn't use the old dmb + return true; //we didn't use the old dmb } } @@ -329,25 +336,31 @@ namespace Tgstation.Server.Host.Components.Watchdog case MonitorActivationReason.ActiveServerCrashed: if(monitorState.ActiveServer.RebootState == Components.Watchdog.RebootState.Shutdown) { - await chat.SendWatchdogMessage("Active server crashed! Exiting due to graceful termination request...", cancellationToken).ConfigureAwait(false); + await chat.SendWatchdogMessage("Active server crashed or exited! Exiting due to graceful termination request...", cancellationToken).ConfigureAwait(false); monitorState.NextAction = MonitorAction.Exit; break; } if (FullRestartDeadInactive()) { - await chat.SendWatchdogMessage("Active server crashed! Inactive server unable to online!", cancellationToken).ConfigureAwait(false); + await chat.SendWatchdogMessage("Active server crashed or exited! Inactive server unable to online!", cancellationToken).ConfigureAwait(false); break; } - await chat.SendWatchdogMessage("Active server crashed! Onlining inactive server...", cancellationToken).ConfigureAwait(false); + await chat.SendWatchdogMessage("Active server crashed or exited! Onlining inactive server...", cancellationToken).ConfigureAwait(false); if (!await MakeInactiveActive().ConfigureAwait(false)) break; monitorState.NextAction = MonitorAction.Continue; monitorState.ActiveServer.ClosePortOnReboot = false; - goto case MonitorActivationReason.InactiveServerCrashed; + goto case MonitorActivationReason.ActiveLaunchParametersUpdated; case MonitorActivationReason.InactiveServerCrashed: + await chat.SendWatchdogMessage("Inactive server crashed or exited! Rebooting...", cancellationToken).ConfigureAwait(false); + goto case MonitorActivationReason.ActiveLaunchParametersUpdated; + case MonitorActivationReason.ActiveLaunchParametersUpdated: + //replace the notification tcs here so that the next loop will read a fresh one + activeParametersUpdated = new TaskCompletionSource(); + using (monitorState.InactiveServer) //it's dead, dispose it when we're done { monitorState.NextAction = MonitorAction.Continue; @@ -442,7 +455,7 @@ namespace Tgstation.Server.Host.Components.Watchdog { logger.LogDebug("Entered MonitorLifetimes"); var iteration = 1; - for(var state = new MonitorState(); state.NextAction != MonitorAction.Exit; ++iteration) + for(var monitorState = new MonitorState(); monitorState.NextAction != MonitorAction.Exit; ++iteration) { logger.LogDebug("New iteration of monitor loop"); try @@ -452,26 +465,27 @@ namespace Tgstation.Server.Host.Components.Watchdog else logger.LogDebug("Bravo is the active server"); - if(state.InactiveServerHasStagedDmb) + if(monitorState.InactiveServerHasStagedDmb) logger.LogDebug("Inactive server has staged .dmb"); - if (state.RebootingInactiveServer) + if (monitorState.RebootingInactiveServer) logger.LogDebug("Inactive server is rebooting"); - state.ActiveServer = AlphaIsActive ? alphaServer : bravoServer; - state.InactiveServer = AlphaIsActive ? bravoServer : alphaServer; + monitorState.ActiveServer = AlphaIsActive ? alphaServer : bravoServer; + monitorState.InactiveServer = AlphaIsActive ? bravoServer : alphaServer; - var activeServerLifetime = state.ActiveServer.Lifetime; - var inactiveServerLifetime = state.InactiveServer.Lifetime; - var activeServerReboot = state.ActiveServer.OnReboot; - var inactiveServerReboot = state.InactiveServer.OnReboot; - var inactiveServerStartup = state.InactiveServer.LaunchResult; + var activeServerLifetime = monitorState.ActiveServer.Lifetime; + var inactiveServerLifetime = monitorState.InactiveServer.Lifetime; + var activeServerReboot = monitorState.ActiveServer.OnReboot; + var inactiveServerReboot = monitorState.InactiveServer.OnReboot; + var inactiveServerStartup = monitorState.InactiveServer.LaunchResult; + var activeLaunchParametersChanged = activeParametersUpdated.Task; var newDmbAvailable = dmbFactory.OnNewerDmb; var cancelTcs = new TaskCompletionSource(); using (cancellationToken.Register(() => cancelTcs.SetCanceled())) { - var toWaitOn = Task.WhenAny(activeServerLifetime, inactiveServerLifetime, activeServerReboot, inactiveServerReboot, newDmbAvailable, cancelTcs.Task); - if (state.RebootingInactiveServer) + var toWaitOn = Task.WhenAny(activeServerLifetime, inactiveServerLifetime, activeServerReboot, inactiveServerReboot, newDmbAvailable, cancelTcs.Task, activeLaunchParametersChanged); + if (monitorState.RebootingInactiveServer) toWaitOn = Task.WhenAny(toWaitOn, inactiveServerStartup); await toWaitOn.ConfigureAwait(false); } @@ -481,7 +495,7 @@ namespace Tgstation.Server.Host.Components.Watchdog { MonitorActivationReason activationReason = default; //multiple things may have happened, handle them one at a time - for (var moreActivationsToProcess = true; moreActivationsToProcess && state.NextAction == MonitorAction.Continue; ) + for (var moreActivationsToProcess = true; moreActivationsToProcess && monitorState.NextAction == MonitorAction.Continue; ) { if (activeServerLifetime?.IsCompleted == true) { @@ -513,31 +527,41 @@ namespace Tgstation.Server.Host.Components.Watchdog activationReason = MonitorActivationReason.NewDmbAvailable; newDmbAvailable = null; } + else if(activeLaunchParametersChanged?.IsCompleted == true) + { + activationReason = MonitorActivationReason.ActiveLaunchParametersUpdated; + activeLaunchParametersChanged = null; + } else moreActivationsToProcess = false; } - await HandlerMonitorWakeup(activationReason, state, cancellationToken).ConfigureAwait(false); + await HandlerMonitorWakeup(activationReason, monitorState, cancellationToken).ConfigureAwait(false); + //writeback alphaServer and bravoServer + alphaServer = AlphaIsActive ? monitorState.ActiveServer : monitorState.InactiveServer; + bravoServer = AlphaIsActive ? monitorState.ActiveServer : monitorState.InactiveServer; } //full reboot required - if (state.NextAction == MonitorAction.Restart) + if (monitorState.NextAction == MonitorAction.Restart) { logger.LogDebug("Next state action is to restart"); DisposeAndNullControllers(); chatTask = chat.SendWatchdogMessage("Restarting entirely due to complications...", cancellationToken); } - for (var retryAttempts = 1; state.NextAction == MonitorAction.Restart; ++retryAttempts) + for (var retryAttempts = 1; monitorState.NextAction == MonitorAction.Restart; ++retryAttempts) { WatchdogLaunchResult result; using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken).ConfigureAwait(false)) + { result = await LaunchNoLock(false, false, false, cancellationToken).ConfigureAwait(false); + if (Running) + monitorState = new MonitorState(); //clean the slate + } - await chatTask.ConfigureAwait(false); - if (Running) - state = new MonitorState(); //clean the slate - else + await chatTask.ConfigureAwait(false); + if(!Running) { logger.LogWarning("Failed to automatically restart the watchdog! Alpha: {0}; Bravo: {1}", result.Alpha.ToString(), result.Bravo.ToString()); var retryDelay = Math.Min(Math.Pow(2, retryAttempts), 3600); //max of one hour @@ -553,7 +577,7 @@ namespace Tgstation.Server.Host.Components.Watchdog } catch (Exception e) { - logger.LogError("Monitor crashed! Iteration: {0}, State: {1}", iteration, JsonConvert.SerializeObject(state)); + logger.LogError("Monitor crashed! Iteration: {0}, State: {1}", iteration, JsonConvert.SerializeObject(monitorState)); await chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Monitor crashed, this should NEVER happen! Please report this, full details in logs! Restarting monitor... Error: {0}", e.Message), cancellationToken).ConfigureAwait(false); } } @@ -579,7 +603,8 @@ namespace Tgstation.Server.Host.Components.Watchdog { ActiveLaunchParameters = launchParameters; if (Running) - await RestartNoLock(true, cancellationToken).ConfigureAwait(false); + //queue an update + activeParametersUpdated.TrySetResult(null); } } From 5b60abeb513b0e0c398eb3a4be1f016ce61b8466 Mon Sep 17 00:00:00 2001 From: Cyberboss Date: Thu, 19 Jul 2018 15:51:47 -0400 Subject: [PATCH 6/7] Dispose useless inactive server sooner --- .../Components/Watchdog/Watchdog.cs | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs index 9d060f4063..9dc9559905 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs @@ -360,17 +360,16 @@ namespace Tgstation.Server.Host.Components.Watchdog case MonitorActivationReason.ActiveLaunchParametersUpdated: //replace the notification tcs here so that the next loop will read a fresh one activeParametersUpdated = new TaskCompletionSource(); + monitorState.InactiveServer.Dispose(); //kill or recycle it + monitorState.NextAction = MonitorAction.Continue; - using (monitorState.InactiveServer) //it's dead, dispose it when we're done + var usedLatestDmb = await RestartInactiveServer().ConfigureAwait(false); + + if (monitorState.NextAction == MonitorAction.Continue) { - monitorState.NextAction = MonitorAction.Continue; - var usedLatestDmb = await RestartInactiveServer().ConfigureAwait(false); - if (monitorState.NextAction == MonitorAction.Continue) - { - monitorState.ActiveServer.ClosePortOnReboot = false; - if (monitorState.InactiveServerHasStagedDmb && !usedLatestDmb) - monitorState.InactiveServerHasStagedDmb = false; //don't try to load it again though - } + monitorState.ActiveServer.ClosePortOnReboot = false; + if (monitorState.InactiveServerHasStagedDmb && !usedLatestDmb) + monitorState.InactiveServerHasStagedDmb = false; //don't try to load it again though } break; case MonitorActivationReason.ActiveServerRebooted: From 63425da29811aafec60500c8d5df2404fbdc63ab Mon Sep 17 00:00:00 2001 From: Cyberboss Date: Thu, 19 Jul 2018 16:17:28 -0400 Subject: [PATCH 7/7] We'll never know if it works till we test it --- .../Components/Watchdog/MonitorAction.cs | 1 + .../Watchdog/MonitorActivationReason.cs | 2 +- .../Components/Watchdog/Watchdog.cs | 79 +++++++++---------- 3 files changed, 41 insertions(+), 41 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Watchdog/MonitorAction.cs b/src/Tgstation.Server.Host/Components/Watchdog/MonitorAction.cs index b5178c7b55..1e0199c88a 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/MonitorAction.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/MonitorAction.cs @@ -7,6 +7,7 @@ { Continue, Restart, + Break, Exit } } \ No newline at end of file diff --git a/src/Tgstation.Server.Host/Components/Watchdog/MonitorActivationReason.cs b/src/Tgstation.Server.Host/Components/Watchdog/MonitorActivationReason.cs index 45349a4be0..bfe0183058 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/MonitorActivationReason.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/MonitorActivationReason.cs @@ -6,8 +6,8 @@ InactiveServerCrashed, ActiveServerRebooted, InactiveServerRebooted, - NewDmbAvailable, InactiveServerStartupComplete, + NewDmbAvailable, ActiveLaunchParametersUpdated } } diff --git a/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs index 9dc9559905..217d6dcddf 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs @@ -330,6 +330,23 @@ namespace Tgstation.Server.Host.Components.Watchdog return usedMostRecentDmb; } + async Task UpdateAndRestartInactiveServer(bool breakAfter) + { + //replace the notification tcs here so that the next loop will read a fresh one + activeParametersUpdated = new TaskCompletionSource(); + monitorState.InactiveServer.Dispose(); //kill or recycle it + monitorState.NextAction = breakAfter ? MonitorAction.Break : MonitorAction.Continue; + + var usedLatestDmb = await RestartInactiveServer().ConfigureAwait(false); + + if (monitorState.NextAction == (breakAfter ? MonitorAction.Break : MonitorAction.Continue)) + { + monitorState.ActiveServer.ClosePortOnReboot = false; + if (monitorState.InactiveServerHasStagedDmb && !usedLatestDmb) + monitorState.InactiveServerHasStagedDmb = false; //don't try to load it again though + } + }; + //reason handling switch (activationReason) { @@ -350,27 +367,13 @@ namespace Tgstation.Server.Host.Components.Watchdog await chat.SendWatchdogMessage("Active server crashed or exited! Onlining inactive server...", cancellationToken).ConfigureAwait(false); if (!await MakeInactiveActive().ConfigureAwait(false)) break; - - monitorState.NextAction = MonitorAction.Continue; + monitorState.ActiveServer.ClosePortOnReboot = false; - goto case MonitorActivationReason.ActiveLaunchParametersUpdated; + await UpdateAndRestartInactiveServer(true).ConfigureAwait(false); + break; case MonitorActivationReason.InactiveServerCrashed: await chat.SendWatchdogMessage("Inactive server crashed or exited! Rebooting...", cancellationToken).ConfigureAwait(false); - goto case MonitorActivationReason.ActiveLaunchParametersUpdated; - case MonitorActivationReason.ActiveLaunchParametersUpdated: - //replace the notification tcs here so that the next loop will read a fresh one - activeParametersUpdated = new TaskCompletionSource(); - monitorState.InactiveServer.Dispose(); //kill or recycle it - monitorState.NextAction = MonitorAction.Continue; - - var usedLatestDmb = await RestartInactiveServer().ConfigureAwait(false); - - if (monitorState.NextAction == MonitorAction.Continue) - { - monitorState.ActiveServer.ClosePortOnReboot = false; - if (monitorState.InactiveServerHasStagedDmb && !usedLatestDmb) - monitorState.InactiveServerHasStagedDmb = false; //don't try to load it again though - } + await UpdateAndRestartInactiveServer(false).ConfigureAwait(false); break; case MonitorActivationReason.ActiveServerRebooted: if (FullRestartDeadInactive()) @@ -381,13 +384,6 @@ namespace Tgstation.Server.Host.Components.Watchdog var rebootState = monitorState.ActiveServer.RebootState; monitorState.ActiveServer.ResetRebootState(); //the DMAPI has already done this internally - /* TODO: This should be handled when ActiveLaunchParameters is SET - - if(LastLaunchParameters != ActiveLaunchParameters && rebootState != Components.Watchdog.RebootState.Shutdown) - //they need a relaunch with active parameters - rebootState = Components.Watchdog.RebootState.Restart; - */ - switch (rebootState) { case Components.Watchdog.RebootState.Normal: @@ -403,15 +399,13 @@ namespace Tgstation.Server.Host.Components.Watchdog return; } - if (monitorState.InactiveServerHasStagedDmb) - { - if (monitorState.InactiveServer.Dmb.CompileJob.Id == monitorState.ActiveServer.Dmb.CompileJob.Id) - //both servers up to date - monitorState.InactiveServerHasStagedDmb = false; - else - //need to load a new dmb in ActiveServer - restartOnceSwapped = true; - } + var sameCompileJob = monitorState.InactiveServer.Dmb.CompileJob.Id == monitorState.ActiveServer.Dmb.CompileJob.Id; + if (sameCompileJob && monitorState.InactiveServerHasStagedDmb) + //both servers up to date + monitorState.InactiveServerHasStagedDmb = false; + if (!sameCompileJob || ActiveLaunchParameters != LastLaunchParameters) + //need a new launch in ActiveServer + restartOnceSwapped = true; if (!await MakeInactiveActive().ConfigureAwait(false)) break; @@ -421,16 +415,16 @@ namespace Tgstation.Server.Host.Components.Watchdog //failing that, just reboot it restartOnceSwapped = !await monitorState.InactiveServer.SetPort(ActiveLaunchParameters.SecondaryPort.Value, cancellationToken).ConfigureAwait(false); - if (restartOnceSwapped) //for one reason or another, - { - monitorState.InactiveServer.Dispose(); - monitorState.InactiveServerHasStagedDmb = await RestartInactiveServer().ConfigureAwait(false); - } + if (restartOnceSwapped) //for one reason or another, + await UpdateAndRestartInactiveServer(true).ConfigureAwait(false); //break because worse case, active server is still booting + else + monitorState.NextAction = MonitorAction.Break; break; case MonitorActivationReason.InactiveServerRebooted: //should never happen but okay logger.LogWarning("Inactive server rebooted, this is a bug in DM code!"); monitorState.RebootingInactiveServer = true; + monitorState.InactiveServer.ResetRebootState(); //the DMAPI has already done this internally monitorState.ActiveServer.ClosePortOnReboot = false; monitorState.NextAction = MonitorAction.Continue; break; @@ -441,7 +435,12 @@ namespace Tgstation.Server.Host.Components.Watchdog monitorState.NextAction = MonitorAction.Continue; break; case MonitorActivationReason.NewDmbAvailable: - throw new NotImplementedException(); + monitorState.InactiveServerHasStagedDmb = true; + await UpdateAndRestartInactiveServer(true).ConfigureAwait(false); //next case does same thing + break; + case MonitorActivationReason.ActiveLaunchParametersUpdated: + await UpdateAndRestartInactiveServer(false).ConfigureAwait(false); + break; } }