From 2ba4002dd78251a08e68263bdf35b105beeff963 Mon Sep 17 00:00:00 2001 From: Cyberboss Date: Tue, 17 Jul 2018 17:15:23 -0400 Subject: [PATCH] InactiveServerStartupComplete --- .../Components/Watchdog/Watchdog.cs | 91 ++++++++++--------- 1 file changed, 48 insertions(+), 43 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs index 20523ca6d2..dede7bdd79 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs @@ -205,64 +205,66 @@ namespace Tgstation.Server.Host.Components.Watchdog switch (activationReason) { case MonitorActivationReason.ActiveServerCrashed: - if (monitorState.RebootingInactiveServer || monitorState.InactiveServerCritFail) + using (monitorState.ActiveServer) //it's dead, dispose it when we're done { - logger.LogInformation("Inactive server is {0}! Restarting monitor...", monitorState.InactiveServerCritFail ? "critically failed" : "still rebooting"); - monitorState.NextAction = MonitorAction.Restart; - break; - } + if (monitorState.RebootingInactiveServer || monitorState.InactiveServerCritFail) + { + logger.LogInformation("Inactive server is {0}! Restarting monitor...", monitorState.InactiveServerCritFail ? "critically failed" : "still rebooting"); + monitorState.NextAction = MonitorAction.Restart; + break; + } - var dasDmbTask = dmbFactory.LockNextDmb(cancellationToken); - var result = await monitorState.InactiveServer.SetPort(ActiveLaunchParameters.PrimaryPort.Value, cancellationToken).ConfigureAwait(false); + var dasDmbTask = dmbFactory.LockNextDmb(cancellationToken); + var result = await monitorState.InactiveServer.SetPort(ActiveLaunchParameters.PrimaryPort.Value, cancellationToken).ConfigureAwait(false); - if (!result) - { - logger.LogWarning("Failed to activate inactive server! Restarting monitor..."); - monitorState.NextAction = MonitorAction.Restart; - break; - } + if (!result) + { + logger.LogWarning("Failed to activate inactive server! Restarting monitor..."); + monitorState.NextAction = MonitorAction.Restart; + break; + } - monitorState.InactiveServerHasStagedDmb = false; - LastLaunchParameters = ActiveLaunchParameters; + monitorState.InactiveServerHasStagedDmb = false; + LastLaunchParameters = ActiveLaunchParameters; - var deadServer = monitorState.ActiveServer; - monitorState.ActiveServer = monitorState.InactiveServer; - monitorState.NextAction = MonitorAction.Continue; + monitorState.ActiveServer = monitorState.InactiveServer; + monitorState.NextAction = MonitorAction.Continue; - try - { - monitorState.InactiveServer = await sessionControllerFactory.LaunchNew(ActiveLaunchParameters, await dasDmbTask.ConfigureAwait(false), null, false, !monitorState.ActiveServer.IsPrimary, false, cancellationToken).ConfigureAwait(false); - } - catch (OperationCanceledException) - { - throw; - } - catch (Exception e) - { - logger.LogError("Exception occurred while recreating crashed server! Attempting backup strategy of running DMB of running server! Exception: {0}", e.ToString()); - //ahh jeez, what do we do here? - //this is our fault, so it should never happen - //try to start it using the active server's dmb as a backup try { - var dmbBackup = dmbFactory.FromCompileJob(monitorState.ActiveServer.Dmb.CompileJob); - monitorState.InactiveServer = await sessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbBackup, null, false, !monitorState.ActiveServer.IsPrimary, false, cancellationToken).ConfigureAwait(false); + monitorState.InactiveServer = await sessionControllerFactory.LaunchNew(ActiveLaunchParameters, await dasDmbTask.ConfigureAwait(false), null, false, !monitorState.ActiveServer.IsPrimary, false, cancellationToken).ConfigureAwait(false); } catch (OperationCanceledException) { throw; } - catch (Exception e2) + catch (Exception e) { - //fuuuuucckkk - logger.LogError("Backup strategy failed! Monitor will restart when active server reboots! This Exception: {0}", e2.ToString()); - monitorState.InactiveServerCritFail = true; - break; + logger.LogError("Exception occurred while recreating crashed server! Attempting backup strategy of running DMB of running server! Exception: {0}", e.ToString()); + //ahh jeez, what do we do here? + //this is our fault, so it should never happen + //try to start it using the active server's dmb as a backup + try + { + var dmbBackup = dmbFactory.FromCompileJob(monitorState.ActiveServer.Dmb.CompileJob); + monitorState.InactiveServer = await sessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbBackup, null, false, !monitorState.ActiveServer.IsPrimary, false, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception e2) + { + //fuuuuucckkk + logger.LogError("Backup strategy failed! Monitor will restart when active server reboots! This Exception: {0}", e2.ToString()); + monitorState.InactiveServerCritFail = true; + break; + } } - } - logger.LogInformation("Successfully relaunched inactive server!"); - monitorState.RebootingInactiveServer = true; + logger.LogInformation("Successfully relaunched inactive server!"); + monitorState.RebootingInactiveServer = true; + } break; case MonitorActivationReason.InactiveServerCrashed: throw new NotImplementedException(); @@ -271,7 +273,10 @@ namespace Tgstation.Server.Host.Components.Watchdog case MonitorActivationReason.InactiveServerRebooted: throw new NotImplementedException(); case MonitorActivationReason.InactiveServerStartupComplete: - throw new NotImplementedException(); + //eziest case of my life + monitorState.RebootingInactiveServer = false; + monitorState.NextAction = MonitorAction.Continue; + break; case MonitorActivationReason.NewDmbAvailable: throw new NotImplementedException(); }