From 8d84ca13a608289730378ea9a771565bdd0ff243 Mon Sep 17 00:00:00 2001 From: Cyberboss Date: Thu, 5 Jul 2018 16:52:47 -0400 Subject: [PATCH] Can't wait forever --- .../Watchdog/MonitorActivationReason.cs | 12 ++ .../Components/Watchdog/MonitorState.cs | 11 ++ .../Components/Watchdog/Watchdog.cs | 187 ++---------------- 3 files changed, 35 insertions(+), 175 deletions(-) create mode 100644 src/Tgstation.Server.Host/Components/Watchdog/MonitorActivationReason.cs create mode 100644 src/Tgstation.Server.Host/Components/Watchdog/MonitorState.cs diff --git a/src/Tgstation.Server.Host/Components/Watchdog/MonitorActivationReason.cs b/src/Tgstation.Server.Host/Components/Watchdog/MonitorActivationReason.cs new file mode 100644 index 0000000000..1c36c6f0d6 --- /dev/null +++ b/src/Tgstation.Server.Host/Components/Watchdog/MonitorActivationReason.cs @@ -0,0 +1,12 @@ +namespace Tgstation.Server.Host.Components.Watchdog +{ + enum MonitorActivationReason + { + ActiveServerCrashed, + InactiveServerCrashed, + ActiveServerRebooted, + InactiveServerRebooted, + NewDmbAvailable, + InactiveServerStartupComplete + } +} diff --git a/src/Tgstation.Server.Host/Components/Watchdog/MonitorState.cs b/src/Tgstation.Server.Host/Components/Watchdog/MonitorState.cs new file mode 100644 index 0000000000..dfb418f9f4 --- /dev/null +++ b/src/Tgstation.Server.Host/Components/Watchdog/MonitorState.cs @@ -0,0 +1,11 @@ +namespace Tgstation.Server.Host.Components.Watchdog +{ + sealed class MonitorState + { + public bool RebootingInactiveServer { get; set; } + public bool InactiveServerHasStagedDmb { get; set; } + + public ISessionController ActiveServer { get; set; } + public ISessionController InactiveServer { get; set; } + } +} diff --git a/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs index 1b2de41de1..5bf055c988 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs @@ -138,9 +138,14 @@ namespace Tgstation.Server.Host.Components.Watchdog return null; } + async Task HandlerMonitorWakeup(MonitorActivationReason activationReason, MonitorState monitorState) + { + + } + async Task MonitorLifetimes(CancellationToken cancellationToken) { - var watchReboot = false; + var state = new MonitorState(); while(true) { try @@ -164,180 +169,12 @@ namespace Tgstation.Server.Host.Components.Watchdog using (await SemaphoreContext.Lock(semaphore, default).ConfigureAwait(false)) { - - var activeServerExited = (alphaServerTask.IsCompleted && AlphaIsActive) || (bravoServerTask.IsCompleted && !AlphaIsActive); - var inactiveServerExited = (alphaServerTask.IsCompleted && !AlphaIsActive) || (bravoServerTask.IsCompleted && AlphaIsActive); - var activeServerReboot = (alphaServerReboot.IsCompleted && AlphaIsActive) || (bravoServerReboot.IsCompleted && !AlphaIsActive); - - if (watchReboot) - { - //waiting on the inactive server to come back - var doContinue = true; - var activeServerGracefulAction = activeServer.RebootState != Components.Watchdog.RebootState.Normal; - var aServerRebooted = alphaServerReboot.IsCompleted || bravoServerReboot.IsCompleted; - if (inactiveServer.LaunchResult.IsCompleted) - { - //glad to have you back - //(cleanup the task here) - LastLaunchResult = await inactiveServer.LaunchResult.ConfigureAwait(false); - if (inactiveServer.Lifetime.IsCompleted) - { - //aww heck you died again, try to bring you back i guess... - //transfer control to the "inactive server died" branch - inactiveServerExited = true; - doContinue = false; - } - else - //reenable port swapping on active server - activeServer.ClosePortOnReboot = true; - } - else if (activeServerGracefulAction && activeServerExited) - { - //ok not so bad, we either kill everything or restart everything - if (activeServer.RebootState == Components.Watchdog.RebootState.Shutdown) - //ez - //let control fall through to the "crash but wanted to shutdown anyway" section - doContinue = false; - else - { - //total restart, forget about trying to make the inactive server the active one since it's not even responding yet - var chatTask = chat.SendWatchdogMessage("Entered state where restart is requested but neither server is ready. Restarting watchdog...", cancellationToken); - await LaunchNoLock(false, cancellationToken).ConfigureAwait(false); - await chatTask.ConfigureAwait(false); - if (!Running) - return; - } - } - //if the above is NOT the case that means something probably happened to the ACTIVE server - //hopefully it was just a reboot, otherwise we're gonna need to do some weird recovery shennanigans - else if (!aServerRebooted) - { - //uh oh... - if (!activeServerExited) - //inactive server died again - //let control fall through to the inactive crash handler - doContinue = false; - else - { - //OH NO - //so now we have a dead active server and an inactive server who is still booting - //best we can do is hope it comes online so we can fix things - var chatTask = chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Active server crashed or exited (Code: {0})! Inactive server is still booting! Waiting...", activeServer.Lifetime.Result), cancellationToken); - - LastLaunchResult = await inactiveServer.LaunchResult.ConfigureAwait(false); - - if(inactiveServer.Lifetime.IsCompleted) - //he's dead too jim. - } - } - else - { - //thank god - //doesn't matter which server rebooted, our action is still the same: - //if it was active, tell it to not switch port - //if it was inactive, ensure it's port is the right one - activeServer.ClosePortOnReboot = false; - await inactiveServer.SetPort(ActiveLaunchParameters.SecondaryPort.Value, cancellationToken).ConfigureAwait(false); - } - - if (doContinue) - continue; - } - - if (activeServerExited) - { - //active server just went down - switch (activeServer.RebootState) - { - case Components.Watchdog.RebootState.Restart: - case Components.Watchdog.RebootState.Normal: - //yeah he died unexpectedly - //bring in other - var chatTask = chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Active server crashed or exited (Code: {0})! Activating backup, relaunching in background...", activeServer.Lifetime.Result), cancellationToken); - - activeServer.Dispose(); - var backupOnline = await inactiveServer.SetPort(ActiveLaunchParameters.PrimaryPort.Value, cancellationToken).ConfigureAwait(false); - if (!backupOnline) - { - //can't bring in backup, reboot everything - await chatTask.ConfigureAwait(false); - await chat.SendWatchdogMessage("Backup server not responding to port switch request! Restarting watchdog...", cancellationToken).ConfigureAwait(false); - inactiveServer.Dispose(); - Running = false; - await LaunchNoLock(false, cancellationToken).ConfigureAwait(false); - if (!Running) - return; - continue; - } - - //make a new server with blackjack and hookers - var nextDmb = await dmbFactory.LockNextDmb(cancellationToken).ConfigureAwait(false); - var newServer = await sessionControllerFactory.LaunchNew(ActiveLaunchParameters, nextDmb, false, AlphaIsActive, false, cancellationToken).ConfigureAwait(false); - - if (AlphaIsActive) - alphaServer = newServer; - else - bravoServer = newServer; - watchReboot = true; - AlphaIsActive = !AlphaIsActive; - inactiveServer.ClosePortOnReboot = false; //actually the active server at this point - break; - case Components.Watchdog.RebootState.Shutdown: - await chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Active server crashed or exited (Code: {0})! Not recovering to to pending shutdown.", activeServer.Lifetime.Result), cancellationToken).ConfigureAwait(false); - DisposeAndNullControllers(); - return; - } - } - else if (inactiveServerExited) - { - //inactive server just went down - //bring it back - var chatTask = chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Inactive server crashed or exited (Code: {0})! Relaunching in background...", inactiveServer.Lifetime.Result), cancellationToken); - var nextDmb = await dmbFactory.LockNextDmb(cancellationToken).ConfigureAwait(false); - var newServer = await sessionControllerFactory.LaunchNew(ActiveLaunchParameters, nextDmb, false, AlphaIsActive, false, cancellationToken).ConfigureAwait(false); - - if (!AlphaIsActive) - alphaServer = newServer; - else - bravoServer = newServer; - - watchReboot = true; - activeServer.ClosePortOnReboot = false; - } - else if (activeServerReboot) - { - //try 5 times to open the inactive server on the primary port - //might fail a couple times due to the active server in the process of closing it's own - var I = 0; - for (; I < 5; ++I) - if (await inactiveServer.SetPort(ActiveLaunchParameters.PrimaryPort.Value, cancellationToken).ConfigureAwait(false)) - break; - - if(I == 5) - { - //uggghhh he's being a bitch - var chatTask = chat.SendWatchdogMessage("Unable to open inactive server port! Reopening active server port...", cancellationToken); - var result = await activeServer.SetPort(ActiveLaunchParameters.PrimaryPort.Value, cancellationToken).ConfigureAwait(false); - if (!result) - { - //ok reboot time - await chatTask.ConfigureAwait(false); - await chat.SendWatchdogMessage("Unable to reopen active server port! Restarting watchdog...", cancellationToken).ConfigureAwait(false); - inactiveServer.Dispose(); - Running = false; - await LaunchNoLock(false, cancellationToken).ConfigureAwait(false); - if (!Running) - return; - } - continue; - } - - LastLaunchParameters = ActiveLaunchParameters; - AlphaIsActive = !AlphaIsActive; - LiveCompileJob = inactiveServer.Dmb.CompileJob; - StagedCompileJob = null; - this. - } + state.ActiveServer = AlphaIsActive ? alphaServer : bravoServer; + state.InactiveServer = !AlphaIsActive ? alphaServer : bravoServer; + MonitorActivationReason activationReason; + if (activeServer.Lifetime.IsCompleted) + activationReason = MonitorActivationReason.ActiveServerCrashed; + else if(inactiveServer.Lifetime.IsCompleted) } } catch (OperationCanceledException) { }