cyberpresidentvanellope/tgstation-server
1
0
Fork 0
mirror of https://github.com/tgstation/tgstation-server.git synced 2026-08-27 23:17:20 +01:00
Code Issues Packages Projects Releases Wiki Activity

Can't wait forever

Browse Source
This commit is contained in:
Cyberboss
2018-07-05 16:52:47 -04:00
parent 02bc572b6b
commit 8d84ca13a6
3 changed files with 35 additions and 175 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Tgstation.Server.Host/Components/Watchdog/MonitorActivationReason.cs
+12
View File
@@ -0,0 +1,12 @@
namespace Tgstation.Server.Host.Components.Watchdog
{
enum MonitorActivationReason
{
ActiveServerCrashed,
InactiveServerCrashed,
ActiveServerRebooted,
InactiveServerRebooted,
NewDmbAvailable,
InactiveServerStartupComplete
}
}
src/Tgstation.Server.Host/Components/Watchdog/MonitorState.cs
+11
View File
@@ -0,0 +1,11 @@
namespace Tgstation.Server.Host.Components.Watchdog
{
sealed class MonitorState
{
public bool RebootingInactiveServer { get; set; }
public bool InactiveServerHasStagedDmb { get; set; }
public ISessionController ActiveServer { get; set; }
public ISessionController InactiveServer { get; set; }
}
}
src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs
+12 -175
View File
@@ -138,9 +138,14 @@ namespace Tgstation.Server.Host.Components.Watchdog
return null;
}
async Task HandlerMonitorWakeup(MonitorActivationReason activationReason, MonitorState monitorState)
{
}
async Task MonitorLifetimes(CancellationToken cancellationToken)
{
var watchReboot = false;
var state = new MonitorState();
while(true)
{
try
@@ -164,180 +169,12 @@ namespace Tgstation.Server.Host.Components.Watchdog
using (await SemaphoreContext.Lock(semaphore, default).ConfigureAwait(false))
{
var activeServerExited = (alphaServerTask.IsCompleted && AlphaIsActive) || (bravoServerTask.IsCompleted && !AlphaIsActive);
var inactiveServerExited = (alphaServerTask.IsCompleted && !AlphaIsActive) || (bravoServerTask.IsCompleted && AlphaIsActive);
var activeServerReboot = (alphaServerReboot.IsCompleted && AlphaIsActive) || (bravoServerReboot.IsCompleted && !AlphaIsActive);
if (watchReboot)
{
//waiting on the inactive server to come back
var doContinue = true;
var activeServerGracefulAction = activeServer.RebootState != Components.Watchdog.RebootState.Normal;
var aServerRebooted = alphaServerReboot.IsCompleted || bravoServerReboot.IsCompleted;
if (inactiveServer.LaunchResult.IsCompleted)
{
//glad to have you back
//(cleanup the task here)
LastLaunchResult = await inactiveServer.LaunchResult.ConfigureAwait(false);
if (inactiveServer.Lifetime.IsCompleted)
{
//aww heck you died again, try to bring you back i guess...
//transfer control to the "inactive server died" branch
inactiveServerExited = true;
doContinue = false;
}
else
//reenable port swapping on active server
activeServer.ClosePortOnReboot = true;
}
else if (activeServerGracefulAction && activeServerExited)
{
//ok not so bad, we either kill everything or restart everything
if (activeServer.RebootState == Components.Watchdog.RebootState.Shutdown)
//ez
//let control fall through to the "crash but wanted to shutdown anyway" section
doContinue = false;
else
{
//total restart, forget about trying to make the inactive server the active one since it's not even responding yet
var chatTask = chat.SendWatchdogMessage("Entered state where restart is requested but neither server is ready. Restarting watchdog...", cancellationToken);
await LaunchNoLock(false, cancellationToken).ConfigureAwait(false);
await chatTask.ConfigureAwait(false);
if (!Running)
return;
}
}
//if the above is NOT the case that means something probably happened to the ACTIVE server
//hopefully it was just a reboot, otherwise we're gonna need to do some weird recovery shennanigans
else if (!aServerRebooted)
{
//uh oh...
if (!activeServerExited)
//inactive server died again
//let control fall through to the inactive crash handler
doContinue = false;
else
{
//OH NO
//so now we have a dead active server and an inactive server who is still booting
//best we can do is hope it comes online so we can fix things
var chatTask = chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Active server crashed or exited (Code: {0})! Inactive server is still booting! Waiting...", activeServer.Lifetime.Result), cancellationToken);
LastLaunchResult = await inactiveServer.LaunchResult.ConfigureAwait(false);
if(inactiveServer.Lifetime.IsCompleted)
//he's dead too jim.
}
}
else
{
//thank god
//doesn't matter which server rebooted, our action is still the same:
//if it was active, tell it to not switch port
//if it was inactive, ensure it's port is the right one
activeServer.ClosePortOnReboot = false;
await inactiveServer.SetPort(ActiveLaunchParameters.SecondaryPort.Value, cancellationToken).ConfigureAwait(false);
}
if (doContinue)
continue;
}
if (activeServerExited)
{
//active server just went down
switch (activeServer.RebootState)
{
case Components.Watchdog.RebootState.Restart:
case Components.Watchdog.RebootState.Normal:
//yeah he died unexpectedly
//bring in other
var chatTask = chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Active server crashed or exited (Code: {0})! Activating backup, relaunching in background...", activeServer.Lifetime.Result), cancellationToken);
activeServer.Dispose();
var backupOnline = await inactiveServer.SetPort(ActiveLaunchParameters.PrimaryPort.Value, cancellationToken).ConfigureAwait(false);
if (!backupOnline)
{
//can't bring in backup, reboot everything
await chatTask.ConfigureAwait(false);
await chat.SendWatchdogMessage("Backup server not responding to port switch request! Restarting watchdog...", cancellationToken).ConfigureAwait(false);
inactiveServer.Dispose();
Running = false;
await LaunchNoLock(false, cancellationToken).ConfigureAwait(false);
if (!Running)
return;
continue;
}
//make a new server with blackjack and hookers
var nextDmb = await dmbFactory.LockNextDmb(cancellationToken).ConfigureAwait(false);
var newServer = await sessionControllerFactory.LaunchNew(ActiveLaunchParameters, nextDmb, false, AlphaIsActive, false, cancellationToken).ConfigureAwait(false);
if (AlphaIsActive)
alphaServer = newServer;
else
bravoServer = newServer;
watchReboot = true;
AlphaIsActive = !AlphaIsActive;
inactiveServer.ClosePortOnReboot = false; //actually the active server at this point
break;
case Components.Watchdog.RebootState.Shutdown:
await chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Active server crashed or exited (Code: {0})! Not recovering to to pending shutdown.", activeServer.Lifetime.Result), cancellationToken).ConfigureAwait(false);
DisposeAndNullControllers();
return;
}
}
else if (inactiveServerExited)
{
//inactive server just went down
//bring it back
var chatTask = chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Inactive server crashed or exited (Code: {0})! Relaunching in background...", inactiveServer.Lifetime.Result), cancellationToken);
var nextDmb = await dmbFactory.LockNextDmb(cancellationToken).ConfigureAwait(false);
var newServer = await sessionControllerFactory.LaunchNew(ActiveLaunchParameters, nextDmb, false, AlphaIsActive, false, cancellationToken).ConfigureAwait(false);
if (!AlphaIsActive)
alphaServer = newServer;
else
bravoServer = newServer;
watchReboot = true;
activeServer.ClosePortOnReboot = false;
}
else if (activeServerReboot)
{
//try 5 times to open the inactive server on the primary port
//might fail a couple times due to the active server in the process of closing it's own
var I = 0;
for (; I < 5; ++I)
if (await inactiveServer.SetPort(ActiveLaunchParameters.PrimaryPort.Value, cancellationToken).ConfigureAwait(false))
break;
if(I == 5)
{
//uggghhh he's being a bitch
var chatTask = chat.SendWatchdogMessage("Unable to open inactive server port! Reopening active server port...", cancellationToken);
var result = await activeServer.SetPort(ActiveLaunchParameters.PrimaryPort.Value, cancellationToken).ConfigureAwait(false);
if (!result)
{
//ok reboot time
await chatTask.ConfigureAwait(false);
await chat.SendWatchdogMessage("Unable to reopen active server port! Restarting watchdog...", cancellationToken).ConfigureAwait(false);
inactiveServer.Dispose();
Running = false;
await LaunchNoLock(false, cancellationToken).ConfigureAwait(false);
if (!Running)
return;
}
continue;
}
LastLaunchParameters = ActiveLaunchParameters;
AlphaIsActive = !AlphaIsActive;
LiveCompileJob = inactiveServer.Dmb.CompileJob;
StagedCompileJob = null;
this.
}
state.ActiveServer = AlphaIsActive ? alphaServer : bravoServer;
state.InactiveServer = !AlphaIsActive ? alphaServer : bravoServer;
MonitorActivationReason activationReason;
if (activeServer.Lifetime.IsCompleted)
activationReason = MonitorActivationReason.ActiveServerCrashed;
else if(inactiveServer.Lifetime.IsCompleted)
}
}
catch (OperationCanceledException) { }