diff --git a/src/DMAPI/tgs/v4/api.dm b/src/DMAPI/tgs/v4/api.dm index cde747ba1f..1736130b67 100644 --- a/src/DMAPI/tgs/v4/api.dm +++ b/src/DMAPI/tgs/v4/api.dm @@ -10,8 +10,7 @@ #define TGS4_TOPIC_EVENT "tgs_event" #define TGS4_TOPIC_INTEROP_RESPONSE "tgs_interop" -#define TGS4_COMM_ONLINE "tgs_on" -#define TGS4_COMM_IDENTIFY "tgs_ident" +#define TGS4_COMM_NEW_PORT "tgs_new_port" #define TGS4_COMM_VALIDATE "tgs_validate" #define TGS4_COMM_SERVER_PRIMED "tgs_prime" #define TGS4_COMM_WORLD_REBOOT "tgs_reboot" @@ -21,8 +20,7 @@ #define TGS4_PARAMETER_COMMAND "tgs_com" #define TGS4_PARAMETER_DATA "tgs_data" -#define TGS4_PARAMETER_NEW_PORT "new_port" -#define TGS4_PARAMETER_NEW_REBOOT_MODE "new_rmode" +#define TGS4_PORT_CRITFAIL_MESSAGE " Must exit to let watchdog reboot..." #define EXPORT_TIMEOUT_DS 200 @@ -111,6 +109,7 @@ /datum/tgs_api/v4/OnInitializationComplete() Export(TGS4_COMM_SERVER_PRIMED) + var/tgs4_secret_sleep_offline_sauce = 24051994 var/old_sleep_offline = world.sleep_offline world.sleep_offline = tgs4_secret_sleep_offline_sauce @@ -156,7 +155,22 @@ if(TGS4_TOPIC_INTEROP_RESPONSE) last_interop_response = json_decode(params[TGS4_PARAMETER_DATA]) return - + if(TGS4_TOPIC_CHANGE_PORT) + var/new_port = text2num(params[TGS4_PARAMETER_DATA]) + if (!(new_port > 0)) + return "Invalid port: [new_port]" + + //the topic still completes, miraculously + //I honestly didn't believe byond could do it + event_handler.HandleEvent(TGS_EVENT_PORT_SWAP, new_port) + if(!world.OpenPort(new_port)) + return "Port change failed!" + if(TGS4_TOPIC_CHANGE_REBOOT_MODE) + var/new_reboot_mode = text2num(params[TGS4_PARAMETER_DATA]) + event_handler.HandleEvent(TGS_EVENT_REBOOT_MODE_CHANGE, reboot_mode, new_reboot_mode) + reboot_mode = new_reboot_mode + return + return "Unknown command: [command]" /datum/tgs_api/v4/proc/Export(command, list/data) @@ -164,6 +178,29 @@ data = list() data[TGS4_PARAMETER_COMMAND] = command var/json = json_encode(data) + + //we need some port open at this point to facilitate return communication + if(!world.port) + if(!world.OpenPort(0)) //open any port + TGS_ERROR_LOG("Unable to open random port to retrieve new port![TGS4_PORT_CRITFAIL_MESSAGE]") + del(world) + + //request a new port + export_lock = FALSE + var/list/new_port_json = Export(TGS4_COMM_NEW_PORT, list("current_port" = "[world.port]")) //stringify this on purpose + + if(!new_port_json) + TGS_ERROR_LOG("No new port response from server![TGS4_PORT_CRITFAIL_MESSAGE]") + del(world) + + var/new_port = new_port_json["port"] + if(!isnum(new_port) || new_port <= 0) + TGS_ERROR_LOG("Malformed new port json ([json_encode(new_port_json)])![TGS4_PORT_CRITFAIL_MESSAGE]") + del(world) + + if(new_port != world.port && !world.OpenPort(new_port)) + TGS_ERROR_LOG("Unable to open port [new_port]![TGS4_PORT_CRITFAIL_MESSAGE]") + del(world) while(export_lock) sleep(1) @@ -189,7 +226,18 @@ if(!result) return - //TODO: Port handling + //okay so the standard TGS4 proceedure is: right before rebooting change the port to whatever was sent to us in the above json's data parameter + + var/port = json[TGS4_PARAMETER_DATA] + if(!isnum(port)) + return //this is valid, server may just want use to reboot + + if(port == 0) + //to byond 0 means any port and "none" means close vOv + port = "none" + + if(!world.OpenPort(port)) + TGS_ERROR_LOG("Unable to set port to [port]!") /datum/tgs_api/v4/InstanceName() return instance_name diff --git a/src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs b/src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs index 2092ed275c..045e495fa0 100644 --- a/src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs +++ b/src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs @@ -212,9 +212,15 @@ namespace Tgstation.Server.Host.Components.Compiler lock (this) { if (!jobLockCounts.TryGetValue(compileJob.Id, out int value)) + { + value = 1; jobLockCounts.Add(compileJob.Id, 1); + } else jobLockCounts[compileJob.Id] = ++value; + + logger.LogTrace("Compile job {0} lock count now: {1}", compileJob.Id, value); + providerSubmitted = true; return newProvider; } diff --git a/src/Tgstation.Server.Host/Components/Interop/Constants.cs b/src/Tgstation.Server.Host/Components/Interop/Constants.cs index 00b516bd94..55b7ff5b73 100644 --- a/src/Tgstation.Server.Host/Components/Interop/Constants.cs +++ b/src/Tgstation.Server.Host/Components/Interop/Constants.cs @@ -18,8 +18,7 @@ public const string DMTopicEvent = "tgs_event"; public const string DMTopicInteropResponse = "tgs_interop"; - public const string DMCommandOnline = "tgs_on"; - public const string DMCommandIdentify = "tgs_ident"; + public const string DMCommandNewPort = "tgs_new_port"; public const string DMCommandApiValidate = "tgs_validate"; public const string DMCommandServerPrimed = "tgs_prime"; public const string DMCommandWorldReboot = "tgs_reboot"; diff --git a/src/Tgstation.Server.Host/Components/Watchdog/ISessionController.cs b/src/Tgstation.Server.Host/Components/Watchdog/ISessionController.cs index 6a703f5fc9..3f2e245eba 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/ISessionController.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/ISessionController.cs @@ -19,6 +19,11 @@ namespace Tgstation.Server.Host.Components.Watchdog /// bool IsPrimary { get; } + /// + /// If the DreamDaemon instance sent a + /// + bool TerminationWasRequested { get; } + /// /// If the DMAPI was validated. This field may only be access once completes /// @@ -34,16 +39,16 @@ namespace Tgstation.Server.Host.Components.Watchdog /// ushort? Port { get; } + /// + /// If the port should be rotated off when the world reboots + /// + bool ClosePortOnReboot { get; set; } + /// /// The current /// RebootState RebootState { get; } - /// - /// If the port should close when /world/Reboot() is called. Defaults to - /// - bool ClosePortOnReboot { get; set; } - /// /// A that completes when the server calls /world/Reboot() /// @@ -63,13 +68,6 @@ namespace Tgstation.Server.Host.Components.Watchdog /// A resulting in the result of /world/Topic() Task SendCommand(string command, CancellationToken cancellationToken); - /// - /// Closes the world's port - /// - /// The for the operation - /// A resulting in if the operation succeeded, otherwise - Task ClosePort(CancellationToken cancellationToken); - /// /// Causes the world to start listening on a /// diff --git a/src/Tgstation.Server.Host/Components/Watchdog/SessionController.cs b/src/Tgstation.Server.Host/Components/Watchdog/SessionController.cs index 6a6448678b..e38b92b9bb 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/SessionController.cs @@ -28,9 +28,6 @@ namespace Tgstation.Server.Host.Components.Watchdog } } - /// - public bool ClosePortOnReboot { get; set; } - /// public bool ApiValidated { @@ -58,7 +55,7 @@ namespace Tgstation.Server.Host.Components.Watchdog get { CheckDisposed(); - if (portClosed) + if (portClosedForReboot) return null; return reattachInformation.Port; } @@ -74,6 +71,12 @@ namespace Tgstation.Server.Host.Components.Watchdog } } + /// + public bool ClosePortOnReboot { get; set; } + + /// + public bool TerminationWasRequested { get; private set; } + /// public Task LaunchResult { get; } @@ -124,13 +127,13 @@ namespace Tgstation.Server.Host.Components.Watchdog readonly ILogger logger; /// - /// The waits on when DreamDaemon currently has it's ports closed + /// The waits on when DreamDaemon currently has it's ports closed /// TaskCompletionSource portAssignmentTcs; /// /// The port to assign DreamDaemon when it queries for it /// - ushort nextPort; + ushort? nextPort; /// /// The that completes when DD tells us about a reboot @@ -140,7 +143,8 @@ namespace Tgstation.Server.Host.Components.Watchdog /// /// If we know DreamDaemon currently has it's port closed /// - bool portClosed; + bool portClosedForReboot; + /// /// If the has been disposed /// @@ -181,7 +185,7 @@ namespace Tgstation.Server.Host.Components.Watchdog interopContext.RegisterHandler(this); - portClosed = false; + portClosedForReboot = false; disposed = false; apiValidated = false; released = false; @@ -265,6 +269,7 @@ namespace Tgstation.Server.Host.Components.Watchdog var query = command.Parameters; object content; + Action postRespond = null; if (query.TryGetValue(Constants.DMParameterCommand, out var method)) { content = new object(); @@ -273,20 +278,39 @@ namespace Tgstation.Server.Host.Components.Watchdog case Constants.DMCommandServerPrimed: //currently unused, maybe in the future break; - case Constants.DMCommandIdentify: + case Constants.DMCommandEndProcess: + TerminationWasRequested = true; + process.Terminate(); + return; + case Constants.DMCommandNewPort: lock (this) - if (portClosed) - content = new Dictionary { { Constants.DMParameterData, nextPort } }; - break; - case Constants.DMCommandOnline: - lock (this) - if (portClosed) - { - reattachInformation.Port = nextPort; - portAssignmentTcs.TrySetResult(true); - portAssignmentTcs = null; - portClosed = false; + { + if (!query.TryGetValue(Constants.DMParameterData, out var stringPort) || !UInt16.TryParse(stringPort, out var currentPort)) { + /////UHHHH + logger.LogWarning("DreamDaemon sent new port command without providing it's own!"); + break; } + + if (!nextPort.HasValue) + //not ready yet, so what we'll do is accept the random port DD opened on for now and change it later when we decide to + reattachInformation.Port = currentPort; + else + { + //nextPort is ready, tell DD to switch to that + //if it fails it'll kill itself + content = new Dictionary { { Constants.DMParameterData, nextPort.Value } }; + reattachInformation.Port = nextPort.Value; + nextPort = null; + + //we'll also get here from SetPort so complete that task + var tmpTcs = portAssignmentTcs; + portAssignmentTcs = null; + if (tmpTcs != null) + postRespond = () => tmpTcs.SetResult(true); + } + + portClosedForReboot = false; + } break; case Constants.DMCommandApiValidate: apiValidated = true; @@ -295,13 +319,11 @@ namespace Tgstation.Server.Host.Components.Watchdog if (ClosePortOnReboot) { content = new Dictionary { { Constants.DMParameterData, 0 } }; - portClosed = true; + portClosedForReboot = true; } - else - ClosePortOnReboot = true; var oldTcs = rebootTcs; rebootTcs = new TaskCompletionSource(); - oldTcs.SetResult(null); + postRespond = () => oldTcs.SetResult(null); break; default: content = new ErrorMessage { Message = "Requested command not supported!" }; @@ -316,6 +338,8 @@ namespace Tgstation.Server.Host.Components.Watchdog if (response != Constants.DMResponseSuccess) logger.LogWarning("Recieved error response while responding to interop: {0}", response); + + postRespond?.Invoke(); } /// @@ -368,48 +392,38 @@ namespace Tgstation.Server.Host.Components.Watchdog } } - async Task SetPortImpl(ushort port, CancellationToken cancellationToken) => await SendCommand(String.Format(CultureInfo.InvariantCulture, "{0}&{1}={2}", byondTopicSender.SanitizeString(Constants.DMTopicChangePort), byondTopicSender.SanitizeString(Constants.DMParameterData), byondTopicSender.SanitizeString(port.ToString(CultureInfo.InvariantCulture))), cancellationToken).ConfigureAwait(false) == Constants.DMResponseSuccess; - /// - public async Task ClosePort(CancellationToken cancellationToken) + public Task SetPort(ushort port, CancellationToken cancellationToken) { CheckDisposed(); - if (portClosed) - return true; - if (await SetPortImpl(0, cancellationToken).ConfigureAwait(false)) - { - portClosed = true; - return true; - } - return false; - } - - /// - public async Task SetPort(ushort port, CancellationToken cancellatonToken) - { - CheckDisposed(); - if (portClosed) - { - Task toWait; - lock (this) - { - if (portAssignmentTcs != null) - { - //someone was trying to change the port before us, ignore them - //shouldn't happen anyway, add logging here - logger.LogWarning("Hey uhhh, this shouldn't happen ok? Pls to tell cyberboss. SessionController.SetPort"); - portAssignmentTcs.TrySetResult(false); - } - nextPort = port; - portAssignmentTcs = new TaskCompletionSource(); - toWait = portAssignmentTcs.Task; - } - return await toWait.ConfigureAwait(false); - } if (port == 0) throw new ArgumentOutOfRangeException(nameof(port), port, "port must not be zero!"); - return await SetPortImpl(port, cancellatonToken).ConfigureAwait(false); + + async Task ImmediateTopicPortChange() + { + var commandResult = await SendCommand(String.Format(CultureInfo.InvariantCulture, "{0}&{1}={2}", byondTopicSender.SanitizeString(Constants.DMTopicChangePort), byondTopicSender.SanitizeString(Constants.DMParameterData), byondTopicSender.SanitizeString(port.ToString(CultureInfo.InvariantCulture))), cancellationToken).ConfigureAwait(false); + + if (commandResult != Constants.DMResponseSuccess) + { + logger.LogWarning("Failed port change! DD says: {0}", commandResult); + return false; + } + + return true; + } + + lock (this) + if (portClosedForReboot) + { + if (portAssignmentTcs != null) + throw new InvalidOperationException("A port change operation is already in progress!"); + nextPort = port; + portAssignmentTcs = new TaskCompletionSource(); + return portAssignmentTcs.Task; + } + else + return ImmediateTopicPortChange(); } /// @@ -417,8 +431,8 @@ namespace Tgstation.Server.Host.Components.Watchdog { if (RebootState == newRebootState) return true; - - return await SendCommand(String.Format(CultureInfo.InvariantCulture, "{0}&{1}={2}", Constants.DMTopicChangeReboot, Constants.DMParameterData, (int)newRebootState), cancellationToken).ConfigureAwait(false) == Constants.DMResponseSuccess; + reattachInformation.RebootState = newRebootState; + return await SendCommand(String.Format(CultureInfo.InvariantCulture, "{0}&{1}={2}", byondTopicSender.SanitizeString(Constants.DMTopicChangeReboot), byondTopicSender.SanitizeString(Constants.DMParameterData), (int)newRebootState), cancellationToken).ConfigureAwait(false) == Constants.DMResponseSuccess; } /// diff --git a/src/Tgstation.Server.Host/Components/Watchdog/SessionControllerFactory.cs b/src/Tgstation.Server.Host/Components/Watchdog/SessionControllerFactory.cs index d1de6c468a..69fbe5cd85 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/SessionControllerFactory.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/SessionControllerFactory.cs @@ -180,7 +180,8 @@ namespace Tgstation.Server.Host.Components.Watchdog //more sanitization here cause it uses the same scheme var parameters = String.Format(CultureInfo.InvariantCulture, "{2}={0}&{3}={1}", byondTopicSender.SanitizeString(application.Version.ToString()), byondTopicSender.SanitizeString(interopJsonFile), byondTopicSender.SanitizeString(Constants.DMParamHostVersion), byondTopicSender.SanitizeString(Constants.DMParamInfoJson)); - var arguments = String.Format(CultureInfo.InvariantCulture, "{0} -port {1} {2}-close -{3} -verbose -public -params \"{4}\"", + //important to run on all ports to allow port changing + var arguments = String.Format(CultureInfo.InvariantCulture, "{0} -port {1} -ports 1-65535 {2}-close -{3} -verbose -public -params \"{4}\"", dmbProvider.DmbName, primaryPort ? launchParameters.PrimaryPort : launchParameters.SecondaryPort, launchParameters.AllowWebClient.Value ? "-webclient " : String.Empty, diff --git a/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs index b43720659f..dcda48d43c 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs @@ -22,7 +22,7 @@ namespace Tgstation.Server.Host.Components.Watchdog /// /// The time in seconds to wait from starting to start . Does not take responsiveness into account /// - const int AlphaBravoStartupSeperationInterval = 3; + const int AlphaBravoStartupSeperationInterval = 10; /// public bool Running { get; private set; } @@ -189,35 +189,7 @@ namespace Tgstation.Server.Host.Components.Watchdog bravoServer = null; Running = false; } - - /// - /// Implementation of . Does not lock - /// - async Task RestartNoLock(bool graceful, CancellationToken cancellationToken) - { - var running = Running; - if (!graceful || !running) - { - Task chatTask; - if (running) - { - chatTask = chat.SendWatchdogMessage("Manual restart triggered...", cancellationToken); - await TerminateNoLock(false, false, cancellationToken).ConfigureAwait(false); - } - else - chatTask = Task.CompletedTask; - var result = await LaunchNoLock(true, !running, false, cancellationToken).ConfigureAwait(false); - await chatTask.ConfigureAwait(false); - return result; - } - var toReboot = AlphaIsActive ? alphaServer : bravoServer; - var other = AlphaIsActive ? bravoServer : alphaServer; - if (toReboot != null) - //todo, log the result - await toReboot.SetRebootState(Components.Watchdog.RebootState.Restart, cancellationToken).ConfigureAwait(false); - return null; - } - + /// /// Implementation of . Does not lock /// @@ -251,7 +223,7 @@ namespace Tgstation.Server.Host.Components.Watchdog /// A representing the running operation async Task HandlerMonitorWakeup(MonitorActivationReason activationReason, MonitorState monitorState, CancellationToken cancellationToken) { - logger.LogInformation("Monitor activation. Reason: {0}", activationReason); + logger.LogDebug("Monitor activation. Reason: {0}", activationReason); //returns true if the inactive server can't be used immediately bool FullRestartDeadInactive() @@ -265,10 +237,11 @@ namespace Tgstation.Server.Host.Components.Watchdog return false; }; - //trys to set inactive server's port to the private port + //trys to set inactive server's port to the public port + //doesn't handle closing active server's port async Task MakeInactiveActive() { - logger.LogInformation("Setting inactive server to port {0}...", ActiveLaunchParameters.PrimaryPort.Value); + logger.LogDebug("Setting inactive server to port {0}...", ActiveLaunchParameters.PrimaryPort.Value); var result = await monitorState.InactiveServer.SetPort(ActiveLaunchParameters.PrimaryPort.Value, cancellationToken).ConfigureAwait(false); if (!result) @@ -278,10 +251,6 @@ namespace Tgstation.Server.Host.Components.Watchdog return false; } - // should always be set for InactiveServer - monitorState.InactiveServer.ClosePortOnReboot = false; - monitorState.ActiveServer.ClosePortOnReboot = true; - //inactive server should always be using active launch parameters LastLaunchParameters = ActiveLaunchParameters; @@ -309,7 +278,7 @@ namespace Tgstation.Server.Host.Components.Watchdog } catch (Exception e) { - logger.LogError("Exception occurred while recreating server! Attempting backup strategy of running DMB of running server! Exception: {0}", e.ToString()); + logger.LogError("Error occurred while recreating server! Attempting backup strategy of running DMB of running server! Exception: {0}", e.ToString()); //ahh jeez, what do we do here? //this is our fault, so it should never happen but //idk maybe a database error while handling the newest dmb? @@ -342,8 +311,6 @@ namespace Tgstation.Server.Host.Components.Watchdog logger.LogInformation("Successfully relaunched inactive server!"); monitorState.RebootingInactiveServer = true; - // should always be set for InactiveServer - monitorState.InactiveServer.ClosePortOnReboot = false; return usedMostRecentDmb; } @@ -360,36 +327,37 @@ namespace Tgstation.Server.Host.Components.Watchdog { monitorState.ActiveServer.ClosePortOnReboot = false; if (monitorState.InactiveServerHasStagedDmb && !usedLatestDmb) - monitorState.InactiveServerHasStagedDmb = false; //don't try to load it again though + monitorState.InactiveServerHasStagedDmb = false; //don't try to load it again though } }; + string ExitWord(ISessionController controller) => controller.TerminationWasRequested ? "exited" : "crashed"; + //reason handling switch (activationReason) { case MonitorActivationReason.ActiveServerCrashed: if(monitorState.ActiveServer.RebootState == Components.Watchdog.RebootState.Shutdown) { - await chat.SendWatchdogMessage("Active server crashed or exited! Exiting due to graceful termination request...", cancellationToken).ConfigureAwait(false); + await chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Active server {0}! Exiting due to graceful termination request...", ExitWord(monitorState.ActiveServer)), cancellationToken).ConfigureAwait(false); monitorState.NextAction = MonitorAction.Exit; break; } if (FullRestartDeadInactive()) { - await chat.SendWatchdogMessage("Active server crashed or exited! Inactive server unable to online!", cancellationToken).ConfigureAwait(false); + await chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Active server {0}! Inactive server unable to online!", ExitWord(monitorState.ActiveServer)), cancellationToken).ConfigureAwait(false); break; } - await chat.SendWatchdogMessage("Active server crashed or exited! Onlining inactive server...", cancellationToken).ConfigureAwait(false); + await chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Active server {0}! Onlining inactive server...", ExitWord(monitorState.ActiveServer)), cancellationToken).ConfigureAwait(false); if (!await MakeInactiveActive().ConfigureAwait(false)) break; - - monitorState.ActiveServer.ClosePortOnReboot = false; + await UpdateAndRestartInactiveServer(true).ConfigureAwait(false); break; case MonitorActivationReason.InactiveServerCrashed: - await chat.SendWatchdogMessage("Inactive server crashed or exited! Rebooting...", cancellationToken).ConfigureAwait(false); + await chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Inactive server {0}! Rebooting...", ExitWord(monitorState.InactiveServer)), cancellationToken).ConfigureAwait(false); await UpdateAndRestartInactiveServer(false).ConfigureAwait(false); break; case MonitorActivationReason.ActiveServerRebooted: @@ -423,22 +391,35 @@ namespace Tgstation.Server.Host.Components.Watchdog //need a new launch in ActiveServer restartOnceSwapped = true; - if (!await MakeInactiveActive().ConfigureAwait(false)) + if (restartOnceSwapped && !monitorState.ActiveServer.ClosePortOnReboot) + //we need to manually restart active server + //it won't listen to us right now so just kill it + monitorState.ActiveServer.Dispose(); + + if ((!restartOnceSwapped && !monitorState.ActiveServer.ClosePortOnReboot) || !await MakeInactiveActive().ConfigureAwait(false)) break; - if(!restartOnceSwapped) + monitorState.ActiveServer.ClosePortOnReboot = true; + + if (!restartOnceSwapped) + { + monitorState.InactiveServer.ClosePortOnReboot = false; //try to reopen inactive server on the private port so it's not pinging all the time //failing that, just reboot it restartOnceSwapped = !await monitorState.InactiveServer.SetPort(ActiveLaunchParameters.SecondaryPort.Value, cancellationToken).ConfigureAwait(false); + } - if (restartOnceSwapped) //for one reason or another, + if (restartOnceSwapped) //for one reason or another await UpdateAndRestartInactiveServer(true).ConfigureAwait(false); //break because worse case, active server is still booting else + { + monitorState.InactiveServer.ClosePortOnReboot = false; monitorState.NextAction = MonitorAction.Break; + } break; case MonitorActivationReason.InactiveServerRebooted: monitorState.RebootingInactiveServer = true; - monitorState.InactiveServer.ResetRebootState(); //the DMAPI has already done this internally + monitorState.InactiveServer.ResetRebootState(); monitorState.ActiveServer.ClosePortOnReboot = false; monitorState.NextAction = MonitorAction.Continue; break; @@ -465,11 +446,12 @@ namespace Tgstation.Server.Host.Components.Watchdog /// A representing the running operation async Task MonitorLifetimes(CancellationToken cancellationToken) { - logger.LogDebug("Entered MonitorLifetimes"); + logger.LogTrace("Entered MonitorLifetimes"); var iteration = 1; for(var monitorState = new MonitorState(); monitorState.NextAction != MonitorAction.Exit; ++iteration) { - logger.LogDebug("New iteration of monitor loop"); + monitorState.NextAction = MonitorAction.Continue; + logger.LogDebug("Iteration {0} of monitor loop", iteration); try { if(AlphaIsActive) @@ -485,6 +467,11 @@ namespace Tgstation.Server.Host.Components.Watchdog monitorState.ActiveServer = AlphaIsActive ? alphaServer : bravoServer; monitorState.InactiveServer = AlphaIsActive ? bravoServer : alphaServer; + if (monitorState.ActiveServer.ClosePortOnReboot) + logger.LogDebug("Active server will close port on reboot"); + if (monitorState.InactiveServer.ClosePortOnReboot) + logger.LogDebug("Inactive server will close port on reboot"); + var activeServerLifetime = monitorState.ActiveServer.Lifetime; var inactiveServerLifetime = monitorState.InactiveServer.Lifetime; var activeServerReboot = monitorState.ActiveServer.OnReboot; @@ -554,7 +541,7 @@ namespace Tgstation.Server.Host.Components.Watchdog //writeback alphaServer and bravoServer alphaServer = AlphaIsActive ? monitorState.ActiveServer : monitorState.InactiveServer; - bravoServer = AlphaIsActive ? monitorState.ActiveServer : monitorState.InactiveServer; + bravoServer = !AlphaIsActive ? monitorState.ActiveServer : monitorState.InactiveServer; } //full reboot required @@ -563,25 +550,28 @@ namespace Tgstation.Server.Host.Components.Watchdog logger.LogDebug("Next state action is to restart"); DisposeAndNullControllers(); chatTask = chat.SendWatchdogMessage("Restarting entirely due to complications...", cancellationToken); - } - for (var retryAttempts = 1; monitorState.NextAction == MonitorAction.Restart; ++retryAttempts) - { - WatchdogLaunchResult result; - using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken).ConfigureAwait(false)) + for (var retryAttempts = 1; monitorState.NextAction == MonitorAction.Restart; ++retryAttempts) { - result = await LaunchNoLock(false, false, false, cancellationToken).ConfigureAwait(false); - if (Running) - monitorState = new MonitorState(); //clean the slate - } + WatchdogLaunchResult result; + using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken).ConfigureAwait(false)) + { + result = await LaunchNoLock(false, false, false, cancellationToken).ConfigureAwait(false); + if (Running) + { + logger.LogDebug("Relaunch successful, resetting monitor state..."); + monitorState = new MonitorState(); //clean the slate + } + } - await chatTask.ConfigureAwait(false); - if(!Running) - { - logger.LogWarning("Failed to automatically restart the watchdog! Alpha: {0}; Bravo: {1}", result.Alpha.ToString(), result.Bravo.ToString()); - var retryDelay = Math.Min(Math.Pow(2, retryAttempts), 3600); //max of one hour - chatTask = chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Failed to restart watchdog (Attempt: {0}), retrying in {1} seconds...", retryAttempts, retryDelay), cancellationToken); - await Task.WhenAll(Task.Delay((int)retryDelay, cancellationToken), chatTask).ConfigureAwait(false); + await chatTask.ConfigureAwait(false); + if (!Running) + { + logger.LogWarning("Failed to automatically restart the watchdog! Alpha: {0}; Bravo: {1}", result.Alpha.ToString(), result.Bravo.ToString()); + var retryDelay = Math.Min(Math.Pow(2, retryAttempts), 3600); //max of one hour + chatTask = chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Failed to restart watchdog (Attempt: {0}), retrying in {1} seconds...", retryAttempts, retryDelay), cancellationToken); + await Task.WhenAll(Task.Delay((int)retryDelay, cancellationToken), chatTask).ConfigureAwait(false); + } } } } @@ -708,6 +698,7 @@ namespace Tgstation.Server.Host.Components.Watchdog //both servers are now running, alpha is the active server, huzzah AlphaIsActive = doReattach ? reattachInfo.AlphaIsActive : true; LastLaunchResult = alphaLrt.Result; + logger.LogInformation("Launched servers successfully"); Running = true; if (startMonitor) @@ -760,7 +751,31 @@ namespace Tgstation.Server.Host.Components.Watchdog public async Task Restart(bool graceful, CancellationToken cancellationToken) { using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken).ConfigureAwait(false)) - return await RestartNoLock(graceful, cancellationToken).ConfigureAwait(false); + { + if (!graceful || !Running) + { + Task chatTask; + if (Running) + { + chatTask = chat.SendWatchdogMessage("Manual restart triggered...", cancellationToken); + await TerminateNoLock(false, false, cancellationToken).ConfigureAwait(false); + } + else + chatTask = Task.CompletedTask; + var result = await LaunchNoLock(true, !Running, false, cancellationToken).ConfigureAwait(false); + await chatTask.ConfigureAwait(false); + return result; + } + var toReboot = AlphaIsActive ? alphaServer : bravoServer; + var other = AlphaIsActive ? bravoServer : alphaServer; + if (toReboot != null) + { + if (!await toReboot.SetRebootState(Components.Watchdog.RebootState.Restart, cancellationToken).ConfigureAwait(false)) + logger.LogWarning("Unable to send reboot state change event!"); + + } + return null; + } } /// diff --git a/src/Tgstation.Server.Host/Core/JobManager.cs b/src/Tgstation.Server.Host/Core/JobManager.cs index 4a9b60419f..27980f3805 100644 --- a/src/Tgstation.Server.Host/Core/JobManager.cs +++ b/src/Tgstation.Server.Host/Core/JobManager.cs @@ -94,8 +94,8 @@ namespace Tgstation.Server.Host.Core } catch (Exception e) { - logger.LogDebug("Job {0} exited with error! Exception: {1}", job.Id, e); job.ExceptionDetails = e is JobException ? e.Message : e.ToString(); + logger.LogDebug("Job {0} exited with error! Exception: {1}", job.Id, job.ExceptionDetails); } job.StoppedAt = DateTimeOffset.Now; await databaseContext.Save(default).ConfigureAwait(false);