From 35e4d8bbeecd7d15e93cad9efd67c7a102b7085b Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Sun, 19 Aug 2018 09:30:36 -0400 Subject: [PATCH 01/12] Log monitor iteration number --- src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs index b43720659f..755ff0c7d4 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs @@ -469,7 +469,7 @@ namespace Tgstation.Server.Host.Components.Watchdog var iteration = 1; for(var monitorState = new MonitorState(); monitorState.NextAction != MonitorAction.Exit; ++iteration) { - logger.LogDebug("New iteration of monitor loop"); + logger.LogDebug("Iteration {0} of monitor loop", iteration); try { if(AlphaIsActive) From 54867868882d6fdcf35518f92403143432d8125d Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Sun, 19 Aug 2018 09:31:08 -0400 Subject: [PATCH 02/12] Reset monitor next action on new loop --- src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs index 755ff0c7d4..66d8171aa4 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs @@ -469,6 +469,7 @@ namespace Tgstation.Server.Host.Components.Watchdog var iteration = 1; for(var monitorState = new MonitorState(); monitorState.NextAction != MonitorAction.Exit; ++iteration) { + monitorState.NextAction = MonitorAction.Continue; logger.LogDebug("Iteration {0} of monitor loop", iteration); try { From ebc00d44b4963ba38cb27ee5e6e7baf9eab1ddc7 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Sun, 19 Aug 2018 09:32:33 -0400 Subject: [PATCH 03/12] Make the reboot section a bit more clear --- .../Components/Watchdog/Watchdog.cs | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs index 66d8171aa4..1f8638bf3d 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs @@ -564,25 +564,25 @@ namespace Tgstation.Server.Host.Components.Watchdog logger.LogDebug("Next state action is to restart"); DisposeAndNullControllers(); chatTask = chat.SendWatchdogMessage("Restarting entirely due to complications...", cancellationToken); - } - for (var retryAttempts = 1; monitorState.NextAction == MonitorAction.Restart; ++retryAttempts) - { - WatchdogLaunchResult result; - using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken).ConfigureAwait(false)) + for (var retryAttempts = 1; monitorState.NextAction == MonitorAction.Restart; ++retryAttempts) { - result = await LaunchNoLock(false, false, false, cancellationToken).ConfigureAwait(false); - if (Running) - monitorState = new MonitorState(); //clean the slate - } + WatchdogLaunchResult result; + using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken).ConfigureAwait(false)) + { + result = await LaunchNoLock(false, false, false, cancellationToken).ConfigureAwait(false); + if (Running) + monitorState = new MonitorState(); //clean the slate + } - await chatTask.ConfigureAwait(false); - if(!Running) - { - logger.LogWarning("Failed to automatically restart the watchdog! Alpha: {0}; Bravo: {1}", result.Alpha.ToString(), result.Bravo.ToString()); - var retryDelay = Math.Min(Math.Pow(2, retryAttempts), 3600); //max of one hour - chatTask = chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Failed to restart watchdog (Attempt: {0}), retrying in {1} seconds...", retryAttempts, retryDelay), cancellationToken); - await Task.WhenAll(Task.Delay((int)retryDelay, cancellationToken), chatTask).ConfigureAwait(false); + await chatTask.ConfigureAwait(false); + if (!Running) + { + logger.LogWarning("Failed to automatically restart the watchdog! Alpha: {0}; Bravo: {1}", result.Alpha.ToString(), result.Bravo.ToString()); + var retryDelay = Math.Min(Math.Pow(2, retryAttempts), 3600); //max of one hour + chatTask = chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Failed to restart watchdog (Attempt: {0}), retrying in {1} seconds...", retryAttempts, retryDelay), cancellationToken); + await Task.WhenAll(Task.Delay((int)retryDelay, cancellationToken), chatTask).ConfigureAwait(false); + } } } } From 190a836b62e8b1f849cfd803ab92719b776b82f3 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Sun, 19 Aug 2018 09:36:23 -0400 Subject: [PATCH 04/12] More CompileJob lock logging --- src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs b/src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs index 2092ed275c..045e495fa0 100644 --- a/src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs +++ b/src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs @@ -212,9 +212,15 @@ namespace Tgstation.Server.Host.Components.Compiler lock (this) { if (!jobLockCounts.TryGetValue(compileJob.Id, out int value)) + { + value = 1; jobLockCounts.Add(compileJob.Id, 1); + } else jobLockCounts[compileJob.Id] = ++value; + + logger.LogTrace("Compile job {0} lock count now: {1}", compileJob.Id, value); + providerSubmitted = true; return newProvider; } From 07486b0660690cab32ae6dc1caf4eeca61c173e8 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Sun, 19 Aug 2018 09:37:48 -0400 Subject: [PATCH 05/12] Add and change some monitor logging --- .../Components/Watchdog/Watchdog.cs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs index 1f8638bf3d..687c4f7741 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs @@ -251,7 +251,7 @@ namespace Tgstation.Server.Host.Components.Watchdog /// A representing the running operation async Task HandlerMonitorWakeup(MonitorActivationReason activationReason, MonitorState monitorState, CancellationToken cancellationToken) { - logger.LogInformation("Monitor activation. Reason: {0}", activationReason); + logger.LogDebug("Monitor activation. Reason: {0}", activationReason); //returns true if the inactive server can't be used immediately bool FullRestartDeadInactive() @@ -268,7 +268,7 @@ namespace Tgstation.Server.Host.Components.Watchdog //trys to set inactive server's port to the private port async Task MakeInactiveActive() { - logger.LogInformation("Setting inactive server to port {0}...", ActiveLaunchParameters.PrimaryPort.Value); + logger.LogDebug("Setting inactive server to port {0}...", ActiveLaunchParameters.PrimaryPort.Value); var result = await monitorState.InactiveServer.SetPort(ActiveLaunchParameters.PrimaryPort.Value, cancellationToken).ConfigureAwait(false); if (!result) @@ -309,7 +309,7 @@ namespace Tgstation.Server.Host.Components.Watchdog } catch (Exception e) { - logger.LogError("Exception occurred while recreating server! Attempting backup strategy of running DMB of running server! Exception: {0}", e.ToString()); + logger.LogError("Error occurred while recreating server! Attempting backup strategy of running DMB of running server! Exception: {0}", e.ToString()); //ahh jeez, what do we do here? //this is our fault, so it should never happen but //idk maybe a database error while handling the newest dmb? @@ -465,7 +465,7 @@ namespace Tgstation.Server.Host.Components.Watchdog /// A representing the running operation async Task MonitorLifetimes(CancellationToken cancellationToken) { - logger.LogDebug("Entered MonitorLifetimes"); + logger.LogTrace("Entered MonitorLifetimes"); var iteration = 1; for(var monitorState = new MonitorState(); monitorState.NextAction != MonitorAction.Exit; ++iteration) { @@ -572,7 +572,10 @@ namespace Tgstation.Server.Host.Components.Watchdog { result = await LaunchNoLock(false, false, false, cancellationToken).ConfigureAwait(false); if (Running) + { + logger.LogDebug("Relaunch successful, resetting monitor state..."); monitorState = new MonitorState(); //clean the slate + } } await chatTask.ConfigureAwait(false); @@ -709,6 +712,7 @@ namespace Tgstation.Server.Host.Components.Watchdog //both servers are now running, alpha is the active server, huzzah AlphaIsActive = doReattach ? reattachInfo.AlphaIsActive : true; LastLaunchResult = alphaLrt.Result; + logger.LogInformation("Launched servers successfully"); Running = true; if (startMonitor) From 7732e6dc9e3adb9f01c04271a9aad2ac270d4bfd Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Sun, 19 Aug 2018 13:15:37 -0400 Subject: [PATCH 06/12] Various things: Fix the whole reboot/port change nonsense Actually add the terminate process command Fix reboot mode changes --- src/DMAPI/tgs/v4/api.dm | 60 +++++++- .../Components/Interop/Constants.cs | 3 +- .../Components/Watchdog/ISessionController.cs | 22 ++- .../Components/Watchdog/SessionController.cs | 130 ++++++++++-------- .../Watchdog/SessionControllerFactory.cs | 3 +- .../Components/Watchdog/Watchdog.cs | 91 ++++++------ 6 files changed, 181 insertions(+), 128 deletions(-) diff --git a/src/DMAPI/tgs/v4/api.dm b/src/DMAPI/tgs/v4/api.dm index cde747ba1f..5810ac0158 100644 --- a/src/DMAPI/tgs/v4/api.dm +++ b/src/DMAPI/tgs/v4/api.dm @@ -10,8 +10,7 @@ #define TGS4_TOPIC_EVENT "tgs_event" #define TGS4_TOPIC_INTEROP_RESPONSE "tgs_interop" -#define TGS4_COMM_ONLINE "tgs_on" -#define TGS4_COMM_IDENTIFY "tgs_ident" +#define TGS4_COMM_NEW_PORT "tgs_new_port" #define TGS4_COMM_VALIDATE "tgs_validate" #define TGS4_COMM_SERVER_PRIMED "tgs_prime" #define TGS4_COMM_WORLD_REBOOT "tgs_reboot" @@ -21,8 +20,7 @@ #define TGS4_PARAMETER_COMMAND "tgs_com" #define TGS4_PARAMETER_DATA "tgs_data" -#define TGS4_PARAMETER_NEW_PORT "new_port" -#define TGS4_PARAMETER_NEW_REBOOT_MODE "new_rmode" +#define TGS4_PORT_CRITFAIL_MESSAGE " Must exit to let watchdog reboot..." #define EXPORT_TIMEOUT_DS 200 @@ -111,6 +109,7 @@ /datum/tgs_api/v4/OnInitializationComplete() Export(TGS4_COMM_SERVER_PRIMED) + var/tgs4_secret_sleep_offline_sauce = 24051994 var/old_sleep_offline = world.sleep_offline world.sleep_offline = tgs4_secret_sleep_offline_sauce @@ -156,7 +155,22 @@ if(TGS4_TOPIC_INTEROP_RESPONSE) last_interop_response = json_decode(params[TGS4_PARAMETER_DATA]) return - + if(TGS4_TOPIC_CHANGE_PORT) + var/new_port = params[TGS4_PARAMETER_DATA] + if (!(isnum(new_port) && new_port > 0)) + return "Invalid port!" + + //the topic still completes, miraculously + //I honestly didn't believe byond could do it + event_handler.HandleEvent(TGS_EVENT_PORT_SWAP, new_port) + if(!world.OpenPort(new_port)) + return "Port change failed!" + if(TGS4_TOPIC_CHANGE_REBOOT_MODE) + var/new_reboot_mode = params[TGS4_PARAMETER_DATA] + event_handler.HandleEvent(TGS_EVENT_REBOOT_MODE_CHANGE, reboot_mode, new_reboot_mode) + reboot_mode = new_reboot_mode + return + return "Unknown command: [command]" /datum/tgs_api/v4/proc/Export(command, list/data) @@ -164,6 +178,29 @@ data = list() data[TGS4_PARAMETER_COMMAND] = command var/json = json_encode(data) + + //we need some port open at this point to facilitate return communication + if(!world.port) + if(!world.OpenPort(0)) //open any port + TGS_ERROR_LOG("Unable to open random port to retrieve new port![TGS4_PORT_CRITFAIL_MESSAGE]") + del(world) + + //request a new port + export_lock = FALSE + var/list/new_port_json = Export(TGS4_COMMAND_NEW_PORT, list("current_port" = "[world.port]")) //stringify this on purpose + + if(!new_port_json) + TGS_ERROR_LOG("No new port response from server![TGS4_PORT_CRITFAIL_MESSAGE]") + del(world) + + var/new_port = new_port_json["port"] + if(!isnum(new_port) || new_port <= 0) + TGS_ERROR_LOG("Malformed new port json ([json_encode(new_port_json)])![TGS4_PORT_CRITFAIL_MESSAGE]") + del(world) + + if(new_port != world.port && !world.OpenPort(new_port)) + TGS_ERROR_LOG("Unable to open port [new_port]![TGS4_PORT_CRITFAIL_MESSAGE]") + del(world) while(export_lock) sleep(1) @@ -189,7 +226,18 @@ if(!result) return - //TODO: Port handling + //okay so the standard TGS4 proceedure is: right before rebooting change the port to whatever was sent to us in the above json's data parameter + + var/port = json[TGS4_PARAMETER_DATA] + if(!isnum(port)) + return //this is valid, server may just want use to reboot + + if(port == 0) + //to byond 0 means any port and "none" means close vOv + port = "none" + + if(!world.OpenPort(port)) + TGS_ERROR_LOG("Unable to set port to [port]!") /datum/tgs_api/v4/InstanceName() return instance_name diff --git a/src/Tgstation.Server.Host/Components/Interop/Constants.cs b/src/Tgstation.Server.Host/Components/Interop/Constants.cs index 00b516bd94..55b7ff5b73 100644 --- a/src/Tgstation.Server.Host/Components/Interop/Constants.cs +++ b/src/Tgstation.Server.Host/Components/Interop/Constants.cs @@ -18,8 +18,7 @@ public const string DMTopicEvent = "tgs_event"; public const string DMTopicInteropResponse = "tgs_interop"; - public const string DMCommandOnline = "tgs_on"; - public const string DMCommandIdentify = "tgs_ident"; + public const string DMCommandNewPort = "tgs_new_port"; public const string DMCommandApiValidate = "tgs_validate"; public const string DMCommandServerPrimed = "tgs_prime"; public const string DMCommandWorldReboot = "tgs_reboot"; diff --git a/src/Tgstation.Server.Host/Components/Watchdog/ISessionController.cs b/src/Tgstation.Server.Host/Components/Watchdog/ISessionController.cs index 6a703f5fc9..3f2e245eba 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/ISessionController.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/ISessionController.cs @@ -19,6 +19,11 @@ namespace Tgstation.Server.Host.Components.Watchdog /// bool IsPrimary { get; } + /// + /// If the DreamDaemon instance sent a + /// + bool TerminationWasRequested { get; } + /// /// If the DMAPI was validated. This field may only be access once completes /// @@ -34,16 +39,16 @@ namespace Tgstation.Server.Host.Components.Watchdog /// ushort? Port { get; } + /// + /// If the port should be rotated off when the world reboots + /// + bool ClosePortOnReboot { get; set; } + /// /// The current /// RebootState RebootState { get; } - /// - /// If the port should close when /world/Reboot() is called. Defaults to - /// - bool ClosePortOnReboot { get; set; } - /// /// A that completes when the server calls /world/Reboot() /// @@ -63,13 +68,6 @@ namespace Tgstation.Server.Host.Components.Watchdog /// A resulting in the result of /world/Topic() Task SendCommand(string command, CancellationToken cancellationToken); - /// - /// Closes the world's port - /// - /// The for the operation - /// A resulting in if the operation succeeded, otherwise - Task ClosePort(CancellationToken cancellationToken); - /// /// Causes the world to start listening on a /// diff --git a/src/Tgstation.Server.Host/Components/Watchdog/SessionController.cs b/src/Tgstation.Server.Host/Components/Watchdog/SessionController.cs index 6a6448678b..a3f0642a9a 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/SessionController.cs @@ -28,9 +28,6 @@ namespace Tgstation.Server.Host.Components.Watchdog } } - /// - public bool ClosePortOnReboot { get; set; } - /// public bool ApiValidated { @@ -58,7 +55,7 @@ namespace Tgstation.Server.Host.Components.Watchdog get { CheckDisposed(); - if (portClosed) + if (portClosedForReboot) return null; return reattachInformation.Port; } @@ -74,6 +71,12 @@ namespace Tgstation.Server.Host.Components.Watchdog } } + /// + public bool ClosePortOnReboot { get; set; } + + /// + public bool TerminationWasRequested { get; private set; } + /// public Task LaunchResult { get; } @@ -130,7 +133,7 @@ namespace Tgstation.Server.Host.Components.Watchdog /// /// The port to assign DreamDaemon when it queries for it /// - ushort nextPort; + ushort? nextPort; /// /// The that completes when DD tells us about a reboot @@ -140,7 +143,8 @@ namespace Tgstation.Server.Host.Components.Watchdog /// /// If we know DreamDaemon currently has it's port closed /// - bool portClosed; + bool portClosedForReboot; + /// /// If the has been disposed /// @@ -181,7 +185,7 @@ namespace Tgstation.Server.Host.Components.Watchdog interopContext.RegisterHandler(this); - portClosed = false; + portClosedForReboot = false; disposed = false; apiValidated = false; released = false; @@ -273,20 +277,38 @@ namespace Tgstation.Server.Host.Components.Watchdog case Constants.DMCommandServerPrimed: //currently unused, maybe in the future break; - case Constants.DMCommandIdentify: + case Constants.DMCommandEndProcess: + TerminationWasRequested = true; + process.Terminate(); + return; + case Constants.DMCommandNewPort: lock (this) - if (portClosed) - content = new Dictionary { { Constants.DMParameterData, nextPort } }; - break; - case Constants.DMCommandOnline: - lock (this) - if (portClosed) - { - reattachInformation.Port = nextPort; - portAssignmentTcs.TrySetResult(true); - portAssignmentTcs = null; - portClosed = false; + { + if (!query.TryGetValue(Constants.DMParameterData, out var stringPort) || !UInt16.TryParse(stringPort, out var currentPort)) { + /////UHHHH + logger.LogWarning("DreamDaemon sent new port command without providing it's own!"); + break; } + + if (!nextPort.HasValue) + //not ready yet, so what we'll do is accept the random port DD opened on for now and change it later when we decide to + reattachInformation.Port = currentPort; + else + { + //nextPort is ready, tell DD to switch to that + //if it fails it'll kill itself + content = new Dictionary { { Constants.DMParameterData, nextPort.Value } }; + reattachInformation.Port = nextPort.Value; + nextPort = null; + + //we'll also get here from SetPort so complete that task + var tmpTcs = portAssignmentTcs; + portAssignmentTcs = null; + tmpTcs?.SetResult(true); + } + + portClosedForReboot = false; + } break; case Constants.DMCommandApiValidate: apiValidated = true; @@ -295,10 +317,8 @@ namespace Tgstation.Server.Host.Components.Watchdog if (ClosePortOnReboot) { content = new Dictionary { { Constants.DMParameterData, 0 } }; - portClosed = true; + portClosedForReboot = true; } - else - ClosePortOnReboot = true; var oldTcs = rebootTcs; rebootTcs = new TaskCompletionSource(); oldTcs.SetResult(null); @@ -368,48 +388,38 @@ namespace Tgstation.Server.Host.Components.Watchdog } } - async Task SetPortImpl(ushort port, CancellationToken cancellationToken) => await SendCommand(String.Format(CultureInfo.InvariantCulture, "{0}&{1}={2}", byondTopicSender.SanitizeString(Constants.DMTopicChangePort), byondTopicSender.SanitizeString(Constants.DMParameterData), byondTopicSender.SanitizeString(port.ToString(CultureInfo.InvariantCulture))), cancellationToken).ConfigureAwait(false) == Constants.DMResponseSuccess; - /// - public async Task ClosePort(CancellationToken cancellationToken) + public Task SetPort(ushort port, CancellationToken cancellationToken) { CheckDisposed(); - if (portClosed) - return true; - if (await SetPortImpl(0, cancellationToken).ConfigureAwait(false)) - { - portClosed = true; - return true; - } - return false; - } - - /// - public async Task SetPort(ushort port, CancellationToken cancellatonToken) - { - CheckDisposed(); - if (portClosed) - { - Task toWait; - lock (this) - { - if (portAssignmentTcs != null) - { - //someone was trying to change the port before us, ignore them - //shouldn't happen anyway, add logging here - logger.LogWarning("Hey uhhh, this shouldn't happen ok? Pls to tell cyberboss. SessionController.SetPort"); - portAssignmentTcs.TrySetResult(false); - } - nextPort = port; - portAssignmentTcs = new TaskCompletionSource(); - toWait = portAssignmentTcs.Task; - } - return await toWait.ConfigureAwait(false); - } if (port == 0) throw new ArgumentOutOfRangeException(nameof(port), port, "port must not be zero!"); - return await SetPortImpl(port, cancellatonToken).ConfigureAwait(false); + + async Task ImmediateTopicPortChange() + { + var commandResult = await SendCommand(String.Format(CultureInfo.InvariantCulture, "{0}&{1}={2}", byondTopicSender.SanitizeString(Constants.DMTopicChangePort), byondTopicSender.SanitizeString(Constants.DMParameterData), byondTopicSender.SanitizeString(port.ToString(CultureInfo.InvariantCulture))), cancellationToken).ConfigureAwait(false); + + if (commandResult != Constants.DMResponseSuccess) + { + logger.LogWarning("Failed port change! DD says: {0}", commandResult); + return false; + } + + return true; + } + + lock (this) + if (portClosedForReboot) + { + if (portAssignmentTcs != null) + throw new InvalidOperationException("A port change operation is already in progress!"); + nextPort = port; + portAssignmentTcs = new TaskCompletionSource(); + return portAssignmentTcs.Task; + } + else + return ImmediateTopicPortChange(); } /// @@ -417,8 +427,8 @@ namespace Tgstation.Server.Host.Components.Watchdog { if (RebootState == newRebootState) return true; - - return await SendCommand(String.Format(CultureInfo.InvariantCulture, "{0}&{1}={2}", Constants.DMTopicChangeReboot, Constants.DMParameterData, (int)newRebootState), cancellationToken).ConfigureAwait(false) == Constants.DMResponseSuccess; + reattachInformation.RebootState = newRebootState; + return await SendCommand(String.Format(CultureInfo.InvariantCulture, "{0}&{1}={2}", byondTopicSender.SanitizeString(Constants.DMTopicChangeReboot), byondTopicSender.SanitizeString(Constants.DMParameterData), (int)newRebootState), cancellationToken).ConfigureAwait(false) == Constants.DMResponseSuccess; } /// diff --git a/src/Tgstation.Server.Host/Components/Watchdog/SessionControllerFactory.cs b/src/Tgstation.Server.Host/Components/Watchdog/SessionControllerFactory.cs index d1de6c468a..69fbe5cd85 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/SessionControllerFactory.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/SessionControllerFactory.cs @@ -180,7 +180,8 @@ namespace Tgstation.Server.Host.Components.Watchdog //more sanitization here cause it uses the same scheme var parameters = String.Format(CultureInfo.InvariantCulture, "{2}={0}&{3}={1}", byondTopicSender.SanitizeString(application.Version.ToString()), byondTopicSender.SanitizeString(interopJsonFile), byondTopicSender.SanitizeString(Constants.DMParamHostVersion), byondTopicSender.SanitizeString(Constants.DMParamInfoJson)); - var arguments = String.Format(CultureInfo.InvariantCulture, "{0} -port {1} {2}-close -{3} -verbose -public -params \"{4}\"", + //important to run on all ports to allow port changing + var arguments = String.Format(CultureInfo.InvariantCulture, "{0} -port {1} -ports 1-65535 {2}-close -{3} -verbose -public -params \"{4}\"", dmbProvider.DmbName, primaryPort ? launchParameters.PrimaryPort : launchParameters.SecondaryPort, launchParameters.AllowWebClient.Value ? "-webclient " : String.Empty, diff --git a/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs index 687c4f7741..59e2e9f8df 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs @@ -189,35 +189,7 @@ namespace Tgstation.Server.Host.Components.Watchdog bravoServer = null; Running = false; } - - /// - /// Implementation of . Does not lock - /// - async Task RestartNoLock(bool graceful, CancellationToken cancellationToken) - { - var running = Running; - if (!graceful || !running) - { - Task chatTask; - if (running) - { - chatTask = chat.SendWatchdogMessage("Manual restart triggered...", cancellationToken); - await TerminateNoLock(false, false, cancellationToken).ConfigureAwait(false); - } - else - chatTask = Task.CompletedTask; - var result = await LaunchNoLock(true, !running, false, cancellationToken).ConfigureAwait(false); - await chatTask.ConfigureAwait(false); - return result; - } - var toReboot = AlphaIsActive ? alphaServer : bravoServer; - var other = AlphaIsActive ? bravoServer : alphaServer; - if (toReboot != null) - //todo, log the result - await toReboot.SetRebootState(Components.Watchdog.RebootState.Restart, cancellationToken).ConfigureAwait(false); - return null; - } - + /// /// Implementation of . Does not lock /// @@ -265,7 +237,8 @@ namespace Tgstation.Server.Host.Components.Watchdog return false; }; - //trys to set inactive server's port to the private port + //trys to set inactive server's port to the public port + //doesn't handle closing active server's port async Task MakeInactiveActive() { logger.LogDebug("Setting inactive server to port {0}...", ActiveLaunchParameters.PrimaryPort.Value); @@ -278,10 +251,6 @@ namespace Tgstation.Server.Host.Components.Watchdog return false; } - // should always be set for InactiveServer - monitorState.InactiveServer.ClosePortOnReboot = false; - monitorState.ActiveServer.ClosePortOnReboot = true; - //inactive server should always be using active launch parameters LastLaunchParameters = ActiveLaunchParameters; @@ -342,8 +311,6 @@ namespace Tgstation.Server.Host.Components.Watchdog logger.LogInformation("Successfully relaunched inactive server!"); monitorState.RebootingInactiveServer = true; - // should always be set for InactiveServer - monitorState.InactiveServer.ClosePortOnReboot = false; return usedMostRecentDmb; } @@ -360,36 +327,37 @@ namespace Tgstation.Server.Host.Components.Watchdog { monitorState.ActiveServer.ClosePortOnReboot = false; if (monitorState.InactiveServerHasStagedDmb && !usedLatestDmb) - monitorState.InactiveServerHasStagedDmb = false; //don't try to load it again though + monitorState.InactiveServerHasStagedDmb = false; //don't try to load it again though } }; + string ExitWord(ISessionController controller) => controller.TerminationWasRequested ? "exited" : "crashed"; + //reason handling switch (activationReason) { case MonitorActivationReason.ActiveServerCrashed: if(monitorState.ActiveServer.RebootState == Components.Watchdog.RebootState.Shutdown) { - await chat.SendWatchdogMessage("Active server crashed or exited! Exiting due to graceful termination request...", cancellationToken).ConfigureAwait(false); + await chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Active server {0}! Exiting due to graceful termination request...", ExitWord(monitorState.ActiveServer)), cancellationToken).ConfigureAwait(false); monitorState.NextAction = MonitorAction.Exit; break; } if (FullRestartDeadInactive()) { - await chat.SendWatchdogMessage("Active server crashed or exited! Inactive server unable to online!", cancellationToken).ConfigureAwait(false); + await chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Active server {0}! Inactive server unable to online!", ExitWord(monitorState.ActiveServer)), cancellationToken).ConfigureAwait(false); break; } - await chat.SendWatchdogMessage("Active server crashed or exited! Onlining inactive server...", cancellationToken).ConfigureAwait(false); + await chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Active server {0}! Onlining inactive server...", ExitWord(monitorState.ActiveServer)), cancellationToken).ConfigureAwait(false); if (!await MakeInactiveActive().ConfigureAwait(false)) break; - - monitorState.ActiveServer.ClosePortOnReboot = false; + await UpdateAndRestartInactiveServer(true).ConfigureAwait(false); break; case MonitorActivationReason.InactiveServerCrashed: - await chat.SendWatchdogMessage("Inactive server crashed or exited! Rebooting...", cancellationToken).ConfigureAwait(false); + await chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Inactive server {0}! Rebooting...", ExitWord(monitorState.InactiveServer)), cancellationToken).ConfigureAwait(false); await UpdateAndRestartInactiveServer(false).ConfigureAwait(false); break; case MonitorActivationReason.ActiveServerRebooted: @@ -423,22 +391,27 @@ namespace Tgstation.Server.Host.Components.Watchdog //need a new launch in ActiveServer restartOnceSwapped = true; - if (!await MakeInactiveActive().ConfigureAwait(false)) + if (!monitorState.ActiveServer.ClosePortOnReboot || !await MakeInactiveActive().ConfigureAwait(false)) break; + monitorState.ActiveServer.ClosePortOnReboot = true; + if(!restartOnceSwapped) //try to reopen inactive server on the private port so it's not pinging all the time //failing that, just reboot it restartOnceSwapped = !await monitorState.InactiveServer.SetPort(ActiveLaunchParameters.SecondaryPort.Value, cancellationToken).ConfigureAwait(false); if (restartOnceSwapped) //for one reason or another, - await UpdateAndRestartInactiveServer(true).ConfigureAwait(false); //break because worse case, active server is still booting + await UpdateAndRestartInactiveServer(true).ConfigureAwait(false); //break because worse case, active server is still booting else + { + monitorState.InactiveServer.ClosePortOnReboot = false; monitorState.NextAction = MonitorAction.Break; + } break; case MonitorActivationReason.InactiveServerRebooted: monitorState.RebootingInactiveServer = true; - monitorState.InactiveServer.ResetRebootState(); //the DMAPI has already done this internally + monitorState.InactiveServer.ResetRebootState(); monitorState.ActiveServer.ClosePortOnReboot = false; monitorState.NextAction = MonitorAction.Continue; break; @@ -765,7 +738,31 @@ namespace Tgstation.Server.Host.Components.Watchdog public async Task Restart(bool graceful, CancellationToken cancellationToken) { using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken).ConfigureAwait(false)) - return await RestartNoLock(graceful, cancellationToken).ConfigureAwait(false); + { + if (!graceful || !Running) + { + Task chatTask; + if (Running) + { + chatTask = chat.SendWatchdogMessage("Manual restart triggered...", cancellationToken); + await TerminateNoLock(false, false, cancellationToken).ConfigureAwait(false); + } + else + chatTask = Task.CompletedTask; + var result = await LaunchNoLock(true, !Running, false, cancellationToken).ConfigureAwait(false); + await chatTask.ConfigureAwait(false); + return result; + } + var toReboot = AlphaIsActive ? alphaServer : bravoServer; + var other = AlphaIsActive ? bravoServer : alphaServer; + if (toReboot != null) + { + if (!await toReboot.SetRebootState(Components.Watchdog.RebootState.Restart, cancellationToken).ConfigureAwait(false)) + logger.LogWarning("Unable to send reboot state change event!"); + + } + return null; + } } /// From 4acc3fec649b6e321b03850385f190575aea8256 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Sun, 19 Aug 2018 13:50:54 -0400 Subject: [PATCH 07/12] Fix DMAPI --- src/DMAPI/tgs/v4/api.dm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DMAPI/tgs/v4/api.dm b/src/DMAPI/tgs/v4/api.dm index 5810ac0158..708ada0964 100644 --- a/src/DMAPI/tgs/v4/api.dm +++ b/src/DMAPI/tgs/v4/api.dm @@ -187,7 +187,7 @@ //request a new port export_lock = FALSE - var/list/new_port_json = Export(TGS4_COMMAND_NEW_PORT, list("current_port" = "[world.port]")) //stringify this on purpose + var/list/new_port_json = Export(TGS4_COMM_NEW_PORT, list("current_port" = "[world.port]")) //stringify this on purpose if(!new_port_json) TGS_ERROR_LOG("No new port response from server![TGS4_PORT_CRITFAIL_MESSAGE]") From 479a646060f44edada31252b7ce4e2efa50dcfc8 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Sun, 19 Aug 2018 13:52:44 -0400 Subject: [PATCH 08/12] Fix release build --- .../Components/Watchdog/SessionController.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Components/Watchdog/SessionController.cs b/src/Tgstation.Server.Host/Components/Watchdog/SessionController.cs index a3f0642a9a..66b039fba4 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/SessionController.cs @@ -127,7 +127,7 @@ namespace Tgstation.Server.Host.Components.Watchdog readonly ILogger logger; /// - /// The waits on when DreamDaemon currently has it's ports closed + /// The waits on when DreamDaemon currently has it's ports closed /// TaskCompletionSource portAssignmentTcs; /// From c64fee50a9cafacbe57c02dd6e6b57f9529b34e8 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Sun, 19 Aug 2018 13:53:50 -0400 Subject: [PATCH 09/12] Fix logspam for JobExceptions --- src/Tgstation.Server.Host/Core/JobManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Core/JobManager.cs b/src/Tgstation.Server.Host/Core/JobManager.cs index 4a9b60419f..27980f3805 100644 --- a/src/Tgstation.Server.Host/Core/JobManager.cs +++ b/src/Tgstation.Server.Host/Core/JobManager.cs @@ -94,8 +94,8 @@ namespace Tgstation.Server.Host.Core } catch (Exception e) { - logger.LogDebug("Job {0} exited with error! Exception: {1}", job.Id, e); job.ExceptionDetails = e is JobException ? e.Message : e.ToString(); + logger.LogDebug("Job {0} exited with error! Exception: {1}", job.Id, job.ExceptionDetails); } job.StoppedAt = DateTimeOffset.Now; await databaseContext.Save(default).ConfigureAwait(false); From f983c27b11f214c57461de6be7a53cb2f72a03cd Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Sun, 19 Aug 2018 14:00:25 -0400 Subject: [PATCH 10/12] Highly increase startup seperation interval --- src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs index 59e2e9f8df..3b918f2cc8 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs @@ -22,7 +22,7 @@ namespace Tgstation.Server.Host.Components.Watchdog /// /// The time in seconds to wait from starting to start . Does not take responsiveness into account /// - const int AlphaBravoStartupSeperationInterval = 3; + const int AlphaBravoStartupSeperationInterval = 10; /// public bool Running { get; private set; } From 8625d726b4e895e410967529e50fa16110fea46b Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Sun, 19 Aug 2018 14:32:33 -0400 Subject: [PATCH 11/12] Fix task continuation from interop handler running before the response is set --- .../Components/Watchdog/SessionController.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Watchdog/SessionController.cs b/src/Tgstation.Server.Host/Components/Watchdog/SessionController.cs index 66b039fba4..e38b92b9bb 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/SessionController.cs @@ -269,6 +269,7 @@ namespace Tgstation.Server.Host.Components.Watchdog var query = command.Parameters; object content; + Action postRespond = null; if (query.TryGetValue(Constants.DMParameterCommand, out var method)) { content = new object(); @@ -304,7 +305,8 @@ namespace Tgstation.Server.Host.Components.Watchdog //we'll also get here from SetPort so complete that task var tmpTcs = portAssignmentTcs; portAssignmentTcs = null; - tmpTcs?.SetResult(true); + if (tmpTcs != null) + postRespond = () => tmpTcs.SetResult(true); } portClosedForReboot = false; @@ -321,7 +323,7 @@ namespace Tgstation.Server.Host.Components.Watchdog } var oldTcs = rebootTcs; rebootTcs = new TaskCompletionSource(); - oldTcs.SetResult(null); + postRespond = () => oldTcs.SetResult(null); break; default: content = new ErrorMessage { Message = "Requested command not supported!" }; @@ -336,6 +338,8 @@ namespace Tgstation.Server.Host.Components.Watchdog if (response != Constants.DMResponseSuccess) logger.LogWarning("Recieved error response while responding to interop: {0}", response); + + postRespond?.Invoke(); } /// From 87623e62ad64376dbaf7d8778d82cce19ab1c58c Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Sun, 19 Aug 2018 18:07:09 -0400 Subject: [PATCH 12/12] Basic updates functional --- src/DMAPI/tgs/v4/api.dm | 8 +++---- .../Components/Watchdog/Watchdog.cs | 23 +++++++++++++++---- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/src/DMAPI/tgs/v4/api.dm b/src/DMAPI/tgs/v4/api.dm index 708ada0964..1736130b67 100644 --- a/src/DMAPI/tgs/v4/api.dm +++ b/src/DMAPI/tgs/v4/api.dm @@ -156,9 +156,9 @@ last_interop_response = json_decode(params[TGS4_PARAMETER_DATA]) return if(TGS4_TOPIC_CHANGE_PORT) - var/new_port = params[TGS4_PARAMETER_DATA] - if (!(isnum(new_port) && new_port > 0)) - return "Invalid port!" + var/new_port = text2num(params[TGS4_PARAMETER_DATA]) + if (!(new_port > 0)) + return "Invalid port: [new_port]" //the topic still completes, miraculously //I honestly didn't believe byond could do it @@ -166,7 +166,7 @@ if(!world.OpenPort(new_port)) return "Port change failed!" if(TGS4_TOPIC_CHANGE_REBOOT_MODE) - var/new_reboot_mode = params[TGS4_PARAMETER_DATA] + var/new_reboot_mode = text2num(params[TGS4_PARAMETER_DATA]) event_handler.HandleEvent(TGS_EVENT_REBOOT_MODE_CHANGE, reboot_mode, new_reboot_mode) reboot_mode = new_reboot_mode return diff --git a/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs index 3b918f2cc8..dcda48d43c 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs @@ -391,18 +391,26 @@ namespace Tgstation.Server.Host.Components.Watchdog //need a new launch in ActiveServer restartOnceSwapped = true; - if (!monitorState.ActiveServer.ClosePortOnReboot || !await MakeInactiveActive().ConfigureAwait(false)) + if (restartOnceSwapped && !monitorState.ActiveServer.ClosePortOnReboot) + //we need to manually restart active server + //it won't listen to us right now so just kill it + monitorState.ActiveServer.Dispose(); + + if ((!restartOnceSwapped && !monitorState.ActiveServer.ClosePortOnReboot) || !await MakeInactiveActive().ConfigureAwait(false)) break; monitorState.ActiveServer.ClosePortOnReboot = true; - if(!restartOnceSwapped) + if (!restartOnceSwapped) + { + monitorState.InactiveServer.ClosePortOnReboot = false; //try to reopen inactive server on the private port so it's not pinging all the time //failing that, just reboot it restartOnceSwapped = !await monitorState.InactiveServer.SetPort(ActiveLaunchParameters.SecondaryPort.Value, cancellationToken).ConfigureAwait(false); + } - if (restartOnceSwapped) //for one reason or another, - await UpdateAndRestartInactiveServer(true).ConfigureAwait(false); //break because worse case, active server is still booting + if (restartOnceSwapped) //for one reason or another + await UpdateAndRestartInactiveServer(true).ConfigureAwait(false); //break because worse case, active server is still booting else { monitorState.InactiveServer.ClosePortOnReboot = false; @@ -459,6 +467,11 @@ namespace Tgstation.Server.Host.Components.Watchdog monitorState.ActiveServer = AlphaIsActive ? alphaServer : bravoServer; monitorState.InactiveServer = AlphaIsActive ? bravoServer : alphaServer; + if (monitorState.ActiveServer.ClosePortOnReboot) + logger.LogDebug("Active server will close port on reboot"); + if (monitorState.InactiveServer.ClosePortOnReboot) + logger.LogDebug("Inactive server will close port on reboot"); + var activeServerLifetime = monitorState.ActiveServer.Lifetime; var inactiveServerLifetime = monitorState.InactiveServer.Lifetime; var activeServerReboot = monitorState.ActiveServer.OnReboot; @@ -528,7 +541,7 @@ namespace Tgstation.Server.Host.Components.Watchdog //writeback alphaServer and bravoServer alphaServer = AlphaIsActive ? monitorState.ActiveServer : monitorState.InactiveServer; - bravoServer = AlphaIsActive ? monitorState.ActiveServer : monitorState.InactiveServer; + bravoServer = !AlphaIsActive ? monitorState.ActiveServer : monitorState.InactiveServer; } //full reboot required