Merge pull request #573 from Cyberboss/KeepOnKeepingOn

Watchdog robustness intensifies

src/DMAPI/tgs/v4/api.dm

@@ -10,8 +10,7 @@
 #define TGS4_TOPIC_EVENT "tgs_event"
 #define TGS4_TOPIC_INTEROP_RESPONSE "tgs_interop"
 
-#define TGS4_COMM_ONLINE "tgs_on"
-#define TGS4_COMM_IDENTIFY "tgs_ident"
+#define TGS4_COMM_NEW_PORT "tgs_new_port"
 #define TGS4_COMM_VALIDATE "tgs_validate"
 #define TGS4_COMM_SERVER_PRIMED "tgs_prime"
 #define TGS4_COMM_WORLD_REBOOT "tgs_reboot"
@@ -21,8 +20,7 @@
 #define TGS4_PARAMETER_COMMAND "tgs_com"
 #define TGS4_PARAMETER_DATA "tgs_data"
 
-#define TGS4_PARAMETER_NEW_PORT "new_port"
-#define TGS4_PARAMETER_NEW_REBOOT_MODE "new_rmode"
+#define TGS4_PORT_CRITFAIL_MESSAGE " Must exit to let watchdog reboot..."
 
 #define EXPORT_TIMEOUT_DS 200
 
@@ -111,6 +109,7 @@
 
 /datum/tgs_api/v4/OnInitializationComplete()
 	Export(TGS4_COMM_SERVER_PRIMED)
+
 	var/tgs4_secret_sleep_offline_sauce = 24051994
 	var/old_sleep_offline = world.sleep_offline
 	world.sleep_offline = tgs4_secret_sleep_offline_sauce
@@ -156,7 +155,22 @@
 		if(TGS4_TOPIC_INTEROP_RESPONSE)
 			last_interop_response = json_decode(params[TGS4_PARAMETER_DATA])
 			return
-	
+		if(TGS4_TOPIC_CHANGE_PORT)
+			var/new_port = text2num(params[TGS4_PARAMETER_DATA])
+			if (!(new_port > 0))
+				return "Invalid port: [new_port]"
+
+			//the topic still completes, miraculously
+			//I honestly didn't believe byond could do it
+			event_handler.HandleEvent(TGS_EVENT_PORT_SWAP, new_port)
+			if(!world.OpenPort(new_port))
+				return "Port change failed!"
+		if(TGS4_TOPIC_CHANGE_REBOOT_MODE)
+			var/new_reboot_mode = text2num(params[TGS4_PARAMETER_DATA])
+			event_handler.HandleEvent(TGS_EVENT_REBOOT_MODE_CHANGE, reboot_mode, new_reboot_mode)
+			reboot_mode = new_reboot_mode
+			return
+
 	return "Unknown command: [command]"
 
 /datum/tgs_api/v4/proc/Export(command, list/data)
@@ -164,6 +178,29 @@
 		data = list()
 	data[TGS4_PARAMETER_COMMAND] = command
 	var/json = json_encode(data)
+	
+	//we need some port open at this point to facilitate return communication
+	if(!world.port)
+		if(!world.OpenPort(0)) //open any port
+			TGS_ERROR_LOG("Unable to open random port to retrieve new port![TGS4_PORT_CRITFAIL_MESSAGE]")
+			del(world)
+
+		//request a new port
+		export_lock = FALSE
+		var/list/new_port_json = Export(TGS4_COMM_NEW_PORT, list("current_port" = "[world.port]"))	//stringify this on purpose
+		
+		if(!new_port_json)
+			TGS_ERROR_LOG("No new port response from server![TGS4_PORT_CRITFAIL_MESSAGE]")
+			del(world)
+
+		var/new_port = new_port_json["port"]
+		if(!isnum(new_port) || new_port <= 0)
+			TGS_ERROR_LOG("Malformed new port json ([json_encode(new_port_json)])![TGS4_PORT_CRITFAIL_MESSAGE]")
+			del(world)
+
+		if(new_port != world.port && !world.OpenPort(new_port))
+			TGS_ERROR_LOG("Unable to open port [new_port]![TGS4_PORT_CRITFAIL_MESSAGE]")
+			del(world)
 
 	while(export_lock)
 		sleep(1)
@@ -189,7 +226,18 @@
 	if(!result)
 		return
 	
-	//TODO: Port handling
+	//okay so the standard TGS4 proceedure is: right before rebooting change the port to whatever was sent to us in the above json's data parameter
+
+	var/port = json[TGS4_PARAMETER_DATA]
+	if(!isnum(port))
+		return	//this is valid, server may just want use to reboot
+
+	if(port == 0)
+		//to byond 0 means any port and "none" means close vOv
+		port = "none"
+
+	if(!world.OpenPort(port))
+		TGS_ERROR_LOG("Unable to set port to [port]!")
 
 /datum/tgs_api/v4/InstanceName()
 	return instance_name

src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs

@@ -212,9 +212,15 @@ namespace Tgstation.Server.Host.Components.Compiler
 				lock (this)
 				{
 					if (!jobLockCounts.TryGetValue(compileJob.Id, out int value))
+					{
+						value = 1;
 						jobLockCounts.Add(compileJob.Id, 1);
+					}
 					else
 						jobLockCounts[compileJob.Id] = ++value;
+
+					logger.LogTrace("Compile job {0} lock count now: {1}", compileJob.Id, value);
+
 					providerSubmitted = true;
 					return newProvider;
 				}

src/Tgstation.Server.Host/Components/Interop/Constants.cs

@@ -18,8 +18,7 @@
 		public const string DMTopicEvent = "tgs_event";
 		public const string DMTopicInteropResponse = "tgs_interop";
 
-		public const string DMCommandOnline = "tgs_on";
-		public const string DMCommandIdentify = "tgs_ident";
+		public const string DMCommandNewPort = "tgs_new_port";
 		public const string DMCommandApiValidate = "tgs_validate";
 		public const string DMCommandServerPrimed = "tgs_prime";
 		public const string DMCommandWorldReboot = "tgs_reboot";

src/Tgstation.Server.Host/Components/Watchdog/ISessionController.cs

@@ -19,6 +19,11 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// </summary>
 		bool IsPrimary { get; }
 
+		/// <summary>
+		/// If the DreamDaemon instance sent a
+		/// </summary>
+		bool TerminationWasRequested { get; }
+
 		/// <summary>
 		/// If the DMAPI was validated. This field may only be access once <see cref="IProcessBase.Lifetime"/> completes
 		/// </summary>
@@ -34,16 +39,16 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// </summary>
 		ushort? Port { get; }
 
+		/// <summary>
+		/// If the port should be rotated off when the world reboots
+		/// </summary>
+		bool ClosePortOnReboot { get; set; }
+
 		/// <summary>
 		/// The current <see cref="RebootState"/>
 		/// </summary>
 		RebootState RebootState { get; }
 
-		/// <summary>
-		/// If the port should close when /world/Reboot() is called. Defaults to <see langword="true"/> 
-		/// </summary>
-		bool ClosePortOnReboot { get; set; }
-
 		/// <summary>
 		/// A <see cref="Task"/> that completes when the server calls /world/Reboot()
 		/// </summary>
@@ -63,13 +68,6 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// <returns>A <see cref="Task{TResult}"/> resulting in the result of /world/Topic()</returns>
 		Task<string> SendCommand(string command, CancellationToken cancellationToken);
 
-		/// <summary>
-		/// Closes the world's port
-		/// </summary>
-		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
-		/// <returns>A <see cref="Task{TResult}"/> resulting in <see langword="true"/> if the operation succeeded, <see langword="false"/> otherwise</returns>
-		Task<bool> ClosePort(CancellationToken cancellationToken);
-
 		/// <summary>
 		/// Causes the world to start listening on a <paramref name="newPort"/>
 		/// </summary>

src/Tgstation.Server.Host/Components/Watchdog/SessionController.cs

@@ -28,9 +28,6 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			}
 		}
 
-		/// <inheritdoc />
-		public bool ClosePortOnReboot { get; set; }
-
 		/// <inheritdoc />
 		public bool ApiValidated
 		{
@@ -58,7 +55,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			get
 			{
 				CheckDisposed();
-				if (portClosed)
+				if (portClosedForReboot)
 					return null;
 				return reattachInformation.Port;
 			}
@@ -74,6 +71,12 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			}
 		}
 
+		/// <inheritdoc />
+		public bool ClosePortOnReboot { get; set; }
+
+		/// <inheritdoc />
+		public bool TerminationWasRequested { get; private set; }
+
 		/// <inheritdoc />
 		public Task<LaunchResult> LaunchResult { get; }
 
@@ -124,13 +127,13 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		readonly ILogger<SessionController> logger;
 
 		/// <summary>
-		/// The <see cref="TaskCompletionSource{TResult}"/> <see cref="SetPortImpl(ushort, CancellationToken)"/> waits on when DreamDaemon currently has it's ports closed
+		/// The <see cref="TaskCompletionSource{TResult}"/> <see cref="SetPort(ushort, CancellationToken)"/> waits on when DreamDaemon currently has it's ports closed
 		/// </summary>
 		TaskCompletionSource<bool> portAssignmentTcs;
 		/// <summary>
 		/// The port to assign DreamDaemon when it queries for it
 		/// </summary>
-		ushort nextPort;
+		ushort? nextPort;
 		
 		/// <summary>
 		/// The <see cref="TaskCompletionSource{TResult}"/> that completes when DD tells us about a reboot
@@ -140,7 +143,8 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// <summary>
 		/// If we know DreamDaemon currently has it's port closed
 		/// </summary>
-		bool portClosed;
+		bool portClosedForReboot;
+
 		/// <summary>
 		/// If the <see cref="SessionController"/> has been disposed
 		/// </summary>
@@ -181,7 +185,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 
 			interopContext.RegisterHandler(this);
 
-			portClosed = false;
+			portClosedForReboot = false;
 			disposed = false;
 			apiValidated = false;
 			released = false;
@@ -265,6 +269,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			var query = command.Parameters;
 
 			object content;
+			Action postRespond = null;
 			if (query.TryGetValue(Constants.DMParameterCommand, out var method))
 			{
 				content = new object();
@@ -273,20 +278,39 @@ namespace Tgstation.Server.Host.Components.Watchdog
 					case Constants.DMCommandServerPrimed:
 						//currently unused, maybe in the future
 						break;
-					case Constants.DMCommandIdentify:
+					case Constants.DMCommandEndProcess:
+						TerminationWasRequested = true;
+						process.Terminate();
+						return;
+					case Constants.DMCommandNewPort:
 						lock (this)
-							if (portClosed)
-								content = new Dictionary<string, int> { { Constants.DMParameterData, nextPort } };
-						break;
-					case Constants.DMCommandOnline:
-						lock (this)
-							if (portClosed)
-							{
-								reattachInformation.Port = nextPort;
-								portAssignmentTcs.TrySetResult(true);
-								portAssignmentTcs = null;
-								portClosed = false;
+						{
+							if (!query.TryGetValue(Constants.DMParameterData, out var stringPort) || !UInt16.TryParse(stringPort, out var currentPort)) {
+								/////UHHHH
+								logger.LogWarning("DreamDaemon sent new port command without providing it's own!");
+								break;
 							}
+							
+							if (!nextPort.HasValue)
+								//not ready yet, so what we'll do is accept the random port DD opened on for now and change it later when we decide to
+								reattachInformation.Port = currentPort;
+							else
+							{
+								//nextPort is ready, tell DD to switch to that
+								//if it fails it'll kill itself
+								content = new Dictionary<string, ushort> { { Constants.DMParameterData, nextPort.Value } };
+								reattachInformation.Port = nextPort.Value;
+								nextPort = null;
+								
+								//we'll also get here from SetPort so complete that task
+								var tmpTcs = portAssignmentTcs;
+								portAssignmentTcs = null;
+								if (tmpTcs != null)
+									postRespond = () => tmpTcs.SetResult(true);
+							}
+
+							portClosedForReboot = false;
+						}
 						break;
 					case Constants.DMCommandApiValidate:
 						apiValidated = true;
@@ -295,13 +319,11 @@ namespace Tgstation.Server.Host.Components.Watchdog
 						if (ClosePortOnReboot)
 						{
 							content = new Dictionary<string, int> { { Constants.DMParameterData, 0 } };
-							portClosed = true;
+							portClosedForReboot = true;
 						}
-						else
-							ClosePortOnReboot = true;
 						var oldTcs = rebootTcs;
 						rebootTcs = new TaskCompletionSource<object>();
-						oldTcs.SetResult(null);
+						postRespond = () => oldTcs.SetResult(null);
 						break;
 					default:
 						content = new ErrorMessage { Message = "Requested command not supported!" };
@@ -316,6 +338,8 @@ namespace Tgstation.Server.Host.Components.Watchdog
 
 			if (response != Constants.DMResponseSuccess)
 				logger.LogWarning("Recieved error response while responding to interop: {0}", response);
+
+			postRespond?.Invoke();
 		}
 
 		/// <summary>
@@ -368,48 +392,38 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			}
 		}
 
-		async Task<bool> SetPortImpl(ushort port, CancellationToken cancellationToken) => await SendCommand(String.Format(CultureInfo.InvariantCulture, "{0}&{1}={2}", byondTopicSender.SanitizeString(Constants.DMTopicChangePort), byondTopicSender.SanitizeString(Constants.DMParameterData), byondTopicSender.SanitizeString(port.ToString(CultureInfo.InvariantCulture))), cancellationToken).ConfigureAwait(false) == Constants.DMResponseSuccess;
-
 		/// <inheritdoc />
-		public async Task<bool> ClosePort(CancellationToken cancellationToken)
+		public Task<bool> SetPort(ushort port, CancellationToken cancellationToken)
 		{
 			CheckDisposed();
-			if (portClosed)
-				return true;
-			if (await SetPortImpl(0, cancellationToken).ConfigureAwait(false))
-			{
-				portClosed = true;
-				return true;
-			}
-			return false;
-		}
-
-		/// <inheritdoc />
-		public async Task<bool> SetPort(ushort port, CancellationToken cancellatonToken)
-		{
-			CheckDisposed();
-			if (portClosed)
-			{
-				Task<bool> toWait;
-				lock (this)
-				{
-					if (portAssignmentTcs != null)
-					{
-						//someone was trying to change the port before us, ignore them
-						//shouldn't happen anyway, add logging here
-						logger.LogWarning("Hey uhhh, this shouldn't happen ok? Pls to tell cyberboss. SessionController.SetPort");
-						portAssignmentTcs.TrySetResult(false);
-					}
-					nextPort = port;
-					portAssignmentTcs = new TaskCompletionSource<bool>();
-					toWait = portAssignmentTcs.Task;
-				}
-				return await toWait.ConfigureAwait(false);
-			}
 
 			if (port == 0)
 				throw new ArgumentOutOfRangeException(nameof(port), port, "port must not be zero!");
-			return await SetPortImpl(port, cancellatonToken).ConfigureAwait(false);
+
+			async Task<bool> ImmediateTopicPortChange()
+			{
+				var commandResult = await SendCommand(String.Format(CultureInfo.InvariantCulture, "{0}&{1}={2}", byondTopicSender.SanitizeString(Constants.DMTopicChangePort), byondTopicSender.SanitizeString(Constants.DMParameterData), byondTopicSender.SanitizeString(port.ToString(CultureInfo.InvariantCulture))), cancellationToken).ConfigureAwait(false);
+
+				if (commandResult != Constants.DMResponseSuccess)
+				{
+					logger.LogWarning("Failed port change! DD says: {0}", commandResult);
+					return false;
+				}
+
+				return true;
+			}
+
+			lock (this)
+				if (portClosedForReboot)
+				{
+					if (portAssignmentTcs != null)
+						throw new InvalidOperationException("A port change operation is already in progress!");
+					nextPort = port;
+					portAssignmentTcs = new TaskCompletionSource<bool>();
+					return portAssignmentTcs.Task;
+				}
+				else
+					return ImmediateTopicPortChange();
 		}
 
 		/// <inheritdoc />
@@ -417,8 +431,8 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		{
 			if (RebootState == newRebootState)
 				return true;
-
-			return await SendCommand(String.Format(CultureInfo.InvariantCulture, "{0}&{1}={2}", Constants.DMTopicChangeReboot, Constants.DMParameterData, (int)newRebootState), cancellationToken).ConfigureAwait(false) == Constants.DMResponseSuccess;
+			reattachInformation.RebootState = newRebootState;
+			return await SendCommand(String.Format(CultureInfo.InvariantCulture, "{0}&{1}={2}", byondTopicSender.SanitizeString(Constants.DMTopicChangeReboot), byondTopicSender.SanitizeString(Constants.DMParameterData), (int)newRebootState), cancellationToken).ConfigureAwait(false) == Constants.DMResponseSuccess;
 		}
 
 		/// <inheritdoc />

src/Tgstation.Server.Host/Components/Watchdog/SessionControllerFactory.cs

@@ -180,7 +180,8 @@ namespace Tgstation.Server.Host.Components.Watchdog
 						//more sanitization here cause it uses the same scheme
 						var parameters = String.Format(CultureInfo.InvariantCulture, "{2}={0}&{3}={1}", byondTopicSender.SanitizeString(application.Version.ToString()), byondTopicSender.SanitizeString(interopJsonFile), byondTopicSender.SanitizeString(Constants.DMParamHostVersion), byondTopicSender.SanitizeString(Constants.DMParamInfoJson));
 
-						var arguments = String.Format(CultureInfo.InvariantCulture, "{0} -port {1} {2}-close -{3} -verbose -public -params \"{4}\"",
+						//important to run on all ports to allow port changing
+						var arguments = String.Format(CultureInfo.InvariantCulture, "{0} -port {1} -ports 1-65535 {2}-close -{3} -verbose -public -params \"{4}\"",
 							dmbProvider.DmbName,
 							primaryPort ? launchParameters.PrimaryPort : launchParameters.SecondaryPort,
 							launchParameters.AllowWebClient.Value ? "-webclient " : String.Empty,

src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs

@@ -22,7 +22,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// <summary>
 		/// The time in seconds to wait from starting <see cref="alphaServer"/> to start <see cref="bravoServer"/>. Does not take responsiveness into account
 		/// </summary>
-		const int AlphaBravoStartupSeperationInterval = 3;
+		const int AlphaBravoStartupSeperationInterval = 10;
 
 		/// <inheritdoc />
 		public bool Running { get; private set; }
@@ -189,35 +189,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			bravoServer = null;
 			Running = false;
 		}
-
-		/// <summary>
-		/// Implementation of <see cref="Restart(bool, CancellationToken)"/>. Does not lock <see cref="semaphore"/>
-		/// </summary>
-		async Task<WatchdogLaunchResult> RestartNoLock(bool graceful, CancellationToken cancellationToken)
-		{
-			var running = Running;
-			if (!graceful || !running)
-			{
-				Task chatTask;
-				if (running)
-				{
-					chatTask = chat.SendWatchdogMessage("Manual restart triggered...", cancellationToken);
-					await TerminateNoLock(false, false, cancellationToken).ConfigureAwait(false);
-				}
-				else
-					chatTask = Task.CompletedTask;
-				var result = await LaunchNoLock(true, !running, false, cancellationToken).ConfigureAwait(false);
-				await chatTask.ConfigureAwait(false);
-				return result;
-			}
-			var toReboot = AlphaIsActive ? alphaServer : bravoServer;
-			var other = AlphaIsActive ? bravoServer : alphaServer;
-			if (toReboot != null)
-				//todo, log the result
-				await toReboot.SetRebootState(Components.Watchdog.RebootState.Restart, cancellationToken).ConfigureAwait(false);
-			return null;
-		}		
-
+		
 		/// <summary>
 		/// Implementation of <see cref="Terminate(bool, CancellationToken)"/>. Does not lock <see cref="semaphore"/>
 		/// </summary>
@@ -251,7 +223,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// <returns>A <see cref="Task"/> representing the running operation</returns>
 		async Task HandlerMonitorWakeup(MonitorActivationReason activationReason, MonitorState monitorState, CancellationToken cancellationToken)
 		{
-			logger.LogInformation("Monitor activation. Reason: {0}", activationReason);
+			logger.LogDebug("Monitor activation. Reason: {0}", activationReason);
 
 			//returns true if the inactive server can't be used immediately
 			bool FullRestartDeadInactive()
@@ -265,10 +237,11 @@ namespace Tgstation.Server.Host.Components.Watchdog
 				return false;
 			};
 
-			//trys to set inactive server's port to the private port
+			//trys to set inactive server's port to the public port
+			//doesn't handle closing active server's port
 			async Task<bool> MakeInactiveActive()
 			{
-				logger.LogInformation("Setting inactive server to port {0}...", ActiveLaunchParameters.PrimaryPort.Value);
+				logger.LogDebug("Setting inactive server to port {0}...", ActiveLaunchParameters.PrimaryPort.Value);
 				var result = await monitorState.InactiveServer.SetPort(ActiveLaunchParameters.PrimaryPort.Value, cancellationToken).ConfigureAwait(false);
 
 				if (!result)
@@ -278,10 +251,6 @@ namespace Tgstation.Server.Host.Components.Watchdog
 					return false;
 				}
 
-				// should always be set for InactiveServer
-				monitorState.InactiveServer.ClosePortOnReboot = false;
-				monitorState.ActiveServer.ClosePortOnReboot = true;
-
 				//inactive server should always be using active launch parameters
 				LastLaunchParameters = ActiveLaunchParameters;
 
@@ -309,7 +278,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 				}
 				catch (Exception e)
 				{
-					logger.LogError("Exception occurred while recreating server! Attempting backup strategy of running DMB of running server! Exception: {0}", e.ToString());
+					logger.LogError("Error occurred while recreating server! Attempting backup strategy of running DMB of running server! Exception: {0}", e.ToString());
 					//ahh jeez, what do we do here?
 					//this is our fault, so it should never happen but
 					//idk maybe a database error while handling the newest dmb?
@@ -342,8 +311,6 @@ namespace Tgstation.Server.Host.Components.Watchdog
 
 				logger.LogInformation("Successfully relaunched inactive server!");
 				monitorState.RebootingInactiveServer = true;
-				// should always be set for InactiveServer
-				monitorState.InactiveServer.ClosePortOnReboot = false;
 				return usedMostRecentDmb;
 			}
 
@@ -360,36 +327,37 @@ namespace Tgstation.Server.Host.Components.Watchdog
 				{
 					monitorState.ActiveServer.ClosePortOnReboot = false;
 					if (monitorState.InactiveServerHasStagedDmb && !usedLatestDmb)
-						monitorState.InactiveServerHasStagedDmb = false;    //don't try to load it again though
+						monitorState.InactiveServerHasStagedDmb = false;	//don't try to load it again though
 				}
 			};
 
+			string ExitWord(ISessionController controller) => controller.TerminationWasRequested ? "exited" : "crashed";
+
 			//reason handling
 			switch (activationReason)
 			{
 				case MonitorActivationReason.ActiveServerCrashed:
 					if(monitorState.ActiveServer.RebootState == Components.Watchdog.RebootState.Shutdown)
 					{
-						await chat.SendWatchdogMessage("Active server crashed or exited! Exiting due to graceful termination request...", cancellationToken).ConfigureAwait(false);
+						await chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Active server {0}! Exiting due to graceful termination request...", ExitWord(monitorState.ActiveServer)), cancellationToken).ConfigureAwait(false);
 						monitorState.NextAction = MonitorAction.Exit;
 						break;
 					}
 
 					if (FullRestartDeadInactive())
 					{
-						await chat.SendWatchdogMessage("Active server crashed or exited! Inactive server unable to online!", cancellationToken).ConfigureAwait(false);
+						await chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Active server {0}! Inactive server unable to online!", ExitWord(monitorState.ActiveServer)), cancellationToken).ConfigureAwait(false);
 						break;
 					}
 
-					await chat.SendWatchdogMessage("Active server crashed or exited! Onlining inactive server...", cancellationToken).ConfigureAwait(false);
+					await chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Active server {0}! Onlining inactive server...", ExitWord(monitorState.ActiveServer)), cancellationToken).ConfigureAwait(false);
 					if (!await MakeInactiveActive().ConfigureAwait(false))
 						break;
-					
-					monitorState.ActiveServer.ClosePortOnReboot = false;
+
 					await UpdateAndRestartInactiveServer(true).ConfigureAwait(false);
 					break;
 				case MonitorActivationReason.InactiveServerCrashed:
-					await chat.SendWatchdogMessage("Inactive server crashed or exited! Rebooting...", cancellationToken).ConfigureAwait(false);
+					await chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Inactive server {0}! Rebooting...", ExitWord(monitorState.InactiveServer)), cancellationToken).ConfigureAwait(false);
 					await UpdateAndRestartInactiveServer(false).ConfigureAwait(false);
 					break;
 				case MonitorActivationReason.ActiveServerRebooted:
@@ -423,22 +391,35 @@ namespace Tgstation.Server.Host.Components.Watchdog
 						//need a new launch in ActiveServer
 						restartOnceSwapped = true;
 
-					if (!await MakeInactiveActive().ConfigureAwait(false))
+					if (restartOnceSwapped && !monitorState.ActiveServer.ClosePortOnReboot)
+						//we need to manually restart active server
+						//it won't listen to us right now so just kill it
+						monitorState.ActiveServer.Dispose();
+
+					if ((!restartOnceSwapped && !monitorState.ActiveServer.ClosePortOnReboot) || !await MakeInactiveActive().ConfigureAwait(false))
 						break;
 
-					if(!restartOnceSwapped)
+					monitorState.ActiveServer.ClosePortOnReboot = true;
+
+					if (!restartOnceSwapped)
+					{
+						monitorState.InactiveServer.ClosePortOnReboot = false;
 						//try to reopen inactive server on the private port so it's not pinging all the time
 						//failing that, just reboot it
 						restartOnceSwapped = !await monitorState.InactiveServer.SetPort(ActiveLaunchParameters.SecondaryPort.Value, cancellationToken).ConfigureAwait(false);
+					}
 
-					if (restartOnceSwapped) //for one reason or another, 
+					if (restartOnceSwapped) //for one reason or another
 						await UpdateAndRestartInactiveServer(true).ConfigureAwait(false);	//break because worse case, active server is still booting
 					else
+					{
+						monitorState.InactiveServer.ClosePortOnReboot = false;
 						monitorState.NextAction = MonitorAction.Break;
+					}
 					break;
 				case MonitorActivationReason.InactiveServerRebooted:
 					monitorState.RebootingInactiveServer = true;
-					monitorState.InactiveServer.ResetRebootState();   //the DMAPI has already done this internally
+					monitorState.InactiveServer.ResetRebootState();
 					monitorState.ActiveServer.ClosePortOnReboot = false;
 					monitorState.NextAction = MonitorAction.Continue;
 					break;
@@ -465,11 +446,12 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// <returns>A <see cref="Task"/> representing the running operation</returns>
 		async Task MonitorLifetimes(CancellationToken cancellationToken)
 		{
-			logger.LogDebug("Entered MonitorLifetimes");
+			logger.LogTrace("Entered MonitorLifetimes");
 			var iteration = 1;
 			for(var monitorState = new MonitorState(); monitorState.NextAction != MonitorAction.Exit; ++iteration)
 			{
-				logger.LogDebug("New iteration of monitor loop");
+				monitorState.NextAction = MonitorAction.Continue;
+				logger.LogDebug("Iteration {0} of monitor loop", iteration);
 				try
 				{
 					if(AlphaIsActive)
@@ -485,6 +467,11 @@ namespace Tgstation.Server.Host.Components.Watchdog
 					monitorState.ActiveServer = AlphaIsActive ? alphaServer : bravoServer;
 					monitorState.InactiveServer = AlphaIsActive ? bravoServer : alphaServer;
 
+					if (monitorState.ActiveServer.ClosePortOnReboot)
+						logger.LogDebug("Active server will close port on reboot");
+					if (monitorState.InactiveServer.ClosePortOnReboot)
+						logger.LogDebug("Inactive server will close port on reboot");
+
 					var activeServerLifetime = monitorState.ActiveServer.Lifetime;
 					var inactiveServerLifetime = monitorState.InactiveServer.Lifetime;
 					var activeServerReboot = monitorState.ActiveServer.OnReboot;
@@ -554,7 +541,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 
 						//writeback alphaServer and bravoServer
 						alphaServer = AlphaIsActive ? monitorState.ActiveServer : monitorState.InactiveServer;
-						bravoServer = AlphaIsActive ? monitorState.ActiveServer : monitorState.InactiveServer;
+						bravoServer = !AlphaIsActive ? monitorState.ActiveServer : monitorState.InactiveServer;
 					}
 
 					//full reboot required
@@ -563,25 +550,28 @@ namespace Tgstation.Server.Host.Components.Watchdog
 						logger.LogDebug("Next state action is to restart");
 						DisposeAndNullControllers();
 						chatTask = chat.SendWatchdogMessage("Restarting entirely due to complications...", cancellationToken);
-					}
 
-					for (var retryAttempts = 1; monitorState.NextAction == MonitorAction.Restart; ++retryAttempts)
-					{
-						WatchdogLaunchResult result;
-						using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken).ConfigureAwait(false))
+						for (var retryAttempts = 1; monitorState.NextAction == MonitorAction.Restart; ++retryAttempts)
 						{
-							result = await LaunchNoLock(false, false, false, cancellationToken).ConfigureAwait(false);
-							if (Running)
-								monitorState = new MonitorState();  //clean the slate
-						}
+							WatchdogLaunchResult result;
+							using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken).ConfigureAwait(false))
+							{
+								result = await LaunchNoLock(false, false, false, cancellationToken).ConfigureAwait(false);
+								if (Running)
+								{
+									logger.LogDebug("Relaunch successful, resetting monitor state...");
+									monitorState = new MonitorState();  //clean the slate
+								}
+							}
 
-						await chatTask.ConfigureAwait(false);
-						if(!Running)
-						{
-							logger.LogWarning("Failed to automatically restart the watchdog! Alpha: {0}; Bravo: {1}", result.Alpha.ToString(), result.Bravo.ToString());
-							var retryDelay = Math.Min(Math.Pow(2, retryAttempts), 3600); //max of one hour
-							chatTask = chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Failed to restart watchdog (Attempt: {0}), retrying in {1} seconds...", retryAttempts, retryDelay), cancellationToken);
-							await Task.WhenAll(Task.Delay((int)retryDelay, cancellationToken), chatTask).ConfigureAwait(false);
+							await chatTask.ConfigureAwait(false);
+							if (!Running)
+							{
+								logger.LogWarning("Failed to automatically restart the watchdog! Alpha: {0}; Bravo: {1}", result.Alpha.ToString(), result.Bravo.ToString());
+								var retryDelay = Math.Min(Math.Pow(2, retryAttempts), 3600); //max of one hour
+								chatTask = chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Failed to restart watchdog (Attempt: {0}), retrying in {1} seconds...", retryAttempts, retryDelay), cancellationToken);
+								await Task.WhenAll(Task.Delay((int)retryDelay, cancellationToken), chatTask).ConfigureAwait(false);
+							}
 						}
 					}
 				}
@@ -708,6 +698,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 						//both servers are now running, alpha is the active server, huzzah
 						AlphaIsActive = doReattach ? reattachInfo.AlphaIsActive : true;
 						LastLaunchResult = alphaLrt.Result;
+						logger.LogInformation("Launched servers successfully");
 						Running = true;
 
 						if (startMonitor)
@@ -760,7 +751,31 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		public async Task<WatchdogLaunchResult> Restart(bool graceful, CancellationToken cancellationToken)
 		{
 			using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken).ConfigureAwait(false))
-				return await RestartNoLock(graceful, cancellationToken).ConfigureAwait(false);
+			{
+				if (!graceful || !Running)
+				{
+					Task chatTask;
+					if (Running)
+					{
+						chatTask = chat.SendWatchdogMessage("Manual restart triggered...", cancellationToken);
+						await TerminateNoLock(false, false, cancellationToken).ConfigureAwait(false);
+					}
+					else
+						chatTask = Task.CompletedTask;
+					var result = await LaunchNoLock(true, !Running, false, cancellationToken).ConfigureAwait(false);
+					await chatTask.ConfigureAwait(false);
+					return result;
+				}
+				var toReboot = AlphaIsActive ? alphaServer : bravoServer;
+				var other = AlphaIsActive ? bravoServer : alphaServer;
+				if (toReboot != null)
+				{
+					if (!await toReboot.SetRebootState(Components.Watchdog.RebootState.Restart, cancellationToken).ConfigureAwait(false))
+						logger.LogWarning("Unable to send reboot state change event!");
+						
+				}
+				return null;
+			}
 		}
 
 		/// <inheritdoc />

src/Tgstation.Server.Host/Core/JobManager.cs

@@ -94,8 +94,8 @@ namespace Tgstation.Server.Host.Core
 					}
 					catch (Exception e)
 					{
-						logger.LogDebug("Job {0} exited with error! Exception: {1}", job.Id, e);
 						job.ExceptionDetails = e is JobException ? e.Message : e.ToString();
+						logger.LogDebug("Job {0} exited with error! Exception: {1}", job.Id, job.ExceptionDetails);
 					}
 					job.StoppedAt = DateTimeOffset.Now;
 					await databaseContext.Save(default).ConfigureAwait(false);