Merge pull request #1551 from tgstation/1550-FixAgeOld [TGSDeploy]

If the DMAPI is available, delay non-basic watchdog swaps until TgsReboot() is called

build/Dockerfile

@@ -43,6 +43,8 @@ COPY tgstation-server.sln ./
 COPY src/Tgstation.Server.Host.Console/Tgstation.Server.Host.Console.csproj src/Tgstation.Server.Host.Console/
 COPY src/Tgstation.Server.Host.Watchdog/Tgstation.Server.Host.Watchdog.csproj src/Tgstation.Server.Host.Watchdog/
 COPY src/Tgstation.Server.Api/Tgstation.Server.Api.csproj src/Tgstation.Server.Api/
+COPY src/Tgstation.Server.Common/Tgstation.Server.Common.csproj src/Tgstation.Server.Common/
+COPY src/Tgstation.Server.Host.Common/Tgstation.Server.Host.Common.csproj src/Tgstation.Server.Host.Common/
 
 RUN dotnet restore -nowarn:MSB3202,nu1503 -p:RestoreUseSkipNonexistentTargets=false
 

build/Version.props

@@ -3,7 +3,7 @@
   <!-- Integration tests will ensure they match across the board -->
   <Import Project="ControlPanelVersion.props" />
   <PropertyGroup>
-    <TgsCoreVersion>5.12.5</TgsCoreVersion>
+    <TgsCoreVersion>5.12.6</TgsCoreVersion>
     <TgsConfigVersion>4.6.0</TgsConfigVersion>
     <TgsApiVersion>9.10.2</TgsApiVersion>
     <TgsApiLibraryVersion>10.4.1</TgsApiLibraryVersion>

src/Tgstation.Server.Host/Components/Deployment/SwappableDmbProvider.cs

@@ -26,6 +26,11 @@ namespace Tgstation.Server.Host.Components.Deployment
 		/// <inheritdoc />
 		public CompileJob CompileJob => baseProvider.CompileJob;
 
+		/// <summary>
+		/// If <see cref="MakeActive(CancellationToken)"/> has been run.
+		/// </summary>
+		public bool Swapped => swapped != 0;
+
 		/// <summary>
 		/// The <see cref="IDmbProvider"/> we are swapping for.
 		/// </summary>
@@ -41,6 +46,11 @@ namespace Tgstation.Server.Host.Components.Deployment
 		/// </summary>
 		readonly ISymlinkFactory symlinkFactory;
 
+		/// <summary>
+		/// Backing field for <see cref="Swapped"/>.
+		/// </summary>
+		volatile int swapped;
+
 		/// <summary>
 		/// Initializes a new instance of the <see cref="SwappableDmbProvider"/> class.
 		/// </summary>
@@ -67,9 +77,14 @@ namespace Tgstation.Server.Host.Components.Deployment
 		/// <returns>A <see cref="Task"/> representing the running operation.</returns>
 		public async Task MakeActive(CancellationToken cancellationToken)
 		{
-			// Note this comment from TGS3:
-			// These next two lines should be atomic but this is the best we can do
-			await ioManager.DeleteDirectory(LiveGameDirectory, cancellationToken);
+			if (Interlocked.Exchange(ref swapped, 1) != 0)
+				throw new InvalidOperationException("Already swapped!");
+
+			if (symlinkFactory.SymlinkedDirectoriesAreDeletedAsFiles)
+				await ioManager.DeleteFile(LiveGameDirectory, cancellationToken);
+			else
+				await ioManager.DeleteDirectory(LiveGameDirectory, cancellationToken);
+
 			await symlinkFactory.CreateSymbolicLink(
 				ioManager.ResolvePath(baseProvider.Directory),
 				ioManager.ResolvePath(LiveGameDirectory),

src/Tgstation.Server.Host/Components/Session/ISessionController.cs

@@ -49,11 +49,21 @@ namespace Tgstation.Server.Host.Components.Session
 		/// </summary>
 		bool ClosePortOnReboot { get; set; }
 
+		/// <summary>
+		/// If the <see cref="ISessionController"/> is currently processing a bridge request from TgsReboot().
+		/// </summary>
+		bool ProcessingRebootBridgeRequest { get; }
+
 		/// <summary>
 		/// The current <see cref="RebootState"/>.
 		/// </summary>
 		RebootState RebootState { get; }
 
+		/// <summary>
+		/// A <see cref="Task"/> that completes when the server calls /world/TgsNew().
+		/// </summary>
+		Task OnStartup { get; }
+
 		/// <summary>
 		/// A <see cref="Task"/> that completes when the server calls /world/TgsReboot().
 		/// </summary>
@@ -113,6 +123,7 @@ namespace Tgstation.Server.Host.Components.Session
 		/// Replace the <see cref="IDmbProvider"/> in use with a given <paramref name="newProvider"/>, disposing the old one.
 		/// </summary>
 		/// <param name="newProvider">The new <see cref="IDmbProvider"/>.</param>
-		void ReplaceDmbProvider(IDmbProvider newProvider);
+		/// <returns>An <see cref="IDisposable"/> to be disposed once certain that the original <see cref="IDmbProvider"/> is no longer in use.</returns>
+		IDisposable ReplaceDmbProvider(IDmbProvider newProvider);
 	}
 }

src/Tgstation.Server.Host/Components/Session/SessionController.cs

@@ -69,6 +69,9 @@ namespace Tgstation.Server.Host.Components.Session
 		/// <inheritdoc />
 		public Task<int> Lifetime { get; }
 
+		/// <inheritdoc />
+		public Task OnStartup => startupTcs.Task;
+
 		/// <inheritdoc />
 		public Task OnReboot => rebootTcs.Task;
 
@@ -78,6 +81,9 @@ namespace Tgstation.Server.Host.Components.Session
 		/// <inheritdoc />
 		public bool DMApiAvailable => ReattachInformation.Dmb.CompileJob.DMApiVersion?.Major == DMApiConstants.InteropVersion.Major;
 
+		/// <inheritdoc />
+		public bool ProcessingRebootBridgeRequest => rebootBridgeRequestsProcessing > 0;
+
 		/// <summary>
 		/// The up to date <see cref="Session.ReattachInformation"/>.
 		/// </summary>
@@ -143,20 +149,35 @@ namespace Tgstation.Server.Host.Components.Session
 		/// </summary>
 		TaskCompletionSource<bool> portAssignmentTcs;
 
+		/// <summary>
+		/// The <see cref="TaskCompletionSource"/> that completes when DD sends a valid startup bridge request.
+		/// </summary>
+		volatile TaskCompletionSource startupTcs;
+
+		/// <summary>
+		/// The <see cref="TaskCompletionSource"/> that completes when DD tells us about a reboot.
+		/// </summary>
+		volatile TaskCompletionSource rebootTcs;
+
+		/// <summary>
+		/// The <see cref="TaskCompletionSource"/> that completes when DD tells us it's primed.
+		/// </summary>
+		volatile TaskCompletionSource primeTcs;
+
+		/// <summary>
+		/// The number of currently active calls to <see cref="ProcessBridgeRequest(BridgeParameters, CancellationToken)"/> from TgsReboot().
+		/// </summary>
+		volatile uint rebootBridgeRequestsProcessing;
+
 		/// <summary>
 		/// The port to assign DreamDaemon when it queries for it.
 		/// </summary>
 		ushort? nextPort;
 
 		/// <summary>
-		/// The <see cref="TaskCompletionSource"/> that completes when DD tells us about a reboot.
+		/// The <see cref="ApiValidationStatus"/> for the <see cref="SessionController"/>.
 		/// </summary>
-		TaskCompletionSource rebootTcs;
-
-		/// <summary>
-		/// The <see cref="TaskCompletionSource"/> that completes when DD tells us it's primed.
-		/// </summary>
-		TaskCompletionSource primeTcs;
+		ApiValidationStatus apiValidationStatus;
 
 		/// <summary>
 		/// If we know DreamDaemon currently has it's port closed.
@@ -168,11 +189,6 @@ namespace Tgstation.Server.Host.Components.Session
 		/// </summary>
 		bool disposed;
 
-		/// <summary>
-		/// The <see cref="ApiValidationStatus"/> for the <see cref="SessionController"/>.
-		/// </summary>
-		ApiValidationStatus apiValidationStatus;
-
 		/// <summary>
 		/// If <see cref="process"/> should be kept alive instead.
 		/// </summary>
@@ -234,6 +250,7 @@ namespace Tgstation.Server.Host.Components.Session
 			apiValidationStatus = ApiValidationStatus.NeverValidated;
 			released = false;
 
+			startupTcs = new TaskCompletionSource();
 			rebootTcs = new TaskCompletionSource();
 			primeTcs = new TaskCompletionSource();
 
@@ -500,11 +517,11 @@ namespace Tgstation.Server.Host.Components.Session
 		public void Resume() => process.Resume();
 
 		/// <inheritdoc />
-		public void ReplaceDmbProvider(IDmbProvider dmbProvider)
+		public IDisposable ReplaceDmbProvider(IDmbProvider dmbProvider)
 		{
 			var oldDmb = ReattachInformation.Dmb;
 			ReattachInformation.Dmb = dmbProvider ?? throw new ArgumentNullException(nameof(dmbProvider));
-			oldDmb.Dispose();
+			return oldDmb;
 		}
 
 		/// <inheritdoc />
@@ -639,9 +656,7 @@ namespace Tgstation.Server.Host.Components.Session
 						parsedChannels);
 					break;
 				case BridgeCommandType.Prime:
-					var oldPrimeTcs = primeTcs;
-					primeTcs = new TaskCompletionSource();
-					oldPrimeTcs.SetResult();
+					Interlocked.Exchange(ref primeTcs, new TaskCompletionSource()).SetResult();
 					break;
 				case BridgeCommandType.Kill:
 					Logger.LogInformation("Bridge requested process termination!");
@@ -722,18 +737,26 @@ namespace Tgstation.Server.Host.Components.Session
 
 					// Load custom commands
 					chatTrackingContext.CustomCommands = parameters.CustomCommands;
+					Interlocked.Exchange(ref startupTcs, new TaskCompletionSource()).SetResult();
 					break;
 				case BridgeCommandType.Reboot:
-					if (ClosePortOnReboot)
+					Interlocked.Increment(ref rebootBridgeRequestsProcessing);
+					try
 					{
-						chatTrackingContext.Active = false;
-						response.NewPort = 0;
-						portClosedForReboot = true;
+						if (ClosePortOnReboot)
+						{
+							chatTrackingContext.Active = false;
+							response.NewPort = 0;
+							portClosedForReboot = true;
+						}
+
+						Interlocked.Exchange(ref rebootTcs, new TaskCompletionSource()).SetResult();
+					}
+					finally
+					{
+						Interlocked.Decrement(ref rebootBridgeRequestsProcessing);
 					}
 
-					var oldRebootTcs = rebootTcs;
-					rebootTcs = new TaskCompletionSource();
-					oldRebootTcs.SetResult();
 					break;
 				case BridgeCommandType.Chunk:
 					return await ProcessChunk<BridgeParameters, BridgeResponse>(ProcessBridgeCommand, BridgeError, parameters.Chunk, cancellationToken);

src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs

@@ -286,6 +286,7 @@ namespace Tgstation.Server.Host.Components.Session
 				await CheckPagerIsNotRunning(cancellationToken);
 
 				string outputFilePath = null;
+				var preserveLogFile = true;
 				if (launchParameters.LogOutput.Value)
 				{
 					var now = DateTimeOffset.UtcNow;
@@ -299,7 +300,10 @@ namespace Tgstation.Server.Host.Components.Session
 					logger.LogInformation("Logging DreamDaemon output to {path}...", outputFilePath);
 				}
 				else if (!byondLock.SupportsCli)
+				{
 					outputFilePath = gameIOManager.ConcatPath(dmbProvider.Directory, $"{Guid.NewGuid()}.dd.log");
+					preserveLogFile = false;
+				}
 
 				var accessIdentifier = cryptographySuite.GetSecureString();
 
@@ -352,9 +356,12 @@ namespace Tgstation.Server.Host.Components.Session
 							assemblyInformationProvider,
 							asyncDelayer,
 							loggerFactory.CreateLogger<SessionController>(),
-							() => !launchParameters.LogOutput.Value
-								? LogDDOutput(process, outputFilePath, byondLock.SupportsCli, default) // DCT: None available
-								: Task.CompletedTask,
+							() => LogDDOutput(
+								process,
+								outputFilePath,
+								byondLock.SupportsCli,
+								preserveLogFile,
+								default), // DCT: None available
 							launchParameters.StartupTimeout,
 							false,
 							apiValidate);
@@ -562,18 +569,20 @@ namespace Tgstation.Server.Host.Components.Session
 		/// Attempts to log DreamDaemon output.
 		/// </summary>
 		/// <param name="process">The DreamDaemon <see cref="IProcess"/>.</param>
-		/// <param name="outputFilePath">The path to the DreamDaemon log file. Will be deleted.</param>
+		/// <param name="outputFilePath">The path to the DreamDaemon log file. Will be deleted if <paramref name="preserveFile"/> is <see langword="false"/>.</param>
 		/// <param name="cliSupported">If DreamDaemon was launched with CLI capabilities.</param>
+		/// <param name="preserveFile">If <see langword="false"/>, <paramref name="outputFilePath"/> will be deleted.</param>
 		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
 		/// <returns>A <see cref="Task"/> representing the running operation.</returns>
-		async Task LogDDOutput(IProcess process, string outputFilePath, bool cliSupported, CancellationToken cancellationToken)
+		async Task LogDDOutput(IProcess process, string outputFilePath, bool cliSupported, bool preserveFile, CancellationToken cancellationToken)
 		{
 			try
 			{
-				string ddOutput;
+				string ddOutput = null;
 				if (cliSupported)
 					ddOutput = await process.GetCombinedOutput(cancellationToken);
-				else
+
+				if (ddOutput == null)
 					try
 					{
 						var dreamDaemonLogBytes = await gameIOManager.ReadAllBytes(
@@ -584,14 +593,16 @@ namespace Tgstation.Server.Host.Components.Session
 					}
 					finally
 					{
-						try
-						{
-							await gameIOManager.DeleteFile(outputFilePath, cancellationToken);
-						}
-						catch (Exception ex)
-						{
-							logger.LogWarning(ex, "Failed to delete DreamDaemon log file {outputFilePath}!", outputFilePath);
-						}
+						if (!preserveFile)
+							try
+							{
+								logger.LogTrace("Deleting temporary log file {path}...", outputFilePath);
+								await gameIOManager.DeleteFile(outputFilePath, cancellationToken);
+							}
+							catch (Exception ex)
+							{
+								logger.LogWarning(ex, "Failed to delete DreamDaemon log file {outputFilePath}!", outputFilePath);
+							}
 					}
 
 				logger.LogTrace(

src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs

@@ -143,21 +143,27 @@ namespace Tgstation.Server.Host.Components.Watchdog
 					gracefulRebootRequired = false;
 					Server.ResetRebootState();
 
-					await HandleEvent(EventType.WorldReboot, Enumerable.Empty<string>(), false, cancellationToken);
-
-					switch (rebootState)
+					var eventTask = HandleEvent(EventType.WorldReboot, Enumerable.Empty<string>(), false, cancellationToken);
+					try
 					{
-						case Session.RebootState.Normal:
-							return await HandleNormalReboot(cancellationToken);
-						case Session.RebootState.Restart:
-							return MonitorAction.Restart;
-						case Session.RebootState.Shutdown:
-							// graceful shutdown time
-							Chat.QueueWatchdogMessage(
-								"Active server rebooted! Shutting down due to graceful termination request...");
-							return MonitorAction.Exit;
-						default:
-							throw new InvalidOperationException($"Invalid reboot state: {rebootState}");
+						switch (rebootState)
+						{
+							case Session.RebootState.Normal:
+								return await HandleNormalReboot(cancellationToken);
+							case Session.RebootState.Restart:
+								return MonitorAction.Restart;
+							case Session.RebootState.Shutdown:
+								// graceful shutdown time
+								Chat.QueueWatchdogMessage(
+									"Active server rebooted! Shutting down due to graceful termination request...");
+								return MonitorAction.Exit;
+							default:
+								throw new InvalidOperationException($"Invalid reboot state: {rebootState}");
+						}
+					}
+					finally
+					{
+						await eventTask;
 					}
 
 				case MonitorActivationReason.ActiveLaunchParametersUpdated:
@@ -170,6 +176,8 @@ namespace Tgstation.Server.Host.Components.Watchdog
 				case MonitorActivationReason.ActiveServerPrimed:
 					await HandleEvent(EventType.WorldPrime, Enumerable.Empty<string>(), false, cancellationToken);
 					break;
+				case MonitorActivationReason.ActiveServerStartup:
+					break; // unused in BasicWatchdog
 				case MonitorActivationReason.Heartbeat:
 				default:
 					throw new InvalidOperationException($"Invalid activation reason: {reason}");

src/Tgstation.Server.Host/Components/Watchdog/MonitorActivationReason.cs

@@ -34,5 +34,11 @@
 		/// Server primed.
 		/// </summary>
 		ActiveServerPrimed,
+
+		/// <summary>
+		/// Server started.
+		/// </summary>
+		/// <remarks>The monitor misses the first startup of a session.</remarks>
+		ActiveServerStartup,
 	}
 }

src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs

@@ -1,5 +1,4 @@
-using System;
-using System.Threading;
+using System.Threading;
 using System.Threading.Tasks;
 
 using Microsoft.Extensions.Logging;
@@ -22,11 +21,6 @@ namespace Tgstation.Server.Host.Components.Watchdog
 	/// </summary>
 	sealed class PosixWatchdog : WindowsWatchdog
 	{
-		/// <summary>
-		/// If the swappable game directory is currently a rename of the compile job.
-		/// </summary>
-		IDmbProvider hardLinkedDmb;
-
 		/// <summary>
 		/// Initializes a new instance of the <see cref="PosixWatchdog"/> class.
 		/// </summary>
@@ -84,77 +78,10 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		}
 
 		/// <inheritdoc />
-		protected override Task ApplyInitialDmb(CancellationToken cancellationToken) => Task.CompletedTask;
-
-		/// <inheritdoc />
-		protected override async Task InitialLink(CancellationToken cancellationToken)
+		protected override Task ApplyInitialDmb(CancellationToken cancellationToken)
 		{
-			// The logic to check for an active live directory is in SwappableDmbProvider, so we just do it again here for safety
-			Logger.LogTrace("Hard linking compile job...");
-
-			// Symlinks are counted as a file on linux??
-			if (await GameIOManager.DirectoryExists(ActiveSwappable.Directory, cancellationToken))
-				await GameIOManager.DeleteDirectory(ActiveSwappable.Directory, cancellationToken);
-			else
-				await GameIOManager.DeleteFile(ActiveSwappable.Directory, cancellationToken);
-
-			// Instead of symlinking to begin with we actually rename the directory
-			await GameIOManager.MoveDirectory(
-				ActiveSwappable.CompileJob.DirectoryName.ToString(),
-				ActiveSwappable.Directory,
-				cancellationToken);
-
-			hardLinkedDmb = ActiveSwappable;
-		}
-
-		/// <inheritdoc />
-		protected override async Task InitController(Task chatTask, ReattachInformation reattachInfo, CancellationToken cancellationToken)
-		{
-			var suspended = false;
-			try
-			{
-				await base.InitController(chatTask, reattachInfo, cancellationToken);
-			}
-			finally
-			{
-				// Then we move it back and apply the symlink
-				if (hardLinkedDmb != null)
-				{
-					try
-					{
-						Logger.LogTrace("Unhardlinking compile job...");
-						Server?.Suspend();
-						suspended = true;
-						var hardLink = hardLinkedDmb.Directory;
-						var originalPosition = hardLinkedDmb.CompileJob.DirectoryName.ToString();
-						await GameIOManager.MoveDirectory(
-							hardLink,
-							originalPosition,
-							default);
-					}
-					catch (Exception ex)
-					{
-						Logger.LogError(
-							ex,
-							"Failed to un-hard link compile job #{compileJobId} ({compileJobDirectory})",
-							hardLinkedDmb.CompileJob.Id,
-							hardLinkedDmb.CompileJob.DirectoryName);
-					}
-
-					hardLinkedDmb = null;
-				}
-			}
-
-			if (reattachInfo != null)
-			{
-				Logger.LogTrace("Skipping symlink due to reattach");
-				return;
-			}
-
-			Logger.LogTrace("Symlinking compile job...");
-			await ActiveSwappable.MakeActive(cancellationToken);
-			if (suspended)
-				Server.Resume();
+			// not necessary to hold initial .dmb on Linux because of based inode deletes
+			return Task.CompletedTask;
 		}
 	}
 }

src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs

@@ -796,6 +796,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 				MonitorAction nextAction = MonitorAction.Continue;
 				Task activeServerLifetime = null,
 					activeServerReboot = null,
+					activeServerStartup = null,
 					serverPrimed = null,
 					activeLaunchParametersChanged = null,
 					newDmbAvailable = null;
@@ -825,12 +826,14 @@ namespace Tgstation.Server.Host.Components.Watchdog
 									TryUpdateTask(ref activeServerLifetime, () => controller.Lifetime);
 									TryUpdateTask(ref activeServerReboot, () => controller.OnReboot);
 									TryUpdateTask(ref serverPrimed, () => controller.OnPrime);
+									TryUpdateTask(ref activeServerStartup, () => controller.OnStartup);
 								}
 								else
 								{
 									activeServerLifetime = controller.Lifetime;
 									activeServerReboot = controller.OnReboot;
 									serverPrimed = controller.OnPrime;
+									activeServerStartup = controller.OnStartup;
 									lastController = controller;
 								}
 
@@ -862,6 +865,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 							var toWaitOn = Task.WhenAny(
 								activeServerLifetime,
 								activeServerReboot,
+								activeServerStartup,
 								heartbeat,
 								newDmbAvailable,
 								cancelTcs.Task,
@@ -908,7 +912,8 @@ namespace Tgstation.Server.Host.Components.Watchdog
 										|| CheckActivationReason(ref newDmbAvailable, MonitorActivationReason.NewDmbAvailable)
 										|| CheckActivationReason(ref activeLaunchParametersChanged, MonitorActivationReason.ActiveLaunchParametersUpdated)
 										|| CheckActivationReason(ref heartbeat, MonitorActivationReason.Heartbeat)
-										|| CheckActivationReason(ref serverPrimed, MonitorActivationReason.ActiveServerPrimed);
+										|| CheckActivationReason(ref serverPrimed, MonitorActivationReason.ActiveServerPrimed)
+										|| CheckActivationReason(ref activeServerStartup, MonitorActivationReason.ActiveServerStartup);
 
 									UpdateMonitoredTasks();
 

src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs

@@ -1,4 +1,6 @@
 using System;
+using System.Collections.Generic;
+using System.Diagnostics;
 using System.Threading;
 using System.Threading.Tasks;
 
@@ -37,11 +39,21 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// </summary>
 		readonly ISymlinkFactory symlinkFactory;
 
+		/// <summary>
+		/// <see cref="List{T}"/> of <see cref="Task"/>s that are waiting to clean up old deployments.
+		/// </summary>
+		readonly List<Task> deploymentCleanupTasks;
+
 		/// <summary>
 		/// The active <see cref="SwappableDmbProvider"/> for <see cref="WatchdogBase.ActiveLaunchParameters"/>.
 		/// </summary>
 		SwappableDmbProvider pendingSwappable;
 
+		/// <summary>
+		/// The <see cref="TaskCompletionSource"/> representing the cleanup of an unused <see cref="IDmbProvider"/>.
+		/// </summary>
+		volatile TaskCompletionSource deploymentCleanupGate;
+
 		/// <summary>
 		/// Initializes a new instance of the <see cref="WindowsWatchdog"/> class.
 		/// </summary>
@@ -98,11 +110,16 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			{
 				GameIOManager = gameIOManager ?? throw new ArgumentNullException(nameof(gameIOManager));
 				this.symlinkFactory = symlinkFactory ?? throw new ArgumentNullException(nameof(symlinkFactory));
+
+				deploymentCleanupTasks = new List<Task>();
 			}
 			catch
 			{
 				// Async dispose is for if we have controllers running, not the case here
-				DisposeAsync().AsTask().GetAwaiter().GetResult();
+				var disposeTask = DisposeAsync();
+				Debug.Assert(disposeTask.IsCompleted, "This should always be true during construction!");
+				disposeTask.GetAwaiter().GetResult();
+
 				throw;
 			}
 		}
@@ -116,6 +133,8 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			ActiveSwappable = null;
 			pendingSwappable?.Dispose();
 			pendingSwappable = null;
+
+			await DrainDeploymentCleanupTasks(true);
 		}
 
 		/// <inheritdoc />
@@ -124,8 +143,68 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			if (pendingSwappable != null)
 			{
 				var updateTask = BeforeApplyDmb(pendingSwappable.CompileJob, cancellationToken);
+
+				if (!pendingSwappable.Swapped)
+				{
+					// IMPORTANT: THE SESSIONCONTROLLER SHOULD STILL BE PROCESSING THE BRIDGE REQUEST SO WE KNOW DD IS SLEEPING
+					// OTHERWISE, IT COULD RETURN TO /world/Reboot() TOO EARLY AND LOAD THE WRONG .DMB
+					if (!Server.ProcessingRebootBridgeRequest)
+					{
+						// integration test logging will catch this
+						Logger.LogError(
+							"The reboot bridge request completed before the watchdog could suspend the server! This can lead to buggy DreamDaemon behaviour and should be reported! To ensure stability, we will need to hard reboot the server");
+						await updateTask;
+						return MonitorAction.Restart;
+					}
+
+					await PerformDmbSwap(pendingSwappable, cancellationToken);
+				}
+
+				var currentCompileJobId = Server.ReattachInformation.Dmb.CompileJob.Id;
+
+				await DrainDeploymentCleanupTasks(false);
+
+				IDisposable lingeringDeployment;
+				var localDeploymentCleanupGate = new TaskCompletionSource();
+				async Task CleanupLingeringDeployment()
+				{
+					var lingeringDeploymentExpirySeconds = ActiveLaunchParameters.StartupTimeout.Value;
+					Logger.LogDebug(
+						"Holding old deployment {compileJobId} for up to {expiry} seconds...",
+						currentCompileJobId,
+						lingeringDeploymentExpirySeconds);
+
+					var timeout = AsyncDelayer.Delay(TimeSpan.FromSeconds(lingeringDeploymentExpirySeconds), cancellationToken);
+
+					var completedTask = await Task.WhenAny(
+						localDeploymentCleanupGate.Task,
+						timeout);
+
+					var timedOut = completedTask == timeout;
+					Logger.Log(
+						timedOut
+							? LogLevel.Warning
+							: LogLevel.Trace,
+						"Releasing old deployment {compileJobId}{afterTimeout}",
+						timedOut
+							? " due to timeout!"
+							: "...");
+
+					lingeringDeployment.Dispose();
+				}
+
+				var oldDeploymentCleanupGate = Interlocked.Exchange(ref deploymentCleanupGate, localDeploymentCleanupGate);
+				oldDeploymentCleanupGate?.TrySetResult();
+
 				Logger.LogTrace("Replacing activeSwappable with pendingSwappable...");
-				Server.ReplaceDmbProvider(pendingSwappable);
+
+				lock (deploymentCleanupTasks)
+				{
+					lingeringDeployment = Server.ReplaceDmbProvider(pendingSwappable);
+					deploymentCleanupTasks.Add(
+						CleanupLingeringDeployment());
+				}
+
 				ActiveSwappable = pendingSwappable;
 				pendingSwappable = null;
 
@@ -173,23 +252,14 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			}
 
 			SwappableDmbProvider windowsProvider = null;
-			bool suspended = false;
 			try
 			{
 				windowsProvider = new SwappableDmbProvider(compileJobProvider, GameIOManager, symlinkFactory);
-
-				Logger.LogDebug("Swapping to compile job {0}...", windowsProvider.CompileJob.Id);
-				try
+				if (ActiveCompileJob.DMApiVersion == null)
 				{
-					Server.Suspend();
-					suspended = true;
+					Logger.LogWarning("Active compile job has no DMAPI! Commencing immediate .dmb swap. Note this behavior is known to be buggy in some DM code contexts. See https://github.com/tgstation/tgstation-server/issues/1550");
+					await PerformDmbSwap(windowsProvider, cancellationToken);
 				}
-				catch (Exception ex)
-				{
-					Logger.LogWarning(ex, "Exception while suspending server!");
-				}
-
-				await windowsProvider.MakeActive(cancellationToken);
 			}
 			catch (Exception ex)
 			{
@@ -199,10 +269,6 @@ namespace Tgstation.Server.Host.Components.Watchdog
 				throw;
 			}
 
-			// Let this throw hard if it fails
-			if (suspended)
-				Server.Resume();
-
 			pendingSwappable?.Dispose();
 			pendingSwappable = windowsProvider;
 		}
@@ -250,15 +316,89 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			await base.SessionStartupPersist(cancellationToken);
 		}
 
+		/// <inheritdoc />
+		protected override async Task<MonitorAction> HandleMonitorWakeup(MonitorActivationReason reason, CancellationToken cancellationToken)
+		{
+			var result = await base.HandleMonitorWakeup(reason, cancellationToken);
+			if (reason == MonitorActivationReason.ActiveServerStartup)
+				await DrainDeploymentCleanupTasks(false);
+
+			return result;
+		}
+
 		/// <summary>
 		/// Create the initial link to the live game directory using <see cref="ActiveSwappable"/>.
 		/// </summary>
 		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
 		/// <returns>A <see cref="Task"/> representing the running operation.</returns>
-		protected virtual Task InitialLink(CancellationToken cancellationToken)
+		Task InitialLink(CancellationToken cancellationToken)
 		{
 			Logger.LogTrace("Symlinking compile job...");
 			return ActiveSwappable.MakeActive(cancellationToken);
 		}
+
+		/// <summary>
+		/// Suspends the <see cref="BasicWatchdog.Server"/> and calls <see cref="SwappableDmbProvider.MakeActive(CancellationToken)"/> on a <paramref name="newProvider"/>.
+		/// </summary>
+		/// <param name="newProvider">The <see cref="SwappableDmbProvider"/> to activate.</param>
+		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
+		/// <returns>A <see cref="ValueTask"/> representing the running operation.</returns>
+		async ValueTask PerformDmbSwap(SwappableDmbProvider newProvider, CancellationToken cancellationToken)
+		{
+			Logger.LogDebug("Swapping to compile job {id}...", newProvider.CompileJob.Id);
+
+			var suspended = false;
+			var server = Server;
+			try
+			{
+				server.Suspend();
+				suspended = true;
+			}
+			catch (Exception ex)
+			{
+				Logger.LogWarning(ex, "Exception while suspending server!");
+			}
+
+			try
+			{
+				await newProvider.MakeActive(cancellationToken);
+			}
+			finally
+			{
+				// Let this throw hard if it fails
+				if (suspended)
+					server.Resume();
+			}
+		}
+
+		/// <summary>
+		/// Asynchronously drain <see cref="deploymentCleanupTasks"/>.
+		/// </summary>
+		/// <param name="blocking">If <see langword="true"/>, all <see cref="Task"/>s will be <see langword="await"/>ed. Otherwise, only <see cref="Task"/>s with <see cref="Task.IsCompleted"/> set will be <see langword="await"/>ed.</param>
+		/// <returns>A <see cref="Task"/> representing the running operation.</returns>
+		Task DrainDeploymentCleanupTasks(bool blocking)
+		{
+			Logger.LogTrace("DrainDeploymentCleanupTasks...");
+			var localDeploymentCleanupGate = Interlocked.Exchange(ref deploymentCleanupGate, null);
+			localDeploymentCleanupGate?.TrySetResult();
+
+			List<Task> localDeploymentCleanupTasks;
+			lock (deploymentCleanupTasks)
+			{
+				var totalActiveTasks = deploymentCleanupTasks.Count;
+				localDeploymentCleanupTasks = new List<Task>(totalActiveTasks);
+				for (var i = totalActiveTasks - 1; i >= 0; --i)
+				{
+					var currentTask = deploymentCleanupTasks[i];
+					if (!blocking && !currentTask.IsCompleted)
+						continue;
+
+					localDeploymentCleanupTasks.Add(currentTask);
+					deploymentCleanupTasks.RemoveAt(i);
+				}
+			}
+
+			return Task.WhenAll(localDeploymentCleanupTasks);
+		}
 	}
 }

src/Tgstation.Server.Host/IO/ISymlinkFactory.cs

@@ -8,6 +8,12 @@ namespace Tgstation.Server.Host.IO
 	/// </summary>
 	interface ISymlinkFactory
 	{
+		/// <summary>
+		/// If directory symlinks must be deleted as files would in the current environment.
+		/// </summary>
+		/// <remarks>This is because Linux symlinked directories must be deleted with <see cref="global::System.IO.File.Delete(string)"/>.</remarks>
+		bool SymlinkedDirectoriesAreDeletedAsFiles { get; }
+
 		/// <summary>
 		/// Create a symbolic link.
 		/// </summary>

src/Tgstation.Server.Host/IO/PosixSymlinkFactory.cs

@@ -12,6 +12,9 @@ namespace Tgstation.Server.Host.IO
 	/// </summary>
 	sealed class PosixSymlinkFactory : ISymlinkFactory
 	{
+		/// <inheritdoc />
+		public bool SymlinkedDirectoriesAreDeletedAsFiles => true;
+
 		/// <inheritdoc />
 		public Task CreateSymbolicLink(string targetPath, string linkPath, CancellationToken cancellationToken) => Task.Factory.StartNew(
 			() =>

src/Tgstation.Server.Host/IO/WindowsSymlinkFactory.cs

@@ -13,6 +13,9 @@ namespace Tgstation.Server.Host.IO
 	/// </summary>
 	sealed class WindowsSymlinkFactory : ISymlinkFactory
 	{
+		/// <inheritdoc />
+		public bool SymlinkedDirectoriesAreDeletedAsFiles => false;
+
 		/// <inheritdoc />
 		public Task CreateSymbolicLink(string targetPath, string linkPath, CancellationToken cancellationToken) => Task.Factory.StartNew(
 			() =>

src/Tgstation.Server.Host/System/PosixProcessFeatures.cs

@@ -48,7 +48,7 @@ namespace Tgstation.Server.Host.System
 		{
 			var result = Syscall.kill(process.Id, Signum.SIGCONT);
 			if (result != 0)
-				throw new UnixIOException(result);
+				throw new UnixIOException(Stdlib.GetLastError());
 		}
 
 		/// <inheritdoc />
@@ -56,7 +56,7 @@ namespace Tgstation.Server.Host.System
 		{
 			var result = Syscall.kill(process.Id, Signum.SIGSTOP);
 			if (result != 0)
-				throw new UnixIOException(result);
+				throw new UnixIOException(Stdlib.GetLastError());
 		}
 
 		/// <inheritdoc />

tests/DMAPI/LongRunning/Test.dm

@@ -128,6 +128,12 @@ var/run_bridge_test
 		var/list/channels = TgsChatChannelInfo()
 		return "[length(channels)]"
 
+	var/legalize_nuclear_bombs = data["shadow_wizard_money_gang"]
+	if(legalize_nuclear_bombs)
+		text2file("I expect this to remain here for a while", "kajigger.txt")
+		kajigger_test = TRUE
+		return "we love casting spells"
+
 	TgsChatBroadcast(new /datum/tgs_message_content("Recieved non-tgs topic: `[T]`"))
 
 	return "feck"
@@ -147,10 +153,18 @@ var/run_bridge_test
 	world.TgsChatBroadcast(new /datum/tgs_message_content("2/3 queued detached chat messages"))
 	world.TgsChatBroadcast(new /datum/tgs_message_content("3/3 queued detached chat messages"))
 
+var/kajigger_test = FALSE
+
 /world/Reboot(reason)
 	TgsChatBroadcast("World Rebooting")
+
+	if(kajigger_test && !fexists("kajigger.txt"))
+		FailTest("TGS STOLE MY KAJIGGER (#1548 regression)")
+
 	TgsReboot()
 
+	..()
+
 /datum/tgs_event_handler/impl/HandleEvent(event_code, ...)
 	set waitfor = FALSE
 
@@ -175,6 +189,7 @@ var/run_bridge_test
 	world.log << "Done sleep, calling Reboot"
 	world.Reboot()
 
+
 /datum/tgs_chat_command/embeds_test
 	name = "embeds_test"
 	help_text = "dumps an embed"

tests/Tgstation.Server.Host.Tests/IO/TestPostWriteHandler.cs

@@ -70,7 +70,8 @@ namespace Tgstation.Server.Host.IO.Tests
 		public void TestThrowsOnUnix()
 		{
 			if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
-				return;
+				Assert.Inconclusive("POSIX only test.");
+
 			var postWriteHandler = new PosixPostWriteHandler(Mock.Of<ILogger<PosixPostWriteHandler>>());
 			var tmpFile = Path.GetTempFileName();
 			File.Delete(tmpFile);

tests/Tgstation.Server.Host.Tests/System/TestSymlinkFactory.cs

@@ -0,0 +1,105 @@
+using System;
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+using Tgstation.Server.Host.IO;
+
+namespace Tgstation.Server.Host.System.Tests
+{
+	[TestClass]
+	public sealed class TestSymlinkFactory
+	{
+		readonly ISymlinkFactory factory = new PlatformIdentifier().IsWindows
+			? new WindowsSymlinkFactory()
+			: new PosixSymlinkFactory();
+
+		[TestMethod]
+		public async Task TestSymlinks()
+		{
+			var cancellationToken = CancellationToken.None;
+			var cwd = Path.GetTempFileName();
+			File.Delete(cwd);
+			Directory.CreateDirectory(cwd);
+			try
+			{
+				var realDir = Path.Combine(cwd, "RealDir");
+				var symDir = Path.Combine(cwd, "SymDir");
+				var realFile = Path.Combine(cwd, "RealFile.txt");
+				var symFile = Path.Combine(realDir, "RealFile.txt");
+
+				var subRealFile = Path.Combine(realDir, "test.txt");
+				var subSymFile = Path.Combine(symDir, "test.txt");
+
+				try
+				{
+					await factory.CreateSymbolicLink(subRealFile, subSymFile, cancellationToken);
+					Assert.Fail("Expected Exception!");
+				}
+				catch
+				{
+				}
+
+				Directory.CreateDirectory(realDir);
+				Directory.CreateDirectory(symDir);
+
+				await File.WriteAllBytesAsync(realFile, Array.Empty<byte>(), cancellationToken);
+				await File.WriteAllBytesAsync(symFile, Array.Empty<byte>(), cancellationToken);
+
+				try
+				{
+					await factory.CreateSymbolicLink(realFile, symFile, cancellationToken);
+					Assert.Fail("Expected Exception!");
+				}
+				catch
+				{
+				}
+
+				Directory.Delete(symDir);
+				File.Delete(symFile);
+
+				await factory.CreateSymbolicLink(realFile, symFile, cancellationToken);
+				Assert.IsTrue(File.Exists(symFile));
+				Assert.IsFalse(Directory.Exists(symFile));
+
+				await File.WriteAllTextAsync(realFile, "test", cancellationToken);
+				var symFileContents = await File.ReadAllTextAsync(symFile, cancellationToken);
+				Assert.AreEqual("test", symFileContents);
+				File.Delete(symFile);
+				File.Delete(realFile);
+
+				try
+				{
+					await factory.CreateSymbolicLink(realDir, symDir, cancellationToken);
+
+					Assert.IsFalse(File.Exists(symDir));
+					Assert.IsTrue(Directory.Exists(symDir));
+
+					await File.WriteAllTextAsync(subRealFile, "test", cancellationToken);
+					Assert.IsTrue(File.Exists(subSymFile));
+
+					File.Delete(subSymFile);
+					Assert.IsFalse(File.Exists(subRealFile));
+				}
+				finally
+				{
+					if (factory.SymlinkedDirectoriesAreDeletedAsFiles)
+						File.Delete(symDir);
+					else
+						Directory.Delete(symDir);
+				}
+
+				if (factory.SymlinkedDirectoriesAreDeletedAsFiles)
+					Assert.IsFalse(File.Exists(symDir));
+				else
+					Assert.IsFalse(Directory.Exists(symDir));
+			}
+			finally
+			{
+				Directory.Delete(cwd, true);
+			}
+		}
+	}
+}

tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs

@@ -102,6 +102,8 @@ namespace Tgstation.Server.Tests.Live.Instance
 		{
 			await StartAndLeaveRunning(cancellationToken);
 
+			await RegressionTest1550(cancellationToken);
+
 			var deleteJobTask = TestDeleteByondInstallErrorCasesAndQueing(cancellationToken);
 
 			SessionController.LogTopicRequests = false;
@@ -127,6 +129,54 @@ namespace Tgstation.Server.Tests.Live.Instance
 			await WaitForJob(restartJob, 15, false, null, cancellationToken);
 		}
 
+		async ValueTask RegressionTest1550(CancellationToken cancellationToken)
+		{
+			// we need to cycle deployments twice because TGS holds the initial deployment
+			await DeployTestDme("LongRunning/long_running_test", DreamDaemonSecurity.Trusted, true, cancellationToken);
+			var currentStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
+
+			Assert.AreEqual(WatchdogStatus.Online, currentStatus.Status);
+			Assert.IsNotNull(currentStatus.StagedCompileJob);
+			var expectedStaged = currentStatus.StagedCompileJob;
+			Assert.AreNotEqual(expectedStaged.Id, currentStatus.ActiveCompileJob.Id);
+
+			await TellWorldToReboot(cancellationToken);
+
+			currentStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
+			Assert.AreEqual(expectedStaged.Id, currentStatus.ActiveCompileJob.Id);
+
+			await Task.Delay(TimeSpan.FromSeconds(3), cancellationToken);
+
+			var topicRequestResult = await TopicClient.SendTopic(
+				IPAddress.Loopback,
+				$"shadow_wizard_money_gang=1",
+				TestLiveServer.DDPort,
+				cancellationToken);
+
+			Assert.IsNotNull(topicRequestResult);
+			Assert.AreEqual("we love casting spells", topicRequestResult.StringData);
+
+			await DeployTestDme("LongRunning/long_running_test", DreamDaemonSecurity.Trusted, true, cancellationToken);
+
+			currentStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
+
+			Assert.AreEqual(WatchdogStatus.Online, currentStatus.Status);
+			Assert.IsNotNull(currentStatus.StagedCompileJob);
+			Assert.AreEqual(expectedStaged.Id, currentStatus.ActiveCompileJob.Id);
+			expectedStaged = currentStatus.StagedCompileJob;
+			Assert.AreNotEqual(expectedStaged.Id, currentStatus.ActiveCompileJob.Id);
+
+			await TellWorldToReboot(cancellationToken);
+
+			currentStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
+			Assert.AreEqual(WatchdogStatus.Online, currentStatus.Status);
+			Assert.IsNull(currentStatus.StagedCompileJob);
+			Assert.AreEqual(expectedStaged.Id, currentStatus.ActiveCompileJob.Id);
+
+			await CheckDMApiFail(currentStatus.ActiveCompileJob, cancellationToken, false);
+			await CheckDMApiFail(expectedStaged, cancellationToken, false);
+		}
+
 		async Task<JobResponse> TestDeleteByondInstallErrorCasesAndQueing(CancellationToken cancellationToken)
 		{
 			var testCustomVersion = new Version(ByondTest.TestVersion.Major, ByondTest.TestVersion.Minor, 1);
@@ -530,10 +580,11 @@ namespace Tgstation.Server.Tests.Live.Instance
 			};
 
 			var json = JsonConvert.SerializeObject(baseTopic, DMApiConstants.SerializerSettings);
-			var topicString = $"tgs_integration_test_tactics3={TopicClient.SanitizeString(json)}";
 
-			var baseSize = topicString.Length;
-			var wrappingSize = baseSize;
+			var baseSize = (int)(DMApiConstants.MaximumTopicRequestLength - 1);
+
+			var topicString = $"tgs_integration_test_tactics3={TopicClient.SanitizeString(json)}";
+			var wrappingSize = topicString.Length;
 
 			while (!cancellationToken.IsCancellationRequested)
 			{
@@ -584,7 +635,7 @@ namespace Tgstation.Server.Tests.Live.Instance
 			System.Console.WriteLine("TEST: Receiving Topic tests topics...");
 
 			// Receive
-			baseSize = 1;
+			baseSize = (int)(DMApiConstants.MaximumTopicResponseLength - 1);
 			nextPow = 0;
 			lastSize = 0;
 			while (!cancellationToken.IsCancellationRequested)
@@ -949,32 +1000,28 @@ namespace Tgstation.Server.Tests.Live.Instance
 		public async Task<DreamDaemonResponse> TellWorldToReboot(CancellationToken cancellationToken)
 		{
 			var daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
+			Assert.IsNotNull(daemonStatus.StagedCompileJob);
 			var initialCompileJob = daemonStatus.ActiveCompileJob;
 
-			try
-			{
-				System.Console.WriteLine("TEST: Sending world reboot topic...");
-				var result = await TopicClient.SendTopic(IPAddress.Loopback, "tgs_integration_test_special_tactics=1", TestLiveServer.DDPort, cancellationToken);
-				Assert.AreEqual("ack", result.StringData);
+			System.Console.WriteLine("TEST: Sending world reboot topic...");
+			var result = await TopicClient.SendTopic(IPAddress.Loopback, "tgs_integration_test_special_tactics=1", TestLiveServer.DDPort, cancellationToken);
+			Assert.AreEqual("ack", result.StringData);
 
-				using (var tempCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
-				using (tempCts.Token.Register(() => System.Console.WriteLine("TEST ERROR: Timeout in TellWorldToReboot!")))
+			using var tempCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
+			var tempToken = tempCts.Token;
+			using (tempToken.Register(() => System.Console.WriteLine("TEST ERROR: Timeout in TellWorldToReboot!")))
+			{
+				tempCts.CancelAfter(TimeSpan.FromMinutes(2));
+
+				do
 				{
-					tempCts.CancelAfter(TimeSpan.FromMinutes(2));
-					var tempToken = tempCts.Token;
-
-					do
-					{
-						await Task.Delay(TimeSpan.FromSeconds(1), tempToken);
-						daemonStatus = await instanceClient.DreamDaemon.Read(tempToken);
-					}
-					while (initialCompileJob.Id == daemonStatus.ActiveCompileJob.Id);
+					await Task.Delay(TimeSpan.FromSeconds(1), tempToken);
+					daemonStatus = await instanceClient.DreamDaemon.Read(tempToken);
 				}
+				while (initialCompileJob.Id == daemonStatus.ActiveCompileJob.Id);
 			}
-			catch (OperationCanceledException)
-			{
-				throw;
-			}
+
+			await Task.Delay(TimeSpan.FromSeconds(3), cancellationToken);
 
 			return daemonStatus;
 		}