Merge pull request #1447 from tgstation/SessionPersistenceFix [TGSDeploy][DMDeploy][NugetDeploy]

Various fixes (5.7.3)

.github/workflows/ci-suite.yml

@@ -631,7 +631,7 @@ jobs:
         with:
           tag_name: dmapi-v${{ env.TGS_DM_VERSION }}
           release_name: tgstation-server DMAPI v${{ env.TGS_DM_VERSION }}
-          body: The TGS DMAPI
+          body: The TGS DMAPI \#tgs-dmapi-release
           commitish: ${{ github.event.head_commit.id }}
 
       - name: Upload DMAPI Artifact

build/Version.props

@@ -3,13 +3,13 @@
   <!-- Integration tests will ensure they match across the board -->
   <Import Project="ControlPanelVersion.props" />
   <PropertyGroup>
-    <TgsCoreVersion>5.7.2</TgsCoreVersion>
+    <TgsCoreVersion>5.7.3</TgsCoreVersion>
     <TgsConfigVersion>4.4.0</TgsConfigVersion>
     <TgsApiVersion>9.9.0</TgsApiVersion>
     <TgsApiLibraryVersion>10.3.0</TgsApiLibraryVersion>
     <TgsClientVersion>11.3.0</TgsClientVersion>
-    <TgsDmapiVersion>6.2.0</TgsDmapiVersion>
-    <TgsInteropVersion>5.4.0</TgsInteropVersion>
+    <TgsDmapiVersion>6.3.0</TgsDmapiVersion>
+    <TgsInteropVersion>5.5.0</TgsInteropVersion>
     <TgsHostWatchdogVersion>1.2.1</TgsHostWatchdogVersion>
     <TgsContainerScriptVersion>1.2.1</TgsContainerScriptVersion>
     <TgsMigratorVersion>1.0.1</TgsMigratorVersion>

src/DMAPI/tgs.dm

@@ -1,6 +1,6 @@
 // tgstation-server DMAPI
 
-#define TGS_DMAPI_VERSION "6.2.0"
+#define TGS_DMAPI_VERSION "6.3.0"
 
 // All functions and datums outside this document are subject to change with any version and should not be relied on.
 

src/DMAPI/tgs/v5/__interop_version.dm

@@ -1 +1 @@
-"5.4.0"
+"5.5.0"

src/DMAPI/tgs/v5/api.dm

@@ -99,7 +99,8 @@
 
 /datum/tgs_api/v5/proc/TopicResponse(error_message = null)
 	var/list/response = list()
-	response[DMAPI5_RESPONSE_ERROR_MESSAGE] = error_message
+	if(error_message)
+		response[DMAPI5_RESPONSE_ERROR_MESSAGE] = error_message
 
 	return json_encode(response)
 
@@ -128,9 +129,12 @@
 
 	switch(command)
 		if(DMAPI5_TOPIC_COMMAND_CHAT_COMMAND)
+			intercepted_message_queue = list()
 			var/result = HandleCustomCommand(topic_parameters[DMAPI5_TOPIC_PARAMETER_CHAT_COMMAND])
 			if(!result)
 				result = TopicResponse("Error running chat command!")
+			result[DMAPI5_TOPIC_RESPONSE_CHAT_RESPONSES] = intercepted_message_queue
+			intercepted_message_queue = null
 			return result
 		if(DMAPI5_TOPIC_COMMAND_EVENT_NOTIFICATION)
 			intercepted_message_queue = list()

src/DMAPI/tgs/v5/serializers.dm

@@ -43,6 +43,13 @@
 	. = ..()
 	.["iconUrl"] = icon_url
 	.["proxyIconUrl"] = proxy_icon_url
+	
+/datum/tgs_chat_embed/footer/_interop_serialize()
+	return list(
+		"text" = text,
+		"iconUrl" = icon_url,
+		"proxyIconUrl" = proxy_icon_url
+	)
 
 /datum/tgs_chat_embed/field/_interop_serialize()
 	return list(

src/Tgstation.Server.Host/Components/Byond/ByondManager.cs

@@ -255,7 +255,7 @@ namespace Tgstation.Server.Host.Components.Byond
 				lock (installedVersions)
 					hasRequestedActiveVersion = installedVersions.ContainsKey(activeVersionString);
 				if (hasRequestedActiveVersion && Version.TryParse(activeVersionString, out var activeVersion))
-					ActiveVersion = activeVersion.Semver();
+					ActiveVersion = activeVersion;
 				else
 				{
 					logger.LogWarning("Failed to load saved active version {0}!", activeVersionString);

src/Tgstation.Server.Host/Components/Chat/ChatManager.cs

@@ -679,7 +679,11 @@ namespace Tgstation.Server.Host.Components.Chat
 				message.User.Channel.IsAdminChannel = mappingChannelRepresentation.IsAdminChannel;
 			}
 
-			var splits = new List<string>(message.Content.Trim().Split(' ', StringSplitOptions.RemoveEmptyEntries));
+			var trimmedMessage = message.Content.Trim();
+			if (trimmedMessage.Length == 0)
+				return;
+
+			var splits = new List<string>(trimmedMessage.Split(' ', StringSplitOptions.RemoveEmptyEntries));
 			var address = splits[0];
 			if (address.Length > 1 && (address.Last() == ':' || address.Last() == ','))
 				address = address[0..^1];

src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs

@@ -335,19 +335,24 @@ namespace Tgstation.Server.Host.Components.Deployment
 			List<string> jobUidsToNotErase = null;
 
 			// find the uids of locked directories
-			await databaseContextFactory.UseContext(async db =>
+			if (jobIdsToSkip.Any())
 			{
-				jobUidsToNotErase = (await db
-						.CompileJobs
-						.AsQueryable()
-						.Where(
-							x => x.Job.Instance.Id == metadata.Id
-							&& jobIdsToSkip.Contains(x.Id.Value))
-						.Select(x => x.DirectoryName.Value)
-						.ToListAsync(cancellationToken))
-					.Select(x => x.ToString())
-					.ToList();
-			});
+				await databaseContextFactory.UseContext(async db =>
+				{
+					jobUidsToNotErase = (await db
+							.CompileJobs
+							.AsQueryable()
+							.Where(
+								x => x.Job.Instance.Id == metadata.Id
+								&& jobIdsToSkip.Contains(x.Id.Value))
+							.Select(x => x.DirectoryName.Value)
+							.ToListAsync(cancellationToken))
+						.Select(x => x.ToString())
+						.ToList();
+				});
+			}
+			else
+				jobUidsToNotErase = new List<string>();
 
 			jobUidsToNotErase.Add(SwappableDmbProvider.LiveGameDirectory);
 
@@ -411,17 +416,20 @@ namespace Tgstation.Server.Host.Components.Deployment
 			}
 
 			lock (jobLockCounts)
-				if (!jobLockCounts.TryGetValue(job.Id.Value, out var currentVal) || currentVal == 1)
-				{
-					jobLockCounts.Remove(job.Id.Value);
-					logger.LogDebug("Cleaning lock-free compile job {0} => {1}", job.Id, job.DirectoryName);
-					cleanupTask = HandleCleanup();
-				}
+				if (jobLockCounts.TryGetValue(job.Id.Value, out var currentVal))
+					if (currentVal == 1)
+					{
+						jobLockCounts.Remove(job.Id.Value);
+						logger.LogDebug("Cleaning lock-free compile job {0} => {1}", job.Id, job.DirectoryName);
+						cleanupTask = HandleCleanup();
+					}
+					else
+					{
+						var decremented = --jobLockCounts[job.Id.Value];
+						logger.LogTrace("Compile job {0} lock count now: {1}", job.Id, decremented);
+					}
 				else
-				{
-					var decremented = --jobLockCounts[job.Id.Value];
-					logger.LogTrace("Compile job {0} lock count now: {1}", job.Id, decremented);
-				}
+					logger.LogError("Extra Dispose of DmbProvider for CompileJob {compileJobId}!", job.Id);
 		}
 
 		/// <summary>

src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs

@@ -744,7 +744,7 @@ namespace Tgstation.Server.Host.Components.Deployment
 						do
 						{
 							var remainingSleepThisInterval = nextInterval - DateTimeOffset.UtcNow;
-							var nextSleepSpan = remainingSleepThisInterval < minimumSleepInterval ? remainingSleepThisInterval : minimumSleepInterval;
+							var nextSleepSpan = remainingSleepThisInterval < minimumSleepInterval ? minimumSleepInterval : remainingSleepThisInterval;
 
 							await Task.Delay(nextSleepSpan, cancellationToken);
 							progressReporter.StageName = currentStage;
@@ -764,6 +764,10 @@ namespace Tgstation.Server.Host.Components.Deployment
 			{
 				logger.LogTrace(ex, "ProgressTask aborted.");
 			}
+			catch (Exception ex)
+			{
+				logger.LogError(ex, "ProgressTask crashed!");
+			}
 		}
 
 		/// <summary>

src/Tgstation.Server.Host/Components/InstanceFactory.cs

@@ -385,6 +385,6 @@ namespace Tgstation.Server.Host.Components
 		/// <summary>
 		/// Test that the <see cref="repositoryFactory"/> is functional.
 		/// </summary>
-		private void CheckSystemCompatibility() => repositoryFactory.CreateInMemory();
+		void CheckSystemCompatibility() => repositoryFactory.CreateInMemory();
 	}
 }

src/Tgstation.Server.Host/Components/InstanceManager.cs

@@ -461,14 +461,27 @@ namespace Tgstation.Server.Host.Components
 				logger.LogDebug("Stopping instance manager...");
 				var instanceFactoryStopTask = instanceFactory.StopAsync(cancellationToken);
 				await jobManager.StopAsync(cancellationToken);
-				await Task.WhenAll(instances.Select(x => x.Value.Instance.StopAsync(cancellationToken)));
+
+				async Task OfflineInstanceImmediate(IInstance instance, CancellationToken cancellationToken)
+				{
+					try
+					{
+						await instance.StopAsync(cancellationToken);
+					}
+					catch (Exception ex)
+					{
+						logger.LogError(ex, "Instance shutdown exception!");
+					}
+				}
+
+				await Task.WhenAll(instances.Select(x => OfflineInstanceImmediate(x.Value.Instance, cancellationToken)));
 				await instanceFactoryStopTask;
 
 				await swarmService.Shutdown(cancellationToken);
 			}
 			catch (Exception ex)
 			{
-				logger.LogError(ex, "Instance manager stop exception!");
+				logger.LogCritical(ex, "Instance manager stop exception!");
 			}
 		}
 
@@ -538,7 +551,7 @@ namespace Tgstation.Server.Host.Components
 		/// <summary>
 		/// Check we have a valid system identity.
 		/// </summary>
-		private void CheckSystemCompatibility()
+		void CheckSystemCompatibility()
 		{
 			using (var systemIdentity = systemIdentityFactory.GetCurrent())
 			{

src/Tgstation.Server.Host/Components/Session/ReattachInformation.cs

@@ -17,6 +17,11 @@ namespace Tgstation.Server.Host.Components.Session
 		/// </summary>
 		public IDmbProvider Dmb { get; set; }
 
+		/// <summary>
+		/// The <see cref="IDmbProvider"/> initially used to launch DreamDaemon. Should be a different <see cref="IDmbProvider"/> than <see cref="Dmb"/>. Should not be set if persisting the initial <see cref="CompileJob"/> isn't necessary.
+		/// </summary>
+		public IDmbProvider InitialDmb { get; set; }
+
 		/// <summary>
 		/// The <see cref="Interop.Bridge.RuntimeInformation"/> for the DMAPI.
 		/// </summary>
@@ -37,13 +42,16 @@ namespace Tgstation.Server.Host.Components.Session
 		/// </summary>
 		/// <param name="copy">The <see cref="Models.ReattachInformation"/> to copy values from.</param>
 		/// <param name="dmb">The value of <see cref="Dmb"/>.</param>
+		/// <param name="initialDmb">The value of <see cref="InitialDmb"/>.</param>
 		/// <param name="topicRequestTimeout">The value of <see cref="TopicRequestTimeout"/>.</param>
 		public ReattachInformation(
 			Models.ReattachInformation copy,
 			IDmbProvider dmb,
+			IDmbProvider initialDmb,
 			TimeSpan topicRequestTimeout) : base(copy)
 		{
 			Dmb = dmb ?? throw new ArgumentNullException(nameof(dmb));
+			InitialDmb = initialDmb;
 			TopicRequestTimeout = topicRequestTimeout;
 
 			runtimeInformationLock = new object();

src/Tgstation.Server.Host/Components/Session/SessionController.cs

@@ -270,22 +270,18 @@ namespace Tgstation.Server.Host.Components.Session
 
 			logger.LogTrace("Disposing...");
 			if (!released)
-			{
 				process.Terminate();
-				byondLock.Dispose();
-			}
 
 			await process.DisposeAsync();
+			byondLock.Dispose();
 			bridgeRegistration?.Dispose();
-			ReattachInformation.Dmb?.Dispose(); // will be null when released
+			ReattachInformation.Dmb.Dispose();
+			ReattachInformation.InitialDmb?.Dispose();
 			chatTrackingContext.Dispose();
 			reattachTopicCts.Dispose();
 
 			if (!released)
-			{
-				// finish the async callback
-				await Lifetime;
-			}
+				await Lifetime; // finish the async callback
 		}
 
 		/// <inheritdoc />
@@ -320,9 +316,20 @@ namespace Tgstation.Server.Host.Components.Session
 						if (parameters.ChatMessage.Text == null)
 							return Error("Missing message field in chatMessage!");
 
+						var anyFailed = false;
+						var parsedChannels = parameters.ChatMessage.ChannelIds.Select(
+							channelString =>
+							{
+								anyFailed |= !UInt64.TryParse(channelString, out var channelId);
+								return channelId;
+							});
+
+						if (anyFailed)
+							return Error("Failed to parse channelIds as U64!");
+
 						chat.QueueMessage(
 							parameters.ChatMessage,
-							parameters.ChatMessage.ChannelIds.Select(UInt64.Parse));
+							parsedChannels);
 						break;
 					case BridgeCommandType.Prime:
 						var oldPrimeTcs = primeTcs;
@@ -441,14 +448,11 @@ namespace Tgstation.Server.Host.Components.Session
 		{
 			CheckDisposed();
 
-			// we still don't want to dispose the dmb yet, even though we're keeping it alive
-			var tmpProvider = ReattachInformation.Dmb;
-			ReattachInformation.Dmb = null;
+			ReattachInformation.Dmb.KeepAlive();
+			ReattachInformation.InitialDmb?.KeepAlive();
+			byondLock.DoNotDeleteThisSession();
 			released = true;
 			await DisposeAsync();
-			byondLock.DoNotDeleteThisSession();
-			tmpProvider.KeepAlive();
-			ReattachInformation.Dmb = tmpProvider;
 		}
 
 		/// <inheritdoc />

src/Tgstation.Server.Host/Components/Session/SessionPersistor.cs

@@ -78,6 +78,7 @@ namespace Tgstation.Server.Host.Components.Session
 			{
 				AccessIdentifier = reattachInformation.AccessIdentifier,
 				CompileJobId = reattachInformation.Dmb.CompileJob.Id.Value,
+				InitialCompileJobId = reattachInformation.InitialDmb?.CompileJob.Id.Value,
 				Port = reattachInformation.Port,
 				ProcessId = reattachInformation.ProcessId,
 				RebootState = reattachInformation.RebootState,
@@ -128,6 +129,7 @@ namespace Tgstation.Server.Host.Components.Session
 					.AsQueryable()
 					.Where(x => x.CompileJob.Job.Instance.Id == metadata.Id)
 					.Include(x => x.CompileJob)
+					.Include(x => x.InitialCompileJob)
 					.ToListAsync(cancellationToken);
 				result = dbReattachInfos.FirstOrDefault();
 				if (result == default)
@@ -191,9 +193,19 @@ namespace Tgstation.Server.Host.Components.Session
 				return null;
 			}
 
+			IDmbProvider initialDmb = null;
+			if (result.InitialCompileJob != null)
+			{
+				logger.LogTrace("Loading initial compile job...");
+				initialDmb = await dmbFactory.FromCompileJob(result.InitialCompileJob, cancellationToken);
+			}
+
+			logger.LogTrace("Retrieved ReattachInformation");
+
 			var info = new ReattachInformation(
 				result,
 				dmb,
+				initialDmb,
 				topicTimeout.Value);
 
 			logger.LogDebug("Reattach information loaded: {info}", info);

src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs

@@ -249,7 +249,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 
 				// Server.AdjustPriority(true);
 				if (!reattachInProgress)
-					await SessionPersistor.Save(Server.ReattachInformation, cancellationToken);
+					await SessionStartupPersist(cancellationToken);
 
 				await CheckLaunchResult(Server, "Server", cancellationToken);
 
@@ -273,6 +273,16 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			}
 		}
 
+		/// <summary>
+		/// Called to save the current <see cref="Server"/> into the <see cref="WatchdogBase.SessionPersistor"/> when initially launched.
+		/// </summary>
+		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
+		/// <returns>A <see cref="Task"/> representing the running operation.</returns>
+		protected virtual Task SessionStartupPersist(CancellationToken cancellationToken)
+		{
+			return SessionPersistor.Save(Server.ReattachInformation, cancellationToken);
+		}
+
 		/// <summary>
 		/// Handler for <see cref="MonitorActivationReason.ActiveServerRebooted"/> when the <see cref="RebootState"/> is <see cref="RebootState.Normal"/>.
 		/// </summary>

src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs

@@ -82,6 +82,9 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		{
 		}
 
+		/// <inheritdoc />
+		protected override Task ApplyInitialDmb(CancellationToken cancellationToken) => Task.CompletedTask;
+
 		/// <inheritdoc />
 		protected override async Task InitialLink(CancellationToken cancellationToken)
 		{
@@ -107,6 +110,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// <inheritdoc />
 		protected override async Task InitController(Task chatTask, ReattachInformation reattachInfo, CancellationToken cancellationToken)
 		{
+			var suspended = false;
 			try
 			{
 				await base.InitController(chatTask, reattachInfo, cancellationToken);
@@ -120,6 +124,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 					{
 						Logger.LogTrace("Unhardlinking compile job...");
 						Server?.Suspend();
+						suspended = true;
 						var hardLink = hardLinkedDmb.Directory;
 						var originalPosition = hardLinkedDmb.CompileJob.DirectoryName.ToString();
 						await GameIOManager.MoveDirectory(
@@ -149,7 +154,8 @@ namespace Tgstation.Server.Host.Components.Watchdog
 
 			Logger.LogTrace("Symlinking compile job...");
 			await ActiveSwappable.MakeActive(cancellationToken);
-			Server.Resume();
+			if (suspended)
+				Server.Resume();
 		}
 	}
 }

src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs

@@ -307,6 +307,8 @@ namespace Tgstation.Server.Host.Components.Watchdog
 					commandResponse.Text = "TGS: Command processed but no DMAPI response returned!";
 				}
 
+				HandleChatResponses(commandResult);
+
 				return commandResponse;
 			}
 		}
@@ -465,20 +467,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 				cancellationToken)
 				;
 
-			if (result?.InteropResponse?.ChatResponses != null)
-				foreach (var response in result.InteropResponse.ChatResponses)
-					Chat.QueueMessage(
-						response,
-						response.ChannelIds
-							.Select(channelIdString =>
-							{
-								if (UInt64.TryParse(channelIdString, out var channelId))
-									return (ulong?)channelId;
-
-								return null;
-							})
-							.Where(nullableChannelId => nullableChannelId.HasValue)
-							.Select(nullableChannelId => nullableChannelId.Value));
+			HandleChatResponses(result);
 		}
 
 		/// <summary>
@@ -727,7 +716,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// </summary>
 		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
 		/// <returns>A <see cref="Task"/> representing the running operation.</returns>
-		private async Task MonitorRestart(CancellationToken cancellationToken)
+		async Task MonitorRestart(CancellationToken cancellationToken)
 		{
 			Logger.LogTrace("Monitor restart!");
 
@@ -781,6 +770,28 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			}
 		}
 
+		/// <summary>
+		/// Check for a new <see cref="IDmbProvider"/>.
+		/// </summary>
+		/// <param name="currentCompileJob">The session's current <see cref="CompileJob"/>.</param>
+		/// <returns>A <see cref="Task"/> that completes if and when a newer <see cref="CompileJob"/> is available.</returns>
+		Task InitialCheckDmbUpdated(CompileJob currentCompileJob)
+		{
+			var factoryTask = DmbFactory.OnNewerDmb;
+
+			var latestCompileJob = DmbFactory.LatestCompileJob();
+			if (latestCompileJob == null)
+				return factoryTask;
+
+			if (latestCompileJob.Id != currentCompileJob.Id)
+			{
+				Logger.LogDebug("Found new CompileJob without waiting");
+				return Task.CompletedTask;
+			}
+
+			return factoryTask;
+		}
+
 		/// <summary>
 		/// The main loop of the watchdog. Ayschronously waits for events to occur and then responds to them.
 		/// </summary>
@@ -803,6 +814,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 					activeLaunchParametersChanged = null,
 					newDmbAvailable = null;
 				ISessionController lastController = null;
+				var ranInitialDmbCheck = false;
 				for (ulong iteration = 1; nextAction != MonitorAction.Exit; ++iteration)
 					using (LogContext.PushProperty("Monitor", iteration))
 						try
@@ -814,19 +826,19 @@ namespace Tgstation.Server.Host.Components.Watchdog
 
 							void UpdateMonitoredTasks()
 							{
-								static void TryUpdateTask(ref Task oldTask, Task newTask)
+								static void TryUpdateTask(ref Task oldTask, Func<Task> newTaskFactory)
 								{
 									if (oldTask?.IsCompleted == true)
 										return;
 
-									oldTask = newTask;
+									oldTask = newTaskFactory();
 								}
 
 								if (lastController == controller)
 								{
-									TryUpdateTask(ref activeServerLifetime, controller.Lifetime);
-									TryUpdateTask(ref activeServerReboot, controller.OnReboot);
-									TryUpdateTask(ref serverPrimed, controller.OnPrime);
+									TryUpdateTask(ref activeServerLifetime, () => controller.Lifetime);
+									TryUpdateTask(ref activeServerReboot, () => controller.OnReboot);
+									TryUpdateTask(ref serverPrimed, () => controller.OnPrime);
 								}
 								else
 								{
@@ -836,8 +848,17 @@ namespace Tgstation.Server.Host.Components.Watchdog
 									lastController = controller;
 								}
 
-								TryUpdateTask(ref activeLaunchParametersChanged, ActiveParametersUpdated.Task);
-								TryUpdateTask(ref newDmbAvailable, DmbFactory.OnNewerDmb);
+								TryUpdateTask(ref activeLaunchParametersChanged, () => ActiveParametersUpdated.Task);
+								TryUpdateTask(
+									ref newDmbAvailable,
+									() =>
+									{
+										var result = ranInitialDmbCheck
+											? DmbFactory.OnNewerDmb
+											: InitialCheckDmbUpdated(controller.CompileJob);
+										ranInitialDmbCheck = true;
+										return result;
+									});
 							}
 
 							UpdateMonitoredTasks();
@@ -1100,5 +1121,29 @@ namespace Tgstation.Server.Host.Components.Watchdog
 
 			return MonitorAction.Continue;
 		}
+
+		/// <summary>
+		/// Handle any <see cref="TopicResponse.ChatResponses"/> in a given topic <paramref name="result"/>.
+		/// </summary>
+		/// <param name="result">The <see cref="CombinedTopicResponse"/>.</param>
+		void HandleChatResponses(CombinedTopicResponse result)
+		{
+			if (result?.InteropResponse?.ChatResponses != null)
+				foreach (var response in result.InteropResponse.ChatResponses)
+					Chat.QueueMessage(
+						response,
+						response.ChannelIds
+							.Select(channelIdString =>
+							{
+								if (UInt64.TryParse(channelIdString, out var channelId))
+									return (ulong?)channelId;
+								else
+									Logger.LogWarning("Could not parse chat response channel ID: {channelID}", channelIdString);
+
+								return null;
+							})
+							.Where(nullableChannelId => nullableChannelId.HasValue)
+							.Select(nullableChannelId => nullableChannelId.Value));
+		}
 	}
 }

src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs

@@ -41,11 +41,6 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// </summary>
 		SwappableDmbProvider pendingSwappable;
 
-		/// <summary>
-		/// The <see cref="IDmbProvider"/> the <see cref="WindowsWatchdog"/> was started with.
-		/// </summary>
-		IDmbProvider startupDmbProvider;
-
 		/// <summary>
 		/// Initializes a new instance of the <see cref="WindowsWatchdog"/> class.
 		/// </summary>
@@ -120,9 +115,6 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			ActiveSwappable = null;
 			pendingSwappable?.Dispose();
 			pendingSwappable = null;
-
-			startupDmbProvider?.Dispose();
-			startupDmbProvider = null;
 		}
 
 		/// <inheritdoc />
@@ -136,6 +128,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 				ActiveSwappable = pendingSwappable;
 				pendingSwappable = null;
 
+				await SessionPersistor.Save(Server.ReattachInformation, cancellationToken);
 				await updateTask;
 			}
 			else
@@ -218,18 +211,12 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		{
 			if (ActiveSwappable != null)
 				throw new InvalidOperationException("Expected activeSwappable to be null!");
-			if (startupDmbProvider != null)
-				throw new InvalidOperationException("Expected startupDmbProvider to be null!");
+			if (pendingSwappable != null)
+				throw new InvalidOperationException("Expected pendingSwappable to be null!");
 
-			Logger.LogTrace("Prep for server launch. pendingSwappable is {0}available", pendingSwappable == null ? "not " : String.Empty);
-
-			// Add another lock to the startup DMB because it'll be used throughout the lifetime of the watchdog
-			startupDmbProvider = await DmbFactory.FromCompileJob(dmbToUse.CompileJob, cancellationToken);
-
-			pendingSwappable ??= new SwappableDmbProvider(dmbToUse, GameIOManager, symlinkFactory);
-			ActiveSwappable = pendingSwappable;
-			pendingSwappable = null;
+			Logger.LogTrace("Prep for server launch");
 
+			ActiveSwappable = new SwappableDmbProvider(dmbToUse, GameIOManager, symlinkFactory);
 			try
 			{
 				await InitialLink(cancellationToken);
@@ -245,6 +232,23 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			return ActiveSwappable;
 		}
 
+		/// <summary>
+		/// Set the <see cref="ReattachInformation.InitialDmb"/> for the <see cref="BasicWatchdog.Server"/>.
+		/// </summary>
+		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
+		/// <returns>A <see cref="Task"/> representing the running operation.</returns>
+		protected virtual async Task ApplyInitialDmb(CancellationToken cancellationToken)
+		{
+			Server.ReattachInformation.InitialDmb = await DmbFactory.FromCompileJob(Server.CompileJob, cancellationToken);
+		}
+
+		/// <inheritdoc />
+		protected override async Task SessionStartupPersist(CancellationToken cancellationToken)
+		{
+			await ApplyInitialDmb(cancellationToken);
+			await base.SessionStartupPersist(cancellationToken);
+		}
+
 		/// <summary>
 		/// Create the initial link to the live game directory using <see cref="ActiveSwappable"/>.
 		/// </summary>

src/Tgstation.Server.Host/Configuration/SessionConfiguration.cs

@@ -13,7 +13,7 @@
 		/// <summary>
 		/// The default value for <see cref="HighPriorityLiveDreamDaemon"/>.
 		/// </summary>
-		private const bool DefaultHighPriorityLiveDreamDaemon = true;
+		const bool DefaultHighPriorityLiveDreamDaemon = true;
 
 		/// <summary>
 		/// If the public DreamDaemon instances are set to be above normal priority processes.

src/Tgstation.Server.Host/Controllers/ChatController.cs

@@ -385,7 +385,7 @@ namespace Tgstation.Server.Host.Controllers
 		/// <param name="model">The <see cref="ChatBotApiBase"/> to validate.</param>
 		/// <param name="forCreation">If the <paramref name="model"/> is being created.</param>
 		/// <returns>An <see cref="IActionResult"/> to respond with or <see langword="null"/>.</returns>
-		private IActionResult StandardModelChecks(ChatBotApiBase model, bool forCreation)
+		IActionResult StandardModelChecks(ChatBotApiBase model, bool forCreation)
 		{
 			if (model.ReconnectionInterval == 0)
 				throw new InvalidOperationException("RecconnectionInterval cannot be zero!");

src/Tgstation.Server.Host/Controllers/JobController.cs

@@ -176,7 +176,7 @@ namespace Tgstation.Server.Host.Controllers
 		/// </summary>
 		/// <param name="jobResponse">The <see cref="JobResponse"/> to augment.</param>
 		/// <returns>A <see cref="Task"/> representing the running operation.</returns>
-		private Task AddJobProgressResponseTransformer(JobResponse jobResponse)
+		Task AddJobProgressResponseTransformer(JobResponse jobResponse)
 		{
 			jobManager.SetJobProgress(jobResponse);
 			return Task.CompletedTask;

src/Tgstation.Server.Host/Core/OpenApiEnumVarNamesExtension.cs

@@ -21,7 +21,7 @@ namespace Tgstation.Server.Host.Core
 		/// Initializes a new instance of the <see cref="OpenApiEnumVarNamesExtension"/> class.
 		/// </summary>
 		/// <param name="enumType">The value of <see cref="enumType"/>,.</param>
-		private OpenApiEnumVarNamesExtension(Type enumType)
+		OpenApiEnumVarNamesExtension(Type enumType)
 		{
 			this.enumType = enumType ?? throw new ArgumentNullException(nameof(enumType));
 		}

src/Tgstation.Server.Host/Database/DatabaseContext.cs

@@ -298,7 +298,7 @@ namespace Tgstation.Server.Host.Database
 			else
 				logger.LogDebug("No migrations to apply");
 
-			wasEmpty |= (await Users.AsQueryable().CountAsync(cancellationToken)) == 0;
+			wasEmpty |= !await Users.AsQueryable().AnyAsync(cancellationToken);
 
 			return wasEmpty;
 		}
@@ -379,22 +379,22 @@ namespace Tgstation.Server.Host.Database
 		/// <summary>
 		/// Used by unit tests to remind us to setup the correct MSSQL migration downgrades.
 		/// </summary>
-		internal static readonly Type MSLatestMigration = typeof(MSAddDreamDaemonLogOutput);
+		internal static readonly Type MSLatestMigration = typeof(MSAddReattachInfoInitialCompileJob);
 
 		/// <summary>
 		/// Used by unit tests to remind us to setup the correct MYSQL migration downgrades.
 		/// </summary>
-		internal static readonly Type MYLatestMigration = typeof(MYAddDreamDaemonLogOutput);
+		internal static readonly Type MYLatestMigration = typeof(MYAddReattachInfoInitialCompileJob);
 
 		/// <summary>
 		/// Used by unit tests to remind us to setup the correct PostgresSQL migration downgrades.
 		/// </summary>
-		internal static readonly Type PGLatestMigration = typeof(PGAddDreamDaemonLogOutput);
+		internal static readonly Type PGLatestMigration = typeof(PGAddReattachInfoInitialCompileJob);
 
 		/// <summary>
 		/// Used by unit tests to remind us to setup the correct SQLite migration downgrades.
 		/// </summary>
-		internal static readonly Type SLLatestMigration = typeof(SLAddDreamDaemonLogOutput);
+		internal static readonly Type SLLatestMigration = typeof(SLAddReattachInfoInitialCompileJob);
 
 		/// <inheritdoc />
 #pragma warning disable CA1502 // Cyclomatic complexity
@@ -425,6 +425,15 @@ namespace Tgstation.Server.Host.Database
 
 			string BadDatabaseType() => throw new ArgumentException($"Invalid DatabaseType: {currentDatabaseType}", nameof(currentDatabaseType));
 
+			if (targetVersion < new Version(5, 7, 3))
+				targetMigration = currentDatabaseType switch
+				{
+					DatabaseType.MySql => nameof(MYAddDreamDaemonLogOutput),
+					DatabaseType.PostgresSql => nameof(PGAddDreamDaemonLogOutput),
+					DatabaseType.SqlServer => nameof(MSAddDreamDaemonLogOutput),
+					DatabaseType.Sqlite => nameof(SLAddDreamDaemonLogOutput),
+					_ => BadDatabaseType(),
+				};
 			if (targetVersion < new Version(5, 7, 0))
 				targetMigration = currentDatabaseType switch
 				{

src/Tgstation.Server.Host/Database/Migrations/20230403050623_MSAddReattachInfoInitialCompileJob.cs

@@ -0,0 +1,58 @@
+using System;
+
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace Tgstation.Server.Host.Database.Migrations
+{
+	/// <summary>
+	/// Adds the InitialCompileJobId to the ReattachInformations table for MSSQL.
+	/// </summary>
+	public partial class MSAddReattachInfoInitialCompileJob : Migration
+	{
+		/// <inheritdoc />
+		protected override void Up(MigrationBuilder migrationBuilder)
+		{
+			if (migrationBuilder == null)
+				throw new ArgumentNullException(nameof(migrationBuilder));
+
+			migrationBuilder.AddColumn<long>(
+				name: "InitialCompileJobId",
+				table: "ReattachInformations",
+				type: "bigint",
+				nullable: true);
+
+			migrationBuilder.CreateIndex(
+				name: "IX_ReattachInformations_InitialCompileJobId",
+				table: "ReattachInformations",
+				column: "InitialCompileJobId");
+
+			migrationBuilder.AddForeignKey(
+				name: "FK_ReattachInformations_CompileJobs_InitialCompileJobId",
+				table: "ReattachInformations",
+				column: "InitialCompileJobId",
+				principalTable: "CompileJobs",
+				principalColumn: "Id");
+		}
+
+		/// <inheritdoc />
+		protected override void Down(MigrationBuilder migrationBuilder)
+		{
+			if (migrationBuilder == null)
+				throw new ArgumentNullException(nameof(migrationBuilder));
+
+			migrationBuilder.DropForeignKey(
+				name: "FK_ReattachInformations_CompileJobs_InitialCompileJobId",
+				table: "ReattachInformations");
+
+			migrationBuilder.DropIndex(
+				name: "IX_ReattachInformations_InitialCompileJobId",
+				table: "ReattachInformations");
+
+			migrationBuilder.DropColumn(
+				name: "InitialCompileJobId",
+				table: "ReattachInformations");
+		}
+	}
+}

src/Tgstation.Server.Host/Database/Migrations/20230403050737_MYAddReattachInfoInitialCompileJob.cs

@@ -0,0 +1,58 @@
+using System;
+
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace Tgstation.Server.Host.Database.Migrations
+{
+	/// <summary>
+	/// Adds the InitialCompileJobId to the ReattachInformations table for MYSQL.
+	/// </summary>
+	public partial class MYAddReattachInfoInitialCompileJob : Migration
+	{
+		/// <inheritdoc />
+		protected override void Up(MigrationBuilder migrationBuilder)
+		{
+			if (migrationBuilder == null)
+				throw new ArgumentNullException(nameof(migrationBuilder));
+
+			migrationBuilder.AddColumn<long>(
+				name: "InitialCompileJobId",
+				table: "ReattachInformations",
+				type: "bigint",
+				nullable: true);
+
+			migrationBuilder.CreateIndex(
+				name: "IX_ReattachInformations_InitialCompileJobId",
+				table: "ReattachInformations",
+				column: "InitialCompileJobId");
+
+			migrationBuilder.AddForeignKey(
+				name: "FK_ReattachInformations_CompileJobs_InitialCompileJobId",
+				table: "ReattachInformations",
+				column: "InitialCompileJobId",
+				principalTable: "CompileJobs",
+				principalColumn: "Id");
+		}
+
+		/// <inheritdoc />
+		protected override void Down(MigrationBuilder migrationBuilder)
+		{
+			if (migrationBuilder == null)
+				throw new ArgumentNullException(nameof(migrationBuilder));
+
+			migrationBuilder.DropForeignKey(
+				name: "FK_ReattachInformations_CompileJobs_InitialCompileJobId",
+				table: "ReattachInformations");
+
+			migrationBuilder.DropIndex(
+				name: "IX_ReattachInformations_InitialCompileJobId",
+				table: "ReattachInformations");
+
+			migrationBuilder.DropColumn(
+				name: "InitialCompileJobId",
+				table: "ReattachInformations");
+		}
+	}
+}

src/Tgstation.Server.Host/Database/Migrations/20230403050832_PGAddReattachInfoInitialCompileJob.cs

@@ -0,0 +1,58 @@
+using System;
+
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace Tgstation.Server.Host.Database.Migrations
+{
+	/// <summary>
+	/// Adds the InitialCompileJobId to the ReattachInformations table for PostgresSQL.
+	/// </summary>
+	public partial class PGAddReattachInfoInitialCompileJob : Migration
+	{
+		/// <inheritdoc />
+		protected override void Up(MigrationBuilder migrationBuilder)
+		{
+			if (migrationBuilder == null)
+				throw new ArgumentNullException(nameof(migrationBuilder));
+
+			migrationBuilder.AddColumn<long>(
+				name: "InitialCompileJobId",
+				table: "ReattachInformations",
+				type: "bigint",
+				nullable: true);
+
+			migrationBuilder.CreateIndex(
+				name: "IX_ReattachInformations_InitialCompileJobId",
+				table: "ReattachInformations",
+				column: "InitialCompileJobId");
+
+			migrationBuilder.AddForeignKey(
+				name: "FK_ReattachInformations_CompileJobs_InitialCompileJobId",
+				table: "ReattachInformations",
+				column: "InitialCompileJobId",
+				principalTable: "CompileJobs",
+				principalColumn: "Id");
+		}
+
+		/// <inheritdoc />
+		protected override void Down(MigrationBuilder migrationBuilder)
+		{
+			if (migrationBuilder == null)
+				throw new ArgumentNullException(nameof(migrationBuilder));
+
+			migrationBuilder.DropForeignKey(
+				name: "FK_ReattachInformations_CompileJobs_InitialCompileJobId",
+				table: "ReattachInformations");
+
+			migrationBuilder.DropIndex(
+				name: "IX_ReattachInformations_InitialCompileJobId",
+				table: "ReattachInformations");
+
+			migrationBuilder.DropColumn(
+				name: "InitialCompileJobId",
+				table: "ReattachInformations");
+		}
+	}
+}

src/Tgstation.Server.Host/Database/Migrations/20230403050941_SLAddReattachInfoInitialCompileJob.cs

@@ -0,0 +1,58 @@
+using System;
+
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace Tgstation.Server.Host.Database.Migrations
+{
+	/// <summary>
+	/// Adds the InitialCompileJobId to the ReattachInformations table for SQLite.
+	/// </summary>
+	public partial class SLAddReattachInfoInitialCompileJob : Migration
+	{
+		/// <inheritdoc />
+		protected override void Up(MigrationBuilder migrationBuilder)
+		{
+			if (migrationBuilder == null)
+				throw new ArgumentNullException(nameof(migrationBuilder));
+
+			migrationBuilder.AddColumn<long>(
+				name: "InitialCompileJobId",
+				table: "ReattachInformations",
+				type: "INTEGER",
+				nullable: true);
+
+			migrationBuilder.CreateIndex(
+				name: "IX_ReattachInformations_InitialCompileJobId",
+				table: "ReattachInformations",
+				column: "InitialCompileJobId");
+
+			migrationBuilder.AddForeignKey(
+				name: "FK_ReattachInformations_CompileJobs_InitialCompileJobId",
+				table: "ReattachInformations",
+				column: "InitialCompileJobId",
+				principalTable: "CompileJobs",
+				principalColumn: "Id");
+		}
+
+		/// <inheritdoc />
+		protected override void Down(MigrationBuilder migrationBuilder)
+		{
+			if (migrationBuilder == null)
+				throw new ArgumentNullException(nameof(migrationBuilder));
+
+			migrationBuilder.DropForeignKey(
+				name: "FK_ReattachInformations_CompileJobs_InitialCompileJobId",
+				table: "ReattachInformations");
+
+			migrationBuilder.DropIndex(
+				name: "IX_ReattachInformations_InitialCompileJobId",
+				table: "ReattachInformations");
+
+			migrationBuilder.DropColumn(
+				name: "InitialCompileJobId",
+				table: "ReattachInformations");
+		}
+	}
+}

src/Tgstation.Server.Host/Database/Migrations/MySqlDatabaseContextModelSnapshot.cs

@@ -502,6 +502,9 @@ namespace Tgstation.Server.Host.Database.Migrations
 				b.Property<long>("CompileJobId")
 					.HasColumnType("bigint");
 
+				b.Property<long?>("InitialCompileJobId")
+					.HasColumnType("bigint");
+
 				b.Property<int>("LaunchSecurityLevel")
 					.HasColumnType("int");
 
@@ -521,6 +524,8 @@ namespace Tgstation.Server.Host.Database.Migrations
 
 				b.HasIndex("CompileJobId");
 
+				b.HasIndex("InitialCompileJobId");
+
 				b.ToTable("ReattachInformations");
 			});
 
@@ -942,7 +947,13 @@ namespace Tgstation.Server.Host.Database.Migrations
 					.OnDelete(DeleteBehavior.Cascade)
 					.IsRequired();
 
+				b.HasOne("Tgstation.Server.Host.Models.CompileJob", "InitialCompileJob")
+					.WithMany()
+					.HasForeignKey("InitialCompileJobId");
+
 				b.Navigation("CompileJob");
+
+				b.Navigation("InitialCompileJob");
 			});
 
 			modelBuilder.Entity("Tgstation.Server.Host.Models.RepositorySettings", b =>

src/Tgstation.Server.Host/Database/Migrations/PostgresSqlDatabaseContextModelSnapshot.cs

@@ -485,6 +485,9 @@ namespace Tgstation.Server.Host.Database.Migrations
 				b.Property<long>("CompileJobId")
 					.HasColumnType("bigint");
 
+				b.Property<long?>("InitialCompileJobId")
+					.HasColumnType("bigint");
+
 				b.Property<int>("LaunchSecurityLevel")
 					.HasColumnType("integer");
 
@@ -504,6 +507,8 @@ namespace Tgstation.Server.Host.Database.Migrations
 
 				b.HasIndex("CompileJobId");
 
+				b.HasIndex("InitialCompileJobId");
+
 				b.ToTable("ReattachInformations");
 			});
 
@@ -903,7 +908,13 @@ namespace Tgstation.Server.Host.Database.Migrations
 					.OnDelete(DeleteBehavior.Cascade)
 					.IsRequired();
 
+				b.HasOne("Tgstation.Server.Host.Models.CompileJob", "InitialCompileJob")
+					.WithMany()
+					.HasForeignKey("InitialCompileJobId");
+
 				b.Navigation("CompileJob");
+
+				b.Navigation("InitialCompileJob");
 			});
 
 			modelBuilder.Entity("Tgstation.Server.Host.Models.RepositorySettings", b =>

src/Tgstation.Server.Host/Database/Migrations/SqlServerDatabaseContextModelSnapshot.cs

@@ -490,6 +490,9 @@ namespace Tgstation.Server.Host.Database.Migrations
 				b.Property<long>("CompileJobId")
 					.HasColumnType("bigint");
 
+				b.Property<long?>("InitialCompileJobId")
+					.HasColumnType("bigint");
+
 				b.Property<int>("LaunchSecurityLevel")
 					.HasColumnType("int");
 
@@ -509,6 +512,8 @@ namespace Tgstation.Server.Host.Database.Migrations
 
 				b.HasIndex("CompileJobId");
 
+				b.HasIndex("InitialCompileJobId");
+
 				b.ToTable("ReattachInformations");
 			});
 
@@ -909,7 +914,13 @@ namespace Tgstation.Server.Host.Database.Migrations
 					.OnDelete(DeleteBehavior.Cascade)
 					.IsRequired();
 
+				b.HasOne("Tgstation.Server.Host.Models.CompileJob", "InitialCompileJob")
+					.WithMany()
+					.HasForeignKey("InitialCompileJobId");
+
 				b.Navigation("CompileJob");
+
+				b.Navigation("InitialCompileJob");
 			});
 
 			modelBuilder.Entity("Tgstation.Server.Host.Models.RepositorySettings", b =>

src/Tgstation.Server.Host/Database/Migrations/SqliteDatabaseContextModelSnapshot.cs

@@ -468,6 +468,9 @@ namespace Tgstation.Server.Host.Database.Migrations
 				b.Property<long>("CompileJobId")
 					.HasColumnType("INTEGER");
 
+				b.Property<long?>("InitialCompileJobId")
+					.HasColumnType("INTEGER");
+
 				b.Property<int>("LaunchSecurityLevel")
 					.HasColumnType("INTEGER");
 
@@ -487,6 +490,8 @@ namespace Tgstation.Server.Host.Database.Migrations
 
 				b.HasIndex("CompileJobId");
 
+				b.HasIndex("InitialCompileJobId");
+
 				b.ToTable("ReattachInformations");
 			});
 
@@ -874,7 +879,13 @@ namespace Tgstation.Server.Host.Database.Migrations
 					.OnDelete(DeleteBehavior.Cascade)
 					.IsRequired();
 
+				b.HasOne("Tgstation.Server.Host.Models.CompileJob", "InitialCompileJob")
+					.WithMany()
+					.HasForeignKey("InitialCompileJobId");
+
 				b.Navigation("CompileJob");
+
+				b.Navigation("InitialCompileJob");
 			});
 
 			modelBuilder.Entity("Tgstation.Server.Host.Models.RepositorySettings", b =>

src/Tgstation.Server.Host/IO/DefaultIOManager.cs

@@ -7,6 +7,7 @@ using System.Net.Http;
 using System.Threading;
 using System.Threading.Tasks;
 
+using Tgstation.Server.Host.Core;
 using Tgstation.Server.Host.System;
 
 namespace Tgstation.Server.Host.IO
@@ -98,13 +99,8 @@ namespace Tgstation.Server.Host.IO
 			src = ResolvePath(src);
 			dest = ResolvePath(dest);
 
-			var allTasks = CopyDirectoryImpl(src, dest, ignore, postCopyCallback, cancellationToken);
-
-			// Special tactics, increase the size of the ThreadPool until we have a 10-1 file-thread ratio.
-			var allFileTasks = allTasks.Skip(1);
-
-			var unityTask = Task.WhenAll(allFileTasks);
-			await unityTask.ConfigureAwait(false);
+			using var semaphore = new SemaphoreSlim(100 * Environment.ProcessorCount);
+			await Task.WhenAll(CopyDirectoryImpl(src, dest, ignore, postCopyCallback, semaphore, cancellationToken));
 		}
 
 		/// <inheritdoc />
@@ -118,24 +114,18 @@ namespace Tgstation.Server.Host.IO
 			if (dest == null)
 				throw new ArgumentNullException(nameof(dest));
 
-			// 0 size buffers prevents unnecessary buffering, async mode just uses the copy buffers See https://github.com/dotnet/runtime/blob/ad8031c813bae48d529ed6d265a2441c4b41fe7b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.Windows.cs#L163-L169
+			// tested to hell and back, these are the optimal buffer sizes
 			using var srcStream = new FileStream(
 				ResolvePath(src),
 				FileMode.Open,
 				FileAccess.Read,
 				FileShare.Read | FileShare.Delete,
-				0,
-				FileOptions.Asynchronous | FileOptions.SequentialScan);
-			using var destStream = new FileStream(
-				ResolvePath(dest),
-				FileMode.Create,
-				FileAccess.Write,
-				FileShare.Read | FileShare.Delete,
-				0,
+				DefaultBufferSize,
 				FileOptions.Asynchronous | FileOptions.SequentialScan);
+			using var destStream = CreateAsyncSequentialWriteStream(dest);
 
 			// value taken from documentation
-			await srcStream.CopyToAsync(destStream, DefaultBufferSize, cancellationToken);
+			await srcStream.CopyToAsync(destStream, 81920, cancellationToken);
 		}
 
 		/// <inheritdoc />
@@ -251,12 +241,12 @@ namespace Tgstation.Server.Host.IO
 		/// <inheritdoc />
 		public async Task WriteAllBytes(string path, byte[] contents, CancellationToken cancellationToken)
 		{
-			using var file = CreateAsyncWriteStream(path);
+			using var file = CreateAsyncSequentialWriteStream(path);
 			await file.WriteAsync(contents, cancellationToken);
 		}
 
 		/// <inheritdoc />
-		public FileStream CreateAsyncWriteStream(string path)
+		public FileStream CreateAsyncSequentialWriteStream(string path)
 		{
 			path = ResolvePath(path);
 			return new FileStream(
@@ -383,6 +373,7 @@ namespace Tgstation.Server.Host.IO
 		/// <param name="dest">The destination directory path.</param>
 		/// <param name="ignore">Files and folders to ignore at the root level.</param>
 		/// <param name="postCopyCallback">The optional callback called for each source/dest file pair post copy.</param>
+		/// <param name="semaphore"><see cref="SemaphoreSlim"/> used to limit degree of parallelism.</param>
 		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
 		/// <returns>A <see cref="IEnumerable{T}"/> of <see cref="Task"/>s representing the running operations. The first <see cref="Task"/> returned is always the necessary call to <see cref="CreateDirectory(string, CancellationToken)"/>.</returns>
 		IEnumerable<Task> CopyDirectoryImpl(
@@ -390,6 +381,7 @@ namespace Tgstation.Server.Host.IO
 			string dest,
 			IEnumerable<string> ignore,
 			Func<string, string, Task> postCopyCallback,
+			SemaphoreSlim semaphore,
 			CancellationToken cancellationToken)
 		{
 			var dir = new DirectoryInfo(src);
@@ -400,7 +392,7 @@ namespace Tgstation.Server.Host.IO
 					continue;
 
 				var checkingSubdirCreationTask = true;
-				foreach (var copyTask in CopyDirectoryImpl(subDirectory.FullName, Path.Combine(dest, subDirectory.Name), null, postCopyCallback, cancellationToken))
+				foreach (var copyTask in CopyDirectoryImpl(subDirectory.FullName, Path.Combine(dest, subDirectory.Name), null, postCopyCallback, semaphore, cancellationToken))
 				{
 					if (subdirCreationTask == null)
 					{
@@ -430,9 +422,8 @@ namespace Tgstation.Server.Host.IO
 
 				async Task CopyThisFile()
 				{
-					// Grab all tasks before firing
 					await subdirCreationTask;
-					await Task.Yield();
+					using var lockContext = await SemaphoreSlimContext.Lock(semaphore, cancellationToken);
 					await CopyFile(sourceFile, destFile, cancellationToken);
 					if (postCopyCallback != null)
 						await postCopyCallback(sourceFile, destFile);

src/Tgstation.Server.Host/IO/IIOManager.cs

@@ -102,11 +102,11 @@ namespace Tgstation.Server.Host.IO
 		Task<IReadOnlyList<string>> GetFiles(string path, CancellationToken cancellationToken);
 
 		/// <summary>
-		/// Creates a <see cref="FileStream"/> for writing.
+		/// Creates an asynchronous <see cref="FileStream"/> for sequential writing.
 		/// </summary>
 		/// <param name="path">The path of the file to write, will be truncated.</param>
 		/// <returns>The open <see cref="FileStream"/>.</returns>
-		FileStream CreateAsyncWriteStream(string path);
+		FileStream CreateAsyncSequentialWriteStream(string path);
 
 		/// <summary>
 		/// Writes some <paramref name="contents"/> to a file at <paramref name="path"/> overwriting previous content.

src/Tgstation.Server.Host/Models/ReattachInformation.cs

@@ -22,5 +22,15 @@ namespace Tgstation.Server.Host.Models
 		/// The <see cref="Api.Models.EntityId.Id"/> of <see cref="CompileJob"/>.
 		/// </summary>
 		public long CompileJobId { get; set; }
+
+		/// <summary>
+		/// The <see cref="Models.CompileJob"/> the server was initially launched with in the case of Windows.
+		/// </summary>
+		public CompileJob InitialCompileJob { get; set; }
+
+		/// <summary>
+		/// The <see cref="Api.Models.EntityId.Id"/> of <see cref="InitialCompileJob"/>.
+		/// </summary>
+		public long? InitialCompileJobId { get; set; }
 	}
 }

src/Tgstation.Server.Host/System/ProcessExecutor.cs

@@ -292,7 +292,7 @@ namespace Tgstation.Server.Host.System
 				return line;
 			}
 
-			using var fileStream = fileRedirect != null ? ioManager.CreateAsyncWriteStream(fileRedirect) : null;
+			using var fileStream = fileRedirect != null ? ioManager.CreateAsyncSequentialWriteStream(fileRedirect) : null;
 			using var writer = fileStream != null ? new StreamWriter(fileStream) : null;
 
 			string text;

tests/DMAPI/ApiFree/Test.dm

@@ -1,2 +1,7 @@
 /world/New()
+	text2file("SUCCESS", "test_success.txt")
 	log << "Hello world!"
+
+/world/Error(exception)
+	fdel("test_success.txt")
+	text2file("Runtime Error: [exception]", "test_fail_reason.txt")

tests/DMAPI/BasicOperation/Test.dm

@@ -1,10 +1,15 @@
 /world/New()
+	text2file("SUCCESS", "test_success.txt")
 	log << "About to call TgsNew()"
 	sleep_offline = FALSE
 	TgsNew(minimum_required_security_level = TGS_SECURITY_SAFE)
 	log << "About to call StartAsync()"
 	StartAsync()
 
+/world/Error(exception)
+	fdel("test_success.txt")
+	text2file("Runtime Error: [exception]", "test_fail_reason.txt")
+
 /proc/StartAsync()
 	set waitfor = FALSE
 	Run()
@@ -14,7 +19,7 @@
 	sleep(50)
 	world.TgsTargetedChatBroadcast("Sample admin-only message", TRUE)
 
-	var/list/world_params = params2list(world.params)
+	var/list/world_params = world.params
 	if(!("test" in world_params) || world_params["test"] != "bababooey")
 		text2file("Expected parameter test=bababooey but did not receive", "test_fail_reason.txt")
 

tests/DMAPI/LongRunning/Test.dm

@@ -2,7 +2,12 @@
 	sleep_offline = FALSE
 	loop_checks = FALSE
 
+/world/Error(exception)
+	fdel("test_success.txt")
+	text2file("Runtime Error: [exception]", "test_fail_reason.txt")
+
 /world/New()
+	text2file("SUCCESS", "test_success.txt")
 	log << "Initial value of sleep_offline: [sleep_offline]"
 	sleep_offline = FALSE
 
@@ -69,7 +74,7 @@
 /datum/tgs_event_handler/impl/HandleEvent(event_code, ...)
 	set waitfor = FALSE
 
-	world.TgsChatBroadcast("Recieved event: [json_encode(args)]")
+	world.TgsChatBroadcast("Recieved event: `[json_encode(args)]`")
 
 /world/Export(url)
 	log << "Export: [url]"

tests/Tgstation.Server.Tests/Instance/ByondTest.cs

@@ -21,7 +21,7 @@ namespace Tgstation.Server.Tests.Instance
 {
 	sealed class ByondTest : JobsRequiredTest
 	{
-		public static readonly Version TestVersion = new (515, 1592);
+		public static readonly Version TestVersion = new (514, 1588);
 
 		readonly IByondClient byondClient;
 

tests/Tgstation.Server.Tests/Instance/DeploymentTest.cs

@@ -60,10 +60,10 @@ namespace Tgstation.Server.Tests.Instance
 
 			var updatedDD = await dreamDaemonClient.Update(new DreamDaemonRequest
 			{
-				StartupTimeout = 5,
+				StartupTimeout = 15,
 				Port = IntegrationTest.DDPort
 			}, cancellationToken);
-			Assert.AreEqual(5U, updatedDD.StartupTimeout);
+			Assert.AreEqual(15U, updatedDD.StartupTimeout);
 			Assert.AreEqual(IntegrationTest.DDPort, updatedDD.Port);
 
 			await ApiAssert.ThrowsException<ConflictException>(() => dreamDaemonClient.Update(new DreamDaemonRequest
@@ -77,7 +77,7 @@ namespace Tgstation.Server.Tests.Instance
 			}, cancellationToken), ErrorCode.PortNotAvailable);
 
 			deployJob = await dreamMakerClient.Compile(cancellationToken);
-			await WaitForJob(deployJob, 30, true, ErrorCode.DreamMakerNeverValidated, cancellationToken);
+			await WaitForJob(deployJob, 40, true, ErrorCode.DreamMakerNeverValidated, cancellationToken);
 
 			const string FailProject = "tests/DMAPI/BuildFail/build_fail";
 			var updated = await dreamMakerClient.Update(new DreamMakerRequest
@@ -88,7 +88,7 @@ namespace Tgstation.Server.Tests.Instance
 			Assert.AreEqual(FailProject, updated.ProjectName);
 
 			deployJob = await dreamMakerClient.Compile(cancellationToken);
-			await WaitForJob(deployJob, 30, true, ErrorCode.DreamMakerExitCode, cancellationToken);
+			await WaitForJob(deployJob, 40, true, ErrorCode.DreamMakerExitCode, cancellationToken);
 
 			await dreamMakerClient.Update(new DreamMakerRequest
 			{
@@ -96,7 +96,7 @@ namespace Tgstation.Server.Tests.Instance
 			}, cancellationToken);
 
 			deployJob = await dreamMakerClient.Compile(cancellationToken);
-			await WaitForJob(deployJob, 30, true, ErrorCode.DreamMakerMissingDme, cancellationToken);
+			await WaitForJob(deployJob, 40, true, ErrorCode.DreamMakerMissingDme, cancellationToken);
 
 			// check that we can change the visibility
 

tests/Tgstation.Server.Tests/Instance/WatchdogTest.cs

@@ -28,6 +28,8 @@ namespace Tgstation.Server.Tests.Instance
 	{
 		readonly IInstanceClient instanceClient;
 
+		bool ranTimeoutTest = false;
+
 		public WatchdogTest(IInstanceClient instanceClient)
 			: base(instanceClient.Jobs)
 		{
@@ -40,7 +42,7 @@ namespace Tgstation.Server.Tests.Instance
 			// Increase startup timeout, disable heartbeats
 			var initialSettings = await instanceClient.DreamDaemon.Update(new DreamDaemonRequest
 			{
-				StartupTimeout = 60,
+				StartupTimeout = 15,
 				HeartbeatSeconds = 0,
 				Port = IntegrationTest.DDPort
 			}, cancellationToken);
@@ -96,7 +98,7 @@ namespace Tgstation.Server.Tests.Instance
 				killTaskStarted.SetResult(null);
 				while (!jobTcs.Task.IsCompleted)
 					KillDD(false);
-			});
+			}, cancellationToken);
 
 			JobResponse job;
 			try
@@ -138,6 +140,7 @@ namespace Tgstation.Server.Tests.Instance
 			Assert.AreEqual(false, daemonStatus.SoftShutdown);
 			Assert.AreEqual(String.Empty, daemonStatus.AdditionalParameters);
 			var initialCompileJob = daemonStatus.ActiveCompileJob;
+			await CheckDMApiFail(daemonStatus.ActiveCompileJob, cancellationToken);
 
 			daemonStatus = await DeployTestDme("BasicOperation/basic_operation_test", DreamDaemonSecurity.Trusted, true, cancellationToken);
 
@@ -239,7 +242,7 @@ namespace Tgstation.Server.Tests.Instance
 				.GetProcess(ddProc.Id);
 
 			// Ensure it's responding to heartbeats
-			await Task.WhenAny(Task.Delay(20000), ourProcessHandler.Lifetime);
+			await Task.WhenAny(Task.Delay(20000, cancellationToken), ourProcessHandler.Lifetime);
 			Assert.IsFalse(ddProc.HasExited);
 
 			await instanceClient.DreamDaemon.Update(new DreamDaemonRequest
@@ -249,7 +252,7 @@ namespace Tgstation.Server.Tests.Instance
 
 			ourProcessHandler.Suspend();
 
-			await Task.WhenAny(ourProcessHandler.Lifetime, Task.Delay(TimeSpan.FromMinutes(1)));
+			await Task.WhenAny(ourProcessHandler.Lifetime, Task.Delay(TimeSpan.FromMinutes(1), cancellationToken));
 
 			var timeout = 20;
 			do
@@ -258,7 +261,10 @@ namespace Tgstation.Server.Tests.Instance
 				var ddStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
 				Assert.AreEqual(1U, ddStatus.HeartbeatSeconds.Value);
 				if (ddStatus.Status.Value == WatchdogStatus.Offline)
+				{
+					await CheckDMApiFail(ddStatus.ActiveCompileJob, cancellationToken);
 					break;
+				}
 
 				if (--timeout == 0)
 					Assert.Fail("DreamDaemon didn't shutdown within the timeout!");
@@ -341,6 +347,7 @@ namespace Tgstation.Server.Tests.Instance
 			Assert.AreNotEqual(initialCompileJob.Id, newerCompileJob.Id);
 			Assert.AreEqual(DreamDaemonSecurity.Ultrasafe, newerCompileJob.MinimumSecurityLevel);
 
+			await CheckDMApiFail(daemonStatus.ActiveCompileJob, cancellationToken);
 			daemonStatus = await TellWorldToReboot(cancellationToken);
 
 			Assert.AreNotEqual(initialCompileJob.Id, daemonStatus.ActiveCompileJob.Id);
@@ -350,6 +357,7 @@ namespace Tgstation.Server.Tests.Instance
 
 			daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
 			Assert.AreEqual(WatchdogStatus.Offline, daemonStatus.Status.Value);
+			await CheckDMApiFail(daemonStatus.ActiveCompileJob, cancellationToken);
 		}
 
 		async Task RunLongRunningTestThenUpdateWithNewDme(CancellationToken cancellationToken)
@@ -381,6 +389,7 @@ namespace Tgstation.Server.Tests.Instance
 			Assert.AreNotEqual(initialCompileJob.Id, newerCompileJob.Id);
 			Assert.AreEqual(DreamDaemonSecurity.Ultrasafe, newerCompileJob.MinimumSecurityLevel);
 
+			await CheckDMApiFail(daemonStatus.ActiveCompileJob, cancellationToken);
 			daemonStatus = await TellWorldToReboot(cancellationToken);
 
 			Assert.AreNotEqual(initialCompileJob.Id, daemonStatus.ActiveCompileJob.Id);
@@ -390,6 +399,7 @@ namespace Tgstation.Server.Tests.Instance
 
 			daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
 			Assert.AreEqual(WatchdogStatus.Offline, daemonStatus.Status.Value);
+			await CheckDMApiFail(daemonStatus.ActiveCompileJob, cancellationToken);
 		}
 
 		async Task RunLongRunningTestThenUpdateWithByondVersionSwitch(CancellationToken cancellationToken)
@@ -434,12 +444,14 @@ namespace Tgstation.Server.Tests.Instance
 
 			Assert.AreEqual(true, daemonStatus.SoftRestart);
 
+			await CheckDMApiFail(daemonStatus.ActiveCompileJob, cancellationToken);
 			daemonStatus = await TellWorldToReboot(cancellationToken);
 
 			Assert.AreEqual(versionToInstall, daemonStatus.ActiveCompileJob.ByondVersion);
 			Assert.IsNull(daemonStatus.StagedCompileJob);
 
 			await instanceClient.DreamDaemon.Shutdown(cancellationToken);
+			await CheckDMApiFail(daemonStatus.ActiveCompileJob, cancellationToken);
 
 			daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
 			Assert.AreEqual(WatchdogStatus.Offline, daemonStatus.Status.Value);
@@ -460,33 +472,30 @@ namespace Tgstation.Server.Tests.Instance
 			Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value);
 			Assert.AreEqual(IntegrationTest.DDPort, daemonStatus.CurrentPort);
 
-			// The measure we use to test dream daemon startup doesn't work on linux currently
-			if (new PlatformIdentifier().IsWindows)
+			// Try killing the DD process to ensure it gets set to the restoring state
+			do
 			{
-				// Try killing the DD process to ensure it gets set to the restoring state
-				do
-				{
-					KillDD(true);
-					await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken);
-					daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
-				}
-				while (daemonStatus.Status == WatchdogStatus.Online);
-				Assert.AreEqual(WatchdogStatus.Restoring, daemonStatus.Status.Value);
-
-				// Kill it again
-				do
-				{
-					KillDD(false);
-					daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
-				}
-				while (daemonStatus.Status == WatchdogStatus.Online || daemonStatus.Status == WatchdogStatus.Restoring);
-				Assert.AreEqual(WatchdogStatus.DelayedRestart, daemonStatus.Status.Value);
-
-				await Task.Delay(TimeSpan.FromSeconds(15), cancellationToken);
-
+				KillDD(true);
+				await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken);
 				daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
-				Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value);
 			}
+			while (daemonStatus.Status == WatchdogStatus.Online);
+			Assert.AreEqual(WatchdogStatus.Restoring, daemonStatus.Status.Value);
+
+			// Kill it again
+			do
+			{
+				KillDD(false);
+				daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
+			}
+			while (daemonStatus.Status == WatchdogStatus.Online || daemonStatus.Status == WatchdogStatus.Restoring);
+			Assert.AreEqual(WatchdogStatus.DelayedRestart, daemonStatus.Status.Value);
+
+			await Task.Delay(TimeSpan.FromSeconds(15), cancellationToken);
+
+			daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
+			Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value);
+			await CheckDMApiFail(daemonStatus.ActiveCompileJob, cancellationToken);
 		}
 
 		static bool KillDD(bool require)
@@ -502,7 +511,7 @@ namespace Tgstation.Server.Tests.Instance
 			return ddProc != null;
 		}
 
-		async Task<DreamDaemonResponse> TellWorldToReboot(CancellationToken cancellationToken)
+		public async Task<DreamDaemonResponse> TellWorldToReboot(CancellationToken cancellationToken)
 		{
 			var daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
 			var initialCompileJob = daemonStatus.ActiveCompileJob;
@@ -529,10 +538,10 @@ namespace Tgstation.Server.Tests.Instance
 
 					do
 					{
-						await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken);
-						daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
+						await Task.Delay(TimeSpan.FromSeconds(1), tempToken);
+						daemonStatus = await instanceClient.DreamDaemon.Read(tempToken);
 					}
-					while (initialCompileJob.Id == daemonStatus.ActiveCompileJob.Id && !tempToken.IsCancellationRequested);
+					while (initialCompileJob.Id == daemonStatus.ActiveCompileJob.Id);
 				}
 			}
 			catch (OperationCanceledException)
@@ -553,13 +562,19 @@ namespace Tgstation.Server.Tests.Instance
 				Timeout = TimeSpan.FromMilliseconds(1),
 			}, cancellationToken);
 
-			Assert.AreEqual(deploymentSecurity, refreshed.ApiValidationSecurityLevel);
-			Assert.AreEqual(requireApi, refreshed.RequireDMApiValidation);
-			Assert.AreEqual(TimeSpan.FromMilliseconds(1), refreshed.Timeout);
+			JobResponse compileJobJob;
+			if (!ranTimeoutTest)
+			{
+				Assert.AreEqual(deploymentSecurity, refreshed.ApiValidationSecurityLevel);
+				Assert.AreEqual(requireApi, refreshed.RequireDMApiValidation);
+				Assert.AreEqual(TimeSpan.FromMilliseconds(1), refreshed.Timeout);
 
-			var compileJobJob = await instanceClient.DreamMaker.Compile(cancellationToken);
+				compileJobJob = await instanceClient.DreamMaker.Compile(cancellationToken);
+
+				await WaitForJob(compileJobJob, 90, true, ErrorCode.DeploymentTimeout, cancellationToken);
+				ranTimeoutTest = true;
+			}
 
-			await WaitForJob(compileJobJob, 90, true, ErrorCode.DeploymentTimeout, cancellationToken);
 			await instanceClient.DreamMaker.Update(new DreamMakerRequest
 			{
 				Timeout = TimeSpan.FromMinutes(5),
@@ -599,9 +614,14 @@ namespace Tgstation.Server.Tests.Instance
 
 		async Task CheckDMApiFail(CompileJobResponse compileJob, CancellationToken cancellationToken)
 		{
-			var failFile = Path.Combine(instanceClient.Metadata.Path, "Game", compileJob.DirectoryName.Value.ToString(), "A", Path.GetDirectoryName(compileJob.DmeName), "test_fail_reason.txt");
+			var gameDir = Path.Combine(instanceClient.Metadata.Path, "Game", compileJob.DirectoryName.Value.ToString(), Path.GetDirectoryName(compileJob.DmeName));
+			var failFile = Path.Combine(gameDir, "test_fail_reason.txt");
 			if (!File.Exists(failFile))
+			{
+				var successFile = Path.Combine(gameDir, "test_success.txt");
+				Assert.IsTrue(File.Exists(successFile));
 				return;
+			}
 
 			var text = await File.ReadAllTextAsync(failFile, cancellationToken);
 			Assert.Fail(text);

tests/Tgstation.Server.Tests/IntegrationTest.cs

@@ -26,6 +26,7 @@ using Tgstation.Server.Api.Models.Request;
 using Tgstation.Server.Api.Models.Response;
 using Tgstation.Server.Api.Rights;
 using Tgstation.Server.Client;
+using Tgstation.Server.Client.Components;
 using Tgstation.Server.Host.Components.Events;
 using Tgstation.Server.Host.Components.Repository;
 using Tgstation.Server.Host.Configuration;
@@ -939,11 +940,13 @@ namespace Tgstation.Server.Tests
 					Assert.AreEqual(WatchdogStatus.Online, dd.Status.Value);
 
 					await instanceClient.DreamDaemon.Shutdown(cancellationToken);
-					await instanceClient.DreamDaemon.Update(new DreamDaemonRequest
+					dd = await instanceClient.DreamDaemon.Update(new DreamDaemonRequest
 					{
 						AutoStart = true
 					}, cancellationToken);
 
+					Assert.AreEqual(WatchdogStatus.Offline, dd.Status);
+
 					await adminClient.Administration.Restart(cancellationToken);
 				}
 
@@ -952,12 +955,8 @@ namespace Tgstation.Server.Tests
 
 				preStartupTime = DateTimeOffset.UtcNow;
 
-				// chat bot start, dd autostart, and entity delete tests
-				serverTask = server.Run(cancellationToken);
-				using (var adminClient = await CreateAdminClient(server.Url, cancellationToken))
+				async Task WaitForInitialJobs(IInstanceClient instanceClient)
 				{
-					var instanceClient = adminClient.Instances.CreateClient(instance);
-
 					var jobs = await instanceClient.Jobs.ListActive(null, cancellationToken);
 					if (!jobs.Any())
 					{
@@ -970,20 +969,83 @@ namespace Tgstation.Server.Tests
 						jobs = getTasks
 							.Select(x => x.Result)
 							.Where(x => x.StartedAt.Value > preStartupTime)
-							.ToList();
+						.ToList();
 					}
 
 					var jrt = new JobsRequiredTest(instanceClient.Jobs);
 					foreach (var job in jobs)
 					{
 						Assert.IsTrue(job.StartedAt.Value >= preStartupTime);
-						await jrt.WaitForJob(job, 140, job.Description.Contains("Reconnect chat bot") ? (bool?)null : (bool?)false, null, cancellationToken);
+						await jrt.WaitForJob(job, 140, job.Description.Contains("Reconnect chat bot") ? null : false, null, cancellationToken);
 					}
+				}
+
+				// chat bot start, dd autostart, and reboot with different initial job test
+				preStartupTime = DateTimeOffset.UtcNow;
+				serverTask = server.Run(cancellationToken);
+				long expectedCompileJobId, expectedStaged;
+				using (var adminClient = await CreateAdminClient(server.Url, cancellationToken))
+				{
+					var instanceClient = adminClient.Instances.CreateClient(instance);
+					await WaitForInitialJobs(instanceClient);
 
 					var dd = await instanceClient.DreamDaemon.Read(cancellationToken);
 
 					Assert.AreEqual(WatchdogStatus.Online, dd.Status.Value);
 
+					var compileJob = await instanceClient.DreamMaker.Compile(cancellationToken);
+					var wdt = new WatchdogTest(instanceClient);
+					await wdt.WaitForJob(compileJob, 30, false, null, cancellationToken);
+
+					dd = await instanceClient.DreamDaemon.Read(cancellationToken);
+					Assert.AreEqual(dd.StagedCompileJob.Job.Id, compileJob.Id);
+
+					expectedCompileJobId = compileJob.Id.Value;
+					dd = await wdt.TellWorldToReboot(cancellationToken);
+
+					while (dd.Status.Value == WatchdogStatus.Restoring)
+					{
+						await Task.Delay(TimeSpan.FromSeconds(1));
+						dd = await instanceClient.DreamDaemon.Read(cancellationToken);
+					}
+
+					Assert.AreEqual(dd.ActiveCompileJob.Job.Id, expectedCompileJobId);
+					Assert.AreEqual(WatchdogStatus.Online, dd.Status.Value);
+
+					expectedCompileJobId = dd.ActiveCompileJob.Id.Value;
+
+					await instanceClient.DreamDaemon.Update(new DreamDaemonRequest
+					{
+						AutoStart = false,
+					}, cancellationToken);
+
+					compileJob = await instanceClient.DreamMaker.Compile(cancellationToken);
+					await wdt.WaitForJob(compileJob, 30, false, null, cancellationToken);
+					expectedStaged = compileJob.Id.Value;
+
+					await adminClient.Administration.Restart(cancellationToken);
+				}
+
+				await Task.WhenAny(serverTask, Task.Delay(TimeSpan.FromMinutes(1), cancellationToken));
+				Assert.IsTrue(serverTask.IsCompleted);
+
+				// post/entity deletion tests
+				serverTask = server.Run(cancellationToken);
+				using (var adminClient = await CreateAdminClient(server.Url, cancellationToken))
+				{
+					var instanceClient = adminClient.Instances.CreateClient(instance);
+					await WaitForInitialJobs(instanceClient);
+
+					var currentDD = await instanceClient.DreamDaemon.Read(cancellationToken);
+					Assert.AreEqual(expectedCompileJobId, currentDD.ActiveCompileJob.Id.Value);
+					Assert.AreEqual(WatchdogStatus.Online, currentDD.Status);
+					Assert.AreEqual(expectedStaged, currentDD.StagedCompileJob.Job.Id.Value);
+
+					var wdt = new WatchdogTest(instanceClient);
+					currentDD = await wdt.TellWorldToReboot(cancellationToken);
+					Assert.AreEqual(expectedStaged, currentDD.ActiveCompileJob.Job.Id.Value);
+					Assert.IsNull(currentDD.StagedCompileJob);
+
 					var repoTest = new RepositoryTest(instanceClient.Repository, instanceClient.Jobs).RunPostTest(cancellationToken);
 					await new ChatTest(instanceClient.ChatBots, adminClient.Instances, instance).RunPostTest(cancellationToken);
 					await repoTest;