Merge pull request #987 from tgstation/986-FixAutostart [DMDeploy][TGSDeploy]

Fix issue with compile job cleaning

build/Version.props

@@ -2,10 +2,10 @@
   <PropertyGroup>
     <!-- This is the authorative version list -->
     <!-- Integration tests will ensure they match across the board -->
-    <TgsCoreVersion>4.2.0</TgsCoreVersion>
+    <TgsCoreVersion>4.2.1</TgsCoreVersion>
     <TgsApiVersion>6.2.0</TgsApiVersion>
     <TgsClientVersion>6.1.0</TgsClientVersion>
-    <TgsDmapiVersion>5.1.0</TgsDmapiVersion>
+    <TgsDmapiVersion>5.1.1</TgsDmapiVersion>
     <TgsControlPanelVersion>0.4.0</TgsControlPanelVersion>
     <TgsHostWatchdogVersion>1.1.0</TgsHostWatchdogVersion>
   </PropertyGroup>

build/prep_deployment.ps1

@@ -1,5 +1,7 @@
 $bf = $env:APPVEYOR_BUILD_FOLDER
-[XML]$versionXML = Get-Content "$bf/build/Version.props"
+$propsPath = "$bf/build/Version.props"
+
+[XML]$versionXML = Get-Content $propsPath
 $env:TGSVersion = $versionXML.Project.PropertyGroup.TgsCoreVersion
 $env:APIVersion = $versionXML.Project.PropertyGroup.TgsApiVersion
 $env:DMVersion = $versionXML.Project.PropertyGroup.TgsDmapiVersion
@@ -11,7 +13,7 @@ if (($env:CONFIGURATION -match "Release") -And ($env:APPVEYOR_REPO_BRANCH -match
     $env:TGSDeploy = "Do it." 
 
     Write-Host "Generating release notes..."
-    dotnet run -p "$bf/tools/ReleaseNotes" $env:TGSVersion
+    dotnet run -p "$bf/tools/ReleaseNotes" $env:TGSVersion $propsPath
     $env:TGSDraftNotes = !($?)
     $releaseNotesPath = "$bf/release_notes.md"
     Write-Host "Reading release notes from $releaseNotesPath..."

src/DMAPI/tgs.dm

@@ -1,6 +1,6 @@
 //tgstation-server DMAPI
 
-#define TGS_DMAPI_VERSION "5.1.0"
+#define TGS_DMAPI_VERSION "5.1.1"
 
 //All functions and datums outside this document are subject to change with any version and should not be relied on
 

src/DMAPI/tgs/core/datum.dm

@@ -6,7 +6,7 @@ TGS_DEFINE_AND_SET_GLOBAL(tgs, null)
 
 /datum/tgs_api/New(datum/tgs_event_handler/event_handler, datum/tgs_version/version)
 	. = ..()
-	src.event_handler = version
+	src.event_handler = event_handler
 	src.version = version
 
 /datum/tgs_api/latest

src/DMAPI/tgs/v5/_defines.dm

@@ -60,10 +60,10 @@
 #define DMAPI5_TOPIC_COMMAND_CHANGE_PORT 2
 #define DMAPI5_TOPIC_COMMAND_CHANGE_REBOOT_STATE 3
 #define DMAPI5_TOPIC_COMMAND_INSTANCE_RENAMED 4
-#define DMAPI5_TOPIC_COMMAND_CHAT_CHANNELS_UPDATE 4
-#define DMAPI5_TOPIC_COMMAND_SERVER_PORT_UPDATE 5
-#define DMAPI5_TOPIC_COMMAND_HEARTBEAT 6
-#define DMAPI5_TOPIC_COMMAND_WATCHDOG_REATTACH 7
+#define DMAPI5_TOPIC_COMMAND_CHAT_CHANNELS_UPDATE 5
+#define DMAPI5_TOPIC_COMMAND_SERVER_PORT_UPDATE 6
+#define DMAPI5_TOPIC_COMMAND_HEARTBEAT 7
+#define DMAPI5_TOPIC_COMMAND_WATCHDOG_REATTACH 8
 
 #define DMAPI5_TOPIC_PARAMETER_COMMAND_TYPE "commandType"
 #define DMAPI5_TOPIC_PARAMETER_CHAT_COMMAND "chatCommand"

src/DMAPI/tgs/v5/api.dm

@@ -16,7 +16,7 @@
 	var/list/chat_channels
 
 /datum/tgs_api/v5/ApiVersion()
-	return new /datum/tgs_version("5.1.0")
+	return new /datum/tgs_version("5.1.1")
 
 /datum/tgs_api/v5/OnWorldNew(minimum_required_security_level)
 	server_port = world.params[DMAPI5_PARAM_SERVER_PORT]

src/Tgstation.Server.Api/Models/ErrorCode.cs

@@ -415,7 +415,7 @@ namespace Tgstation.Server.Api.Models
 		/// <summary>
 		/// Attempted to start the watchdog with a corrupted <see cref="CompileJob"/>.
 		/// </summary>
-		[Description("Cannot launch with active compile job as it is corrupted!")]
+		[Description("Cannot launch active compile job as it is missing or corrupted!")]
 		WatchdogCompileJobCorrupted,
 
 		/// <summary>

src/Tgstation.Server.Host/Components/Byond/WindowsByondInstaller.cs

@@ -119,7 +119,10 @@ namespace Tgstation.Server.Host.Components.Byond
 						IProcess directXInstaller;
 						try
 						{
-							directXInstaller = processExecutor.LaunchProcess(IOManager.ConcatPath(rbdx, "DXSETUP.exe"), rbdx, "/silent", noShellExecute: true);
+							directXInstaller = processExecutor.LaunchProcess(
+								IOManager.ConcatPath(rbdx, "DXSETUP.exe"),
+								rbdx, "/silent",
+								noShellExecute: true);
 						}
 						catch (Exception e)
 						{

src/Tgstation.Server.Host/Components/Chat/ChatManager.cs

@@ -646,7 +646,8 @@ namespace Tgstation.Server.Host.Components.Chat
 		public async Task StopAsync(CancellationToken cancellationToken)
 		{
 			handlerCts.Cancel();
-			await chatHandler.ConfigureAwait(false);
+			if (chatHandler != null)
+				await chatHandler.ConfigureAwait(false);
 			await Task.WhenAll(providers.Select(x => x.Value).Select(x => x.Disconnect(cancellationToken))).ConfigureAwait(false);
 		}
 

src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs

@@ -118,7 +118,7 @@ namespace Tgstation.Server.Host.Components.Deployment
 				if (!jobLockCounts.TryGetValue(job.Id, out var currentVal) || currentVal == 1)
 				{
 					jobLockCounts.Remove(job.Id);
-					logger.LogDebug("Cleaning compile job {0} => {1}", job.Id, job.DirectoryName);
+					logger.LogDebug("Cleaning lock-free compile job {0} => {1}", job.Id, job.DirectoryName);
 					cleanupTask = HandleCleanup();
 				}
 				else
@@ -247,7 +247,7 @@ namespace Tgstation.Server.Host.Components.Deployment
 
 		/// <inheritdoc />
 		#pragma warning disable CA1506 // TODO: Decomplexify
-		public async Task CleanUnusedCompileJobs(CompileJob exceptThisOne, CancellationToken cancellationToken)
+		public async Task CleanUnusedCompileJobs(CancellationToken cancellationToken)
 		{
 			List<long> jobIdsToSkip;
 
@@ -260,12 +260,18 @@ namespace Tgstation.Server.Host.Components.Deployment
 			// find the uids of locked directories
 			await databaseContextFactory.UseContext(async db =>
 			{
-				jobUidsToNotErase = await db.CompileJobs.Where(x => x.Job.Instance.Id == instance.Id && jobIdsToSkip.Contains(x.Id)).Select(x => x.DirectoryName.Value.ToString().ToUpperInvariant()).ToListAsync(cancellationToken).ConfigureAwait(false);
+				jobUidsToNotErase = (await db.CompileJobs.Where(
+					x => x.Job.Instance.Id == instance.Id && jobIdsToSkip.Contains(x.Id))
+					.Select(x => x.DirectoryName.Value)
+					.ToListAsync(cancellationToken)
+					.ConfigureAwait(false))
+					.Select(x => x.ToString())
+					.ToList();
 			}).ConfigureAwait(false);
 
-			// add the other exemption
-			if (exceptThisOne != null)
-				jobUidsToNotErase.Add(exceptThisOne.DirectoryName.Value.ToString().ToUpperInvariant());
+			jobUidsToNotErase.Add(WindowsSwappableDmbProvider.LiveGameDirectory);
+
+			logger.LogTrace("We will not clean the following directories: {0}", String.Join(", ", jobUidsToNotErase));
 
 			// cleanup
 			var gameDirectory = ioManager.ResolvePath();
@@ -275,8 +281,9 @@ namespace Tgstation.Server.Host.Components.Deployment
 			var tasks = directories.Select(async x =>
 			{
 				var nameOnly = ioManager.GetFileName(x);
-				if (jobUidsToNotErase.Contains(nameOnly.ToUpperInvariant()))
+				if (jobUidsToNotErase.Contains(nameOnly))
 					return;
+				logger.LogDebug("Cleaning unused game folder: {0}...", nameOnly);
 				try
 				{
 					++deleting;
@@ -292,10 +299,7 @@ namespace Tgstation.Server.Host.Components.Deployment
 				}
 			}).ToList();
 			if (deleting > 0)
-			{
-				logger.LogDebug("Cleaning unused game folders: {0}...", String.Join(", ", directories));
-				await Task.WhenAll().ConfigureAwait(false);
-			}
+				await Task.WhenAll(tasks).ConfigureAwait(false);
 		}
 		#pragma warning restore CA1506
 	}

src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs

@@ -244,6 +244,7 @@ namespace Tgstation.Server.Host.Components.Deployment
 						ADirectoryName)),
 				$"-clean {job.DmeName}.{DmeExtension}",
 				true,
+				true,
 				true);
 			int exitCode;
 			using (cancellationToken.Register(() => dm.Terminate()))
@@ -381,7 +382,7 @@ namespace Tgstation.Server.Host.Components.Deployment
 			await chatManager.SendUpdateMessage(
 				String.Format(
 					CultureInfo.InvariantCulture,
-					"*Deployment Triggered*{0}Revision: {1}{2}{3}{0} BYOND Version: {4}.{5}",
+					"*Deployment Triggered*{0}Revision: {1}{2}{3}{0}BYOND Version: {4}.{5}",
 					Environment.NewLine,
 					commitInsert,
 					testmergeInsert,

src/Tgstation.Server.Host/Components/Deployment/IDmbFactory.cs

@@ -37,11 +37,10 @@ namespace Tgstation.Server.Host.Components.Deployment
 		Task<IDmbProvider> FromCompileJob(CompileJob compileJob, CancellationToken cancellationToken);
 
 		/// <summary>
-		/// Deletes all compile jobs that are inactive in the Game folder <paramref name="exceptThisOne"/>
+		/// Deletes all compile jobs that are inactive in the Game folder.
 		/// </summary>
-		/// <param name="exceptThisOne">An optional compile job to not delete</param>
 		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
 		/// <returns>A <see cref="Task"/> representing the running operation</returns>
-		Task CleanUnusedCompileJobs(CompileJob exceptThisOne, CancellationToken cancellationToken);
+		Task CleanUnusedCompileJobs(CancellationToken cancellationToken);
 	}
 }

src/Tgstation.Server.Host/Components/Deployment/WindowsSwappableDmbProvider.cs

@@ -14,7 +14,7 @@ namespace Tgstation.Server.Host.Components.Deployment
 		/// <summary>
 		/// The directory where the <see cref="baseProvider"/> is symlinked to.
 		/// </summary>
-		const string LiveGameDirectory = "Live";
+		public const string LiveGameDirectory = "Live";
 
 		/// <inheritdoc />
 		public string DmbName => baseProvider.DmbName;

src/Tgstation.Server.Host/Components/EventConsumer.cs

@@ -30,14 +30,13 @@ namespace Tgstation.Server.Host.Components
 		}
 
 		/// <inheritdoc />
-		public async Task<bool> HandleEvent(EventType eventType, IEnumerable<string> parameters, CancellationToken cancellationToken)
+		public async Task HandleEvent(EventType eventType, IEnumerable<string> parameters, CancellationToken cancellationToken)
 		{
 			if (watchdog == null)
 				throw new InvalidOperationException("EventConsumer used without watchdog set!");
 
-			if (!await configuration.HandleEvent(eventType, parameters, cancellationToken).ConfigureAwait(false))
-				return false;
-			return await watchdog.HandleEvent(eventType, parameters, cancellationToken).ConfigureAwait(false);
+			await configuration.HandleEvent(eventType, parameters, cancellationToken).ConfigureAwait(false);
+			await watchdog.HandleEvent(eventType, parameters, cancellationToken).ConfigureAwait(false);
 		}
 
 		/// <summary>

src/Tgstation.Server.Host/Components/IEventConsumer.cs

@@ -15,7 +15,7 @@ namespace Tgstation.Server.Host.Components
 		/// <param name="eventType">The <see cref="EventType"/></param>
 		/// <param name="parameters">The parameters for <paramref name="eventType"/></param>
 		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
-		/// <returns>A <see cref="Task{TResult}"/> resulting in <see langword="true"/> if more <see cref="IEventConsumer"/> should run, <see langword="false"/> otherwise</returns>
-		Task<bool> HandleEvent(EventType eventType, IEnumerable<string> parameters, CancellationToken cancellationToken);
+		/// <returns>A <see cref="Task"/> representing the running operation.</returns>
+		Task HandleEvent(EventType eventType, IEnumerable<string> parameters, CancellationToken cancellationToken);
 	}
 }

src/Tgstation.Server.Host/Components/Instance.cs

@@ -15,7 +15,6 @@ using Tgstation.Server.Host.Components.Repository;
 using Tgstation.Server.Host.Components.Watchdog;
 using Tgstation.Server.Host.Core;
 using Tgstation.Server.Host.Database;
-using Tgstation.Server.Host.Extensions;
 using Tgstation.Server.Host.Jobs;
 using Tgstation.Server.Host.Models;
 
@@ -623,12 +622,7 @@ namespace Tgstation.Server.Host.Components
 			// dependent on so many things, its just safer this way
 			await Watchdog.StartAsync(cancellationToken).ConfigureAwait(false);
 
-			CompileJob latestCompileJob = null;
-			await databaseContextFactory.UseContext(async db =>
-			{
-				latestCompileJob = await db.MostRecentCompletedCompileJobOrDefault(metadata, cancellationToken).ConfigureAwait(false);
-			}).ConfigureAwait(false);
-			await dmbFactory.CleanUnusedCompileJobs(latestCompileJob, cancellationToken).ConfigureAwait(false);
+			await dmbFactory.CleanUnusedCompileJobs(cancellationToken).ConfigureAwait(false);
 		}
 
 		/// <inheritdoc />

src/Tgstation.Server.Host/Components/InstanceFactory.cs

@@ -235,6 +235,7 @@ namespace Tgstation.Server.Host.Components
 						bridgeRegistrar,
 						serverPortProvider,
 						loggerFactory,
+						loggerFactory.CreateLogger<SessionControllerFactory>(),
 						metadata.CloneMetadata());
 
 					var dmbFactory = new DmbFactory(databaseContextFactory, gameIoManager, loggerFactory.CreateLogger<DmbFactory>(), metadata.CloneMetadata());

src/Tgstation.Server.Host/Components/Interop/DMApiConstants.cs

@@ -33,7 +33,7 @@ namespace Tgstation.Server.Host.Components.Interop
 		/// <summary>
 		/// The DMAPI <see cref="Version"/> being used.
 		/// </summary>
-		public static readonly Version Version = new Version(5, 1, 0);
+		public static readonly Version Version = new Version(5, 1, 1);
 
 		/// <summary>
 		/// <see cref="JsonSerializerSettings"/> for use when communicating with the DMAPI.

src/Tgstation.Server.Host/Components/Repository/Repository.cs

@@ -163,8 +163,8 @@ namespace Tgstation.Server.Host.Components.Repository
 				if (remote.EndsWith(item, StringComparison.OrdinalIgnoreCase))
 					remote = remote.Substring(0, remote.LastIndexOf(item, StringComparison.OrdinalIgnoreCase));
 			var splits = remote.Split('/');
-			name = splits[splits.Length - 1];
-			owner = splits[splits.Length - 2].Split('.')[0];
+			name = splits.Last();
+			owner = splits[^2].Split('.').First();
 
 			logger.LogTrace("GetRepositoryOwnerName({0}) => {1} / {2}", remote, owner, name);
 		}
@@ -622,18 +622,14 @@ namespace Tgstation.Server.Host.Components.Repository
 			cancellationToken.ThrowIfCancellationRequested();
 			try
 			{
-				if (!await eventConsumer.HandleEvent(
+				await eventConsumer.HandleEvent(
 					EventType.RepoPreSynchronize,
 					new List<string>
 					{
 						ioMananger.ResolvePath()
 					},
 					cancellationToken)
-					.ConfigureAwait(false))
-				{
-					logger.LogDebug("Aborted synchronize due to event handler response!");
-					return false;
-				}
+					.ConfigureAwait(false);
 			}
 			finally
 			{

src/Tgstation.Server.Host/Components/Session/SessionController.cs

@@ -522,7 +522,6 @@ namespace Tgstation.Server.Host.Components.Session
 
 			var json = JsonConvert.SerializeObject(parameters, DMApiConstants.SerializerSettings);
 			logger.LogTrace("Topic request: {0}", json);
-			Exception caughtException;
 			try
 			{
 				var commandString = String.Format(CultureInfo.InvariantCulture,
@@ -558,19 +557,20 @@ namespace Tgstation.Server.Host.Components.Session
 
 				return new CombinedTopicResponse(topicResponse, interopResponse);
 			}
-			catch (OperationCanceledException e)
+			catch (OperationCanceledException)
 			{
+				logger.LogTrace(
+					"Topic request {0}!",
+					cancellationToken.IsCancellationRequested
+						? "aborted"
+						: "timed out");
 				cancellationToken.ThrowIfCancellationRequested();
-				caughtException = e;
 			}
 			catch (Exception e)
 			{
-				caughtException = e;
+				logger.LogWarning("Send command exception:{0}{1}", Environment.NewLine, e);
 			}
 
-			if (caughtException == null)
-				logger.LogWarning("Send command exception:{0}{1}", Environment.NewLine, caughtException.Message);
-
 			return null;
 		}
 

src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs

@@ -15,6 +15,7 @@ using Tgstation.Server.Host.Components.Interop;
 using Tgstation.Server.Host.Components.Interop.Bridge;
 using Tgstation.Server.Host.Core;
 using Tgstation.Server.Host.IO;
+using Tgstation.Server.Host.Jobs;
 using Tgstation.Server.Host.Security;
 using Tgstation.Server.Host.System;
 
@@ -83,6 +84,11 @@ namespace Tgstation.Server.Host.Components.Session
 		/// </summary>
 		readonly ILoggerFactory loggerFactory;
 
+		/// <summary>
+		/// The <see cref="ILogger"/> for the <see cref="SessionControllerFactory"/>
+		/// </summary>
+		readonly ILogger<SessionControllerFactory> logger;
+
 		/// <summary>
 		/// The <see cref="Api.Models.Instance"/> for the <see cref="SessionControllerFactory"/>
 		/// </summary>
@@ -120,6 +126,7 @@ namespace Tgstation.Server.Host.Components.Session
 		/// <param name="bridgeRegistrar">The value of <see cref="bridgeRegistrar"/>.</param>
 		/// <param name="serverPortProvider">The value of <see cref="serverPortProvider"/>.</param>
 		/// <param name="loggerFactory">The value of <see cref="loggerFactory"/></param>
+		/// <param name="logger">The value of <see cref="logger"/>.</param>
 		public SessionControllerFactory(
 			IProcessExecutor processExecutor,
 			IByondManager byond,
@@ -133,6 +140,7 @@ namespace Tgstation.Server.Host.Components.Session
 			IBridgeRegistrar bridgeRegistrar,
 			IServerPortProvider serverPortProvider,
 			ILoggerFactory loggerFactory,
+			ILogger<SessionControllerFactory> logger,
 			Api.Models.Instance instance)
 		{
 			this.processExecutor = processExecutor ?? throw new ArgumentNullException(nameof(processExecutor));
@@ -148,6 +156,7 @@ namespace Tgstation.Server.Host.Components.Session
 			this.bridgeRegistrar = bridgeRegistrar ?? throw new ArgumentNullException(nameof(bridgeRegistrar));
 			this.serverPortProvider = serverPortProvider ?? throw new ArgumentNullException(nameof(serverPortProvider));
 			this.loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory));
+			this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
 		}
 
 		/// <inheritdoc />
@@ -192,6 +201,8 @@ namespace Tgstation.Server.Host.Components.Session
 					if (launchParameters.SecurityLevel == DreamDaemonSecurity.Trusted)
 						await byondLock.TrustDmbPath(ioManager.ConcatPath(basePath, dmbProvider.DmbName), cancellationToken).ConfigureAwait(false);
 
+					CheckPagerIsNotRunning();
+
 					var accessIdentifier = cryptographySuite.GetSecureString();
 
 					// set command line options
@@ -213,7 +224,24 @@ namespace Tgstation.Server.Host.Components.Session
 					var noShellExecute = !platformIdentifier.IsWindows;
 
 					// launch dd
-					var process = processExecutor.LaunchProcess(byondLock.DreamDaemonPath, basePath, arguments, noShellExecute: noShellExecute);
+					var process = processExecutor.LaunchProcess(
+						byondLock.DreamDaemonPath,
+						basePath,
+						arguments,
+						noShellExecute,
+						noShellExecute,
+						noShellExecute: noShellExecute);
+
+					if (noShellExecute)
+					{
+						// Log DD output
+						_ = process.Lifetime.ContinueWith(
+								x => logger.LogTrace(
+									"DreamDaemon Output:{0}{1}",
+									Environment.NewLine, process.GetCombinedOutput()),
+								TaskScheduler.Current);
+					}
+
 					try
 					{
 						networkPromptReaper.RegisterProcess(process);
@@ -371,5 +399,14 @@ namespace Tgstation.Server.Host.Components.Session
 				securityLevel,
 				apiValidateOnly);
 		}
+
+		/// <summary>
+		/// Make sure the BYOND pager is not running.
+		/// </summary>
+		void CheckPagerIsNotRunning()
+		{
+			if (platformIdentifier.IsWindows && processExecutor.IsProcessWithNameRunning("byond"))
+				throw new JobException("Cannot start DreamDaemon headless with the BYOND pager running!");
+		}
 	}
 }

src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs

@@ -11,6 +11,7 @@ using System.Threading.Tasks;
 using Tgstation.Server.Api.Models;
 using Tgstation.Server.Host.Core;
 using Tgstation.Server.Host.IO;
+using Tgstation.Server.Host.Jobs;
 using Tgstation.Server.Host.Security;
 using Tgstation.Server.Host.System;
 
@@ -442,12 +443,12 @@ namespace Tgstation.Server.Host.Components.StaticFiles
 		public Task StopAsync(CancellationToken cancellationToken) => EnsureDirectories(cancellationToken);
 
 		/// <inheritdoc />
-		public async Task<bool> HandleEvent(EventType eventType, IEnumerable<string> parameters, CancellationToken cancellationToken)
+		public async Task HandleEvent(EventType eventType, IEnumerable<string> parameters, CancellationToken cancellationToken)
 		{
 			await EnsureDirectories(cancellationToken).ConfigureAwait(false);
 
 			if (!EventTypeScriptFileNameMap.TryGetValue(eventType, out var scriptName))
-				return true;
+				return;
 
 			// always execute in serial
 			using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken).ConfigureAwait(false))
@@ -456,17 +457,22 @@ namespace Tgstation.Server.Host.Components.StaticFiles
 				var resolvedScriptsDir = ioManager.ResolvePath(EventScriptsSubdirectory);
 
 				foreach (var I in files.Select(x => ioManager.GetFileName(x)).Where(x => x.StartsWith(scriptName, StringComparison.Ordinal)))
-					using (var script = processExecutor.LaunchProcess(ioManager.ConcatPath(resolvedScriptsDir, I), resolvedScriptsDir, String.Join(' ', parameters), noShellExecute: true))
+					using (var script = processExecutor.LaunchProcess(
+						ioManager.ConcatPath(resolvedScriptsDir, I),
+						resolvedScriptsDir,
+						String.Join(' ', parameters),
+						true,
+						true,
+						true))
 					using (cancellationToken.Register(() => script.Terminate()))
 					{
 						var exitCode = await script.Lifetime.ConfigureAwait(false);
 						cancellationToken.ThrowIfCancellationRequested();
+						var scriptOutput = script.GetCombinedOutput();
 						if (exitCode != 0)
-							return false;
+							throw new JobException($"Script {I} exited with code {exitCode}:{Environment.NewLine}{scriptOutput}");
 					}
 			}
-
-			return true;
 		}
 
 		/// <inheritdoc />

src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs

@@ -727,19 +727,15 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		}
 
 		/// <inheritdoc />
-		public async Task<bool> HandleEvent(EventType eventType, IEnumerable<string> parameters, CancellationToken cancellationToken)
+		public async Task HandleEvent(EventType eventType, IEnumerable<string> parameters, CancellationToken cancellationToken)
 		{
-			if (!Running)
-				return true;
-
-			var notification = new EventNotification(eventType, parameters);
-
 			var activeServer = GetActiveController();
 
 			// Server may have ended
 			if (activeServer == null)
-				return true;
+				return;
 
+			var notification = new EventNotification(eventType, parameters);
 			var result = await activeServer.SendCommand(
 				new TopicParameters(notification),
 				cancellationToken)
@@ -762,8 +758,6 @@ namespace Tgstation.Server.Host.Components.Watchdog
 								.Select(nullableChannelId => nullableChannelId.Value),
 							cancellationToken)))
 					.ConfigureAwait(false);
-
-			return true;
 		}
 
 		/// <inheritdoc />

src/Tgstation.Server.Host/Database/Design/DesignTimeDbContextFactoryHelpers.cs

@@ -1,8 +1,5 @@
-using Microsoft.Extensions.Configuration;
-using Microsoft.Extensions.Options;
+using Microsoft.Extensions.Options;
 using Tgstation.Server.Host.Configuration;
-using Tgstation.Server.Host.IO;
-using Tgstation.Server.Host.System;
 
 namespace Tgstation.Server.Host.Database.Design
 {
@@ -11,31 +8,21 @@ namespace Tgstation.Server.Host.Database.Design
 	/// </summary>
 	static class DesignTimeDbContextFactoryHelpers
 	{
-		/// <summary>
-		/// Path to the json file to use for migrations configuration
-		/// </summary>
-		const string RootJson = "appsettings.json";
-
-		/// <summary>
-		/// Path to the development json file to use for migrations configuration
-		/// </summary>
-		const string DevJson = "appsettings.Development.json";
-
 		/// <summary>
 		/// Get the <see cref="IOptions{TOptions}"/> for the <see cref="DatabaseConfiguration"/>
 		/// </summary>
+		/// <param name="databaseType">The <see cref="DatabaseConfiguration.DatabaseType"/>.</param>
+		/// <param name="connectionString">The <see cref="DatabaseConfiguration.ConnectionString"/>.</param>
 		/// <returns>The <see cref="IOptions{TOptions}"/> for the <see cref="DatabaseConfiguration"/></returns>
-		public static IOptions<DatabaseConfiguration> GetDbContextOptions()
+		public static IOptions<DatabaseConfiguration> GetDbContextOptions(DatabaseType databaseType, string connectionString)
 		{
-			var builder = new ConfigurationBuilder();
-			var assemblyInfoProvider = new AssemblyInformationProvider();
-			var ioManager = new DefaultIOManager();
-			builder.SetBasePath(ioManager.GetDirectoryName(assemblyInfoProvider.Path));
-			builder.AddJsonFile(RootJson);
-			builder.AddJsonFile(DevJson);
-			var configuration = builder.Build();
-			var dbConfig = configuration.GetSection(DatabaseConfiguration.Section).Get<DatabaseConfiguration>();
-			dbConfig.DesignTime = true;
+			var dbConfig = new DatabaseConfiguration
+			{
+				DesignTime = true,
+				DatabaseType = databaseType,
+				ConnectionString = connectionString
+			};
+
 			return Options.Create(dbConfig);
 		}
 	}

src/Tgstation.Server.Host/Database/Design/MySqlDesignTimeDbContextFactory.cs

@@ -2,6 +2,7 @@
 using Microsoft.EntityFrameworkCore;
 using Microsoft.EntityFrameworkCore.Design;
 using Microsoft.Extensions.Logging;
+using Tgstation.Server.Host.Configuration;
 using Tgstation.Server.Host.Models;
 using Tgstation.Server.Host.Security;
 using Tgstation.Server.Host.System;
@@ -17,7 +18,9 @@ namespace Tgstation.Server.Host.Database.Design
 			using var loggerFactory = new LoggerFactory();
 			return new MySqlDatabaseContext(
 				new DbContextOptions<MySqlDatabaseContext>(),
-				DesignTimeDbContextFactoryHelpers.GetDbContextOptions(),
+				DesignTimeDbContextFactoryHelpers.GetDbContextOptions(
+					DatabaseType.MariaDB,
+					"Server=127.0.0.1;User Id=root;Password=fake;Database=TGS_Design"),
 				new DatabaseSeeder(
 				new CryptographySuite(
 				new PasswordHasher<User>()),

src/Tgstation.Server.Host/Database/Design/SqlServerDesignTimeDbContextFactory.cs

@@ -2,6 +2,7 @@
 using Microsoft.EntityFrameworkCore;
 using Microsoft.EntityFrameworkCore.Design;
 using Microsoft.Extensions.Logging;
+using Tgstation.Server.Host.Configuration;
 using Tgstation.Server.Host.Models;
 using Tgstation.Server.Host.Security;
 using Tgstation.Server.Host.System;
@@ -17,7 +18,9 @@ namespace Tgstation.Server.Host.Database.Design
 			using var loggerFactory = new LoggerFactory();
 			return new SqlServerDatabaseContext(
 				new DbContextOptions<SqlServerDatabaseContext>(),
-				DesignTimeDbContextFactoryHelpers.GetDbContextOptions(),
+				DesignTimeDbContextFactoryHelpers.GetDbContextOptions(
+					DatabaseType.SqlServer,
+					"Data Source=fake;Initial Catalog=TGS_Design;Integrated Security=True;Application Name=tgstation-server"),
 				new DatabaseSeeder(
 				new CryptographySuite(
 				new PasswordHasher<User>()),

src/Tgstation.Server.Host/Database/Design/SqliteDesignTimeDbContextFactory.cs

@@ -2,6 +2,7 @@
 using Microsoft.EntityFrameworkCore;
 using Microsoft.EntityFrameworkCore.Design;
 using Microsoft.Extensions.Logging;
+using Tgstation.Server.Host.Configuration;
 using Tgstation.Server.Host.Models;
 using Tgstation.Server.Host.Security;
 using Tgstation.Server.Host.System;
@@ -17,7 +18,9 @@ namespace Tgstation.Server.Host.Database.Design
 			using var loggerFactory = new LoggerFactory();
 			return new SqliteDatabaseContext(
 				new DbContextOptions<SqliteDatabaseContext>(),
-				DesignTimeDbContextFactoryHelpers.GetDbContextOptions(),
+				DesignTimeDbContextFactoryHelpers.GetDbContextOptions(
+					DatabaseType.Sqlite,
+					"Data Source=tgs_design.sqlite3;Mode=ReadWriteCreate"),
 				new DatabaseSeeder(
 				new CryptographySuite(
 				new PasswordHasher<User>()),

src/Tgstation.Server.Host/System/IProcessExecutor.cs

@@ -23,5 +23,12 @@
 		/// <param name="id">The <see cref="IProcess.Id"/></param>
 		/// <returns>The <see cref="IProcess"/> represented by <paramref name="id"/> on success, <see langword="null"/> on failure</returns>
 		IProcess GetProcess(int id);
+
+		/// <summary>
+		/// Check if a <see cref="IProcess"/> with a given <paramref name="name"/> is running.
+		/// </summary>
+		/// <param name="name">The name of the process without the extension.</param>
+		/// <returns><see langword="true"/> if the process is running, <see langword="false"/> otherwise.</returns>
+		bool IsProcessWithNameRunning(string name);
 	}
 }

src/Tgstation.Server.Host/System/Process.cs

@@ -117,7 +117,7 @@ namespace Tgstation.Server.Host.System
 		{
 			if (outputStringBuilder == null)
 				throw new InvalidOperationException("Output reading was not enabled!");
-			return errorStringBuilder.ToString().TrimStart(Environment.NewLine.ToCharArray());
+			return outputStringBuilder.ToString().TrimStart(Environment.NewLine.ToCharArray());
 		}
 
 		/// <inheritdoc />

src/Tgstation.Server.Host/System/ProcessExecutor.cs

@@ -1,5 +1,6 @@
 using Microsoft.Extensions.Logging;
 using System;
+using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
 
@@ -102,9 +103,30 @@ namespace Tgstation.Server.Host.System
 		}
 
 		/// <inheritdoc />
-		public IProcess LaunchProcess(string fileName, string workingDirectory, string arguments, bool readOutput, bool readError, bool noShellExecute)
+		public IProcess LaunchProcess(
+			string fileName,
+			string workingDirectory,
+			string arguments,
+			bool readOutput,
+			bool readError,
+			bool noShellExecute)
 		{
-			logger.LogDebug("Launching process in {0}: {1} {2}", workingDirectory, fileName, arguments);
+			if (fileName == null)
+				throw new ArgumentNullException(nameof(fileName));
+			if (workingDirectory == null)
+				throw new ArgumentNullException(nameof(workingDirectory));
+			if (arguments == null)
+				throw new ArgumentNullException(nameof(arguments));
+
+			if (!noShellExecute && (readOutput || readError))
+				throw new InvalidOperationException("Requesting output/error reading requires noShellExecute to be true!");
+
+			logger.LogDebug(
+				"{0}aunching process in {1}: {2} {3}",
+				noShellExecute ? "L" : "Shell l",
+				workingDirectory,
+				fileName,
+				arguments);
 			var handle = new global::System.Diagnostics.Process();
 			try
 			{
@@ -112,9 +134,12 @@ namespace Tgstation.Server.Host.System
 				handle.StartInfo.Arguments = arguments;
 				handle.StartInfo.WorkingDirectory = workingDirectory;
 
-				handle.StartInfo.UseShellExecute = !(noShellExecute || readOutput || readError);
+				handle.StartInfo.UseShellExecute = !noShellExecute;
 
 				StringBuilder outputStringBuilder = null, errorStringBuilder = null, combinedStringBuilder = null;
+
+				TaskCompletionSource<object> outputReadTcs = null;
+				TaskCompletionSource<object> errorReadTcs = null;
 				if (readOutput || readError)
 				{
 					combinedStringBuilder = new StringBuilder();
@@ -122,8 +147,15 @@ namespace Tgstation.Server.Host.System
 					{
 						outputStringBuilder = new StringBuilder();
 						handle.StartInfo.RedirectStandardOutput = true;
+						outputReadTcs = new TaskCompletionSource<object>();
 						handle.OutputDataReceived += (sender, e) =>
 						{
+							if (e.Data == null)
+							{
+								outputReadTcs.SetResult(null);
+								return;
+							}
+
 							combinedStringBuilder.Append(Environment.NewLine);
 							combinedStringBuilder.Append(e.Data);
 							outputStringBuilder.Append(Environment.NewLine);
@@ -135,8 +167,15 @@ namespace Tgstation.Server.Host.System
 					{
 						errorStringBuilder = new StringBuilder();
 						handle.StartInfo.RedirectStandardError = true;
+						errorReadTcs = new TaskCompletionSource<object>();
 						handle.ErrorDataReceived += (sender, e) =>
 						{
+							if (e.Data == null)
+							{
+								errorReadTcs.SetResult(null);
+								return;
+							}
+
 							combinedStringBuilder.Append(Environment.NewLine);
 							combinedStringBuilder.Append(e.Data);
 							errorStringBuilder.Append(Environment.NewLine);
@@ -148,16 +187,30 @@ namespace Tgstation.Server.Host.System
 				var lifetimeTask = AttachExitHandler(handle);
 
 				handle.Start();
+
+				static async Task<int> AddToLifetimeTask(Task<int> originalTask, TaskCompletionSource<object> tcs)
+				{
+					var exitCode = await originalTask.ConfigureAwait(false);
+					await tcs.Task.ConfigureAwait(false);
+					return exitCode;
+				}
+
 				try
 				{
 					if (readOutput)
+					{
 						handle.BeginOutputReadLine();
+						lifetimeTask = AddToLifetimeTask(lifetimeTask, outputReadTcs);
+					}
 				}
 				catch (InvalidOperationException) { }
 				try
 				{
 					if (readError)
+					{
 						handle.BeginErrorReadLine();
+						lifetimeTask = AddToLifetimeTask(lifetimeTask, errorReadTcs);
+					}
 				}
 				catch (InvalidOperationException) { }
 
@@ -176,5 +229,15 @@ namespace Tgstation.Server.Host.System
 				throw;
 			}
 		}
+
+		/// <inheritdoc />
+		public bool IsProcessWithNameRunning(string name)
+		{
+			var procs = global::System.Diagnostics.Process.GetProcessesByName(name);
+			foreach (var proc in procs)
+				proc.Dispose();
+
+			return procs.Any();
+		}
 	}
 }

src/Tgstation.Server.Host/Tgstation.Server.Host.csproj

@@ -48,7 +48,7 @@
 
   <ItemGroup>
     <PackageReference Include="BetterWin32Errors" Version="0.2.0" />
-    <PackageReference Include="Byond.TopicSender" Version="4.0.0" />
+    <PackageReference Include="Byond.TopicSender" Version="5.0.0" />
     <PackageReference Include="Cyberboss.AspNetCore.AsyncInitializer" Version="1.2.0" />
     <PackageReference Include="Cyberboss.SmartIrc4net.Standard" Version="0.4.6" />
     <PackageReference Include="Discord.Net.WebSocket" Version="2.2.0" />
@@ -112,19 +112,10 @@
     <ClientApp Include="ClientApp\src\**\*;ClientApp\public\**\*;ClientApp\tsconfig.json;ClientApp\package.json;ClientApp\package-lock.json" />
     <AdditionalFiles Include="../../build/stylecop.json" />
   </ItemGroup>
-
-  <ItemGroup>
-    <None Update="appsettings.Development.json">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-    </None>
-    <None Update="appsettings.json">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-    </None>
-  </ItemGroup>
-
+  
   <ItemGroup>
     <Content Update="appsettings.Development.json">
-      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
     <Content Update="appsettings.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>

tests/DMAPI/BasicOperation/Test.dm

@@ -26,6 +26,11 @@
 	world.log << "You really shouldn't be able to read this"
 
 /world/Topic(T, Addr, Master, Keys)
+	world.log << "Topic: [T]"
+	. =  HandleTopic(T)
+	world.log << "Response: [.]"
+
+/world/proc/HandleTopic(T)
 	TGS_TOPIC
 
 /world/Reboot(reason)

tests/DMAPI/LongRunning/Test.dm

@@ -12,6 +12,11 @@
 	world.TgsInitializationComplete()
 
 /world/Topic(T, Addr, Master, Keys)
+	world.log << "Topic: [T]"
+	. =  HandleTopic(T)
+	world.log << "Response: [.]"
+
+/world/proc/HandleTopic(T)
 	TGS_TOPIC
 
 	world.sleep_offline = FALSE

tests/Tgstation.Server.Tests/Instance/ByondTest.cs

@@ -25,7 +25,7 @@ namespace Tgstation.Server.Tests.Instance
 		public async Task Run(CancellationToken cancellationToken)
 		{
 			await TestNoVersion(cancellationToken).ConfigureAwait(false);
-			await TestInstall511(cancellationToken).ConfigureAwait(false);
+			await TestInstallStable(cancellationToken).ConfigureAwait(false);
 			await TestInstallFakeVersion(cancellationToken).ConfigureAwait(false);
 		}
 
@@ -40,11 +40,11 @@ namespace Tgstation.Server.Tests.Instance
 			await WaitForJob(test.InstallJob, 60, true, cancellationToken).ConfigureAwait(false);
 		}
 
-		async Task TestInstall511(CancellationToken cancellationToken)
+		async Task TestInstallStable(CancellationToken cancellationToken)
 		{
 			var newModel = new Api.Models.Byond
 			{
-				Version = new Version(511, 1385)
+				Version = new Version(513, 1514)
 			};
 			var test = await byondClient.SetActiveVersion(newModel, cancellationToken).ConfigureAwait(false);
 			Assert.IsNotNull(test.InstallJob);
@@ -57,7 +57,7 @@ namespace Tgstation.Server.Tests.Instance
 			if (new PlatformIdentifier().IsWindows)
 				dreamMaker += ".exe";
 
-			var dreamMakerDir = Path.Combine(metadata.Path, "Byond", "511.1385", "byond", "bin");
+			var dreamMakerDir = Path.Combine(metadata.Path, "Byond", newModel.Version.ToString(), "byond", "bin");
 
 			Assert.IsTrue(Directory.Exists(dreamMakerDir), $"Directory {dreamMakerDir} does not exist!");
 			Assert.IsTrue(File.Exists(Path.Combine(dreamMakerDir, dreamMaker)), $"Missing DreamMaker executable! Dir contents: {String.Join(", ", Directory.GetFileSystemEntries(dreamMakerDir))}");

tests/Tgstation.Server.Tests/Instance/WatchdogTest.cs

@@ -42,11 +42,15 @@ namespace Tgstation.Server.Tests.Instance
 			}, cancellationToken), ErrorCode.DreamDaemonDoubleSoft);
 
 			await RunBasicTest(cancellationToken);
-			await RunHeartbeatTest(cancellationToken);
 
 			// await RunLongRunningTestThenUpdate(cancellationToken);
 			// await RunLongRunningTestThenUpdateWithByondVersionSwitch(cancellationToken);
 
+			// Remove this deploy when the above tests are reenabled
+			await DeployTestDme("LongRunning/long_running_test", DreamDaemonSecurity.Trusted, cancellationToken);
+
+			await RunHeartbeatTest(cancellationToken);
+
 			await StartAndLeaveRunning(cancellationToken);
 		}
 
@@ -89,21 +93,29 @@ namespace Tgstation.Server.Tests.Instance
 
 			await WaitForJob(startJob, 10, false, cancellationToken);
 
-			await instanceClient.DreamDaemon.Update(new DreamDaemon
-			{
-				SoftShutdown = true
-			}, cancellationToken);
-
 			// lock on to DD and pause it so it can't heartbeat
 			var ddProcs = System.Diagnostics.Process.GetProcessesByName("DreamDaemon").ToList();
 			if (ddProcs.Count != 1)
 				Assert.Inconclusive($"Incorrect number of DD processes: {ddProcs.Count}");
 
-			var pid = ddProcs.Single().Id;
+			using var ddProc = ddProcs.Single();
 			using var ourProcessHandler = new ProcessExecutor(
-				new PlatformIdentifier().IsWindows ? (IProcessSuspender)new WindowsProcessSuspender(Mock.Of<ILogger<WindowsProcessSuspender>>()) : new PosixProcessSuspender(Mock.Of<ILogger<PosixProcessSuspender>>()),
+				new PlatformIdentifier().IsWindows
+					? (IProcessSuspender)new WindowsProcessSuspender(Mock.Of<ILogger<WindowsProcessSuspender>>())
+					: new PosixProcessSuspender(Mock.Of<ILogger<PosixProcessSuspender>>()),
 				Mock.Of<ILogger<ProcessExecutor>>(),
-				LoggerFactory.Create(x => { })).GetProcess(pid);
+				LoggerFactory.Create(x => { }))
+				.GetProcess(ddProc.Id);
+
+			// Ensure it's responding to heartbeats
+			await Task.WhenAny(Task.Delay(20000), ourProcessHandler.Lifetime);
+			Assert.IsFalse(ddProc.HasExited);
+
+			await instanceClient.DreamDaemon.Update(new DreamDaemon
+			{
+				SoftShutdown = true
+			}, cancellationToken);
+
 			ourProcessHandler.Suspend();
 
 			await Task.WhenAny(ourProcessHandler.Lifetime, Task.Delay(TimeSpan.FromSeconds(20)));
@@ -121,6 +133,12 @@ namespace Tgstation.Server.Tests.Instance
 					Assert.Fail("DreamDaemon didn't shutdown within the timeout!");
 			}
 			while (timeout > 0);
+
+			// disable heartbeats
+			await instanceClient.DreamDaemon.Update(new DreamDaemon
+			{
+				HeartbeatSeconds = 0,
+			}, cancellationToken);
 		}
 
 		async Task RunLongRunningTestThenUpdate(CancellationToken cancellationToken)

tests/Tgstation.Server.Tests/IntegrationTest.cs

@@ -1,4 +1,3 @@
-using Discord.WebSocket;
 using Microsoft.Extensions.Logging;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 using Moq;
@@ -16,7 +15,8 @@ using Tgstation.Server.Api;
 using Tgstation.Server.Api.Models;
 using Tgstation.Server.Client;
 using Tgstation.Server.Host;
-using Tgstation.Server.Host.Components.Chat.Providers;
+using Tgstation.Server.Host.Extensions;
+using Tgstation.Server.Host.System;
 using Tgstation.Server.Tests.Instance;
 
 namespace Tgstation.Server.Tests
@@ -28,13 +28,9 @@ namespace Tgstation.Server.Tests
 		readonly IServerClientFactory clientFactory = new ServerClientFactory(new ProductHeaderValue(Assembly.GetExecutingAssembly().GetName().Name, Assembly.GetExecutingAssembly().GetName().Version.ToString()));
 
 		[TestMethod]
-		public async Task TestServerUpdate()
+		public async Task TestUpdateProtocol()
 		{
 			using var server = new TestingServer();
-
-			if (server.DatabaseType == "Sqlite")
-				Assert.Inconclusive("Cannot run this test on SQLite yet!");
-
 			using var serverCts = new CancellationTokenSource();
 			var cancellationToken = serverCts.Token;
 			var serverTask = server.Run(cancellationToken);
@@ -101,14 +97,22 @@ namespace Tgstation.Server.Tests
 
 		static void TerminateAllDDs()
 		{
-			foreach (var proc in Process.GetProcessesByName("DreamDaemon"))
+			foreach (var proc in System.Diagnostics.Process.GetProcessesByName("DreamDaemon"))
 				using (proc)
 					proc.Kill();
 		}
 
 		[TestMethod]
-		public async Task TestFullStandardOperation()
+		public async Task TestServer()
 		{
+			var procs = System.Diagnostics.Process.GetProcessesByName("byond");
+			if(procs.Any())
+			{
+				foreach (var proc in procs)
+					proc.Dispose();
+				Assert.Inconclusive("Cannot run server test because DreamDaemon will not start headless while the BYOND pager is running!");
+			}	
+
 			using var server = new TestingServer();
 			using var serverCts = new CancellationTokenSource();
 			var cancellationToken = serverCts.Token;
@@ -191,20 +195,35 @@ namespace Tgstation.Server.Tests
 				await Task.WhenAny(serverTask, Task.Delay(30000, cancellationToken));
 				Assert.IsTrue(serverTask.IsCompleted);
 
+				var preStartupTime = DateTimeOffset.Now;
+
 				serverTask = server.Run(cancellationToken);
 				using (var adminClient = await CreateAdminClient())
 				{
 					var instanceClient = adminClient.Instances.CreateClient(instance);
 
-					// reattach job
 					var jobs = await instanceClient.Jobs.ListActive(cancellationToken);
-					if (jobs.Any())
+					if (!jobs.Any())
 					{
-						Assert.AreEqual(1, jobs.Count);
+						var entities = await instanceClient.Jobs.List(cancellationToken);
+						var getTasks = entities
+							.Select(e => instanceClient.Jobs.GetId(e, cancellationToken))
+							.ToList();
 
-						await new JobsRequiredTest(instanceClient.Jobs).WaitForJob(jobs.Single(), 40, false, cancellationToken);
+						await Task.WhenAll(getTasks);
+						jobs = getTasks
+							.Select(x => x.Result)
+							.Where(x => x.StartedAt.Value > preStartupTime)
+							.ToList();
 					}
 
+					Assert.AreEqual(1, jobs.Count);
+
+					var reattachJob = jobs.Single();
+					Assert.IsTrue(reattachJob.StartedAt.Value >= preStartupTime);
+
+					await new JobsRequiredTest(instanceClient.Jobs).WaitForJob(reattachJob, 40, false, cancellationToken);
+
 					var dd = await instanceClient.DreamDaemon.Read(cancellationToken);
 					Assert.IsTrue(dd.Running.Value);
 
@@ -220,20 +239,34 @@ namespace Tgstation.Server.Tests
 				await Task.WhenAny(serverTask, Task.Delay(30000, cancellationToken));
 				Assert.IsTrue(serverTask.IsCompleted);
 
+				preStartupTime = DateTimeOffset.Now;
 				serverTask = server.Run(cancellationToken);
 				using (var adminClient = await CreateAdminClient())
 				{
 					var instanceClient = adminClient.Instances.CreateClient(instance);
 
-					// launch job
 					var jobs = await instanceClient.Jobs.ListActive(cancellationToken);
-					if (jobs.Any())
+					if (!jobs.Any())
 					{
-						Assert.AreEqual(1, jobs.Count);
+						var entities = await instanceClient.Jobs.List(cancellationToken);
+						var getTasks = entities
+							.Select(e => instanceClient.Jobs.GetId(e, cancellationToken))
+							.ToList();
 
-						await new JobsRequiredTest(instanceClient.Jobs).WaitForJob(jobs.Single(), 40, false, cancellationToken);
+						await Task.WhenAll(getTasks);
+						jobs = getTasks
+							.Select(x => x.Result)
+							.Where(x => x.StartedAt.Value > preStartupTime)
+							.ToList();
 					}
 
+					Assert.AreEqual(1, jobs.Count);
+
+					var launchJob = jobs.Single();
+					Assert.IsTrue(launchJob.StartedAt.Value >= preStartupTime);
+
+					await new JobsRequiredTest(instanceClient.Jobs).WaitForJob(launchJob, 40, false, cancellationToken);
+
 					var dd = await instanceClient.DreamDaemon.Read(cancellationToken);
 
 					Assert.IsTrue(dd.Running.Value);
@@ -262,5 +295,24 @@ namespace Tgstation.Server.Tests
 				TerminateAllDDs();
 			}
 		}
+
+		[TestMethod]
+		public async Task TestScriptExecution()
+		{
+			var platformIdentifier = new PlatformIdentifier();
+			var processExecutor = new ProcessExecutor(
+				Mock.Of<IProcessSuspender>(),
+				Mock.Of<ILogger<ProcessExecutor>>(),
+				LoggerFactory.Create(x => { }));
+
+			using var process = processExecutor.LaunchProcess("test." + platformIdentifier.ScriptFileExtension, ".", String.Empty, true, true, true);
+			using var cts = new CancellationTokenSource();
+			cts.CancelAfter(3000);
+			var exitCode = await process.Lifetime.WithToken(cts.Token);
+
+			Assert.AreEqual(0, exitCode);
+			Assert.AreEqual(String.Empty, process.GetErrorOutput().Trim());
+			Assert.AreEqual("Hello World!", process.GetStandardOutput().Trim());
+		}
 	}
 }

tests/Tgstation.Server.Tests/Tgstation.Server.Tests.csproj

@@ -26,4 +26,13 @@
     </Reference>
   </ItemGroup>
 
+  <ItemGroup>
+    <None Update="test.bat">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Update="test.sh">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+
 </Project>

tests/Tgstation.Server.Tests/test.bat

@@ -0,0 +1,3 @@
+@echo off
+
+echo Hello World!

tests/Tgstation.Server.Tests/test.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+echo Hello World!

tools/ReleaseNotes/Program.cs

@@ -1,7 +1,6 @@
 using Octokit;
 using System;
 using System.Collections.Generic;
-using System.Globalization;
 using System.IO;
 using System.Linq;
 using System.Text;
@@ -33,7 +32,11 @@ namespace ReleaseNotes
 				return 2;
 			}
 
-			var doNotCloseMilestone = args.Length >= 2 && args[1].ToUpperInvariant() == "--NO-CLOSE";
+			var doNotCloseMilestone = args.Length > 1 && args[1].ToUpperInvariant() == "--NO-CLOSE";
+
+			string propsPath = "../../../../../build/Version.props";
+			if (args.Length > 1 && !doNotCloseMilestone)
+				propsPath = args[1];
 
 			const string ReleaseNotesEnvVar = "TGS4_RELEASE_NOTES_TOKEN";
 			var githubToken = Environment.GetEnvironmentVariable(ReleaseNotesEnvVar);
@@ -73,7 +76,7 @@ namespace ReleaseNotes
 
 				Task<Milestone> milestoneTask = null;
 				var milestoneTaskLock = new object();
-				var releaseDictionary = new Dictionary<int, List<string>>();
+				var releaseDictionary = new Dictionary<string, List<Tuple<string, int>>>(StringComparer.OrdinalIgnoreCase);
 				var authorizedUsers = new Dictionary<long, Task<bool>>();
 
 				bool postControlPanelMessage = false;
@@ -94,74 +97,84 @@ namespace ReleaseNotes
 						if (milestoneTask == null)
 							milestoneTask = GetMilestone();
 
-					if (!fullPR.Merged)
-						return;
+					// if (!fullPR.Merged)
+						//return;
 
 					async Task BuildNotesFromComment(string comment, User user)
 					{
+						async Task CommitNotes(string component, List<string> notes)
+						{
+							Task<bool> authTask;
+							TaskCompletionSource<bool> ourTcs = null;
+							lock (authorizedUsers)
+							{
+								if (!authorizedUsers.TryGetValue(user.Id, out authTask))
+								{
+									ourTcs = new TaskCompletionSource<bool>();
+									authTask = ourTcs.Task;
+									authorizedUsers.Add(user.Id, authTask);
+								}
+							}
+
+							if (ourTcs != null)
+								try
+								{
+									//check if the user has access
+									var perm = String.IsNullOrWhiteSpace(githubToken)
+										? PermissionLevel.Write
+										: (await client.Repository.Collaborator.ReviewPermission(RepoOwner, RepoName, user.Login).ConfigureAwait(false)).Permission;
+									ourTcs.SetResult(perm == PermissionLevel.Write || perm == PermissionLevel.Admin);
+								}
+								catch
+								{
+									ourTcs.SetResult(false);
+									throw;
+								}
+
+							var authorized = await authTask.ConfigureAwait(false);
+							if (!authorized)
+								return;
+
+							lock (releaseDictionary)
+							{
+								foreach (var I in notes)
+									Console.WriteLine(component + " #" + fullPR.Number + " - " + I + " (@" + user.Login + ")");
+
+								var tupleSelector = notes.Select(note => Tuple.Create(note, fullPR.Number));
+								if (releaseDictionary.TryGetValue(component, out var currentValues))
+									currentValues.AddRange(tupleSelector);
+								else
+									releaseDictionary.Add(component, tupleSelector.ToList());
+							}
+						}
+
 						var commentSplits = comment.Split('\n');
-						var notesOpen = false;
+						string targetComponent = null;
 						var notes = new List<string>();
 						foreach (var line in commentSplits)
 						{
 							var trimmedLine = line.Trim();
-							if (!notesOpen)
+							if (targetComponent == null)
 							{
-								notesOpen = trimmedLine.StartsWith(":cl:", StringComparison.Ordinal);
+								if (trimmedLine.StartsWith(":cl:", StringComparison.Ordinal))
+								{
+									targetComponent = trimmedLine.Substring(4).Trim();
+									if (targetComponent.Length == 0)
+										targetComponent = "Core";
+								}
 								continue;
 							}
 							if (trimmedLine.StartsWith("/:cl:", StringComparison.Ordinal))
 							{
-								notesOpen = false;
+								await CommitNotes(targetComponent, notes);
+								targetComponent = null;
+								notes.Clear();
 								continue;
 							}
 							if (trimmedLine.Length == 0)
 								continue;
 							notes.Add(trimmedLine);
 						}
-						if (notesOpen || notes.Count == 0)
-							return;
-
-						Task<bool> authTask;
-						TaskCompletionSource<bool> ourTcs = null;
-						lock (authorizedUsers)
-						{
-							if (!authorizedUsers.TryGetValue(user.Id, out authTask))
-							{
-								ourTcs = new TaskCompletionSource<bool>();
-								authTask = ourTcs.Task;
-								authorizedUsers.Add(user.Id, authTask);
-							}
-						}
-
-						if (ourTcs != null)
-							try
-							{
-								//check if the user has access
-								var perm = String.IsNullOrWhiteSpace(githubToken)
-									? PermissionLevel.Write
-									: (await client.Repository.Collaborator.ReviewPermission(RepoOwner, RepoName, user.Login).ConfigureAwait(false)).Permission;
-								ourTcs.SetResult(perm == PermissionLevel.Write || perm == PermissionLevel.Admin);
-							}
-							catch
-							{
-								ourTcs.SetResult(false);
-								throw;
-							}
-
-						var authorized = await authTask.ConfigureAwait(false);
-						if (!authorized)
-							return;
-
-						lock (releaseDictionary)
-						{
-							foreach (var I in notes)
-								Console.WriteLine("#" + fullPR.Number + " - " + I + " (@" + user.Login + ")");
-							if (releaseDictionary.TryGetValue(fullPR.Number, out var currentValues))
-								currentValues.AddRange(notes);
-							else
-								releaseDictionary.Add(fullPR.Number, notes);
-						}
 					}
 
 					var comments = await client.Issue.Comment.GetAllForIssue(RepoOwner, RepoName, fullPR.Number).ConfigureAwait(false);
@@ -205,7 +218,7 @@ namespace ReleaseNotes
 				//trim away all the lines that don't start with #
 
 				string keepThisRelease;
-				if (version.Build == 0)
+				if (version.Build <= 1)
 					keepThisRelease = "# ";
 				else
 					keepThisRelease = "## ";
@@ -223,7 +236,7 @@ namespace ReleaseNotes
 				switch (releasingSuite)
 				{
 					case 4:
-						var doc = XDocument.Load("../../../../../build/Version.props");
+						var doc = XDocument.Load(propsPath);
 						var project = doc.Root;
 						var xmlNamespace = project.GetDefaultNamespace();
 						var versionsPropertyGroup = project.Elements().First();
@@ -294,17 +307,25 @@ namespace ReleaseNotes
 				}
 
 				foreach (var I in releaseDictionary.OrderBy(kvp => kvp.Key))
-					foreach (var note in I.Value)
+				{
+					newNotes.Append(Environment.NewLine);
+					newNotes.Append("#### ");
+					newNotes.Append(I.Key);
+
+
+					foreach (var noteTuple in I.Value)
 					{
 						newNotes.Append(Environment.NewLine);
 						newNotes.Append("- ");
-						newNotes.Append(note);
+						newNotes.Append(noteTuple.Item1);
 						newNotes.Append(" (#");
-						newNotes.Append(I.Key);
+						newNotes.Append(noteTuple.Item2);
 						newNotes.Append(')');
 					}
 
-				newNotes.Append(Environment.NewLine);
+					newNotes.Append(Environment.NewLine);
+				}
+
 				newNotes.Append(Environment.NewLine);
 
 				if (version != new Version(4, 1, 0))