Merge pull request #1506 from tgstation/LinuxLoggingFix [TGSDeploy]

Test logging DreamDaemon output works

build/Version.props

@@ -3,7 +3,7 @@
   <!-- Integration tests will ensure they match across the board -->
   <Import Project="ControlPanelVersion.props" />
   <PropertyGroup>
-    <TgsCoreVersion>5.12.3</TgsCoreVersion>
+    <TgsCoreVersion>5.12.4</TgsCoreVersion>
     <TgsConfigVersion>4.6.0</TgsConfigVersion>
     <TgsApiVersion>9.10.2</TgsApiVersion>
     <TgsApiLibraryVersion>10.4.1</TgsApiLibraryVersion>

src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs

@@ -814,8 +814,6 @@ namespace Tgstation.Server.Host.Components.Deployment
 			using (var provider = new TemporaryDmbProvider(ioManager.ResolvePath(job.DirectoryName.ToString()), String.Concat(job.DmeName, DmbExtension), job))
 			await using (var controller = await sessionControllerFactory.LaunchNew(provider, byondLock, launchParameters, true, cancellationToken))
 			{
-				controller.AdjustPriority(false);
-
 				var launchResult = await controller.LaunchResult;
 
 				if (launchResult.StartupTime.HasValue)

src/Tgstation.Server.Host/Components/Session/SessionController.cs

@@ -29,6 +29,11 @@ namespace Tgstation.Server.Host.Components.Session
 	/// <inheritdoc />
 	sealed class SessionController : Chunker, ISessionController, IBridgeHandler, IChannelSink
 	{
+		/// <summary>
+		/// If calls to <see cref="SendTopicRequest(TopicParameters, CancellationToken)"/> should be trace logged.
+		/// </summary>
+		internal static bool LogTopicRequests { get; set; } = true;
+
 		/// <inheritdoc />
 		public DMApiParameters DMApiParameters => ReattachInformation;
 
@@ -365,7 +370,7 @@ namespace Tgstation.Server.Host.Components.Session
 
 				void LogCombinedResponse()
 				{
-					if (combinedResponse != null)
+					if (LogTopicRequests && combinedResponse != null)
 						Logger.LogTrace("Topic response: {topicString}", combinedResponse.ByondTopicResponse.StringData ?? "(NO STRING DATA)");
 				}
 
@@ -768,7 +773,8 @@ namespace Tgstation.Server.Host.Components.Session
 			parameters.AccessIdentifier = ReattachInformation.AccessIdentifier;
 
 			var fullCommandString = GenerateQueryString(parameters, out var json);
-			Logger.LogTrace("Topic request: {json}", json);
+			if (LogTopicRequests)
+				Logger.LogTrace("Topic request: {json}", json);
 			var fullCommandByteCount = Encoding.UTF8.GetByteCount(fullCommandString);
 			var topicPriority = parameters.IsPriority;
 			if (fullCommandByteCount <= DMApiConstants.MaximumTopicRequestLength)

src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs

@@ -507,9 +507,7 @@ namespace Tgstation.Server.Host.Components.Session
 				VisibilityWord(launchParameters.Visibility.Value),
 				!byondLock.SupportsCli
 					? $" -logself -log {logFilePath}"
-					: !platformIdentifier.IsWindows // Just use stdout on if CLI is supported
-						? " -logself"
-						: String.Empty, // Windows doesn't output anything to dd.exe if -logself is set?
+					: String.Empty, // DD doesn't output anything if -logself is set???
 				launchParameters.StartProfiler.Value
 					? " -profile"
 					: String.Empty,

src/Tgstation.Server.Host/Core/ServerUpdater.cs

@@ -87,78 +87,18 @@ namespace Tgstation.Server.Host.Core
 		}
 
 		/// <inheritdoc />
-		public async Task<ServerUpdateResult> BeginUpdate(ISwarmService swarmService, Version newVersion, CancellationToken cancellationToken)
+		public async Task<ServerUpdateResult> BeginUpdate(ISwarmService swarmService, Version version, CancellationToken cancellationToken)
 		{
 			if (swarmService == null)
 				throw new ArgumentNullException(nameof(swarmService));
 
-			if (newVersion == null)
-				throw new ArgumentNullException(nameof(newVersion));
+			if (version == null)
+				throw new ArgumentNullException(nameof(version));
 
 			if (!swarmService.ExpectedNumberOfNodesConnected)
 				return ServerUpdateResult.SwarmIntegrityCheckFailed;
 
-			logger.LogDebug("Looking for GitHub releases version {version}...", newVersion);
-			var gitHubClient = gitHubClientFactory.CreateClient();
-			var releases = await gitHubClient
-				.Repository
-				.Release
-				.GetAll(updatesConfiguration.GitHubRepositoryId)
-				.WithToken(cancellationToken);
-
-			logger.LogTrace("Received {releaseCount} total releases from GitHub", releases.Count);
-
-			var filteredReleases = releases
-				.Where(x => x.TagName.StartsWith(updatesConfiguration.GitTagPrefix, StringComparison.InvariantCulture))
-				.ToList();
-
-			logger.LogTrace(
-				"Filtered to {releaseCount} releases matching the configured tag prefix of \"{tagPrefix}\"",
-				filteredReleases.Count,
-				updatesConfiguration.GitTagPrefix);
-
-			foreach (var release in filteredReleases)
-				if (Version.TryParse(
-					release.TagName.Replace(
-						updatesConfiguration.GitTagPrefix, String.Empty, StringComparison.Ordinal),
-					out var version))
-				{
-					if (version == newVersion)
-					{
-						var asset = release.Assets.Where(x => x.Name.Equals(updatesConfiguration.UpdatePackageAssetName, StringComparison.Ordinal)).FirstOrDefault();
-						if (asset == default)
-							continue;
-
-						serverUpdateOperation = new ServerUpdateOperation
-						{
-							TargetVersion = version,
-							UpdateZipUrl = new Uri(asset.BrowserDownloadUrl),
-							SwarmService = swarmService,
-						};
-
-						try
-						{
-							if (!serverControl.TryStartUpdate(this, version))
-								return ServerUpdateResult.UpdateInProgress;
-						}
-						finally
-						{
-							serverUpdateOperation = null;
-						}
-
-						return ServerUpdateResult.Started;
-					}
-				}
-				else
-					logger.LogDebug("Unparsable release tag: {releaseTag}", release.TagName);
-
-			if (updatesConfiguration.DumpReleasesOnNotFound)
-				logger.LogInformation(
-					"Found releases:{newline}\t{releases}",
-					Environment.NewLine,
-					String.Join($"{Environment.NewLine}\t", releases.Select(x => x.TagName).OrderBy(x => x)));
-
-			return ServerUpdateResult.ReleaseMissing;
+			return await BeginUpdateImpl(swarmService, version, false, cancellationToken);
 		}
 
 		/// <inheritdoc />
@@ -272,5 +212,84 @@ namespace Tgstation.Server.Host.Core
 
 			return false;
 		}
+
+		/// <summary>
+		/// Start the process of downloading and applying an update to a new server version. Doesn't perform argument checking.
+		/// </summary>
+		/// <param name="swarmService">The <see cref="ISwarmService"/> to use to coordinate the update.</param>
+		/// <param name="newVersion">The TGS <see cref="Version"/> to update to.</param>
+		/// <param name="recursed">If this is a recursive call.</param>
+		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
+		/// <returns>A <see cref="Task{TResult}"/> resulting in the <see cref="ServerUpdateResult"/>.</returns>
+		async Task<ServerUpdateResult> BeginUpdateImpl(ISwarmService swarmService, Version newVersion, bool recursed, CancellationToken cancellationToken)
+		{
+			logger.LogDebug("Looking for GitHub releases version {version}...", newVersion);
+			var gitHubClient = gitHubClientFactory.CreateClient();
+			var releases = await gitHubClient
+				.Repository
+				.Release
+				.GetAll(updatesConfiguration.GitHubRepositoryId)
+				.WithToken(cancellationToken);
+
+			logger.LogTrace("Received {releaseCount} total releases from GitHub", releases.Count);
+
+			var filteredReleases = releases
+				.Where(x => x.TagName.StartsWith(updatesConfiguration.GitTagPrefix, StringComparison.InvariantCulture))
+				.ToList();
+
+			logger.LogTrace(
+				"Filtered to {releaseCount} releases matching the configured tag prefix of \"{tagPrefix}\"",
+				filteredReleases.Count,
+				updatesConfiguration.GitTagPrefix);
+
+			foreach (var release in filteredReleases)
+				if (Version.TryParse(
+					release.TagName.Replace(
+						updatesConfiguration.GitTagPrefix, String.Empty, StringComparison.Ordinal),
+					out var version))
+				{
+					if (version == newVersion)
+					{
+						var asset = release.Assets.Where(x => x.Name.Equals(updatesConfiguration.UpdatePackageAssetName, StringComparison.Ordinal)).FirstOrDefault();
+						if (asset == default)
+							continue;
+
+						serverUpdateOperation = new ServerUpdateOperation
+						{
+							TargetVersion = version,
+							UpdateZipUrl = new Uri(asset.BrowserDownloadUrl),
+							SwarmService = swarmService,
+						};
+
+						try
+						{
+							if (!serverControl.TryStartUpdate(this, version))
+								return ServerUpdateResult.UpdateInProgress;
+						}
+						finally
+						{
+							serverUpdateOperation = null;
+						}
+
+						return ServerUpdateResult.Started;
+					}
+				}
+				else
+					logger.LogDebug("Unparsable release tag: {releaseTag}", release.TagName);
+
+			if (updatesConfiguration.DumpReleasesOnNotFound)
+				logger.LogInformation(
+					"Found releases:{newline}\t{releases}",
+					Environment.NewLine,
+					String.Join($"{Environment.NewLine}\t", releases.Select(x => x.TagName).OrderBy(x => x)));
+
+			if (!recursed)
+			{
+				logger.LogWarning("We didn't find the requested release, but GitHub has been known to just not give full results when querying all releases. We'll try one more time.");
+				return await BeginUpdateImpl(swarmService, newVersion, true, cancellationToken);
+			}
+
+			return ServerUpdateResult.ReleaseMissing;
+		}
 	}
 }

tests/DMAPI/test_setup.dm

@@ -1,4 +1,5 @@
 /world/New()
+	log << "Starting test..."
 	text2file("SUCCESS", "test_success.txt")
 	world.RunTest()
 

tests/Tgstation.Server.Tests/Live/DummyChatProvider.cs

@@ -86,8 +86,6 @@ namespace Tgstation.Server.Tests.Live
 			this.commands = commands ?? throw new ArgumentNullException(nameof(commands));
 			this.random = random ?? throw new ArgumentNullException(nameof(random));
 
-			logger.LogTrace("Base channel ID {baseChannelId}", channelIdAllocator);
-
 			knownChannels = new Dictionary<ulong, ChannelRepresentation>();
 			randomMessageCts = new CancellationTokenSource();
 			randomMessageTask = RandomMessageLoop(this.randomMessageCts.Token);
@@ -95,7 +93,6 @@ namespace Tgstation.Server.Tests.Live
 
 		public override async ValueTask DisposeAsync()
 		{
-			Logger.LogTrace("DisposeAsync Child");
 			this.randomMessageCts.Cancel();
 			this.randomMessageCts.Dispose();
 			await this.randomMessageTask;
@@ -107,8 +104,6 @@ namespace Tgstation.Server.Tests.Live
 			if (message == null)
 				throw new ArgumentNullException(nameof(message));
 
-			Logger.LogTrace("SendMessage");
-
 			Assert.IsTrue(knownChannels.ContainsKey(channelId));
 
 			cancellationToken.ThrowIfCancellationRequested();
@@ -131,8 +126,6 @@ namespace Tgstation.Server.Tests.Live
 			if (gitHubRepo == null)
 				throw new ArgumentNullException(nameof(gitHubRepo));
 
-			Logger.LogTrace("SendUpdateMessage");
-
 			Assert.IsTrue(knownChannels.ContainsKey(channelId));
 
 			cancellationToken.ThrowIfCancellationRequested();
@@ -155,7 +148,6 @@ namespace Tgstation.Server.Tests.Live
 
 		protected override Task Connect(CancellationToken cancellationToken)
 		{
-			Logger.LogTrace("Connect");
 			cancellationToken.ThrowIfCancellationRequested();
 
 			// 30% chance to fail AFTER initial connection
@@ -169,7 +161,6 @@ namespace Tgstation.Server.Tests.Live
 
 		protected override Task DisconnectImpl(CancellationToken cancellationToken)
 		{
-			Logger.LogTrace("DisconnectImpl");
 			cancellationToken.ThrowIfCancellationRequested();
 			connected = false;
 
@@ -181,7 +172,6 @@ namespace Tgstation.Server.Tests.Live
 		protected override Task<Dictionary<ChatChannel, IEnumerable<ChannelRepresentation>>> MapChannelsImpl(IEnumerable<ChatChannel> channels, CancellationToken cancellationToken)
 		{
 			channels = channels.ToList();
-			Logger.LogTrace("MapChannels: [{channels}]", String.Join(", ", channels.Select(channel => channel.IrcChannel ?? channel.DiscordChannelId?.ToString() ?? throw new InvalidOperationException("BAD CHANNEL"))));
 
 			cancellationToken.ThrowIfCancellationRequested();
 
@@ -202,7 +192,6 @@ namespace Tgstation.Server.Tests.Live
 
 		private ChannelRepresentation CreateChannel(ChatChannel channel)
 		{
-
 			ulong channelId;
 			if (channel.DiscordChannelId.HasValue)
 				channelId = channel.DiscordChannelId.Value;
@@ -235,7 +224,6 @@ namespace Tgstation.Server.Tests.Live
 
 		async Task RandomMessageLoop(CancellationToken cancellationToken)
 		{
-			Logger.LogTrace("RandomMessageLoop");
 			try
 			{
 				for (var i = 0UL; !cancellationToken.IsCancellationRequested; ++i)
@@ -337,11 +325,7 @@ namespace Tgstation.Server.Tests.Live
 							content = $"{content} {commands.ElementAt(random.Next(0, commands.Count)).Name}";
 						// 40% chance to attempt a custom chat command in long_running_test
 						else
-							// equal chance for each
-							if (random.Next(0, 100) > 50)
-								content = $"{content} embeds_test";
-							else
-								content = $"{content} response_overload_test";
+							content = $"{content} embeds_test"; // NEVER send the response_overload_test, it causes so much havoc in CI and we test it manually
 
 					EnqueueMessage(new Message
 					{
@@ -353,7 +337,6 @@ namespace Tgstation.Server.Tests.Live
 			}
 			catch (OperationCanceledException)
 			{
-				Logger.LogTrace("RandomMessageLoop cancelled");
 			}
 		}
 	}

tests/Tgstation.Server.Tests/Live/Instance/DeploymentTest.cs

@@ -20,13 +20,16 @@ namespace Tgstation.Server.Tests.Live.Instance
 		readonly IDreamDaemonClient dreamDaemonClient;
 		readonly IInstanceClient instanceClient;
 
+		readonly bool lowPriorityDeployments;
+
 		Task vpTest;
 
-		public DeploymentTest(IInstanceClient instanceClient, IJobsClient jobsClient) : base(jobsClient)
+		public DeploymentTest(IInstanceClient instanceClient, IJobsClient jobsClient, bool lowPriorityDeployments) : base(jobsClient)
 		{
 			this.instanceClient = instanceClient ?? throw new ArgumentNullException(nameof(instanceClient));
 			dreamMakerClient = instanceClient.DreamMaker;
 			dreamDaemonClient = instanceClient.DreamDaemon;
+			this.lowPriorityDeployments = lowPriorityDeployments;
 		}
 
 		public async Task RunPreRepoClone(CancellationToken cancellationToken)
@@ -34,7 +37,9 @@ namespace Tgstation.Server.Tests.Live.Instance
 			Assert.IsNull(vpTest);
 			vpTest = TestVisibilityPermission(cancellationToken);
 			var deployJob = await dreamMakerClient.Compile(cancellationToken);
-			deployJob = await WaitForJob(deployJob, 30, true, null, cancellationToken);
+			var deploymentJobWaitTask = WaitForJob(deployJob, 30, true, null, cancellationToken);
+			await CheckDreamDaemonPriority(deploymentJobWaitTask, cancellationToken);
+			deployJob = await deploymentJobWaitTask;
 			Assert.IsTrue(deployJob.ErrorCode == ErrorCode.RepoCloning || deployJob.ErrorCode == ErrorCode.RepoMissing);
 
 			var dmSettings = await dreamMakerClient.Read(cancellationToken);
@@ -42,6 +47,65 @@ namespace Tgstation.Server.Tests.Live.Instance
 			Assert.AreEqual(null, dmSettings.ProjectName);
 		}
 
+		async Task CheckDreamDaemonPriority(Task deploymentJobWaitTask, CancellationToken cancellationToken)
+		{
+			// this doesn't check dm's priority, but it really should
+
+			while (!deploymentJobWaitTask.IsCompleted)
+			{
+				var ddProcessName = new PlatformIdentifier().IsWindows && ByondTest.TestVersion >= new Version(515, 1598)
+					? "dd"
+					: "DreamDaemon";
+
+				var allProcesses = System.Diagnostics.Process.GetProcessesByName(ddProcessName);
+				if (allProcesses.Length == 0)
+					continue;
+
+				if (allProcesses.Length > 1)
+					Assert.Fail("Multiple DreamDaemon-like processes running!");
+
+				using var process = allProcesses[0];
+
+				int processId;
+				try
+				{
+					processId = process.Id;
+				}
+				catch
+				{
+					return; // vOv
+				}
+
+				bool good = false;
+				while (!process.HasExited)
+				{
+					// we need to constantly reacquire the handle to invalidate caches
+					using var localProcess = System.Diagnostics.Process.GetProcessById(processId);
+					if (lowPriorityDeployments)
+					{
+						if (localProcess.PriorityClass == System.Diagnostics.ProcessPriorityClass.BelowNormal)
+						{
+							good = true;
+							break;
+						}
+						else
+							Assert.AreEqual(System.Diagnostics.ProcessPriorityClass.Normal, localProcess.PriorityClass);
+					}
+					else
+					{
+						good = true;
+						Assert.AreEqual(System.Diagnostics.ProcessPriorityClass.Normal, localProcess.PriorityClass, "DreamDaemon's process priority changed when it shouldn't have!");
+						await Task.Delay(1, cancellationToken);
+					}
+				}
+
+				if (!good)
+					Assert.Fail("Did not detect DreamDaemon lowering its process priority!");
+
+				break;
+			}
+		}
+
 		public async Task RunPostRepoClone(Task byondTask, CancellationToken cancellationToken)
 		{
 			Assert.IsNotNull(vpTest);
@@ -80,6 +144,11 @@ namespace Tgstation.Server.Tests.Live.Instance
 			}
 
 			var deployJobTask = CompileAfterByondInstall();
+			var deployJob = await deployJobTask;
+			var deploymentJobWaitTask = WaitForJob(deployJob, 40, true, ErrorCode.DreamMakerNeverValidated, cancellationToken);
+
+			await CheckDreamDaemonPriority(deploymentJobWaitTask, cancellationToken);
+
 			await Task.WhenAll(
 				ApiAssert.ThrowsException<ConflictException>(() => dreamDaemonClient.Update(new DreamDaemonRequest
 				{
@@ -89,11 +158,7 @@ namespace Tgstation.Server.Tests.Live.Instance
 				{
 					ApiValidationPort = TestLiveServer.DDPort
 				}, cancellationToken), ErrorCode.PortNotAvailable),
-				deployJobTask);
-
-			var deployJob = await deployJobTask;
-
-			await WaitForJob(deployJob, 40, true, ErrorCode.DreamMakerNeverValidated, cancellationToken);
+				deploymentJobWaitTask);
 
 			const string FailProject = "tests/DMAPI/BuildFail/build_fail";
 			var updated = await dreamMakerClient.Update(new DreamMakerRequest

tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs

@@ -23,13 +23,13 @@ namespace Tgstation.Server.Tests.Live.Instance
 			this.serverPort = serverPort;
 		}
 
-		public async Task RunTests(CancellationToken cancellationToken)
+		public async Task RunTests(CancellationToken cancellationToken, bool highPrioDD, bool lowPrioDeployment)
 		{
 			var byondTest = new ByondTest(instanceClient.Byond, instanceClient.Jobs, instanceClient.Metadata);
 			var chatTest = new ChatTest(instanceClient.ChatBots, instanceManagerClient, instanceClient.Jobs, instanceClient.Metadata);
 			var configTest = new ConfigurationTest(instanceClient.Configuration, instanceClient.Metadata);
 			var repoTest = new RepositoryTest(instanceClient.Repository, instanceClient.Jobs);
-			var dmTest = new DeploymentTest(instanceClient, instanceClient.Jobs);
+			var dmTest = new DeploymentTest(instanceClient, instanceClient.Jobs, lowPrioDeployment);
 
 			var byondTask = byondTest.Run(cancellationToken, out var firstInstall);
 			var chatTask = chatTest.RunPreWatchdog(cancellationToken);
@@ -45,7 +45,7 @@ namespace Tgstation.Server.Tests.Live.Instance
 			await dmTask;
 			await byondTask;
 
-			await new WatchdogTest(instanceClient, instanceManager, serverPort).Run(cancellationToken);
+			await new WatchdogTest(instanceClient, instanceManager, serverPort, highPrioDD).Run(cancellationToken);
 		}
 	}
 }

tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs

@@ -26,12 +26,12 @@ using Tgstation.Server.Client.Components;
 using Tgstation.Server.Host.Components;
 using Tgstation.Server.Host.Components.Chat;
 using Tgstation.Server.Host.Components.Interop;
+using Tgstation.Server.Host.Components.Session;
 using Tgstation.Server.Host.Components.Watchdog;
 using Tgstation.Server.Host.Controllers;
 using Tgstation.Server.Host.Extensions;
 using Tgstation.Server.Host.IO;
 using Tgstation.Server.Host.System;
-using Tgstation.Server.Tests.Live;
 
 namespace Tgstation.Server.Tests.Live.Instance
 {
@@ -40,15 +40,17 @@ namespace Tgstation.Server.Tests.Live.Instance
 		readonly IInstanceClient instanceClient;
 		readonly InstanceManager instanceManager;
 		readonly ushort serverPort;
+		readonly bool highPrioDD;
 
 		bool ranTimeoutTest = false;
 
-		public WatchdogTest(IInstanceClient instanceClient, InstanceManager instanceManager, ushort serverPort)
+		public WatchdogTest(IInstanceClient instanceClient, InstanceManager instanceManager, ushort serverPort, bool highPrioDD)
 			: base(instanceClient.Jobs)
 		{
 			this.instanceClient = instanceClient ?? throw new ArgumentNullException(nameof(instanceClient));
 			this.instanceManager = instanceManager ?? throw new ArgumentNullException(nameof(instanceManager));
 			this.serverPort = serverPort;
+			this.highPrioDD = highPrioDD;
 		}
 
 		public async Task Run(CancellationToken cancellationToken)
@@ -61,7 +63,8 @@ namespace Tgstation.Server.Tests.Live.Instance
 				{
 					StartupTimeout = 15,
 					HeartbeatSeconds = 0,
-					Port = TestLiveServer.DDPort
+					Port = TestLiveServer.DDPort,
+					LogOutput = false,
 				}, cancellationToken),
 				ApiAssert.ThrowsException<ApiConflictException>(() => instanceClient.DreamDaemon.Update(new DreamDaemonRequest
 				{
@@ -101,9 +104,11 @@ namespace Tgstation.Server.Tests.Live.Instance
 
 			var deleteJobTask = TestDeleteByondInstallErrorCasesAndQueing(cancellationToken);
 
+			SessionController.LogTopicRequests = false;
 			await WhiteBoxChatCommandTest(cancellationToken);
 			await SendChatOverloadCommand(cancellationToken);
 			await ValidateTopicLimits(cancellationToken);
+			SessionController.LogTopicRequests = true;
 
 			// This one fucks with the access_identifer, run it in isolation
 			await WhiteBoxValidateBridgeRequestLimitAndTestChunking(cancellationToken);
@@ -264,7 +269,9 @@ namespace Tgstation.Server.Tests.Live.Instance
 			await WaitForJob(startJob, 40, false, null, cancellationToken);
 
 			daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
+
 			Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value);
+			CheckDDPriority(cancellationToken);
 			Assert.AreEqual(false, daemonStatus.SoftRestart);
 			Assert.AreEqual(false, daemonStatus.SoftShutdown);
 			Assert.AreEqual(string.Empty, daemonStatus.AdditionalParameters);
@@ -321,7 +328,9 @@ namespace Tgstation.Server.Tests.Live.Instance
 			await WaitForJob(startJob, 40, false, null, cancellationToken);
 
 			daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
+
 			Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value);
+			CheckDDPriority(cancellationToken);
 			Assert.AreEqual(false, daemonStatus.SoftRestart);
 			Assert.AreEqual(false, daemonStatus.SoftShutdown);
 
@@ -330,11 +339,12 @@ namespace Tgstation.Server.Tests.Live.Instance
 			daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
 			Assert.AreEqual(WatchdogStatus.Offline, daemonStatus.Status.Value);
 
-			await CheckDMApiFail(daemonStatus.ActiveCompileJob, cancellationToken);
+			await CheckDMApiFail(daemonStatus.ActiveCompileJob, cancellationToken, false);
 
 			daemonStatus = await instanceClient.DreamDaemon.Update(new DreamDaemonRequest
 			{
-				AdditionalParameters = string.Empty
+				AdditionalParameters = string.Empty,
+				LogOutput = true,
 			}, cancellationToken);
 			Assert.AreEqual(string.Empty, daemonStatus.AdditionalParameters);
 		}
@@ -353,6 +363,8 @@ namespace Tgstation.Server.Tests.Live.Instance
 
 			await WaitForJob(startJob, 40, false, null, cancellationToken);
 
+			CheckDDPriority(cancellationToken);
+
 			// lock on to DD and pause it so it can't heartbeat
 			var ddProcs = System.Diagnostics.Process.GetProcessesByName("DreamDaemon").Where(x => !x.HasExited).ToList();
 			if (ddProcs.Count != 1)
@@ -676,6 +688,28 @@ namespace Tgstation.Server.Tests.Live.Instance
 			Assert.AreEqual("Footer text", embedsResponse.Embed.Footer?.Text);
 		}
 
+		void CheckDDPriority(CancellationToken cancellationToken)
+		{
+			var ddProcessName = new PlatformIdentifier().IsWindows && ByondTest.TestVersion >= new Version(515, 1598)
+				? "dd"
+				: "DreamDaemon";
+
+			var allProcesses = System.Diagnostics.Process.GetProcessesByName(ddProcessName);
+			if (allProcesses.Length == 0)
+				Assert.Fail("Expected DreamDaemon to be running here");
+
+			if (allProcesses.Length > 1)
+				Assert.Fail("Multiple DreamDaemon-like processes running!");
+
+			using var process = allProcesses[0];
+
+			Assert.AreEqual(
+				highPrioDD
+					? System.Diagnostics.ProcessPriorityClass.AboveNormal
+					: System.Diagnostics.ProcessPriorityClass.Normal,
+				process.PriorityClass);
+		}
+
 		async Task RunLongRunningTestThenUpdate(CancellationToken cancellationToken)
 		{
 			System.Console.WriteLine("TEST: WATCHDOG LONG RUNNING WITH UPDATE TEST");
@@ -697,6 +731,7 @@ namespace Tgstation.Server.Tests.Live.Instance
 			daemonStatus = await DeployTestDme(DmeName, DreamDaemonSecurity.Safe, true, cancellationToken);
 
 			Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value);
+			CheckDDPriority(cancellationToken);
 
 			Assert.AreEqual(initialCompileJob.Id, daemonStatus.ActiveCompileJob.Id);
 			var newerCompileJob = daemonStatus.StagedCompileJob;
@@ -737,7 +772,9 @@ namespace Tgstation.Server.Tests.Live.Instance
 
 			daemonStatus = await DeployTestDme("LongRunning/long_running_test_copy", DreamDaemonSecurity.Safe, true, cancellationToken);
 
+
 			Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value);
+			CheckDDPriority(cancellationToken);
 
 			Assert.AreEqual(initialCompileJob.Id, daemonStatus.ActiveCompileJob.Id);
 			var newerCompileJob = daemonStatus.StagedCompileJob;
@@ -774,6 +811,8 @@ namespace Tgstation.Server.Tests.Live.Instance
 
 			await WaitForJob(startJob, 70, false, null, cancellationToken);
 
+			CheckDDPriority(cancellationToken);
+
 			var byondInstallJobTask = instanceClient.Byond.SetActiveVersion(
 				new ByondVersionRequest
 				{
@@ -828,7 +867,9 @@ namespace Tgstation.Server.Tests.Live.Instance
 			await WaitForJob(startJob, 40, false, null, cancellationToken);
 
 			var daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
+
 			Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value);
+			CheckDDPriority(cancellationToken);
 			Assert.AreEqual(TestLiveServer.DDPort, daemonStatus.CurrentPort);
 
 			// Try killing the DD process to ensure it gets set to the restoring state
@@ -985,7 +1026,7 @@ namespace Tgstation.Server.Tests.Live.Instance
 			while (timeout > 0);
 		}
 
-		async Task CheckDMApiFail(CompileJobResponse compileJob, CancellationToken cancellationToken)
+		async Task CheckDMApiFail(CompileJobResponse compileJob, CancellationToken cancellationToken, bool checkLogs = true)
 		{
 			var gameDir = Path.Combine(instanceClient.Metadata.Path, "Game", compileJob.DirectoryName.Value.ToString(), Path.GetDirectoryName(compileJob.DmeName));
 			var failFile = Path.Combine(gameDir, "test_fail_reason.txt");
@@ -993,11 +1034,29 @@ namespace Tgstation.Server.Tests.Live.Instance
 			{
 				var successFile = Path.Combine(gameDir, "test_success.txt");
 				Assert.IsTrue(File.Exists(successFile));
-				return;
+			}
+			else
+			{
+				var text = await File.ReadAllTextAsync(failFile, cancellationToken);
+				Assert.Fail(text);
 			}
 
-			var text = await File.ReadAllTextAsync(failFile, cancellationToken);
-			Assert.Fail(text);
+			if (!checkLogs)
+				return;
+
+			var daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
+			if (daemonStatus.Status != WatchdogStatus.Offline)
+				return;
+
+			var outerLogsDir = Path.Combine(instanceClient.Metadata.Path, "Diagnostics", "DreamDaemonLogs");
+			var logsDir = new DirectoryInfo(outerLogsDir).GetDirectories().OrderByDescending(x => x.CreationTime).FirstOrDefault();
+			Assert.IsNotNull(logsDir);
+
+			var logfile = logsDir.GetFiles().OrderByDescending(x => x.CreationTime).FirstOrDefault();
+			Assert.IsNotNull(logfile);
+
+			var logtext = await File.ReadAllTextAsync(logfile.FullName, cancellationToken);
+			Assert.IsFalse(String.IsNullOrWhiteSpace(logtext));
 		}
 	}
 }

tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs

@@ -32,6 +32,9 @@ namespace Tgstation.Server.Tests.Live
 
 		public bool DumpOpenApiSpecpath { get; }
 
+		public bool HighPriorityDreamDaemon { get; }
+		public bool LowPriorityDeployments { get; }
+
 		public bool RestartRequested => RealServer.RestartRequested;
 
 		readonly List<string> args;
@@ -72,34 +75,45 @@ namespace Tgstation.Server.Tests.Live
 			var gitHubAccessToken = Environment.GetEnvironmentVariable("TGS_TEST_GITHUB_TOKEN");
 			var dumpOpenAPISpecPathEnvVar = Environment.GetEnvironmentVariable("TGS_TEST_DUMP_API_SPEC");
 
-			if (string.IsNullOrEmpty(DatabaseType))
+			if (String.IsNullOrEmpty(DatabaseType))
 				Assert.Inconclusive("No database type configured in env var TGS_TEST_DATABASE_TYPE!");
 
-			if (string.IsNullOrEmpty(connectionString))
+			if (String.IsNullOrEmpty(connectionString))
 				Assert.Inconclusive("No connection string configured in env var TGS_TEST_CONNECTION_STRING!");
 
-			if (string.IsNullOrEmpty(gitHubAccessToken))
+			if (String.IsNullOrEmpty(gitHubAccessToken))
 				Console.WriteLine("WARNING: No GitHub access token configured, test may fail due to rate limits!");
 
-			DumpOpenApiSpecpath = !string.IsNullOrEmpty(dumpOpenAPISpecPathEnvVar);
+			DumpOpenApiSpecpath = !String.IsNullOrEmpty(dumpOpenAPISpecPathEnvVar);
+
+			// neither of these should really matter but it's better that we test them
+			// high prio DD might help with some topic flakiness actually
+			// github doesn't allow nicing on linux though
+			var runningInGitHubActions = !String.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("GITHUB_RUN_ID"));
+			var windows = new Host.System.PlatformIdentifier().IsWindows;
+			var nicingAllowed = windows || !runningInGitHubActions;
+			HighPriorityDreamDaemon = nicingAllowed;
+			LowPriorityDeployments = nicingAllowed;
 
 			args = new List<string>()
 			{
-				string.Format(CultureInfo.InvariantCulture, "Database:DropDatabase={0}", true), // Replaced after first Run
-				string.Format(CultureInfo.InvariantCulture, "General:ConfigVersion={0}", GeneralConfiguration.CurrentConfigVersion),
-				string.Format(CultureInfo.InvariantCulture, "General:ApiPort={0}", port),
-				string.Format(CultureInfo.InvariantCulture, "Database:DatabaseType={0}", DatabaseType),
-				string.Format(CultureInfo.InvariantCulture, "Database:ConnectionString={0}", connectionString),
-				string.Format(CultureInfo.InvariantCulture, "General:SetupWizardMode={0}", SetupWizardMode.Never),
-				string.Format(CultureInfo.InvariantCulture, "General:MinimumPasswordLength={0}", 10),
-				string.Format(CultureInfo.InvariantCulture, "General:InstanceLimit={0}", 11),
-				string.Format(CultureInfo.InvariantCulture, "General:UserLimit={0}", 150),
-				string.Format(CultureInfo.InvariantCulture, "General:UserGroupLimit={0}", 47),
-				string.Format(CultureInfo.InvariantCulture, "General:HostApiDocumentation={0}", DumpOpenApiSpecpath),
-				string.Format(CultureInfo.InvariantCulture, "FileLogging:Directory={0}", Path.Combine(Directory, "Logs")),
-				string.Format(CultureInfo.InvariantCulture, "FileLogging:LogLevel={0}", "Trace"),
-				string.Format(CultureInfo.InvariantCulture, "General:ValidInstancePaths:0={0}", Directory),
-				"General:ByondTopicTimeout=3000"
+				String.Format(CultureInfo.InvariantCulture, "Database:DropDatabase={0}", true), // Replaced after first Run
+				String.Format(CultureInfo.InvariantCulture, "General:ConfigVersion={0}", GeneralConfiguration.CurrentConfigVersion),
+				String.Format(CultureInfo.InvariantCulture, "General:ApiPort={0}", port),
+				String.Format(CultureInfo.InvariantCulture, "Database:DatabaseType={0}", DatabaseType),
+				String.Format(CultureInfo.InvariantCulture, "Database:ConnectionString={0}", connectionString),
+				String.Format(CultureInfo.InvariantCulture, "General:SetupWizardMode={0}", SetupWizardMode.Never),
+				String.Format(CultureInfo.InvariantCulture, "General:MinimumPasswordLength={0}", 10),
+				String.Format(CultureInfo.InvariantCulture, "General:InstanceLimit={0}", 11),
+				String.Format(CultureInfo.InvariantCulture, "General:UserLimit={0}", 150),
+				String.Format(CultureInfo.InvariantCulture, "General:UserGroupLimit={0}", 47),
+				String.Format(CultureInfo.InvariantCulture, "General:HostApiDocumentation={0}", DumpOpenApiSpecpath),
+				String.Format(CultureInfo.InvariantCulture, "FileLogging:Directory={0}", Path.Combine(Directory, "Logs")),
+				String.Format(CultureInfo.InvariantCulture, "FileLogging:LogLevel={0}", "Trace"),
+				String.Format(CultureInfo.InvariantCulture, "General:ValidInstancePaths:0={0}", Directory),
+				"General:ByondTopicTimeout=3000",
+				$"Session:HighPriorityLiveDreamDaemon={HighPriorityDreamDaemon}",
+				$"Session:LowPriorityDeploymentProcesses={LowPriorityDeployments}",
 			};
 
 			if (dumpOnMissingUpdate)

tests/Tgstation.Server.Tests/Live/TestLiveServer.cs

@@ -730,6 +730,22 @@ namespace Tgstation.Server.Tests.Live
 		[TestMethod]
 		public async Task TestStandardTgsOperation()
 		{
+			using(var currentProcess = System.Diagnostics.Process.GetCurrentProcess())
+			{
+				var currentPriorityClass = currentProcess.PriorityClass;
+				if (currentPriorityClass != ProcessPriorityClass.Normal)
+				{
+					// attempt to adjust it
+					Console.WriteLine($"TEST PROCESS PRIORITY: Attempting to normalize process priority from {currentPriorityClass}...");
+					currentProcess.PriorityClass = ProcessPriorityClass.Normal;
+				}
+			}
+
+			using (var currentProcess = System.Diagnostics.Process.GetCurrentProcess())
+			{
+				Assert.AreEqual(ProcessPriorityClass.Normal, currentProcess.PriorityClass);
+			}
+
 			const int MaximumTestMinutes = 30;
 			using var hardCancellationTokenSource = new CancellationTokenSource(TimeSpan.FromMinutes(MaximumTestMinutes));
 			var hardCancellationToken = hardCancellationTokenSource.Token;
@@ -864,7 +880,7 @@ namespace Tgstation.Server.Tests.Live
 
 					Assert.IsTrue(Directory.Exists(instanceClient.Metadata.Path));
 
-					var instanceTests = FailFast(new InstanceTest(instanceClient, adminClient.Instances, GetInstanceManager(), (ushort)server.Url.Port).RunTests(cancellationToken));
+					var instanceTests = FailFast(new InstanceTest(instanceClient, adminClient.Instances, GetInstanceManager(), (ushort)server.Url.Port).RunTests(cancellationToken, server.HighPriorityDreamDaemon, server.LowPriorityDeployments));
 
 					await Task.WhenAll(rootTest, adminTest, instancesTest, instanceTests, usersTest);
 
@@ -1020,7 +1036,7 @@ namespace Tgstation.Server.Tests.Live
 					Assert.AreEqual(WatchdogStatus.Online, dd.Status.Value);
 
 					var compileJob = await instanceClient.DreamMaker.Compile(cancellationToken);
-					var wdt = new WatchdogTest(instanceClient, GetInstanceManager(), (ushort)server.Url.Port);
+					var wdt = new WatchdogTest(instanceClient, GetInstanceManager(), (ushort)server.Url.Port, server.HighPriorityDreamDaemon);
 					await wdt.WaitForJob(compileJob, 30, false, null, cancellationToken);
 
 					dd = await instanceClient.DreamDaemon.Read(cancellationToken);
@@ -1067,7 +1083,7 @@ namespace Tgstation.Server.Tests.Live
 					Assert.AreEqual(WatchdogStatus.Online, currentDD.Status);
 					Assert.AreEqual(expectedStaged, currentDD.StagedCompileJob.Job.Id.Value);
 
-					var wdt = new WatchdogTest(instanceClient, GetInstanceManager(), (ushort)server.Url.Port);
+					var wdt = new WatchdogTest(instanceClient, GetInstanceManager(), (ushort)server.Url.Port, server.HighPriorityDreamDaemon);
 					currentDD = await wdt.TellWorldToReboot(cancellationToken);
 					Assert.AreEqual(expectedStaged, currentDD.ActiveCompileJob.Job.Id.Value);
 					Assert.IsNull(currentDD.StagedCompileJob);