diff --git a/tests/Tgstation.Server.Tests/Live/Instance/DeploymentTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/DeploymentTest.cs index b56d30efd1..81e5f5343c 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/DeploymentTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/DeploymentTest.cs @@ -20,13 +20,16 @@ namespace Tgstation.Server.Tests.Live.Instance readonly IDreamDaemonClient dreamDaemonClient; readonly IInstanceClient instanceClient; + readonly bool lowPriorityDeployments; + Task vpTest; - public DeploymentTest(IInstanceClient instanceClient, IJobsClient jobsClient) : base(jobsClient) + public DeploymentTest(IInstanceClient instanceClient, IJobsClient jobsClient, bool lowPriorityDeployments) : base(jobsClient) { this.instanceClient = instanceClient ?? throw new ArgumentNullException(nameof(instanceClient)); dreamMakerClient = instanceClient.DreamMaker; dreamDaemonClient = instanceClient.DreamDaemon; + this.lowPriorityDeployments = lowPriorityDeployments; } public async Task RunPreRepoClone(CancellationToken cancellationToken) @@ -34,7 +37,9 @@ namespace Tgstation.Server.Tests.Live.Instance Assert.IsNull(vpTest); vpTest = TestVisibilityPermission(cancellationToken); var deployJob = await dreamMakerClient.Compile(cancellationToken); - deployJob = await WaitForJob(deployJob, 30, true, null, cancellationToken); + var deploymentJobWaitTask = WaitForJob(deployJob, 30, true, null, cancellationToken); + await CheckDreamDaemonPriority(deploymentJobWaitTask, cancellationToken); + deployJob = await deploymentJobWaitTask; Assert.IsTrue(deployJob.ErrorCode == ErrorCode.RepoCloning || deployJob.ErrorCode == ErrorCode.RepoMissing); var dmSettings = await dreamMakerClient.Read(cancellationToken); @@ -42,6 +47,65 @@ namespace Tgstation.Server.Tests.Live.Instance Assert.AreEqual(null, dmSettings.ProjectName); } + async Task CheckDreamDaemonPriority(Task deploymentJobWaitTask, CancellationToken cancellationToken) + { + while (!deploymentJobWaitTask.IsCompleted) + { + var ddProcessName = new PlatformIdentifier().IsWindows && ByondTest.TestVersion >= new Version(515, 1598) + ? "dd" + : "DreamDaemon"; + + var allProcesses = System.Diagnostics.Process.GetProcessesByName(ddProcessName); + if (allProcesses.Length == 0) + continue; + + if (allProcesses.Length > 1) + Assert.Fail("Multiple DreamDaemon-like processes running!"); + + using var process = allProcesses[0]; + + int processId; + try + { + processId = process.Id; + } + catch + { + return; // vOv + } + + bool good = false; + while (!process.HasExited) + { + // we need to constantly reacquire the handle to invalidate caches + using var localProcess = System.Diagnostics.Process.GetProcessById(processId); + if (lowPriorityDeployments) + { + if (localProcess.PriorityClass == System.Diagnostics.ProcessPriorityClass.BelowNormal) + { + good = true; + break; + } + } + else + { + good = true; + if (localProcess.PriorityClass != System.Diagnostics.ProcessPriorityClass.Normal) + { + Assert.Fail("DreamDaemon's process priority changed when it shouldn't have!"); + } + + await Task.Delay(1, cancellationToken); + } + } + + if (!good) + Assert.Fail("Did not detect DreamDaemon lowering its process priority!"); + + break; + } + } + public async Task RunPostRepoClone(Task byondTask, CancellationToken cancellationToken) { Assert.IsNotNull(vpTest); @@ -80,6 +144,11 @@ namespace Tgstation.Server.Tests.Live.Instance } var deployJobTask = CompileAfterByondInstall(); + var deployJob = await deployJobTask; + var deploymentJobWaitTask = WaitForJob(deployJob, 40, true, ErrorCode.DreamMakerNeverValidated, cancellationToken); + + await CheckDreamDaemonPriority(deploymentJobWaitTask, cancellationToken); + await Task.WhenAll( ApiAssert.ThrowsException(() => dreamDaemonClient.Update(new DreamDaemonRequest { @@ -89,11 +158,7 @@ namespace Tgstation.Server.Tests.Live.Instance { ApiValidationPort = TestLiveServer.DDPort }, cancellationToken), ErrorCode.PortNotAvailable), - deployJobTask); - - var deployJob = await deployJobTask; - - await WaitForJob(deployJob, 40, true, ErrorCode.DreamMakerNeverValidated, cancellationToken); + deploymentJobWaitTask); const string FailProject = "tests/DMAPI/BuildFail/build_fail"; var updated = await dreamMakerClient.Update(new DreamMakerRequest diff --git a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs index fd502926a9..0eecae4997 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs @@ -23,13 +23,13 @@ namespace Tgstation.Server.Tests.Live.Instance this.serverPort = serverPort; } - public async Task RunTests(CancellationToken cancellationToken) + public async Task RunTests(CancellationToken cancellationToken, bool highPrioDD, bool lowPrioDeployment) { var byondTest = new ByondTest(instanceClient.Byond, instanceClient.Jobs, instanceClient.Metadata); var chatTest = new ChatTest(instanceClient.ChatBots, instanceManagerClient, instanceClient.Jobs, instanceClient.Metadata); var configTest = new ConfigurationTest(instanceClient.Configuration, instanceClient.Metadata); var repoTest = new RepositoryTest(instanceClient.Repository, instanceClient.Jobs); - var dmTest = new DeploymentTest(instanceClient, instanceClient.Jobs); + var dmTest = new DeploymentTest(instanceClient, instanceClient.Jobs, lowPrioDeployment); var byondTask = byondTest.Run(cancellationToken, out var firstInstall); var chatTask = chatTest.RunPreWatchdog(cancellationToken); @@ -45,7 +45,7 @@ namespace Tgstation.Server.Tests.Live.Instance await dmTask; await byondTask; - await new WatchdogTest(instanceClient, instanceManager, serverPort).Run(cancellationToken); + await new WatchdogTest(instanceClient, instanceManager, serverPort, highPrioDD).Run(cancellationToken); } } } diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index 976358f699..603a8744c6 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -39,15 +39,17 @@ namespace Tgstation.Server.Tests.Live.Instance readonly IInstanceClient instanceClient; readonly InstanceManager instanceManager; readonly ushort serverPort; + readonly bool highPrioDD; bool ranTimeoutTest = false; - public WatchdogTest(IInstanceClient instanceClient, InstanceManager instanceManager, ushort serverPort) + public WatchdogTest(IInstanceClient instanceClient, InstanceManager instanceManager, ushort serverPort, bool highPrioDD) : base(instanceClient.Jobs) { this.instanceClient = instanceClient ?? throw new ArgumentNullException(nameof(instanceClient)); this.instanceManager = instanceManager ?? throw new ArgumentNullException(nameof(instanceManager)); this.serverPort = serverPort; + this.highPrioDD = highPrioDD; } public async Task Run(CancellationToken cancellationToken) @@ -264,7 +266,9 @@ namespace Tgstation.Server.Tests.Live.Instance await WaitForJob(startJob, 40, false, null, cancellationToken); daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); + Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value); + CheckDDPriority(cancellationToken); Assert.AreEqual(false, daemonStatus.SoftRestart); Assert.AreEqual(false, daemonStatus.SoftShutdown); Assert.AreEqual(string.Empty, daemonStatus.AdditionalParameters); @@ -321,7 +325,9 @@ namespace Tgstation.Server.Tests.Live.Instance await WaitForJob(startJob, 40, false, null, cancellationToken); daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); + Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value); + CheckDDPriority(cancellationToken); Assert.AreEqual(false, daemonStatus.SoftRestart); Assert.AreEqual(false, daemonStatus.SoftShutdown); @@ -354,6 +360,8 @@ namespace Tgstation.Server.Tests.Live.Instance await WaitForJob(startJob, 40, false, null, cancellationToken); + CheckDDPriority(cancellationToken); + // lock on to DD and pause it so it can't heartbeat var ddProcs = System.Diagnostics.Process.GetProcessesByName("DreamDaemon").Where(x => !x.HasExited).ToList(); if (ddProcs.Count != 1) @@ -677,6 +685,28 @@ namespace Tgstation.Server.Tests.Live.Instance Assert.AreEqual("Footer text", embedsResponse.Embed.Footer?.Text); } + void CheckDDPriority(CancellationToken cancellationToken) + { + var ddProcessName = new PlatformIdentifier().IsWindows && ByondTest.TestVersion >= new Version(515, 1598) + ? "dd" + : "DreamDaemon"; + + var allProcesses = System.Diagnostics.Process.GetProcessesByName(ddProcessName); + if (allProcesses.Length == 0) + Assert.Fail("Expected DreamDaemon to be running here"); + + if (allProcesses.Length > 1) + Assert.Fail("Multiple DreamDaemon-like processes running!"); + + using var process = allProcesses[0]; + + Assert.AreEqual( + highPrioDD + ? System.Diagnostics.ProcessPriorityClass.AboveNormal + : System.Diagnostics.ProcessPriorityClass.Normal, + process.PriorityClass); + } + async Task RunLongRunningTestThenUpdate(CancellationToken cancellationToken) { System.Console.WriteLine("TEST: WATCHDOG LONG RUNNING WITH UPDATE TEST"); @@ -698,6 +728,7 @@ namespace Tgstation.Server.Tests.Live.Instance daemonStatus = await DeployTestDme(DmeName, DreamDaemonSecurity.Safe, true, cancellationToken); Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value); + CheckDDPriority(cancellationToken); Assert.AreEqual(initialCompileJob.Id, daemonStatus.ActiveCompileJob.Id); var newerCompileJob = daemonStatus.StagedCompileJob; @@ -738,7 +769,9 @@ namespace Tgstation.Server.Tests.Live.Instance daemonStatus = await DeployTestDme("LongRunning/long_running_test_copy", DreamDaemonSecurity.Safe, true, cancellationToken); + Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value); + CheckDDPriority(cancellationToken); Assert.AreEqual(initialCompileJob.Id, daemonStatus.ActiveCompileJob.Id); var newerCompileJob = daemonStatus.StagedCompileJob; @@ -775,6 +808,8 @@ namespace Tgstation.Server.Tests.Live.Instance await WaitForJob(startJob, 70, false, null, cancellationToken); + CheckDDPriority(cancellationToken); + var byondInstallJobTask = instanceClient.Byond.SetActiveVersion( new ByondVersionRequest { @@ -829,7 +864,9 @@ namespace Tgstation.Server.Tests.Live.Instance await WaitForJob(startJob, 40, false, null, cancellationToken); var daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); + Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value); + CheckDDPriority(cancellationToken); Assert.AreEqual(TestLiveServer.DDPort, daemonStatus.CurrentPort); // Try killing the DD process to ensure it gets set to the restoring state diff --git a/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs b/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs index c6fcd0ea82..62c4ceaa3e 100644 --- a/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs +++ b/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs @@ -32,6 +32,9 @@ namespace Tgstation.Server.Tests.Live public bool DumpOpenApiSpecpath { get; } + public bool HighPriorityDreamDaemon { get; } + public bool LowPriorityDeployments { get; } + public bool RestartRequested => RealServer.RestartRequested; readonly List args; @@ -72,34 +75,41 @@ namespace Tgstation.Server.Tests.Live var gitHubAccessToken = Environment.GetEnvironmentVariable("TGS_TEST_GITHUB_TOKEN"); var dumpOpenAPISpecPathEnvVar = Environment.GetEnvironmentVariable("TGS_TEST_DUMP_API_SPEC"); - if (string.IsNullOrEmpty(DatabaseType)) + if (String.IsNullOrEmpty(DatabaseType)) Assert.Inconclusive("No database type configured in env var TGS_TEST_DATABASE_TYPE!"); - if (string.IsNullOrEmpty(connectionString)) + if (String.IsNullOrEmpty(connectionString)) Assert.Inconclusive("No connection string configured in env var TGS_TEST_CONNECTION_STRING!"); - if (string.IsNullOrEmpty(gitHubAccessToken)) + if (String.IsNullOrEmpty(gitHubAccessToken)) Console.WriteLine("WARNING: No GitHub access token configured, test may fail due to rate limits!"); - DumpOpenApiSpecpath = !string.IsNullOrEmpty(dumpOpenAPISpecPathEnvVar); + DumpOpenApiSpecpath = !String.IsNullOrEmpty(dumpOpenAPISpecPathEnvVar); + + // neither of these should really matter but it's better that we test them + // high prio DD might help with some topic flakiness actually + HighPriorityDreamDaemon = true; + LowPriorityDeployments = true; args = new List() { - string.Format(CultureInfo.InvariantCulture, "Database:DropDatabase={0}", true), // Replaced after first Run - string.Format(CultureInfo.InvariantCulture, "General:ConfigVersion={0}", GeneralConfiguration.CurrentConfigVersion), - string.Format(CultureInfo.InvariantCulture, "General:ApiPort={0}", port), - string.Format(CultureInfo.InvariantCulture, "Database:DatabaseType={0}", DatabaseType), - string.Format(CultureInfo.InvariantCulture, "Database:ConnectionString={0}", connectionString), - string.Format(CultureInfo.InvariantCulture, "General:SetupWizardMode={0}", SetupWizardMode.Never), - string.Format(CultureInfo.InvariantCulture, "General:MinimumPasswordLength={0}", 10), - string.Format(CultureInfo.InvariantCulture, "General:InstanceLimit={0}", 11), - string.Format(CultureInfo.InvariantCulture, "General:UserLimit={0}", 150), - string.Format(CultureInfo.InvariantCulture, "General:UserGroupLimit={0}", 47), - string.Format(CultureInfo.InvariantCulture, "General:HostApiDocumentation={0}", DumpOpenApiSpecpath), - string.Format(CultureInfo.InvariantCulture, "FileLogging:Directory={0}", Path.Combine(Directory, "Logs")), - string.Format(CultureInfo.InvariantCulture, "FileLogging:LogLevel={0}", "Trace"), - string.Format(CultureInfo.InvariantCulture, "General:ValidInstancePaths:0={0}", Directory), - "General:ByondTopicTimeout=3000" + String.Format(CultureInfo.InvariantCulture, "Database:DropDatabase={0}", true), // Replaced after first Run + String.Format(CultureInfo.InvariantCulture, "General:ConfigVersion={0}", GeneralConfiguration.CurrentConfigVersion), + String.Format(CultureInfo.InvariantCulture, "General:ApiPort={0}", port), + String.Format(CultureInfo.InvariantCulture, "Database:DatabaseType={0}", DatabaseType), + String.Format(CultureInfo.InvariantCulture, "Database:ConnectionString={0}", connectionString), + String.Format(CultureInfo.InvariantCulture, "General:SetupWizardMode={0}", SetupWizardMode.Never), + String.Format(CultureInfo.InvariantCulture, "General:MinimumPasswordLength={0}", 10), + String.Format(CultureInfo.InvariantCulture, "General:InstanceLimit={0}", 11), + String.Format(CultureInfo.InvariantCulture, "General:UserLimit={0}", 150), + String.Format(CultureInfo.InvariantCulture, "General:UserGroupLimit={0}", 47), + String.Format(CultureInfo.InvariantCulture, "General:HostApiDocumentation={0}", DumpOpenApiSpecpath), + String.Format(CultureInfo.InvariantCulture, "FileLogging:Directory={0}", Path.Combine(Directory, "Logs")), + String.Format(CultureInfo.InvariantCulture, "FileLogging:LogLevel={0}", "Trace"), + String.Format(CultureInfo.InvariantCulture, "General:ValidInstancePaths:0={0}", Directory), + "General:ByondTopicTimeout=3000", + $"Session:HighPriorityLiveDreamDaemon={HighPriorityDreamDaemon}", + $"Session:LowPriorityDeploymentProcesses={LowPriorityDeployments}", }; if (dumpOnMissingUpdate) diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index 1ade1e1c5a..10569c914c 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -864,7 +864,7 @@ namespace Tgstation.Server.Tests.Live Assert.IsTrue(Directory.Exists(instanceClient.Metadata.Path)); - var instanceTests = FailFast(new InstanceTest(instanceClient, adminClient.Instances, GetInstanceManager(), (ushort)server.Url.Port).RunTests(cancellationToken)); + var instanceTests = FailFast(new InstanceTest(instanceClient, adminClient.Instances, GetInstanceManager(), (ushort)server.Url.Port).RunTests(cancellationToken, server.HighPriorityDreamDaemon, server.LowPriorityDeployments)); await Task.WhenAll(rootTest, adminTest, instancesTest, instanceTests, usersTest); @@ -1020,7 +1020,7 @@ namespace Tgstation.Server.Tests.Live Assert.AreEqual(WatchdogStatus.Online, dd.Status.Value); var compileJob = await instanceClient.DreamMaker.Compile(cancellationToken); - var wdt = new WatchdogTest(instanceClient, GetInstanceManager(), (ushort)server.Url.Port); + var wdt = new WatchdogTest(instanceClient, GetInstanceManager(), (ushort)server.Url.Port, server.HighPriorityDreamDaemon); await wdt.WaitForJob(compileJob, 30, false, null, cancellationToken); dd = await instanceClient.DreamDaemon.Read(cancellationToken); @@ -1067,7 +1067,7 @@ namespace Tgstation.Server.Tests.Live Assert.AreEqual(WatchdogStatus.Online, currentDD.Status); Assert.AreEqual(expectedStaged, currentDD.StagedCompileJob.Job.Id.Value); - var wdt = new WatchdogTest(instanceClient, GetInstanceManager(), (ushort)server.Url.Port); + var wdt = new WatchdogTest(instanceClient, GetInstanceManager(), (ushort)server.Url.Port, server.HighPriorityDreamDaemon); currentDD = await wdt.TellWorldToReboot(cancellationToken); Assert.AreEqual(expectedStaged, currentDD.ActiveCompileJob.Job.Id.Value); Assert.IsNull(currentDD.StagedCompileJob);