cyberpresidentvanellope/tgstation-server
1
0
Fork 0
mirror of https://github.com/tgstation/tgstation-server.git synced 2026-08-27 23:17:20 +01:00
Code Issues Packages Projects Releases Wiki Activity

Regression test for DreamDaemon priority issues

Browse Source
This commit is contained in:
Dominion
2023-06-02 10:36:37 -04:00
parent 5832b08c29
commit a110dda407
5 changed files with 145 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/Tgstation.Server.Tests/Live/Instance/DeploymentTest.cs
+72 -7
View File
@@ -20,13 +20,16 @@ namespace Tgstation.Server.Tests.Live.Instance
readonly IDreamDaemonClient dreamDaemonClient;
readonly IInstanceClient instanceClient;
readonly bool lowPriorityDeployments;
Task vpTest;
public DeploymentTest(IInstanceClient instanceClient, IJobsClient jobsClient) : base(jobsClient)
public DeploymentTest(IInstanceClient instanceClient, IJobsClient jobsClient, bool lowPriorityDeployments) : base(jobsClient)
{
this.instanceClient = instanceClient ?? throw new ArgumentNullException(nameof(instanceClient));
dreamMakerClient = instanceClient.DreamMaker;
dreamDaemonClient = instanceClient.DreamDaemon;
this.lowPriorityDeployments = lowPriorityDeployments;
}
public async Task RunPreRepoClone(CancellationToken cancellationToken)
@@ -34,7 +37,9 @@ namespace Tgstation.Server.Tests.Live.Instance
Assert.IsNull(vpTest);
vpTest = TestVisibilityPermission(cancellationToken);
var deployJob = await dreamMakerClient.Compile(cancellationToken);
deployJob = await WaitForJob(deployJob, 30, true, null, cancellationToken);
var deploymentJobWaitTask = WaitForJob(deployJob, 30, true, null, cancellationToken);
await CheckDreamDaemonPriority(deploymentJobWaitTask, cancellationToken);
deployJob = await deploymentJobWaitTask;
Assert.IsTrue(deployJob.ErrorCode == ErrorCode.RepoCloning || deployJob.ErrorCode == ErrorCode.RepoMissing);
var dmSettings = await dreamMakerClient.Read(cancellationToken);
@@ -42,6 +47,65 @@ namespace Tgstation.Server.Tests.Live.Instance
Assert.AreEqual(null, dmSettings.ProjectName);
}
async Task CheckDreamDaemonPriority(Task deploymentJobWaitTask, CancellationToken cancellationToken)
{
while (!deploymentJobWaitTask.IsCompleted)
{
var ddProcessName = new PlatformIdentifier().IsWindows && ByondTest.TestVersion >= new Version(515, 1598)
? "dd"
: "DreamDaemon";
var allProcesses = System.Diagnostics.Process.GetProcessesByName(ddProcessName);
if (allProcesses.Length == 0)
continue;
if (allProcesses.Length > 1)
Assert.Fail("Multiple DreamDaemon-like processes running!");
using var process = allProcesses[0];
int processId;
try
{
processId = process.Id;
}
catch
{
return; // vOv
}
bool good = false;
while (!process.HasExited)
{
// we need to constantly reacquire the handle to invalidate caches
using var localProcess = System.Diagnostics.Process.GetProcessById(processId);
if (lowPriorityDeployments)
{
if (localProcess.PriorityClass == System.Diagnostics.ProcessPriorityClass.BelowNormal)
{
good = true;
break;
}
}
else
{
good = true;
if (localProcess.PriorityClass != System.Diagnostics.ProcessPriorityClass.Normal)
{
Assert.Fail("DreamDaemon's process priority changed when it shouldn't have!");
}
await Task.Delay(1, cancellationToken);
}
}
if (!good)
Assert.Fail("Did not detect DreamDaemon lowering its process priority!");
break;
}
}
public async Task RunPostRepoClone(Task byondTask, CancellationToken cancellationToken)
{
Assert.IsNotNull(vpTest);
@@ -80,6 +144,11 @@ namespace Tgstation.Server.Tests.Live.Instance
}
var deployJobTask = CompileAfterByondInstall();
var deployJob = await deployJobTask;
var deploymentJobWaitTask = WaitForJob(deployJob, 40, true, ErrorCode.DreamMakerNeverValidated, cancellationToken);
await CheckDreamDaemonPriority(deploymentJobWaitTask, cancellationToken);
await Task.WhenAll(
ApiAssert.ThrowsException<ConflictException>(() => dreamDaemonClient.Update(new DreamDaemonRequest
{
@@ -89,11 +158,7 @@ namespace Tgstation.Server.Tests.Live.Instance
{
ApiValidationPort = TestLiveServer.DDPort
}, cancellationToken), ErrorCode.PortNotAvailable),
deployJobTask);
var deployJob = await deployJobTask;
await WaitForJob(deployJob, 40, true, ErrorCode.DreamMakerNeverValidated, cancellationToken);
deploymentJobWaitTask);
const string FailProject = "tests/DMAPI/BuildFail/build_fail";
var updated = await dreamMakerClient.Update(new DreamMakerRequest
tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs
+3 -3
View File
@@ -23,13 +23,13 @@ namespace Tgstation.Server.Tests.Live.Instance
this.serverPort = serverPort;
}
public async Task RunTests(CancellationToken cancellationToken)
public async Task RunTests(CancellationToken cancellationToken, bool highPrioDD, bool lowPrioDeployment)
{
var byondTest = new ByondTest(instanceClient.Byond, instanceClient.Jobs, instanceClient.Metadata);
var chatTest = new ChatTest(instanceClient.ChatBots, instanceManagerClient, instanceClient.Jobs, instanceClient.Metadata);
var configTest = new ConfigurationTest(instanceClient.Configuration, instanceClient.Metadata);
var repoTest = new RepositoryTest(instanceClient.Repository, instanceClient.Jobs);
var dmTest = new DeploymentTest(instanceClient, instanceClient.Jobs);
var dmTest = new DeploymentTest(instanceClient, instanceClient.Jobs, lowPrioDeployment);
var byondTask = byondTest.Run(cancellationToken, out var firstInstall);
var chatTask = chatTest.RunPreWatchdog(cancellationToken);
@@ -45,7 +45,7 @@ namespace Tgstation.Server.Tests.Live.Instance
await dmTask;
await byondTask;
await new WatchdogTest(instanceClient, instanceManager, serverPort).Run(cancellationToken);
await new WatchdogTest(instanceClient, instanceManager, serverPort, highPrioDD).Run(cancellationToken);
}
}
}
tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs
+38 -1
View File
@@ -39,15 +39,17 @@ namespace Tgstation.Server.Tests.Live.Instance
readonly IInstanceClient instanceClient;
readonly InstanceManager instanceManager;
readonly ushort serverPort;
readonly bool highPrioDD;
bool ranTimeoutTest = false;
public WatchdogTest(IInstanceClient instanceClient, InstanceManager instanceManager, ushort serverPort)
public WatchdogTest(IInstanceClient instanceClient, InstanceManager instanceManager, ushort serverPort, bool highPrioDD)
: base(instanceClient.Jobs)
{
this.instanceClient = instanceClient ?? throw new ArgumentNullException(nameof(instanceClient));
this.instanceManager = instanceManager ?? throw new ArgumentNullException(nameof(instanceManager));
this.serverPort = serverPort;
this.highPrioDD = highPrioDD;
}
public async Task Run(CancellationToken cancellationToken)
@@ -264,7 +266,9 @@ namespace Tgstation.Server.Tests.Live.Instance
await WaitForJob(startJob, 40, false, null, cancellationToken);
daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value);
CheckDDPriority(cancellationToken);
Assert.AreEqual(false, daemonStatus.SoftRestart);
Assert.AreEqual(false, daemonStatus.SoftShutdown);
Assert.AreEqual(string.Empty, daemonStatus.AdditionalParameters);
@@ -321,7 +325,9 @@ namespace Tgstation.Server.Tests.Live.Instance
await WaitForJob(startJob, 40, false, null, cancellationToken);
daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value);
CheckDDPriority(cancellationToken);
Assert.AreEqual(false, daemonStatus.SoftRestart);
Assert.AreEqual(false, daemonStatus.SoftShutdown);
@@ -354,6 +360,8 @@ namespace Tgstation.Server.Tests.Live.Instance
await WaitForJob(startJob, 40, false, null, cancellationToken);
CheckDDPriority(cancellationToken);
// lock on to DD and pause it so it can't heartbeat
var ddProcs = System.Diagnostics.Process.GetProcessesByName("DreamDaemon").Where(x => !x.HasExited).ToList();
if (ddProcs.Count != 1)
@@ -677,6 +685,28 @@ namespace Tgstation.Server.Tests.Live.Instance
Assert.AreEqual("Footer text", embedsResponse.Embed.Footer?.Text);
}
void CheckDDPriority(CancellationToken cancellationToken)
{
var ddProcessName = new PlatformIdentifier().IsWindows && ByondTest.TestVersion >= new Version(515, 1598)
? "dd"
: "DreamDaemon";
var allProcesses = System.Diagnostics.Process.GetProcessesByName(ddProcessName);
if (allProcesses.Length == 0)
Assert.Fail("Expected DreamDaemon to be running here");
if (allProcesses.Length > 1)
Assert.Fail("Multiple DreamDaemon-like processes running!");
using var process = allProcesses[0];
Assert.AreEqual(
highPrioDD
? System.Diagnostics.ProcessPriorityClass.AboveNormal
: System.Diagnostics.ProcessPriorityClass.Normal,
process.PriorityClass);
}
async Task RunLongRunningTestThenUpdate(CancellationToken cancellationToken)
{
System.Console.WriteLine("TEST: WATCHDOG LONG RUNNING WITH UPDATE TEST");
@@ -698,6 +728,7 @@ namespace Tgstation.Server.Tests.Live.Instance
daemonStatus = await DeployTestDme(DmeName, DreamDaemonSecurity.Safe, true, cancellationToken);
Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value);
CheckDDPriority(cancellationToken);
Assert.AreEqual(initialCompileJob.Id, daemonStatus.ActiveCompileJob.Id);
var newerCompileJob = daemonStatus.StagedCompileJob;
@@ -738,7 +769,9 @@ namespace Tgstation.Server.Tests.Live.Instance
daemonStatus = await DeployTestDme("LongRunning/long_running_test_copy", DreamDaemonSecurity.Safe, true, cancellationToken);
Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value);
CheckDDPriority(cancellationToken);
Assert.AreEqual(initialCompileJob.Id, daemonStatus.ActiveCompileJob.Id);
var newerCompileJob = daemonStatus.StagedCompileJob;
@@ -775,6 +808,8 @@ namespace Tgstation.Server.Tests.Live.Instance
await WaitForJob(startJob, 70, false, null, cancellationToken);
CheckDDPriority(cancellationToken);
var byondInstallJobTask = instanceClient.Byond.SetActiveVersion(
new ByondVersionRequest
{
@@ -829,7 +864,9 @@ namespace Tgstation.Server.Tests.Live.Instance
await WaitForJob(startJob, 40, false, null, cancellationToken);
var daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value);
CheckDDPriority(cancellationToken);
Assert.AreEqual(TestLiveServer.DDPort, daemonStatus.CurrentPort);
// Try killing the DD process to ensure it gets set to the restoring state
tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs
+29 -19
View File
@@ -32,6 +32,9 @@ namespace Tgstation.Server.Tests.Live
public bool DumpOpenApiSpecpath { get; }
public bool HighPriorityDreamDaemon { get; }
public bool LowPriorityDeployments { get; }
public bool RestartRequested => RealServer.RestartRequested;
readonly List<string> args;
@@ -72,34 +75,41 @@ namespace Tgstation.Server.Tests.Live
var gitHubAccessToken = Environment.GetEnvironmentVariable("TGS_TEST_GITHUB_TOKEN");
var dumpOpenAPISpecPathEnvVar = Environment.GetEnvironmentVariable("TGS_TEST_DUMP_API_SPEC");
if (string.IsNullOrEmpty(DatabaseType))
if (String.IsNullOrEmpty(DatabaseType))
Assert.Inconclusive("No database type configured in env var TGS_TEST_DATABASE_TYPE!");
if (string.IsNullOrEmpty(connectionString))
if (String.IsNullOrEmpty(connectionString))
Assert.Inconclusive("No connection string configured in env var TGS_TEST_CONNECTION_STRING!");
if (string.IsNullOrEmpty(gitHubAccessToken))
if (String.IsNullOrEmpty(gitHubAccessToken))
Console.WriteLine("WARNING: No GitHub access token configured, test may fail due to rate limits!");
DumpOpenApiSpecpath = !string.IsNullOrEmpty(dumpOpenAPISpecPathEnvVar);
DumpOpenApiSpecpath = !String.IsNullOrEmpty(dumpOpenAPISpecPathEnvVar);
// neither of these should really matter but it's better that we test them
// high prio DD might help with some topic flakiness actually
HighPriorityDreamDaemon = true;
LowPriorityDeployments = true;
args = new List<string>()
{
string.Format(CultureInfo.InvariantCulture, "Database:DropDatabase={0}", true), // Replaced after first Run
string.Format(CultureInfo.InvariantCulture, "General:ConfigVersion={0}", GeneralConfiguration.CurrentConfigVersion),
string.Format(CultureInfo.InvariantCulture, "General:ApiPort={0}", port),
string.Format(CultureInfo.InvariantCulture, "Database:DatabaseType={0}", DatabaseType),
string.Format(CultureInfo.InvariantCulture, "Database:ConnectionString={0}", connectionString),
string.Format(CultureInfo.InvariantCulture, "General:SetupWizardMode={0}", SetupWizardMode.Never),
string.Format(CultureInfo.InvariantCulture, "General:MinimumPasswordLength={0}", 10),
string.Format(CultureInfo.InvariantCulture, "General:InstanceLimit={0}", 11),
string.Format(CultureInfo.InvariantCulture, "General:UserLimit={0}", 150),
string.Format(CultureInfo.InvariantCulture, "General:UserGroupLimit={0}", 47),
string.Format(CultureInfo.InvariantCulture, "General:HostApiDocumentation={0}", DumpOpenApiSpecpath),
string.Format(CultureInfo.InvariantCulture, "FileLogging:Directory={0}", Path.Combine(Directory, "Logs")),
string.Format(CultureInfo.InvariantCulture, "FileLogging:LogLevel={0}", "Trace"),
string.Format(CultureInfo.InvariantCulture, "General:ValidInstancePaths:0={0}", Directory),
"General:ByondTopicTimeout=3000"
String.Format(CultureInfo.InvariantCulture, "Database:DropDatabase={0}", true), // Replaced after first Run
String.Format(CultureInfo.InvariantCulture, "General:ConfigVersion={0}", GeneralConfiguration.CurrentConfigVersion),
String.Format(CultureInfo.InvariantCulture, "General:ApiPort={0}", port),
String.Format(CultureInfo.InvariantCulture, "Database:DatabaseType={0}", DatabaseType),
String.Format(CultureInfo.InvariantCulture, "Database:ConnectionString={0}", connectionString),
String.Format(CultureInfo.InvariantCulture, "General:SetupWizardMode={0}", SetupWizardMode.Never),
String.Format(CultureInfo.InvariantCulture, "General:MinimumPasswordLength={0}", 10),
String.Format(CultureInfo.InvariantCulture, "General:InstanceLimit={0}", 11),
String.Format(CultureInfo.InvariantCulture, "General:UserLimit={0}", 150),
String.Format(CultureInfo.InvariantCulture, "General:UserGroupLimit={0}", 47),
String.Format(CultureInfo.InvariantCulture, "General:HostApiDocumentation={0}", DumpOpenApiSpecpath),
String.Format(CultureInfo.InvariantCulture, "FileLogging:Directory={0}", Path.Combine(Directory, "Logs")),
String.Format(CultureInfo.InvariantCulture, "FileLogging:LogLevel={0}", "Trace"),
String.Format(CultureInfo.InvariantCulture, "General:ValidInstancePaths:0={0}", Directory),
"General:ByondTopicTimeout=3000",
$"Session:HighPriorityLiveDreamDaemon={HighPriorityDreamDaemon}",
$"Session:LowPriorityDeploymentProcesses={LowPriorityDeployments}",
};
if (dumpOnMissingUpdate)
tests/Tgstation.Server.Tests/Live/TestLiveServer.cs
+3 -3
View File
@@ -864,7 +864,7 @@ namespace Tgstation.Server.Tests.Live
Assert.IsTrue(Directory.Exists(instanceClient.Metadata.Path));
var instanceTests = FailFast(new InstanceTest(instanceClient, adminClient.Instances, GetInstanceManager(), (ushort)server.Url.Port).RunTests(cancellationToken));
var instanceTests = FailFast(new InstanceTest(instanceClient, adminClient.Instances, GetInstanceManager(), (ushort)server.Url.Port).RunTests(cancellationToken, server.HighPriorityDreamDaemon, server.LowPriorityDeployments));
await Task.WhenAll(rootTest, adminTest, instancesTest, instanceTests, usersTest);
@@ -1020,7 +1020,7 @@ namespace Tgstation.Server.Tests.Live
Assert.AreEqual(WatchdogStatus.Online, dd.Status.Value);
var compileJob = await instanceClient.DreamMaker.Compile(cancellationToken);
var wdt = new WatchdogTest(instanceClient, GetInstanceManager(), (ushort)server.Url.Port);
var wdt = new WatchdogTest(instanceClient, GetInstanceManager(), (ushort)server.Url.Port, server.HighPriorityDreamDaemon);
await wdt.WaitForJob(compileJob, 30, false, null, cancellationToken);
dd = await instanceClient.DreamDaemon.Read(cancellationToken);
@@ -1067,7 +1067,7 @@ namespace Tgstation.Server.Tests.Live
Assert.AreEqual(WatchdogStatus.Online, currentDD.Status);
Assert.AreEqual(expectedStaged, currentDD.StagedCompileJob.Job.Id.Value);
var wdt = new WatchdogTest(instanceClient, GetInstanceManager(), (ushort)server.Url.Port);
var wdt = new WatchdogTest(instanceClient, GetInstanceManager(), (ushort)server.Url.Port, server.HighPriorityDreamDaemon);
currentDD = await wdt.TellWorldToReboot(cancellationToken);
Assert.AreEqual(expectedStaged, currentDD.ActiveCompileJob.Job.Id.Value);
Assert.IsNull(currentDD.StagedCompileJob);