mirror of
https://github.com/tgstation/tgstation-server.git
synced 2026-07-19 03:53:02 +01:00
Improve live test runtimes
Workstation improvement of 10.2 to 6.4 minutes.
This commit is contained in:
Jordan Dominion
@@ -6,9 +6,11 @@
|
||||
log << "Initial value of sleep_offline: [sleep_offline]"
|
||||
sleep_offline = FALSE
|
||||
|
||||
// Intentionally slow down startup for testing purposes
|
||||
for(var/i in 1 to 10000000)
|
||||
dab()
|
||||
if(params["slow_start"])
|
||||
// Intentionally slow down startup for health check testing purposes
|
||||
for(var/i in 1 to 10000000)
|
||||
dab()
|
||||
|
||||
TgsNew(new /datum/tgs_event_handler/impl, TGS_SECURITY_SAFE)
|
||||
|
||||
var/sec = TgsSecurityLevel()
|
||||
@@ -189,7 +191,7 @@ var/run_bridge_test
|
||||
var/its_sad = data["im_out_of_memes"]
|
||||
if(its_sad)
|
||||
TestLegacyBridge()
|
||||
return "yeah gimmie a sec"
|
||||
return "all gucci"
|
||||
|
||||
TgsChatBroadcast(new /datum/tgs_message_content("Received non-tgs topic: `[T]`"))
|
||||
|
||||
@@ -252,9 +254,9 @@ var/received_health_check = FALSE
|
||||
|
||||
/proc/RebootAsync()
|
||||
set waitfor = FALSE
|
||||
world.TgsChatBroadcast(new /datum/tgs_message_content("Rebooting after 3 seconds"));
|
||||
world.TgsChatBroadcast(new /datum/tgs_message_content("Rebooting after 1 seconds"));
|
||||
world.log << "About to sleep. sleep_offline: [world.sleep_offline]"
|
||||
sleep(30)
|
||||
sleep(10)
|
||||
world.log << "Done sleep, calling Reboot"
|
||||
world.Reboot()
|
||||
|
||||
@@ -362,10 +364,6 @@ var/suppress_bridge_spam = FALSE
|
||||
api.access_identifier = old_ai
|
||||
|
||||
/proc/TestLegacyBridge()
|
||||
set waitfor = FALSE
|
||||
|
||||
sleep(10)
|
||||
|
||||
var/datum/tgs_api/v5/api = TGS_READ_GLOBAL(tgs)
|
||||
if(api.interop_version.suite != 5)
|
||||
FailTest("Legacy bridge test not required anymore?")
|
||||
|
||||
@@ -3,6 +3,7 @@ using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using Microsoft.VisualStudio.TestTools.UnitTesting;
|
||||
|
||||
using Tgstation.Server.Api.Models;
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using Microsoft.VisualStudio.TestTools.UnitTesting;
|
||||
|
||||
using Tgstation.Server.Api.Models;
|
||||
|
||||
@@ -45,11 +45,11 @@ namespace Tgstation.Server.Tests.Live.Instance
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var testVersion = await EngineTest.GetEdgeVersion(EngineType.Byond, fileDownloader, cancellationToken);
|
||||
var engineTest = new EngineTest(instanceClient.Engine, instanceClient.Jobs, fileDownloader, instanceClient.Metadata, testVersion.Engine.Value);
|
||||
var chatTest = new ChatTest(instanceClient.ChatBots, instanceManagerClient, instanceClient.Jobs, instanceClient.Metadata);
|
||||
await using var engineTest = new EngineTest(instanceClient.Engine, instanceClient.Jobs, fileDownloader, instanceClient.Metadata, testVersion.Engine.Value);
|
||||
await using var chatTest = new ChatTest(instanceClient.ChatBots, instanceManagerClient, instanceClient.Jobs, instanceClient.Metadata);
|
||||
var configTest = new ConfigurationTest(instanceClient.Configuration, instanceClient.Metadata);
|
||||
var repoTest = new RepositoryTest(instanceClient.Repository, instanceClient.Jobs);
|
||||
var dmTest = new DeploymentTest(instanceClient, instanceClient.Jobs, dmPort, ddPort, lowPrioDeployment, testVersion.Engine.Value);
|
||||
await using var repoTest = new RepositoryTest(instanceClient.Repository, instanceClient.Jobs);
|
||||
await using var dmTest = new DeploymentTest(instanceClient, instanceClient.Jobs, dmPort, ddPort, lowPrioDeployment, testVersion.Engine.Value);
|
||||
|
||||
var byondTask = engineTest.Run(cancellationToken, out var firstInstall);
|
||||
var chatTask = chatTest.RunPreWatchdog(cancellationToken);
|
||||
@@ -66,15 +66,15 @@ namespace Tgstation.Server.Tests.Live.Instance
|
||||
await configTest.SetupDMApiTests(true, cancellationToken);
|
||||
await byondTask;
|
||||
|
||||
await new WatchdogTest(
|
||||
await using var wdt = new WatchdogTest(
|
||||
testVersion,
|
||||
instanceClient,
|
||||
instanceManager,
|
||||
serverPort,
|
||||
highPrioDD,
|
||||
ddPort,
|
||||
usingBasicWatchdog)
|
||||
.Run(cancellationToken);
|
||||
usingBasicWatchdog);
|
||||
await wdt.Run(cancellationToken);
|
||||
}
|
||||
|
||||
public static async ValueTask<IEngineInstallationData> DownloadEngineVersion(
|
||||
@@ -197,7 +197,7 @@ namespace Tgstation.Server.Tests.Live.Instance
|
||||
ReconnectionInterval = 1,
|
||||
}, cancellationToken);
|
||||
|
||||
var jrt = new JobsRequiredTest(instanceClient.Jobs);
|
||||
await using var jrt = new JobsRequiredTest(instanceClient.Jobs);
|
||||
|
||||
EngineInstallResponse installJob2;
|
||||
await using (var stableBytesMs = await TestingUtils.ExtractMemoryStreamFromInstallationData(
|
||||
@@ -257,7 +257,8 @@ namespace Tgstation.Server.Tests.Live.Instance
|
||||
|
||||
await configSetupTask;
|
||||
|
||||
await new WatchdogTest(compatVersion, instanceClient, instanceManager, serverPort, highPrioDD, ddPort, usingBasicWatchdog).Run(cancellationToken);
|
||||
await using var wdt = new WatchdogTest(compatVersion, instanceClient, instanceManager, serverPort, highPrioDD, ddPort, usingBasicWatchdog);
|
||||
await wdt.Run(cancellationToken);
|
||||
|
||||
await instanceManagerClient.Update(new InstanceUpdateRequest
|
||||
{
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Reflection;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
@@ -13,16 +14,65 @@ using Tgstation.Server.Client.Components;
|
||||
|
||||
namespace Tgstation.Server.Tests.Live.Instance
|
||||
{
|
||||
class JobsRequiredTest
|
||||
class JobsRequiredTest : IAsyncDisposable
|
||||
{
|
||||
protected IJobsClient JobsClient { get; }
|
||||
|
||||
readonly IApiClient apiClient;
|
||||
|
||||
IAsyncDisposable hubConnection;
|
||||
readonly Task hubConnectionTask;
|
||||
readonly CancellationTokenSource cancellationTokenSource;
|
||||
|
||||
readonly ConcurrentDictionary<long, TaskCompletionSource<JobResponse>> registry;
|
||||
|
||||
public JobsRequiredTest(IJobsClient jobsClient)
|
||||
{
|
||||
JobsClient = jobsClient ?? throw new ArgumentNullException(nameof(jobsClient));
|
||||
apiClient = (IApiClient)jobsClient.GetType().GetProperty("ApiClient", BindingFlags.NonPublic | BindingFlags.Instance).GetValue(jobsClient);
|
||||
registry = new ConcurrentDictionary<long, TaskCompletionSource<JobResponse>>();
|
||||
|
||||
cancellationTokenSource = new CancellationTokenSource();
|
||||
hubConnectionTask = CreateHubConnection();
|
||||
}
|
||||
|
||||
async Task CreateHubConnection()
|
||||
{
|
||||
var receiver = new JobReceiver
|
||||
{
|
||||
Callback = job => Register(job),
|
||||
};
|
||||
|
||||
hubConnection = await apiClient.CreateHubConnection<IJobsHub>(receiver, null, null, cancellationTokenSource.Token);
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
cancellationTokenSource.Cancel();
|
||||
cancellationTokenSource.Dispose();
|
||||
await hubConnectionTask;
|
||||
if (hubConnection != null)
|
||||
await hubConnection.DisposeAsync();
|
||||
}
|
||||
|
||||
Task<JobResponse> Register(JobResponse updatedJob)
|
||||
{
|
||||
var tcs = registry.AddOrUpdate(updatedJob.Id.Value,
|
||||
_ =>
|
||||
{
|
||||
var tcs = new TaskCompletionSource<JobResponse>();
|
||||
if (updatedJob.StoppedAt.HasValue)
|
||||
tcs.SetResult(updatedJob);
|
||||
return tcs;
|
||||
},
|
||||
(_, oldTcs) =>
|
||||
{
|
||||
if (updatedJob.StoppedAt.HasValue)
|
||||
oldTcs.TrySetResult(updatedJob);
|
||||
return oldTcs;
|
||||
});
|
||||
|
||||
return tcs.Task;
|
||||
}
|
||||
|
||||
class JobReceiver : IJobsHub
|
||||
@@ -42,47 +92,21 @@ namespace Tgstation.Server.Tests.Live.Instance
|
||||
Assert.IsNotNull(originalJob.JobCode);
|
||||
|
||||
var job = originalJob;
|
||||
var registryTask = Register(job);
|
||||
await Task.WhenAny(
|
||||
registryTask,
|
||||
Task.Delay(TimeSpan.FromSeconds(timeout), cancellationToken));
|
||||
|
||||
if (!registryTask.IsCompleted)
|
||||
// one last get in case SignalR dropped the ball
|
||||
job = await JobsClient.GetId(job, cancellationToken);
|
||||
else
|
||||
job = await registryTask;
|
||||
|
||||
if (!job.StoppedAt.HasValue)
|
||||
{
|
||||
var tcs = new TaskCompletionSource();
|
||||
var receiver = new JobReceiver
|
||||
{
|
||||
Callback = updatedJob =>
|
||||
{
|
||||
if (updatedJob.Id != job.Id)
|
||||
return;
|
||||
|
||||
job = updatedJob;
|
||||
if (updatedJob.StoppedAt.HasValue)
|
||||
tcs.TrySetResult();
|
||||
},
|
||||
};
|
||||
|
||||
JobResponse firstCheck;
|
||||
await using (var hubConnection = await apiClient.CreateHubConnection<IJobsHub>(receiver, null, null, cancellationToken))
|
||||
{
|
||||
// initial GET after connecting
|
||||
firstCheck = await JobsClient.GetId(job, cancellationToken);
|
||||
if (!firstCheck.StoppedAt.HasValue)
|
||||
{
|
||||
firstCheck = null;
|
||||
await Task.WhenAny(
|
||||
tcs.Task,
|
||||
Task.Delay(TimeSpan.FromSeconds(timeout), cancellationToken));
|
||||
}
|
||||
}
|
||||
|
||||
if (firstCheck != null)
|
||||
job = firstCheck;
|
||||
else if (!job.StoppedAt.HasValue)
|
||||
// one last get in case SignalR dropped the ball
|
||||
job = await JobsClient.GetId(job, cancellationToken);
|
||||
|
||||
if (!job.StoppedAt.HasValue)
|
||||
{
|
||||
await JobsClient.Cancel(job, cancellationToken);
|
||||
Assert.Fail($"Job ID {job.Id} \"{job.Description}\" timed out!");
|
||||
}
|
||||
await JobsClient.Cancel(job, cancellationToken);
|
||||
Assert.Fail($"Job ID {job.Id} \"{job.Description}\" timed out!");
|
||||
}
|
||||
|
||||
if (expectFailure.HasValue && expectFailure.Value ^ job.ExceptionDetails != null)
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
using Byond.TopicSender;
|
||||
|
||||
using Microsoft.AspNetCore.DataProtection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.VisualStudio.TestTools.UnitTesting;
|
||||
|
||||
@@ -68,6 +67,7 @@ namespace Tgstation.Server.Tests.Live.Instance
|
||||
readonly bool watchdogRestartsProcess;
|
||||
|
||||
bool ranTimeoutTest = false;
|
||||
const string BaseAdditionalParameters = "expect_chat_channels=1&expect_static_files=1";
|
||||
|
||||
public WatchdogTest(EngineVersion testVersion, IInstanceClient instanceClient, InstanceManager instanceManager, ushort serverPort, bool highPrioDD, ushort ddPort, bool watchdogRestartsProcess)
|
||||
: base(instanceClient.Jobs)
|
||||
@@ -88,8 +88,20 @@ namespace Tgstation.Server.Tests.Live.Instance
|
||||
DisconnectTimeout = TimeSpan.FromSeconds(30)
|
||||
}, loggerFactory.CreateLogger($"WatchdogTest.TopicClient.{instanceClient.Metadata.Name}"));
|
||||
}
|
||||
|
||||
public async Task Run(CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await RunInt(cancellationToken);
|
||||
}
|
||||
catch
|
||||
{
|
||||
System.Console.WriteLine($"WATCHDOG TEST FAILING INSTANCE ID {instanceClient.Metadata.Id.Value}");
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
async Task RunInt(CancellationToken cancellationToken)
|
||||
{
|
||||
System.Console.WriteLine($"TEST: START WATCHDOG TESTS {instanceClient.Metadata.Name}");
|
||||
|
||||
@@ -125,7 +137,7 @@ namespace Tgstation.Server.Tests.Live.Instance
|
||||
Port = ddPort,
|
||||
MapThreads = 2,
|
||||
LogOutput = false,
|
||||
AdditionalParameters = "expect_chat_channels=1&expect_static_files=1"
|
||||
AdditionalParameters = BaseAdditionalParameters
|
||||
}, cancellationToken).AsTask(),
|
||||
CheckByondVersions(),
|
||||
ApiAssert.ThrowsException<ApiConflictException, DreamDaemonResponse>(() => instanceClient.DreamDaemon.Update(new DreamDaemonRequest
|
||||
@@ -176,7 +188,6 @@ namespace Tgstation.Server.Tests.Live.Instance
|
||||
var ddUpdateTask = instanceClient.DreamDaemon.Update(new DreamDaemonRequest
|
||||
{
|
||||
SecurityLevel = useTrusted ? DreamDaemonSecurity.Trusted : DreamDaemonSecurity.Safe,
|
||||
AdditionalParameters = "expect_chat_channels=1&expect_static_files=1",
|
||||
}, cancellationToken);
|
||||
var currentStatus = await DeployTestDme("long_running_test_rooted", DreamDaemonSecurity.Trusted, true, cancellationToken);
|
||||
await ddUpdateTask;
|
||||
@@ -294,24 +305,25 @@ namespace Tgstation.Server.Tests.Live.Instance
|
||||
|
||||
async ValueTask RegressionTest1550(CancellationToken cancellationToken)
|
||||
{
|
||||
// Previous test, StartAndLeaveRunning, has SoftRestart set. We don't want that.
|
||||
var restartJob = await instanceClient.DreamDaemon.Restart(cancellationToken);
|
||||
await WaitForJob(restartJob, 10, false, null, cancellationToken);
|
||||
|
||||
// we need to cycle deployments twice because TGS holds the initial deployment
|
||||
var currentStatus = await DeployTestDme("LongRunning/long_running_test", DreamDaemonSecurity.Trusted, true, cancellationToken);
|
||||
|
||||
Assert.AreEqual(WatchdogStatus.Online, currentStatus.Status);
|
||||
Assert.IsNotNull(currentStatus.StagedCompileJob);
|
||||
ValidateSessionId(currentStatus, false);
|
||||
ValidateSessionId(currentStatus, true);
|
||||
var expectedStaged = currentStatus.StagedCompileJob;
|
||||
Assert.AreNotEqual(expectedStaged.Id, currentStatus.ActiveCompileJob.Id);
|
||||
|
||||
Assert.AreEqual(watchdogRestartsProcess, currentStatus.SoftRestart);
|
||||
Assert.IsFalse(currentStatus.SoftShutdown.Value);
|
||||
Assert.IsFalse(currentStatus.SoftShutdown);
|
||||
|
||||
currentStatus = await TellWorldToReboot(true, cancellationToken);
|
||||
ValidateSessionId(currentStatus, watchdogRestartsProcess);
|
||||
ValidateSessionId(currentStatus, true);
|
||||
Assert.AreEqual(expectedStaged.Id, currentStatus.ActiveCompileJob.Id);
|
||||
|
||||
await Task.Delay(TimeSpan.FromSeconds(3), cancellationToken);
|
||||
|
||||
var topicRequestResult = await topicClient.SendTopic(
|
||||
IPAddress.Loopback,
|
||||
$"shadow_wizard_money_gang=1",
|
||||
@@ -322,6 +334,7 @@ namespace Tgstation.Server.Tests.Live.Instance
|
||||
Assert.AreEqual("we love casting spells", topicRequestResult.StringData);
|
||||
|
||||
currentStatus = await DeployTestDme("LongRunning/long_running_test", DreamDaemonSecurity.Trusted, true, cancellationToken);
|
||||
Assert.AreEqual(watchdogRestartsProcess, currentStatus.SoftRestart);
|
||||
ValidateSessionId(currentStatus, false);
|
||||
|
||||
Assert.AreEqual(WatchdogStatus.Online, currentStatus.Status);
|
||||
@@ -511,11 +524,8 @@ namespace Tgstation.Server.Tests.Live.Instance
|
||||
|
||||
Assert.IsTrue(job.ErrorCode == ErrorCode.GameServerOffline || job.ErrorCode == ErrorCode.GCoreFailure, $"{job.ErrorCode}: {job.ExceptionDetails}");
|
||||
|
||||
await Task.Delay(TimeSpan.FromSeconds(20), cancellationToken);
|
||||
|
||||
var ddStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
|
||||
ValidateSessionId(ddStatus, true);
|
||||
Assert.AreEqual(WatchdogStatus.Online, ddStatus.Status.Value);
|
||||
var restartJob2 = await instanceClient.DreamDaemon.Restart(cancellationToken);
|
||||
await WaitForJob(restartJob2, 20, false, null, cancellationToken);
|
||||
}
|
||||
|
||||
async Task TestDMApiFreeDeploy(CancellationToken cancellationToken)
|
||||
@@ -612,7 +622,7 @@ namespace Tgstation.Server.Tests.Live.Instance
|
||||
Assert.AreEqual(false, daemonStatus.SoftRestart);
|
||||
Assert.AreEqual(false, daemonStatus.SoftShutdown);
|
||||
|
||||
await GracefulWatchdogShutdown(60, cancellationToken);
|
||||
await GracefulWatchdogShutdown(cancellationToken);
|
||||
|
||||
daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
|
||||
Assert.AreEqual(WatchdogStatus.Offline, daemonStatus.Status.Value);
|
||||
@@ -691,20 +701,15 @@ namespace Tgstation.Server.Tests.Live.Instance
|
||||
{
|
||||
System.Console.WriteLine("TEST: WATCHDOG HEALTH CHECK TEST");
|
||||
|
||||
// Check reverse mapping
|
||||
var status = await instanceClient.DreamDaemon.Update(new DreamDaemonRequest
|
||||
{
|
||||
DumpOnHealthCheckRestart = !checkDump,
|
||||
}, cancellationToken);
|
||||
|
||||
// enable health checks
|
||||
status = await instanceClient.DreamDaemon.Update(new DreamDaemonRequest
|
||||
var status = await instanceClient.DreamDaemon.Update(new DreamDaemonRequest
|
||||
{
|
||||
HealthCheckSeconds = 1,
|
||||
DumpOnHealthCheckRestart = checkDump,
|
||||
}, cancellationToken);
|
||||
|
||||
Assert.AreEqual(checkDump, status.DumpOnHealthCheckRestart);
|
||||
Assert.AreEqual(1U, status.HealthCheckSeconds.Value);
|
||||
|
||||
var startJob = await StartDD(cancellationToken);
|
||||
|
||||
@@ -731,7 +736,7 @@ namespace Tgstation.Server.Tests.Live.Instance
|
||||
.GetProcess(ddProc.Id);
|
||||
|
||||
// Ensure it's responding to health checks
|
||||
await Task.WhenAny(Task.Delay(20000, cancellationToken), ourProcessHandler.Lifetime);
|
||||
await Task.WhenAny(Task.Delay(6000, cancellationToken), ourProcessHandler.Lifetime);
|
||||
Assert.IsFalse(ddProc.HasExited);
|
||||
|
||||
// check DD agrees
|
||||
@@ -744,7 +749,7 @@ namespace Tgstation.Server.Tests.Live.Instance
|
||||
Assert.IsNotNull(topicRequestResult);
|
||||
Assert.AreEqual(TopicResponseType.StringResponse, topicRequestResult.ResponseType);
|
||||
Assert.IsNotNull(topicRequestResult.StringData);
|
||||
Assert.AreEqual(topicRequestResult.StringData, "received health check");
|
||||
Assert.AreEqual("received health check", topicRequestResult.StringData);
|
||||
|
||||
var ddStatus = await instanceClient.DreamDaemon.Update(new DreamDaemonRequest
|
||||
{
|
||||
@@ -755,11 +760,11 @@ namespace Tgstation.Server.Tests.Live.Instance
|
||||
ourProcessHandler.Suspend();
|
||||
|
||||
await Task.WhenAny(ourProcessHandler.Lifetime, Task.Delay(TimeSpan.FromMinutes(1), cancellationToken));
|
||||
Assert.IsTrue(ourProcessHandler.Lifetime.IsCompleted);
|
||||
|
||||
var timeout = 60;
|
||||
var timeout = 20;
|
||||
do
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken);
|
||||
ddStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
|
||||
Assert.AreEqual(1U, ddStatus.HealthCheckSeconds.Value);
|
||||
if (ddStatus.Status.Value == WatchdogStatus.Offline)
|
||||
@@ -770,6 +775,8 @@ namespace Tgstation.Server.Tests.Live.Instance
|
||||
|
||||
if (--timeout == 0)
|
||||
Assert.Fail("DreamDaemon didn't shutdown within the timeout!");
|
||||
|
||||
await Task.Delay(TimeSpan.FromSeconds(3), cancellationToken);
|
||||
}
|
||||
while (timeout > 0);
|
||||
|
||||
@@ -816,7 +823,6 @@ namespace Tgstation.Server.Tests.Live.Instance
|
||||
throw;
|
||||
}
|
||||
}
|
||||
await Task.Delay(TimeSpan.FromSeconds(3), cts.Token);
|
||||
|
||||
return await instanceClient.DreamDaemon.Start(cancellationToken);
|
||||
}
|
||||
@@ -855,7 +861,7 @@ namespace Tgstation.Server.Tests.Live.Instance
|
||||
BridgeController.LogContent = true;
|
||||
|
||||
// Time for DD to revert the bridge access identifier change
|
||||
await Task.Delay(TimeSpan.FromSeconds(3), cancellationToken);
|
||||
await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken);
|
||||
}
|
||||
|
||||
async Task ValidateTopicLimits(CancellationToken cancellationToken)
|
||||
@@ -1249,15 +1255,18 @@ namespace Tgstation.Server.Tests.Live.Instance
|
||||
public async Task StartAndLeaveRunning(CancellationToken cancellationToken)
|
||||
{
|
||||
System.Console.WriteLine("TEST: WATCHDOG STARTING ENDLESS");
|
||||
await DeployTestDme("LongRunning/long_running_test", DreamDaemonSecurity.Trusted, true, cancellationToken);
|
||||
|
||||
var startJob = await StartDD(cancellationToken);
|
||||
|
||||
await WaitForJob(startJob, 40, false, null, cancellationToken);
|
||||
|
||||
var daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
|
||||
var daemonStatus = await instanceClient.DreamDaemon.Update(new DreamDaemonRequest
|
||||
{
|
||||
AdditionalParameters = "slow_start=1",
|
||||
},
|
||||
cancellationToken);
|
||||
|
||||
Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value);
|
||||
Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status);
|
||||
Assert.IsTrue(daemonStatus.SoftRestart);
|
||||
CheckDDPriority();
|
||||
Assert.AreEqual(ddPort, daemonStatus.CurrentPort);
|
||||
|
||||
@@ -1265,6 +1274,9 @@ namespace Tgstation.Server.Tests.Live.Instance
|
||||
bool firstTime = true;
|
||||
do
|
||||
{
|
||||
if(!firstTime)
|
||||
Assert.IsFalse(daemonStatus.SoftRestart);
|
||||
|
||||
ValidateSessionId(daemonStatus, true);
|
||||
KillDD(firstTime);
|
||||
firstTime = false;
|
||||
@@ -1272,7 +1284,7 @@ namespace Tgstation.Server.Tests.Live.Instance
|
||||
daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
|
||||
}
|
||||
while (daemonStatus.Status == WatchdogStatus.Online);
|
||||
Assert.AreEqual(WatchdogStatus.Restoring, daemonStatus.Status.Value);
|
||||
Assert.AreEqual(WatchdogStatus.Restoring, daemonStatus.Status);
|
||||
|
||||
// Kill it again
|
||||
do
|
||||
@@ -1281,13 +1293,18 @@ namespace Tgstation.Server.Tests.Live.Instance
|
||||
daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
|
||||
}
|
||||
while (daemonStatus.Status == WatchdogStatus.Online || daemonStatus.Status == WatchdogStatus.Restoring);
|
||||
Assert.AreEqual(WatchdogStatus.DelayedRestart, daemonStatus.Status.Value);
|
||||
Assert.AreEqual(WatchdogStatus.DelayedRestart, daemonStatus.Status);
|
||||
|
||||
await Task.Delay(TimeSpan.FromSeconds(15), cancellationToken);
|
||||
|
||||
daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
|
||||
daemonStatus = await instanceClient.DreamDaemon.Update(new DreamDaemonRequest
|
||||
{
|
||||
AdditionalParameters = String.Empty,
|
||||
},
|
||||
cancellationToken);
|
||||
ValidateSessionId(daemonStatus, true);
|
||||
Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value);
|
||||
Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status);
|
||||
Assert.IsTrue(daemonStatus.SoftRestart);
|
||||
await CheckDMApiFail(daemonStatus.ActiveCompileJob, cancellationToken);
|
||||
}
|
||||
|
||||
@@ -1305,12 +1322,12 @@ namespace Tgstation.Server.Tests.Live.Instance
|
||||
}
|
||||
|
||||
public Task<DreamDaemonResponse> TellWorldToReboot(bool waitForOnlineIfRestoring, CancellationToken cancellationToken)
|
||||
=> TellWorldToReboot2(instanceClient, topicClient, FindTopicPort(), waitForOnlineIfRestoring ? true : testVersion.Engine.Value == EngineType.OpenDream, cancellationToken);
|
||||
=> TellWorldToReboot2(instanceClient, topicClient, FindTopicPort(), waitForOnlineIfRestoring || testVersion.Engine.Value == EngineType.OpenDream, cancellationToken);
|
||||
public static async Task<DreamDaemonResponse> TellWorldToReboot2(IInstanceClient instanceClient, ITopicClient topicClient, ushort topicPort, bool waitForOnlineIfRestoring, CancellationToken cancellationToken)
|
||||
{
|
||||
var daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
|
||||
Assert.IsNotNull(daemonStatus.StagedCompileJob);
|
||||
var initialCompileJob = daemonStatus.ActiveCompileJob;
|
||||
var initialSession = daemonStatus.ActiveCompileJob;
|
||||
|
||||
System.Console.WriteLine("TEST: Sending world reboot topic...");
|
||||
|
||||
@@ -1328,11 +1345,9 @@ namespace Tgstation.Server.Tests.Live.Instance
|
||||
await Task.Delay(TimeSpan.FromSeconds(1), tempToken);
|
||||
daemonStatus = await instanceClient.DreamDaemon.Read(tempToken);
|
||||
}
|
||||
while (initialCompileJob.Id == daemonStatus.ActiveCompileJob.Id);
|
||||
while (initialSession.Id == daemonStatus.ActiveCompileJob.Id);
|
||||
}
|
||||
|
||||
await Task.Delay(TimeSpan.FromSeconds(3), cancellationToken);
|
||||
|
||||
if (waitForOnlineIfRestoring && daemonStatus.Status == WatchdogStatus.Restoring)
|
||||
{
|
||||
do
|
||||
@@ -1353,7 +1368,7 @@ namespace Tgstation.Server.Tests.Live.Instance
|
||||
ApiValidationSecurityLevel = deploymentSecurity,
|
||||
ProjectName = dmeName.Contains("rooted") ? dmeName : $"tests/DMAPI/{dmeName}",
|
||||
RequireDMApiValidation = requireApi,
|
||||
Timeout = TimeSpan.FromMilliseconds(1),
|
||||
Timeout = !ranTimeoutTest ? TimeSpan.FromMilliseconds(1) : TimeSpan.FromMinutes(5),
|
||||
}, cancellationToken);
|
||||
|
||||
JobResponse compileJobJob;
|
||||
@@ -1366,29 +1381,33 @@ namespace Tgstation.Server.Tests.Live.Instance
|
||||
compileJobJob = await instanceClient.DreamMaker.Compile(cancellationToken);
|
||||
|
||||
await WaitForJob(compileJobJob, 90, true, ErrorCode.DeploymentTimeout, cancellationToken);
|
||||
|
||||
await instanceClient.DreamMaker.Update(new DreamMakerRequest
|
||||
{
|
||||
Timeout = TimeSpan.FromMinutes(5),
|
||||
}, cancellationToken);
|
||||
ranTimeoutTest = true;
|
||||
}
|
||||
|
||||
await instanceClient.DreamMaker.Update(new DreamMakerRequest
|
||||
{
|
||||
Timeout = TimeSpan.FromMinutes(5),
|
||||
}, cancellationToken);
|
||||
|
||||
compileJobJob = await instanceClient.DreamMaker.Compile(cancellationToken);
|
||||
await WaitForJob(compileJobJob, 90, false, null, cancellationToken);
|
||||
|
||||
// annoying but, with signalR instant job updates, this running task can get queued before the task that processes the watchdog's monitor activation
|
||||
for (var i = 0; i < 10; ++i)
|
||||
await Task.Yield();
|
||||
|
||||
var ddInfo = await instanceClient.DreamDaemon.Read(cancellationToken);
|
||||
var targetJob = ddInfo.StagedCompileJob ?? ddInfo.ActiveCompileJob;
|
||||
Assert.IsNotNull(targetJob);
|
||||
if (requireApi)
|
||||
{
|
||||
var targetJob = ddInfo.StagedCompileJob ?? ddInfo.ActiveCompileJob;
|
||||
Assert.IsNotNull(targetJob);
|
||||
Assert.IsNotNull(targetJob.DMApiVersion);
|
||||
}
|
||||
else
|
||||
Assert.IsNull(targetJob.DMApiVersion);
|
||||
|
||||
return ddInfo;
|
||||
}
|
||||
|
||||
async Task GracefulWatchdogShutdown(uint timeout, CancellationToken cancellationToken)
|
||||
async Task GracefulWatchdogShutdown(CancellationToken cancellationToken)
|
||||
{
|
||||
await instanceClient.DreamDaemon.Update(new DreamDaemonRequest
|
||||
{
|
||||
@@ -1398,9 +1417,10 @@ namespace Tgstation.Server.Tests.Live.Instance
|
||||
var newStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
|
||||
Assert.IsTrue(newStatus.SoftShutdown.Value || newStatus.Status.Value == WatchdogStatus.Offline);
|
||||
|
||||
var timeout = 20;
|
||||
do
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken);
|
||||
await Task.Delay(TimeSpan.FromSeconds(3), cancellationToken);
|
||||
var ddStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
|
||||
if (ddStatus.Status.Value == WatchdogStatus.Offline)
|
||||
break;
|
||||
@@ -1446,9 +1466,10 @@ namespace Tgstation.Server.Tests.Live.Instance
|
||||
|
||||
async ValueTask TestLegacyBridgeEndpoint(CancellationToken cancellationToken)
|
||||
{
|
||||
System.Console.WriteLine("TEST: TestLegacyBridgeEndpoint");
|
||||
var result = await topicClient.SendTopic(IPAddress.Loopback, "im_out_of_memes=1", FindTopicPort(), cancellationToken);
|
||||
Assert.AreEqual("yeah gimmie a sec", result.StringData);
|
||||
await Task.Delay(TimeSpan.FromSeconds(10), cancellationToken);
|
||||
Assert.IsNotNull(result);
|
||||
Assert.AreEqual("all gucci", result.StringData);
|
||||
await CheckDMApiFail((await instanceClient.DreamDaemon.Read(cancellationToken)).ActiveCompileJob, cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1049,7 +1049,7 @@ namespace Tgstation.Server.Tests.Live
|
||||
|
||||
var ioManager = new Host.IO.DefaultIOManager();
|
||||
var repoPath = ioManager.ConcatPath(instance.Path, "Repository");
|
||||
var jobsTest = new JobsRequiredTest(instanceClient.Jobs);
|
||||
await using var jobsTest = new JobsRequiredTest(instanceClient.Jobs);
|
||||
var postWriteHandler = (Host.IO.IPostWriteHandler)(new PlatformIdentifier().IsWindows
|
||||
? new Host.IO.WindowsPostWriteHandler()
|
||||
: new Host.IO.PosixPostWriteHandler(loggerFactory.CreateLogger<Host.IO.PosixPostWriteHandler>()));
|
||||
@@ -1590,7 +1590,7 @@ namespace Tgstation.Server.Tests.Live
|
||||
.ToList();
|
||||
}
|
||||
|
||||
var jrt = new JobsRequiredTest(instanceClient.Jobs);
|
||||
await using var jrt = new JobsRequiredTest(instanceClient.Jobs);
|
||||
foreach (var job in jobs)
|
||||
{
|
||||
Assert.IsTrue(job.StartedAt.Value >= preStartupTime);
|
||||
@@ -1660,11 +1660,11 @@ namespace Tgstation.Server.Tests.Live
|
||||
.ToList();
|
||||
}
|
||||
|
||||
var jrt = new JobsRequiredTest(instanceClient.Jobs);
|
||||
await using var jrt = new JobsRequiredTest(instanceClient.Jobs);
|
||||
foreach (var job in jobs)
|
||||
{
|
||||
Assert.IsTrue(job.StartedAt.Value >= preStartupTime);
|
||||
await jrt.WaitForJob(job, 140, job.Description.Contains("Reconnect chat bot") ? null : false, null, cancellationToken);
|
||||
await jrt.WaitForJob(job, 140, job.JobCode == JobCode.ReconnectChatBot ? null : false, null, cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1684,7 +1684,7 @@ namespace Tgstation.Server.Tests.Live
|
||||
Assert.AreEqual(WatchdogStatus.Online, dd.Status.Value);
|
||||
|
||||
var compileJob = await instanceClient.DreamMaker.Compile(cancellationToken);
|
||||
var wdt = new WatchdogTest(edgeVersion, instanceClient, GetInstanceManager(), (ushort)server.ApiUrl.Port, server.HighPriorityDreamDaemon, mainDDPort, server.UsingBasicWatchdog);
|
||||
await using var wdt = new WatchdogTest(edgeVersion, instanceClient, GetInstanceManager(), (ushort)server.ApiUrl.Port, server.HighPriorityDreamDaemon, mainDDPort, server.UsingBasicWatchdog);
|
||||
await wdt.WaitForJob(compileJob, 30, false, null, cancellationToken);
|
||||
|
||||
dd = await instanceClient.DreamDaemon.Read(cancellationToken);
|
||||
@@ -1727,13 +1727,15 @@ namespace Tgstation.Server.Tests.Live
|
||||
Assert.AreEqual(WatchdogStatus.Online, currentDD.Status);
|
||||
Assert.AreEqual(expectedStaged, currentDD.StagedCompileJob.Job.Id.Value);
|
||||
|
||||
var wdt = new WatchdogTest(edgeVersion, instanceClient, GetInstanceManager(), (ushort)server.ApiUrl.Port, server.HighPriorityDreamDaemon, mainDDPort, server.UsingBasicWatchdog);
|
||||
currentDD = await wdt.TellWorldToReboot(true, cancellationToken);
|
||||
await using var wdt = new WatchdogTest(edgeVersion, instanceClient, GetInstanceManager(), (ushort)server.ApiUrl.Port, server.HighPriorityDreamDaemon, mainDDPort, server.UsingBasicWatchdog);
|
||||
currentDD = await wdt.TellWorldToReboot(false, cancellationToken);
|
||||
Assert.AreEqual(expectedStaged, currentDD.ActiveCompileJob.Job.Id.Value);
|
||||
Assert.IsNull(currentDD.StagedCompileJob);
|
||||
|
||||
var repoTest = new RepositoryTest(instanceClient.Repository, instanceClient.Jobs).RunPostTest(cancellationToken);
|
||||
await new ChatTest(instanceClient.ChatBots, adminClient.Instances, instanceClient.Jobs, instance).RunPostTest(cancellationToken);
|
||||
await using var repoTestObj = new RepositoryTest(instanceClient.Repository, instanceClient.Jobs);
|
||||
var repoTest = repoTestObj.RunPostTest(cancellationToken);
|
||||
await using var chatTestObj = new ChatTest(instanceClient.ChatBots, adminClient.Instances, instanceClient.Jobs, instance);
|
||||
await chatTestObj.RunPostTest(cancellationToken);
|
||||
await repoTest;
|
||||
|
||||
await DummyChatProvider.RandomDisconnections(false, cancellationToken);
|
||||
|
||||
Reference in New Issue
Block a user