diff --git a/tests/DMAPI/LongRunning/Test.dm b/tests/DMAPI/LongRunning/Test.dm index 0bff755db9..90164c7e80 100644 --- a/tests/DMAPI/LongRunning/Test.dm +++ b/tests/DMAPI/LongRunning/Test.dm @@ -6,9 +6,11 @@ log << "Initial value of sleep_offline: [sleep_offline]" sleep_offline = FALSE - // Intentionally slow down startup for testing purposes - for(var/i in 1 to 10000000) - dab() + if(params["slow_start"]) + // Intentionally slow down startup for health check testing purposes + for(var/i in 1 to 10000000) + dab() + TgsNew(new /datum/tgs_event_handler/impl, TGS_SECURITY_SAFE) var/sec = TgsSecurityLevel() @@ -189,7 +191,7 @@ var/run_bridge_test var/its_sad = data["im_out_of_memes"] if(its_sad) TestLegacyBridge() - return "yeah gimmie a sec" + return "all gucci" TgsChatBroadcast(new /datum/tgs_message_content("Received non-tgs topic: `[T]`")) @@ -252,9 +254,9 @@ var/received_health_check = FALSE /proc/RebootAsync() set waitfor = FALSE - world.TgsChatBroadcast(new /datum/tgs_message_content("Rebooting after 3 seconds")); + world.TgsChatBroadcast(new /datum/tgs_message_content("Rebooting after 1 seconds")); world.log << "About to sleep. sleep_offline: [world.sleep_offline]" - sleep(30) + sleep(10) world.log << "Done sleep, calling Reboot" world.Reboot() @@ -362,10 +364,6 @@ var/suppress_bridge_spam = FALSE api.access_identifier = old_ai /proc/TestLegacyBridge() - set waitfor = FALSE - - sleep(10) - var/datum/tgs_api/v5/api = TGS_READ_GLOBAL(tgs) if(api.interop_version.suite != 5) FailTest("Legacy bridge test not required anymore?") diff --git a/tests/Tgstation.Server.Tests/Live/Instance/ChatTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/ChatTest.cs index 0963b72b4f..abb699815e 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/ChatTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/ChatTest.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; + using Microsoft.VisualStudio.TestTools.UnitTesting; using Tgstation.Server.Api.Models; diff --git a/tests/Tgstation.Server.Tests/Live/Instance/DeploymentTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/DeploymentTest.cs index 8b5fab58c9..d0d074e6e8 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/DeploymentTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/DeploymentTest.cs @@ -1,6 +1,7 @@ using System; using System.Threading; using System.Threading.Tasks; + using Microsoft.VisualStudio.TestTools.UnitTesting; using Tgstation.Server.Api.Models; diff --git a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs index 255a2e8a83..fc5177e355 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs @@ -45,11 +45,11 @@ namespace Tgstation.Server.Tests.Live.Instance CancellationToken cancellationToken) { var testVersion = await EngineTest.GetEdgeVersion(EngineType.Byond, fileDownloader, cancellationToken); - var engineTest = new EngineTest(instanceClient.Engine, instanceClient.Jobs, fileDownloader, instanceClient.Metadata, testVersion.Engine.Value); - var chatTest = new ChatTest(instanceClient.ChatBots, instanceManagerClient, instanceClient.Jobs, instanceClient.Metadata); + await using var engineTest = new EngineTest(instanceClient.Engine, instanceClient.Jobs, fileDownloader, instanceClient.Metadata, testVersion.Engine.Value); + await using var chatTest = new ChatTest(instanceClient.ChatBots, instanceManagerClient, instanceClient.Jobs, instanceClient.Metadata); var configTest = new ConfigurationTest(instanceClient.Configuration, instanceClient.Metadata); - var repoTest = new RepositoryTest(instanceClient.Repository, instanceClient.Jobs); - var dmTest = new DeploymentTest(instanceClient, instanceClient.Jobs, dmPort, ddPort, lowPrioDeployment, testVersion.Engine.Value); + await using var repoTest = new RepositoryTest(instanceClient.Repository, instanceClient.Jobs); + await using var dmTest = new DeploymentTest(instanceClient, instanceClient.Jobs, dmPort, ddPort, lowPrioDeployment, testVersion.Engine.Value); var byondTask = engineTest.Run(cancellationToken, out var firstInstall); var chatTask = chatTest.RunPreWatchdog(cancellationToken); @@ -66,15 +66,15 @@ namespace Tgstation.Server.Tests.Live.Instance await configTest.SetupDMApiTests(true, cancellationToken); await byondTask; - await new WatchdogTest( + await using var wdt = new WatchdogTest( testVersion, instanceClient, instanceManager, serverPort, highPrioDD, ddPort, - usingBasicWatchdog) - .Run(cancellationToken); + usingBasicWatchdog); + await wdt.Run(cancellationToken); } public static async ValueTask DownloadEngineVersion( @@ -197,7 +197,7 @@ namespace Tgstation.Server.Tests.Live.Instance ReconnectionInterval = 1, }, cancellationToken); - var jrt = new JobsRequiredTest(instanceClient.Jobs); + await using var jrt = new JobsRequiredTest(instanceClient.Jobs); EngineInstallResponse installJob2; await using (var stableBytesMs = await TestingUtils.ExtractMemoryStreamFromInstallationData( @@ -257,7 +257,8 @@ namespace Tgstation.Server.Tests.Live.Instance await configSetupTask; - await new WatchdogTest(compatVersion, instanceClient, instanceManager, serverPort, highPrioDD, ddPort, usingBasicWatchdog).Run(cancellationToken); + await using var wdt = new WatchdogTest(compatVersion, instanceClient, instanceManager, serverPort, highPrioDD, ddPort, usingBasicWatchdog); + await wdt.Run(cancellationToken); await instanceManagerClient.Update(new InstanceUpdateRequest { diff --git a/tests/Tgstation.Server.Tests/Live/Instance/JobsRequiredTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/JobsRequiredTest.cs index bfe8c98d82..e7c7cb610c 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/JobsRequiredTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/JobsRequiredTest.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Concurrent; using System.Reflection; using System.Threading; using System.Threading.Tasks; @@ -13,16 +14,65 @@ using Tgstation.Server.Client.Components; namespace Tgstation.Server.Tests.Live.Instance { - class JobsRequiredTest + class JobsRequiredTest : IAsyncDisposable { protected IJobsClient JobsClient { get; } readonly IApiClient apiClient; + IAsyncDisposable hubConnection; + readonly Task hubConnectionTask; + readonly CancellationTokenSource cancellationTokenSource; + + readonly ConcurrentDictionary> registry; + public JobsRequiredTest(IJobsClient jobsClient) { JobsClient = jobsClient ?? throw new ArgumentNullException(nameof(jobsClient)); apiClient = (IApiClient)jobsClient.GetType().GetProperty("ApiClient", BindingFlags.NonPublic | BindingFlags.Instance).GetValue(jobsClient); + registry = new ConcurrentDictionary>(); + + cancellationTokenSource = new CancellationTokenSource(); + hubConnectionTask = CreateHubConnection(); + } + + async Task CreateHubConnection() + { + var receiver = new JobReceiver + { + Callback = job => Register(job), + }; + + hubConnection = await apiClient.CreateHubConnection(receiver, null, null, cancellationTokenSource.Token); + } + + public async ValueTask DisposeAsync() + { + cancellationTokenSource.Cancel(); + cancellationTokenSource.Dispose(); + await hubConnectionTask; + if (hubConnection != null) + await hubConnection.DisposeAsync(); + } + + Task Register(JobResponse updatedJob) + { + var tcs = registry.AddOrUpdate(updatedJob.Id.Value, + _ => + { + var tcs = new TaskCompletionSource(); + if (updatedJob.StoppedAt.HasValue) + tcs.SetResult(updatedJob); + return tcs; + }, + (_, oldTcs) => + { + if (updatedJob.StoppedAt.HasValue) + oldTcs.TrySetResult(updatedJob); + return oldTcs; + }); + + return tcs.Task; } class JobReceiver : IJobsHub @@ -42,47 +92,21 @@ namespace Tgstation.Server.Tests.Live.Instance Assert.IsNotNull(originalJob.JobCode); var job = originalJob; + var registryTask = Register(job); + await Task.WhenAny( + registryTask, + Task.Delay(TimeSpan.FromSeconds(timeout), cancellationToken)); + + if (!registryTask.IsCompleted) + // one last get in case SignalR dropped the ball + job = await JobsClient.GetId(job, cancellationToken); + else + job = await registryTask; + if (!job.StoppedAt.HasValue) { - var tcs = new TaskCompletionSource(); - var receiver = new JobReceiver - { - Callback = updatedJob => - { - if (updatedJob.Id != job.Id) - return; - - job = updatedJob; - if (updatedJob.StoppedAt.HasValue) - tcs.TrySetResult(); - }, - }; - - JobResponse firstCheck; - await using (var hubConnection = await apiClient.CreateHubConnection(receiver, null, null, cancellationToken)) - { - // initial GET after connecting - firstCheck = await JobsClient.GetId(job, cancellationToken); - if (!firstCheck.StoppedAt.HasValue) - { - firstCheck = null; - await Task.WhenAny( - tcs.Task, - Task.Delay(TimeSpan.FromSeconds(timeout), cancellationToken)); - } - } - - if (firstCheck != null) - job = firstCheck; - else if (!job.StoppedAt.HasValue) - // one last get in case SignalR dropped the ball - job = await JobsClient.GetId(job, cancellationToken); - - if (!job.StoppedAt.HasValue) - { - await JobsClient.Cancel(job, cancellationToken); - Assert.Fail($"Job ID {job.Id} \"{job.Description}\" timed out!"); - } + await JobsClient.Cancel(job, cancellationToken); + Assert.Fail($"Job ID {job.Id} \"{job.Description}\" timed out!"); } if (expectFailure.HasValue && expectFailure.Value ^ job.ExceptionDetails != null) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index bcde5714ff..a5e6f12187 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -1,6 +1,5 @@ using Byond.TopicSender; -using Microsoft.AspNetCore.DataProtection; using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -68,6 +67,7 @@ namespace Tgstation.Server.Tests.Live.Instance readonly bool watchdogRestartsProcess; bool ranTimeoutTest = false; + const string BaseAdditionalParameters = "expect_chat_channels=1&expect_static_files=1"; public WatchdogTest(EngineVersion testVersion, IInstanceClient instanceClient, InstanceManager instanceManager, ushort serverPort, bool highPrioDD, ushort ddPort, bool watchdogRestartsProcess) : base(instanceClient.Jobs) @@ -88,8 +88,20 @@ namespace Tgstation.Server.Tests.Live.Instance DisconnectTimeout = TimeSpan.FromSeconds(30) }, loggerFactory.CreateLogger($"WatchdogTest.TopicClient.{instanceClient.Metadata.Name}")); } - public async Task Run(CancellationToken cancellationToken) + { + try + { + await RunInt(cancellationToken); + } + catch + { + System.Console.WriteLine($"WATCHDOG TEST FAILING INSTANCE ID {instanceClient.Metadata.Id.Value}"); + throw; + } + } + + async Task RunInt(CancellationToken cancellationToken) { System.Console.WriteLine($"TEST: START WATCHDOG TESTS {instanceClient.Metadata.Name}"); @@ -125,7 +137,7 @@ namespace Tgstation.Server.Tests.Live.Instance Port = ddPort, MapThreads = 2, LogOutput = false, - AdditionalParameters = "expect_chat_channels=1&expect_static_files=1" + AdditionalParameters = BaseAdditionalParameters }, cancellationToken).AsTask(), CheckByondVersions(), ApiAssert.ThrowsException(() => instanceClient.DreamDaemon.Update(new DreamDaemonRequest @@ -176,7 +188,6 @@ namespace Tgstation.Server.Tests.Live.Instance var ddUpdateTask = instanceClient.DreamDaemon.Update(new DreamDaemonRequest { SecurityLevel = useTrusted ? DreamDaemonSecurity.Trusted : DreamDaemonSecurity.Safe, - AdditionalParameters = "expect_chat_channels=1&expect_static_files=1", }, cancellationToken); var currentStatus = await DeployTestDme("long_running_test_rooted", DreamDaemonSecurity.Trusted, true, cancellationToken); await ddUpdateTask; @@ -294,24 +305,25 @@ namespace Tgstation.Server.Tests.Live.Instance async ValueTask RegressionTest1550(CancellationToken cancellationToken) { + // Previous test, StartAndLeaveRunning, has SoftRestart set. We don't want that. + var restartJob = await instanceClient.DreamDaemon.Restart(cancellationToken); + await WaitForJob(restartJob, 10, false, null, cancellationToken); + // we need to cycle deployments twice because TGS holds the initial deployment var currentStatus = await DeployTestDme("LongRunning/long_running_test", DreamDaemonSecurity.Trusted, true, cancellationToken); Assert.AreEqual(WatchdogStatus.Online, currentStatus.Status); Assert.IsNotNull(currentStatus.StagedCompileJob); - ValidateSessionId(currentStatus, false); + ValidateSessionId(currentStatus, true); var expectedStaged = currentStatus.StagedCompileJob; Assert.AreNotEqual(expectedStaged.Id, currentStatus.ActiveCompileJob.Id); - Assert.AreEqual(watchdogRestartsProcess, currentStatus.SoftRestart); - Assert.IsFalse(currentStatus.SoftShutdown.Value); + Assert.IsFalse(currentStatus.SoftShutdown); currentStatus = await TellWorldToReboot(true, cancellationToken); - ValidateSessionId(currentStatus, watchdogRestartsProcess); + ValidateSessionId(currentStatus, true); Assert.AreEqual(expectedStaged.Id, currentStatus.ActiveCompileJob.Id); - await Task.Delay(TimeSpan.FromSeconds(3), cancellationToken); - var topicRequestResult = await topicClient.SendTopic( IPAddress.Loopback, $"shadow_wizard_money_gang=1", @@ -322,6 +334,7 @@ namespace Tgstation.Server.Tests.Live.Instance Assert.AreEqual("we love casting spells", topicRequestResult.StringData); currentStatus = await DeployTestDme("LongRunning/long_running_test", DreamDaemonSecurity.Trusted, true, cancellationToken); + Assert.AreEqual(watchdogRestartsProcess, currentStatus.SoftRestart); ValidateSessionId(currentStatus, false); Assert.AreEqual(WatchdogStatus.Online, currentStatus.Status); @@ -511,11 +524,8 @@ namespace Tgstation.Server.Tests.Live.Instance Assert.IsTrue(job.ErrorCode == ErrorCode.GameServerOffline || job.ErrorCode == ErrorCode.GCoreFailure, $"{job.ErrorCode}: {job.ExceptionDetails}"); - await Task.Delay(TimeSpan.FromSeconds(20), cancellationToken); - - var ddStatus = await instanceClient.DreamDaemon.Read(cancellationToken); - ValidateSessionId(ddStatus, true); - Assert.AreEqual(WatchdogStatus.Online, ddStatus.Status.Value); + var restartJob2 = await instanceClient.DreamDaemon.Restart(cancellationToken); + await WaitForJob(restartJob2, 20, false, null, cancellationToken); } async Task TestDMApiFreeDeploy(CancellationToken cancellationToken) @@ -612,7 +622,7 @@ namespace Tgstation.Server.Tests.Live.Instance Assert.AreEqual(false, daemonStatus.SoftRestart); Assert.AreEqual(false, daemonStatus.SoftShutdown); - await GracefulWatchdogShutdown(60, cancellationToken); + await GracefulWatchdogShutdown(cancellationToken); daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.AreEqual(WatchdogStatus.Offline, daemonStatus.Status.Value); @@ -691,20 +701,15 @@ namespace Tgstation.Server.Tests.Live.Instance { System.Console.WriteLine("TEST: WATCHDOG HEALTH CHECK TEST"); - // Check reverse mapping - var status = await instanceClient.DreamDaemon.Update(new DreamDaemonRequest - { - DumpOnHealthCheckRestart = !checkDump, - }, cancellationToken); - // enable health checks - status = await instanceClient.DreamDaemon.Update(new DreamDaemonRequest + var status = await instanceClient.DreamDaemon.Update(new DreamDaemonRequest { HealthCheckSeconds = 1, DumpOnHealthCheckRestart = checkDump, }, cancellationToken); Assert.AreEqual(checkDump, status.DumpOnHealthCheckRestart); + Assert.AreEqual(1U, status.HealthCheckSeconds.Value); var startJob = await StartDD(cancellationToken); @@ -731,7 +736,7 @@ namespace Tgstation.Server.Tests.Live.Instance .GetProcess(ddProc.Id); // Ensure it's responding to health checks - await Task.WhenAny(Task.Delay(20000, cancellationToken), ourProcessHandler.Lifetime); + await Task.WhenAny(Task.Delay(6000, cancellationToken), ourProcessHandler.Lifetime); Assert.IsFalse(ddProc.HasExited); // check DD agrees @@ -744,7 +749,7 @@ namespace Tgstation.Server.Tests.Live.Instance Assert.IsNotNull(topicRequestResult); Assert.AreEqual(TopicResponseType.StringResponse, topicRequestResult.ResponseType); Assert.IsNotNull(topicRequestResult.StringData); - Assert.AreEqual(topicRequestResult.StringData, "received health check"); + Assert.AreEqual("received health check", topicRequestResult.StringData); var ddStatus = await instanceClient.DreamDaemon.Update(new DreamDaemonRequest { @@ -755,11 +760,11 @@ namespace Tgstation.Server.Tests.Live.Instance ourProcessHandler.Suspend(); await Task.WhenAny(ourProcessHandler.Lifetime, Task.Delay(TimeSpan.FromMinutes(1), cancellationToken)); + Assert.IsTrue(ourProcessHandler.Lifetime.IsCompleted); - var timeout = 60; + var timeout = 20; do { - await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken); ddStatus = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.AreEqual(1U, ddStatus.HealthCheckSeconds.Value); if (ddStatus.Status.Value == WatchdogStatus.Offline) @@ -770,6 +775,8 @@ namespace Tgstation.Server.Tests.Live.Instance if (--timeout == 0) Assert.Fail("DreamDaemon didn't shutdown within the timeout!"); + + await Task.Delay(TimeSpan.FromSeconds(3), cancellationToken); } while (timeout > 0); @@ -816,7 +823,6 @@ namespace Tgstation.Server.Tests.Live.Instance throw; } } - await Task.Delay(TimeSpan.FromSeconds(3), cts.Token); return await instanceClient.DreamDaemon.Start(cancellationToken); } @@ -855,7 +861,7 @@ namespace Tgstation.Server.Tests.Live.Instance BridgeController.LogContent = true; // Time for DD to revert the bridge access identifier change - await Task.Delay(TimeSpan.FromSeconds(3), cancellationToken); + await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken); } async Task ValidateTopicLimits(CancellationToken cancellationToken) @@ -1249,15 +1255,18 @@ namespace Tgstation.Server.Tests.Live.Instance public async Task StartAndLeaveRunning(CancellationToken cancellationToken) { System.Console.WriteLine("TEST: WATCHDOG STARTING ENDLESS"); - await DeployTestDme("LongRunning/long_running_test", DreamDaemonSecurity.Trusted, true, cancellationToken); - var startJob = await StartDD(cancellationToken); await WaitForJob(startJob, 40, false, null, cancellationToken); - var daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); + var daemonStatus = await instanceClient.DreamDaemon.Update(new DreamDaemonRequest + { + AdditionalParameters = "slow_start=1", + }, + cancellationToken); - Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value); + Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status); + Assert.IsTrue(daemonStatus.SoftRestart); CheckDDPriority(); Assert.AreEqual(ddPort, daemonStatus.CurrentPort); @@ -1265,6 +1274,9 @@ namespace Tgstation.Server.Tests.Live.Instance bool firstTime = true; do { + if(!firstTime) + Assert.IsFalse(daemonStatus.SoftRestart); + ValidateSessionId(daemonStatus, true); KillDD(firstTime); firstTime = false; @@ -1272,7 +1284,7 @@ namespace Tgstation.Server.Tests.Live.Instance daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); } while (daemonStatus.Status == WatchdogStatus.Online); - Assert.AreEqual(WatchdogStatus.Restoring, daemonStatus.Status.Value); + Assert.AreEqual(WatchdogStatus.Restoring, daemonStatus.Status); // Kill it again do @@ -1281,13 +1293,18 @@ namespace Tgstation.Server.Tests.Live.Instance daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); } while (daemonStatus.Status == WatchdogStatus.Online || daemonStatus.Status == WatchdogStatus.Restoring); - Assert.AreEqual(WatchdogStatus.DelayedRestart, daemonStatus.Status.Value); + Assert.AreEqual(WatchdogStatus.DelayedRestart, daemonStatus.Status); await Task.Delay(TimeSpan.FromSeconds(15), cancellationToken); - daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); + daemonStatus = await instanceClient.DreamDaemon.Update(new DreamDaemonRequest + { + AdditionalParameters = String.Empty, + }, + cancellationToken); ValidateSessionId(daemonStatus, true); - Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value); + Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status); + Assert.IsTrue(daemonStatus.SoftRestart); await CheckDMApiFail(daemonStatus.ActiveCompileJob, cancellationToken); } @@ -1305,12 +1322,12 @@ namespace Tgstation.Server.Tests.Live.Instance } public Task TellWorldToReboot(bool waitForOnlineIfRestoring, CancellationToken cancellationToken) - => TellWorldToReboot2(instanceClient, topicClient, FindTopicPort(), waitForOnlineIfRestoring ? true : testVersion.Engine.Value == EngineType.OpenDream, cancellationToken); + => TellWorldToReboot2(instanceClient, topicClient, FindTopicPort(), waitForOnlineIfRestoring || testVersion.Engine.Value == EngineType.OpenDream, cancellationToken); public static async Task TellWorldToReboot2(IInstanceClient instanceClient, ITopicClient topicClient, ushort topicPort, bool waitForOnlineIfRestoring, CancellationToken cancellationToken) { var daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.IsNotNull(daemonStatus.StagedCompileJob); - var initialCompileJob = daemonStatus.ActiveCompileJob; + var initialSession = daemonStatus.ActiveCompileJob; System.Console.WriteLine("TEST: Sending world reboot topic..."); @@ -1328,11 +1345,9 @@ namespace Tgstation.Server.Tests.Live.Instance await Task.Delay(TimeSpan.FromSeconds(1), tempToken); daemonStatus = await instanceClient.DreamDaemon.Read(tempToken); } - while (initialCompileJob.Id == daemonStatus.ActiveCompileJob.Id); + while (initialSession.Id == daemonStatus.ActiveCompileJob.Id); } - await Task.Delay(TimeSpan.FromSeconds(3), cancellationToken); - if (waitForOnlineIfRestoring && daemonStatus.Status == WatchdogStatus.Restoring) { do @@ -1353,7 +1368,7 @@ namespace Tgstation.Server.Tests.Live.Instance ApiValidationSecurityLevel = deploymentSecurity, ProjectName = dmeName.Contains("rooted") ? dmeName : $"tests/DMAPI/{dmeName}", RequireDMApiValidation = requireApi, - Timeout = TimeSpan.FromMilliseconds(1), + Timeout = !ranTimeoutTest ? TimeSpan.FromMilliseconds(1) : TimeSpan.FromMinutes(5), }, cancellationToken); JobResponse compileJobJob; @@ -1366,29 +1381,33 @@ namespace Tgstation.Server.Tests.Live.Instance compileJobJob = await instanceClient.DreamMaker.Compile(cancellationToken); await WaitForJob(compileJobJob, 90, true, ErrorCode.DeploymentTimeout, cancellationToken); + + await instanceClient.DreamMaker.Update(new DreamMakerRequest + { + Timeout = TimeSpan.FromMinutes(5), + }, cancellationToken); ranTimeoutTest = true; } - await instanceClient.DreamMaker.Update(new DreamMakerRequest - { - Timeout = TimeSpan.FromMinutes(5), - }, cancellationToken); - compileJobJob = await instanceClient.DreamMaker.Compile(cancellationToken); await WaitForJob(compileJobJob, 90, false, null, cancellationToken); + // annoying but, with signalR instant job updates, this running task can get queued before the task that processes the watchdog's monitor activation + for (var i = 0; i < 10; ++i) + await Task.Yield(); + var ddInfo = await instanceClient.DreamDaemon.Read(cancellationToken); + var targetJob = ddInfo.StagedCompileJob ?? ddInfo.ActiveCompileJob; + Assert.IsNotNull(targetJob); if (requireApi) - { - var targetJob = ddInfo.StagedCompileJob ?? ddInfo.ActiveCompileJob; - Assert.IsNotNull(targetJob); Assert.IsNotNull(targetJob.DMApiVersion); - } + else + Assert.IsNull(targetJob.DMApiVersion); return ddInfo; } - async Task GracefulWatchdogShutdown(uint timeout, CancellationToken cancellationToken) + async Task GracefulWatchdogShutdown(CancellationToken cancellationToken) { await instanceClient.DreamDaemon.Update(new DreamDaemonRequest { @@ -1398,9 +1417,10 @@ namespace Tgstation.Server.Tests.Live.Instance var newStatus = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.IsTrue(newStatus.SoftShutdown.Value || newStatus.Status.Value == WatchdogStatus.Offline); + var timeout = 20; do { - await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken); + await Task.Delay(TimeSpan.FromSeconds(3), cancellationToken); var ddStatus = await instanceClient.DreamDaemon.Read(cancellationToken); if (ddStatus.Status.Value == WatchdogStatus.Offline) break; @@ -1446,9 +1466,10 @@ namespace Tgstation.Server.Tests.Live.Instance async ValueTask TestLegacyBridgeEndpoint(CancellationToken cancellationToken) { + System.Console.WriteLine("TEST: TestLegacyBridgeEndpoint"); var result = await topicClient.SendTopic(IPAddress.Loopback, "im_out_of_memes=1", FindTopicPort(), cancellationToken); - Assert.AreEqual("yeah gimmie a sec", result.StringData); - await Task.Delay(TimeSpan.FromSeconds(10), cancellationToken); + Assert.IsNotNull(result); + Assert.AreEqual("all gucci", result.StringData); await CheckDMApiFail((await instanceClient.DreamDaemon.Read(cancellationToken)).ActiveCompileJob, cancellationToken); } } diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index 14b26d8ec8..a419fc648c 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -1049,7 +1049,7 @@ namespace Tgstation.Server.Tests.Live var ioManager = new Host.IO.DefaultIOManager(); var repoPath = ioManager.ConcatPath(instance.Path, "Repository"); - var jobsTest = new JobsRequiredTest(instanceClient.Jobs); + await using var jobsTest = new JobsRequiredTest(instanceClient.Jobs); var postWriteHandler = (Host.IO.IPostWriteHandler)(new PlatformIdentifier().IsWindows ? new Host.IO.WindowsPostWriteHandler() : new Host.IO.PosixPostWriteHandler(loggerFactory.CreateLogger())); @@ -1590,7 +1590,7 @@ namespace Tgstation.Server.Tests.Live .ToList(); } - var jrt = new JobsRequiredTest(instanceClient.Jobs); + await using var jrt = new JobsRequiredTest(instanceClient.Jobs); foreach (var job in jobs) { Assert.IsTrue(job.StartedAt.Value >= preStartupTime); @@ -1660,11 +1660,11 @@ namespace Tgstation.Server.Tests.Live .ToList(); } - var jrt = new JobsRequiredTest(instanceClient.Jobs); + await using var jrt = new JobsRequiredTest(instanceClient.Jobs); foreach (var job in jobs) { Assert.IsTrue(job.StartedAt.Value >= preStartupTime); - await jrt.WaitForJob(job, 140, job.Description.Contains("Reconnect chat bot") ? null : false, null, cancellationToken); + await jrt.WaitForJob(job, 140, job.JobCode == JobCode.ReconnectChatBot ? null : false, null, cancellationToken); } } @@ -1684,7 +1684,7 @@ namespace Tgstation.Server.Tests.Live Assert.AreEqual(WatchdogStatus.Online, dd.Status.Value); var compileJob = await instanceClient.DreamMaker.Compile(cancellationToken); - var wdt = new WatchdogTest(edgeVersion, instanceClient, GetInstanceManager(), (ushort)server.ApiUrl.Port, server.HighPriorityDreamDaemon, mainDDPort, server.UsingBasicWatchdog); + await using var wdt = new WatchdogTest(edgeVersion, instanceClient, GetInstanceManager(), (ushort)server.ApiUrl.Port, server.HighPriorityDreamDaemon, mainDDPort, server.UsingBasicWatchdog); await wdt.WaitForJob(compileJob, 30, false, null, cancellationToken); dd = await instanceClient.DreamDaemon.Read(cancellationToken); @@ -1727,13 +1727,15 @@ namespace Tgstation.Server.Tests.Live Assert.AreEqual(WatchdogStatus.Online, currentDD.Status); Assert.AreEqual(expectedStaged, currentDD.StagedCompileJob.Job.Id.Value); - var wdt = new WatchdogTest(edgeVersion, instanceClient, GetInstanceManager(), (ushort)server.ApiUrl.Port, server.HighPriorityDreamDaemon, mainDDPort, server.UsingBasicWatchdog); - currentDD = await wdt.TellWorldToReboot(true, cancellationToken); + await using var wdt = new WatchdogTest(edgeVersion, instanceClient, GetInstanceManager(), (ushort)server.ApiUrl.Port, server.HighPriorityDreamDaemon, mainDDPort, server.UsingBasicWatchdog); + currentDD = await wdt.TellWorldToReboot(false, cancellationToken); Assert.AreEqual(expectedStaged, currentDD.ActiveCompileJob.Job.Id.Value); Assert.IsNull(currentDD.StagedCompileJob); - var repoTest = new RepositoryTest(instanceClient.Repository, instanceClient.Jobs).RunPostTest(cancellationToken); - await new ChatTest(instanceClient.ChatBots, adminClient.Instances, instanceClient.Jobs, instance).RunPostTest(cancellationToken); + await using var repoTestObj = new RepositoryTest(instanceClient.Repository, instanceClient.Jobs); + var repoTest = repoTestObj.RunPostTest(cancellationToken); + await using var chatTestObj = new ChatTest(instanceClient.ChatBots, adminClient.Instances, instanceClient.Jobs, instance); + await chatTestObj.RunPostTest(cancellationToken); await repoTest; await DummyChatProvider.RandomDisconnections(false, cancellationToken);