From f34a5342768f40cc448b4526b997464d65d4d27b Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Tue, 21 Nov 2023 08:33:33 -0500 Subject: [PATCH 01/82] Update Postgres EFCore connector --- src/Tgstation.Server.Host/Tgstation.Server.Host.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index 9890be89d0..5be73fa4ee 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -95,7 +95,7 @@ - + From 18fc529c37539d66d3af1f6436c507416268a4ef Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Thu, 23 Nov 2023 16:23:41 -0500 Subject: [PATCH 02/82] Fix a race condition that caused settings changes to not return API responses with `SoftRestart = true` --- .../Components/Watchdog/IWatchdog.cs | 4 ++-- .../Components/Watchdog/WatchdogBase.cs | 8 +++++--- .../Controllers/DreamDaemonController.cs | 13 +++++++++---- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Watchdog/IWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/IWatchdog.cs index a54cde8e83..7600dc9a36 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/IWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/IWatchdog.cs @@ -62,8 +62,8 @@ namespace Tgstation.Server.Host.Components.Watchdog /// /// The new . May be modified. /// The for the operation. - /// A representing the running operation. - ValueTask ChangeSettings(DreamDaemonLaunchParameters launchParameters, CancellationToken cancellationToken); + /// A resulting in if a reboot is required, otherwise. + ValueTask ChangeSettings(DreamDaemonLaunchParameters launchParameters, CancellationToken cancellationToken); /// /// Restarts the watchdog. diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs index 33b5ca0720..33961aa1f6 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs @@ -267,18 +267,20 @@ namespace Tgstation.Server.Host.Components.Watchdog } /// - public async ValueTask ChangeSettings(DreamDaemonLaunchParameters launchParameters, CancellationToken cancellationToken) + public async ValueTask ChangeSettings(DreamDaemonLaunchParameters launchParameters, CancellationToken cancellationToken) { using (await SemaphoreSlimContext.Lock(synchronizationSemaphore, cancellationToken)) { bool match = launchParameters.CanApplyWithoutReboot(ActiveLaunchParameters); ActiveLaunchParameters = launchParameters; - if (match || Status == WatchdogStatus.Offline) - return; + if (match || Status == WatchdogStatus.Offline || Status == WatchdogStatus.DelayedRestart) + return false; var oldTcs = Interlocked.Exchange(ref activeParametersUpdated, new TaskCompletionSource()); oldTcs.SetResult(); } + + return true; } /// diff --git a/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs b/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs index 5a3ff51f0e..ca84dac467 100644 --- a/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs +++ b/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs @@ -106,7 +106,7 @@ namespace Tgstation.Server.Host.Controllers [TgsAuthorize(DreamDaemonRights.ReadMetadata | DreamDaemonRights.ReadRevision)] [ProducesResponseType(typeof(DreamDaemonResponse), 200)] [ProducesResponseType(typeof(ErrorMessageResponse), 410)] - public ValueTask Read(CancellationToken cancellationToken) => ReadImpl(null, cancellationToken); + public ValueTask Read(CancellationToken cancellationToken) => ReadImpl(null, false, cancellationToken); /// /// Stops the Watchdog if it's running. @@ -231,7 +231,7 @@ namespace Tgstation.Server.Host.Controllers // run this second because current may be modified by it // slight race condition with request cancellation, but I CANNOT be assed right now - await watchdog.ChangeSettings(current, cancellationToken); + var rebootRequired = await watchdog.ChangeSettings(current, cancellationToken); var rebootState = watchdog.RebootState; var oldSoftRestart = rebootState == RebootState.Restart; @@ -243,7 +243,7 @@ namespace Tgstation.Server.Host.Controllers else if ((oldSoftRestart && model.SoftRestart == false) || (oldSoftShutdown && model.SoftShutdown == false)) await watchdog.ResetRebootState(cancellationToken); - return await ReadImpl(current, cancellationToken); + return await ReadImpl(current, rebootRequired, cancellationToken); }); } #pragma warning restore CA1506 @@ -305,9 +305,10 @@ namespace Tgstation.Server.Host.Controllers /// Implementation of . /// /// The to operate on if any. + /// If there was a settings change made that forced a switch to . /// The for the operation. /// A resulting in the of the operation. - ValueTask ReadImpl(DreamDaemonSettings settings, CancellationToken cancellationToken) + ValueTask ReadImpl(DreamDaemonSettings settings, bool knownForcedReboot, CancellationToken cancellationToken) => WithComponentInstance(async instance => { var dd = instance.Watchdog; @@ -360,6 +361,10 @@ namespace Tgstation.Server.Host.Controllers result.Visibility = settings.Visibility.Value; result.SoftRestart = rstate == RebootState.Restart; result.SoftShutdown = rstate == RebootState.Shutdown; + + if (rstate == RebootState.Normal && knownForcedReboot) + result.SoftRestart = true; + result.StartupTimeout = settings.StartupTimeout.Value; result.HealthCheckSeconds = settings.HealthCheckSeconds.Value; result.DumpOnHealthCheckRestart = settings.DumpOnHealthCheckRestart.Value; From 0973174c10b8f3c657ae24fc57bf4e583f25337c Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Thu, 23 Nov 2023 16:24:48 -0500 Subject: [PATCH 03/82] Improve live test runtimes Workstation improvement of 10.2 to 6.4 minutes. --- tests/DMAPI/LongRunning/Test.dm | 18 ++- .../Live/Instance/ChatTest.cs | 1 + .../Live/Instance/DeploymentTest.cs | 1 + .../Live/Instance/InstanceTest.cs | 19 +-- .../Live/Instance/JobsRequiredTest.cs | 104 ++++++++------ .../Live/Instance/WatchdogTest.cs | 133 ++++++++++-------- .../Live/TestLiveServer.cs | 20 +-- 7 files changed, 172 insertions(+), 124 deletions(-) diff --git a/tests/DMAPI/LongRunning/Test.dm b/tests/DMAPI/LongRunning/Test.dm index 0bff755db9..90164c7e80 100644 --- a/tests/DMAPI/LongRunning/Test.dm +++ b/tests/DMAPI/LongRunning/Test.dm @@ -6,9 +6,11 @@ log << "Initial value of sleep_offline: [sleep_offline]" sleep_offline = FALSE - // Intentionally slow down startup for testing purposes - for(var/i in 1 to 10000000) - dab() + if(params["slow_start"]) + // Intentionally slow down startup for health check testing purposes + for(var/i in 1 to 10000000) + dab() + TgsNew(new /datum/tgs_event_handler/impl, TGS_SECURITY_SAFE) var/sec = TgsSecurityLevel() @@ -189,7 +191,7 @@ var/run_bridge_test var/its_sad = data["im_out_of_memes"] if(its_sad) TestLegacyBridge() - return "yeah gimmie a sec" + return "all gucci" TgsChatBroadcast(new /datum/tgs_message_content("Received non-tgs topic: `[T]`")) @@ -252,9 +254,9 @@ var/received_health_check = FALSE /proc/RebootAsync() set waitfor = FALSE - world.TgsChatBroadcast(new /datum/tgs_message_content("Rebooting after 3 seconds")); + world.TgsChatBroadcast(new /datum/tgs_message_content("Rebooting after 1 seconds")); world.log << "About to sleep. sleep_offline: [world.sleep_offline]" - sleep(30) + sleep(10) world.log << "Done sleep, calling Reboot" world.Reboot() @@ -362,10 +364,6 @@ var/suppress_bridge_spam = FALSE api.access_identifier = old_ai /proc/TestLegacyBridge() - set waitfor = FALSE - - sleep(10) - var/datum/tgs_api/v5/api = TGS_READ_GLOBAL(tgs) if(api.interop_version.suite != 5) FailTest("Legacy bridge test not required anymore?") diff --git a/tests/Tgstation.Server.Tests/Live/Instance/ChatTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/ChatTest.cs index 0963b72b4f..abb699815e 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/ChatTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/ChatTest.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; + using Microsoft.VisualStudio.TestTools.UnitTesting; using Tgstation.Server.Api.Models; diff --git a/tests/Tgstation.Server.Tests/Live/Instance/DeploymentTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/DeploymentTest.cs index 8b5fab58c9..d0d074e6e8 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/DeploymentTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/DeploymentTest.cs @@ -1,6 +1,7 @@ using System; using System.Threading; using System.Threading.Tasks; + using Microsoft.VisualStudio.TestTools.UnitTesting; using Tgstation.Server.Api.Models; diff --git a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs index 255a2e8a83..fc5177e355 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs @@ -45,11 +45,11 @@ namespace Tgstation.Server.Tests.Live.Instance CancellationToken cancellationToken) { var testVersion = await EngineTest.GetEdgeVersion(EngineType.Byond, fileDownloader, cancellationToken); - var engineTest = new EngineTest(instanceClient.Engine, instanceClient.Jobs, fileDownloader, instanceClient.Metadata, testVersion.Engine.Value); - var chatTest = new ChatTest(instanceClient.ChatBots, instanceManagerClient, instanceClient.Jobs, instanceClient.Metadata); + await using var engineTest = new EngineTest(instanceClient.Engine, instanceClient.Jobs, fileDownloader, instanceClient.Metadata, testVersion.Engine.Value); + await using var chatTest = new ChatTest(instanceClient.ChatBots, instanceManagerClient, instanceClient.Jobs, instanceClient.Metadata); var configTest = new ConfigurationTest(instanceClient.Configuration, instanceClient.Metadata); - var repoTest = new RepositoryTest(instanceClient.Repository, instanceClient.Jobs); - var dmTest = new DeploymentTest(instanceClient, instanceClient.Jobs, dmPort, ddPort, lowPrioDeployment, testVersion.Engine.Value); + await using var repoTest = new RepositoryTest(instanceClient.Repository, instanceClient.Jobs); + await using var dmTest = new DeploymentTest(instanceClient, instanceClient.Jobs, dmPort, ddPort, lowPrioDeployment, testVersion.Engine.Value); var byondTask = engineTest.Run(cancellationToken, out var firstInstall); var chatTask = chatTest.RunPreWatchdog(cancellationToken); @@ -66,15 +66,15 @@ namespace Tgstation.Server.Tests.Live.Instance await configTest.SetupDMApiTests(true, cancellationToken); await byondTask; - await new WatchdogTest( + await using var wdt = new WatchdogTest( testVersion, instanceClient, instanceManager, serverPort, highPrioDD, ddPort, - usingBasicWatchdog) - .Run(cancellationToken); + usingBasicWatchdog); + await wdt.Run(cancellationToken); } public static async ValueTask DownloadEngineVersion( @@ -197,7 +197,7 @@ namespace Tgstation.Server.Tests.Live.Instance ReconnectionInterval = 1, }, cancellationToken); - var jrt = new JobsRequiredTest(instanceClient.Jobs); + await using var jrt = new JobsRequiredTest(instanceClient.Jobs); EngineInstallResponse installJob2; await using (var stableBytesMs = await TestingUtils.ExtractMemoryStreamFromInstallationData( @@ -257,7 +257,8 @@ namespace Tgstation.Server.Tests.Live.Instance await configSetupTask; - await new WatchdogTest(compatVersion, instanceClient, instanceManager, serverPort, highPrioDD, ddPort, usingBasicWatchdog).Run(cancellationToken); + await using var wdt = new WatchdogTest(compatVersion, instanceClient, instanceManager, serverPort, highPrioDD, ddPort, usingBasicWatchdog); + await wdt.Run(cancellationToken); await instanceManagerClient.Update(new InstanceUpdateRequest { diff --git a/tests/Tgstation.Server.Tests/Live/Instance/JobsRequiredTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/JobsRequiredTest.cs index bfe8c98d82..e7c7cb610c 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/JobsRequiredTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/JobsRequiredTest.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Concurrent; using System.Reflection; using System.Threading; using System.Threading.Tasks; @@ -13,16 +14,65 @@ using Tgstation.Server.Client.Components; namespace Tgstation.Server.Tests.Live.Instance { - class JobsRequiredTest + class JobsRequiredTest : IAsyncDisposable { protected IJobsClient JobsClient { get; } readonly IApiClient apiClient; + IAsyncDisposable hubConnection; + readonly Task hubConnectionTask; + readonly CancellationTokenSource cancellationTokenSource; + + readonly ConcurrentDictionary> registry; + public JobsRequiredTest(IJobsClient jobsClient) { JobsClient = jobsClient ?? throw new ArgumentNullException(nameof(jobsClient)); apiClient = (IApiClient)jobsClient.GetType().GetProperty("ApiClient", BindingFlags.NonPublic | BindingFlags.Instance).GetValue(jobsClient); + registry = new ConcurrentDictionary>(); + + cancellationTokenSource = new CancellationTokenSource(); + hubConnectionTask = CreateHubConnection(); + } + + async Task CreateHubConnection() + { + var receiver = new JobReceiver + { + Callback = job => Register(job), + }; + + hubConnection = await apiClient.CreateHubConnection(receiver, null, null, cancellationTokenSource.Token); + } + + public async ValueTask DisposeAsync() + { + cancellationTokenSource.Cancel(); + cancellationTokenSource.Dispose(); + await hubConnectionTask; + if (hubConnection != null) + await hubConnection.DisposeAsync(); + } + + Task Register(JobResponse updatedJob) + { + var tcs = registry.AddOrUpdate(updatedJob.Id.Value, + _ => + { + var tcs = new TaskCompletionSource(); + if (updatedJob.StoppedAt.HasValue) + tcs.SetResult(updatedJob); + return tcs; + }, + (_, oldTcs) => + { + if (updatedJob.StoppedAt.HasValue) + oldTcs.TrySetResult(updatedJob); + return oldTcs; + }); + + return tcs.Task; } class JobReceiver : IJobsHub @@ -42,47 +92,21 @@ namespace Tgstation.Server.Tests.Live.Instance Assert.IsNotNull(originalJob.JobCode); var job = originalJob; + var registryTask = Register(job); + await Task.WhenAny( + registryTask, + Task.Delay(TimeSpan.FromSeconds(timeout), cancellationToken)); + + if (!registryTask.IsCompleted) + // one last get in case SignalR dropped the ball + job = await JobsClient.GetId(job, cancellationToken); + else + job = await registryTask; + if (!job.StoppedAt.HasValue) { - var tcs = new TaskCompletionSource(); - var receiver = new JobReceiver - { - Callback = updatedJob => - { - if (updatedJob.Id != job.Id) - return; - - job = updatedJob; - if (updatedJob.StoppedAt.HasValue) - tcs.TrySetResult(); - }, - }; - - JobResponse firstCheck; - await using (var hubConnection = await apiClient.CreateHubConnection(receiver, null, null, cancellationToken)) - { - // initial GET after connecting - firstCheck = await JobsClient.GetId(job, cancellationToken); - if (!firstCheck.StoppedAt.HasValue) - { - firstCheck = null; - await Task.WhenAny( - tcs.Task, - Task.Delay(TimeSpan.FromSeconds(timeout), cancellationToken)); - } - } - - if (firstCheck != null) - job = firstCheck; - else if (!job.StoppedAt.HasValue) - // one last get in case SignalR dropped the ball - job = await JobsClient.GetId(job, cancellationToken); - - if (!job.StoppedAt.HasValue) - { - await JobsClient.Cancel(job, cancellationToken); - Assert.Fail($"Job ID {job.Id} \"{job.Description}\" timed out!"); - } + await JobsClient.Cancel(job, cancellationToken); + Assert.Fail($"Job ID {job.Id} \"{job.Description}\" timed out!"); } if (expectFailure.HasValue && expectFailure.Value ^ job.ExceptionDetails != null) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index bcde5714ff..a5e6f12187 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -1,6 +1,5 @@ using Byond.TopicSender; -using Microsoft.AspNetCore.DataProtection; using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -68,6 +67,7 @@ namespace Tgstation.Server.Tests.Live.Instance readonly bool watchdogRestartsProcess; bool ranTimeoutTest = false; + const string BaseAdditionalParameters = "expect_chat_channels=1&expect_static_files=1"; public WatchdogTest(EngineVersion testVersion, IInstanceClient instanceClient, InstanceManager instanceManager, ushort serverPort, bool highPrioDD, ushort ddPort, bool watchdogRestartsProcess) : base(instanceClient.Jobs) @@ -88,8 +88,20 @@ namespace Tgstation.Server.Tests.Live.Instance DisconnectTimeout = TimeSpan.FromSeconds(30) }, loggerFactory.CreateLogger($"WatchdogTest.TopicClient.{instanceClient.Metadata.Name}")); } - public async Task Run(CancellationToken cancellationToken) + { + try + { + await RunInt(cancellationToken); + } + catch + { + System.Console.WriteLine($"WATCHDOG TEST FAILING INSTANCE ID {instanceClient.Metadata.Id.Value}"); + throw; + } + } + + async Task RunInt(CancellationToken cancellationToken) { System.Console.WriteLine($"TEST: START WATCHDOG TESTS {instanceClient.Metadata.Name}"); @@ -125,7 +137,7 @@ namespace Tgstation.Server.Tests.Live.Instance Port = ddPort, MapThreads = 2, LogOutput = false, - AdditionalParameters = "expect_chat_channels=1&expect_static_files=1" + AdditionalParameters = BaseAdditionalParameters }, cancellationToken).AsTask(), CheckByondVersions(), ApiAssert.ThrowsException(() => instanceClient.DreamDaemon.Update(new DreamDaemonRequest @@ -176,7 +188,6 @@ namespace Tgstation.Server.Tests.Live.Instance var ddUpdateTask = instanceClient.DreamDaemon.Update(new DreamDaemonRequest { SecurityLevel = useTrusted ? DreamDaemonSecurity.Trusted : DreamDaemonSecurity.Safe, - AdditionalParameters = "expect_chat_channels=1&expect_static_files=1", }, cancellationToken); var currentStatus = await DeployTestDme("long_running_test_rooted", DreamDaemonSecurity.Trusted, true, cancellationToken); await ddUpdateTask; @@ -294,24 +305,25 @@ namespace Tgstation.Server.Tests.Live.Instance async ValueTask RegressionTest1550(CancellationToken cancellationToken) { + // Previous test, StartAndLeaveRunning, has SoftRestart set. We don't want that. + var restartJob = await instanceClient.DreamDaemon.Restart(cancellationToken); + await WaitForJob(restartJob, 10, false, null, cancellationToken); + // we need to cycle deployments twice because TGS holds the initial deployment var currentStatus = await DeployTestDme("LongRunning/long_running_test", DreamDaemonSecurity.Trusted, true, cancellationToken); Assert.AreEqual(WatchdogStatus.Online, currentStatus.Status); Assert.IsNotNull(currentStatus.StagedCompileJob); - ValidateSessionId(currentStatus, false); + ValidateSessionId(currentStatus, true); var expectedStaged = currentStatus.StagedCompileJob; Assert.AreNotEqual(expectedStaged.Id, currentStatus.ActiveCompileJob.Id); - Assert.AreEqual(watchdogRestartsProcess, currentStatus.SoftRestart); - Assert.IsFalse(currentStatus.SoftShutdown.Value); + Assert.IsFalse(currentStatus.SoftShutdown); currentStatus = await TellWorldToReboot(true, cancellationToken); - ValidateSessionId(currentStatus, watchdogRestartsProcess); + ValidateSessionId(currentStatus, true); Assert.AreEqual(expectedStaged.Id, currentStatus.ActiveCompileJob.Id); - await Task.Delay(TimeSpan.FromSeconds(3), cancellationToken); - var topicRequestResult = await topicClient.SendTopic( IPAddress.Loopback, $"shadow_wizard_money_gang=1", @@ -322,6 +334,7 @@ namespace Tgstation.Server.Tests.Live.Instance Assert.AreEqual("we love casting spells", topicRequestResult.StringData); currentStatus = await DeployTestDme("LongRunning/long_running_test", DreamDaemonSecurity.Trusted, true, cancellationToken); + Assert.AreEqual(watchdogRestartsProcess, currentStatus.SoftRestart); ValidateSessionId(currentStatus, false); Assert.AreEqual(WatchdogStatus.Online, currentStatus.Status); @@ -511,11 +524,8 @@ namespace Tgstation.Server.Tests.Live.Instance Assert.IsTrue(job.ErrorCode == ErrorCode.GameServerOffline || job.ErrorCode == ErrorCode.GCoreFailure, $"{job.ErrorCode}: {job.ExceptionDetails}"); - await Task.Delay(TimeSpan.FromSeconds(20), cancellationToken); - - var ddStatus = await instanceClient.DreamDaemon.Read(cancellationToken); - ValidateSessionId(ddStatus, true); - Assert.AreEqual(WatchdogStatus.Online, ddStatus.Status.Value); + var restartJob2 = await instanceClient.DreamDaemon.Restart(cancellationToken); + await WaitForJob(restartJob2, 20, false, null, cancellationToken); } async Task TestDMApiFreeDeploy(CancellationToken cancellationToken) @@ -612,7 +622,7 @@ namespace Tgstation.Server.Tests.Live.Instance Assert.AreEqual(false, daemonStatus.SoftRestart); Assert.AreEqual(false, daemonStatus.SoftShutdown); - await GracefulWatchdogShutdown(60, cancellationToken); + await GracefulWatchdogShutdown(cancellationToken); daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.AreEqual(WatchdogStatus.Offline, daemonStatus.Status.Value); @@ -691,20 +701,15 @@ namespace Tgstation.Server.Tests.Live.Instance { System.Console.WriteLine("TEST: WATCHDOG HEALTH CHECK TEST"); - // Check reverse mapping - var status = await instanceClient.DreamDaemon.Update(new DreamDaemonRequest - { - DumpOnHealthCheckRestart = !checkDump, - }, cancellationToken); - // enable health checks - status = await instanceClient.DreamDaemon.Update(new DreamDaemonRequest + var status = await instanceClient.DreamDaemon.Update(new DreamDaemonRequest { HealthCheckSeconds = 1, DumpOnHealthCheckRestart = checkDump, }, cancellationToken); Assert.AreEqual(checkDump, status.DumpOnHealthCheckRestart); + Assert.AreEqual(1U, status.HealthCheckSeconds.Value); var startJob = await StartDD(cancellationToken); @@ -731,7 +736,7 @@ namespace Tgstation.Server.Tests.Live.Instance .GetProcess(ddProc.Id); // Ensure it's responding to health checks - await Task.WhenAny(Task.Delay(20000, cancellationToken), ourProcessHandler.Lifetime); + await Task.WhenAny(Task.Delay(6000, cancellationToken), ourProcessHandler.Lifetime); Assert.IsFalse(ddProc.HasExited); // check DD agrees @@ -744,7 +749,7 @@ namespace Tgstation.Server.Tests.Live.Instance Assert.IsNotNull(topicRequestResult); Assert.AreEqual(TopicResponseType.StringResponse, topicRequestResult.ResponseType); Assert.IsNotNull(topicRequestResult.StringData); - Assert.AreEqual(topicRequestResult.StringData, "received health check"); + Assert.AreEqual("received health check", topicRequestResult.StringData); var ddStatus = await instanceClient.DreamDaemon.Update(new DreamDaemonRequest { @@ -755,11 +760,11 @@ namespace Tgstation.Server.Tests.Live.Instance ourProcessHandler.Suspend(); await Task.WhenAny(ourProcessHandler.Lifetime, Task.Delay(TimeSpan.FromMinutes(1), cancellationToken)); + Assert.IsTrue(ourProcessHandler.Lifetime.IsCompleted); - var timeout = 60; + var timeout = 20; do { - await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken); ddStatus = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.AreEqual(1U, ddStatus.HealthCheckSeconds.Value); if (ddStatus.Status.Value == WatchdogStatus.Offline) @@ -770,6 +775,8 @@ namespace Tgstation.Server.Tests.Live.Instance if (--timeout == 0) Assert.Fail("DreamDaemon didn't shutdown within the timeout!"); + + await Task.Delay(TimeSpan.FromSeconds(3), cancellationToken); } while (timeout > 0); @@ -816,7 +823,6 @@ namespace Tgstation.Server.Tests.Live.Instance throw; } } - await Task.Delay(TimeSpan.FromSeconds(3), cts.Token); return await instanceClient.DreamDaemon.Start(cancellationToken); } @@ -855,7 +861,7 @@ namespace Tgstation.Server.Tests.Live.Instance BridgeController.LogContent = true; // Time for DD to revert the bridge access identifier change - await Task.Delay(TimeSpan.FromSeconds(3), cancellationToken); + await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken); } async Task ValidateTopicLimits(CancellationToken cancellationToken) @@ -1249,15 +1255,18 @@ namespace Tgstation.Server.Tests.Live.Instance public async Task StartAndLeaveRunning(CancellationToken cancellationToken) { System.Console.WriteLine("TEST: WATCHDOG STARTING ENDLESS"); - await DeployTestDme("LongRunning/long_running_test", DreamDaemonSecurity.Trusted, true, cancellationToken); - var startJob = await StartDD(cancellationToken); await WaitForJob(startJob, 40, false, null, cancellationToken); - var daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); + var daemonStatus = await instanceClient.DreamDaemon.Update(new DreamDaemonRequest + { + AdditionalParameters = "slow_start=1", + }, + cancellationToken); - Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value); + Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status); + Assert.IsTrue(daemonStatus.SoftRestart); CheckDDPriority(); Assert.AreEqual(ddPort, daemonStatus.CurrentPort); @@ -1265,6 +1274,9 @@ namespace Tgstation.Server.Tests.Live.Instance bool firstTime = true; do { + if(!firstTime) + Assert.IsFalse(daemonStatus.SoftRestart); + ValidateSessionId(daemonStatus, true); KillDD(firstTime); firstTime = false; @@ -1272,7 +1284,7 @@ namespace Tgstation.Server.Tests.Live.Instance daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); } while (daemonStatus.Status == WatchdogStatus.Online); - Assert.AreEqual(WatchdogStatus.Restoring, daemonStatus.Status.Value); + Assert.AreEqual(WatchdogStatus.Restoring, daemonStatus.Status); // Kill it again do @@ -1281,13 +1293,18 @@ namespace Tgstation.Server.Tests.Live.Instance daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); } while (daemonStatus.Status == WatchdogStatus.Online || daemonStatus.Status == WatchdogStatus.Restoring); - Assert.AreEqual(WatchdogStatus.DelayedRestart, daemonStatus.Status.Value); + Assert.AreEqual(WatchdogStatus.DelayedRestart, daemonStatus.Status); await Task.Delay(TimeSpan.FromSeconds(15), cancellationToken); - daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); + daemonStatus = await instanceClient.DreamDaemon.Update(new DreamDaemonRequest + { + AdditionalParameters = String.Empty, + }, + cancellationToken); ValidateSessionId(daemonStatus, true); - Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value); + Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status); + Assert.IsTrue(daemonStatus.SoftRestart); await CheckDMApiFail(daemonStatus.ActiveCompileJob, cancellationToken); } @@ -1305,12 +1322,12 @@ namespace Tgstation.Server.Tests.Live.Instance } public Task TellWorldToReboot(bool waitForOnlineIfRestoring, CancellationToken cancellationToken) - => TellWorldToReboot2(instanceClient, topicClient, FindTopicPort(), waitForOnlineIfRestoring ? true : testVersion.Engine.Value == EngineType.OpenDream, cancellationToken); + => TellWorldToReboot2(instanceClient, topicClient, FindTopicPort(), waitForOnlineIfRestoring || testVersion.Engine.Value == EngineType.OpenDream, cancellationToken); public static async Task TellWorldToReboot2(IInstanceClient instanceClient, ITopicClient topicClient, ushort topicPort, bool waitForOnlineIfRestoring, CancellationToken cancellationToken) { var daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.IsNotNull(daemonStatus.StagedCompileJob); - var initialCompileJob = daemonStatus.ActiveCompileJob; + var initialSession = daemonStatus.ActiveCompileJob; System.Console.WriteLine("TEST: Sending world reboot topic..."); @@ -1328,11 +1345,9 @@ namespace Tgstation.Server.Tests.Live.Instance await Task.Delay(TimeSpan.FromSeconds(1), tempToken); daemonStatus = await instanceClient.DreamDaemon.Read(tempToken); } - while (initialCompileJob.Id == daemonStatus.ActiveCompileJob.Id); + while (initialSession.Id == daemonStatus.ActiveCompileJob.Id); } - await Task.Delay(TimeSpan.FromSeconds(3), cancellationToken); - if (waitForOnlineIfRestoring && daemonStatus.Status == WatchdogStatus.Restoring) { do @@ -1353,7 +1368,7 @@ namespace Tgstation.Server.Tests.Live.Instance ApiValidationSecurityLevel = deploymentSecurity, ProjectName = dmeName.Contains("rooted") ? dmeName : $"tests/DMAPI/{dmeName}", RequireDMApiValidation = requireApi, - Timeout = TimeSpan.FromMilliseconds(1), + Timeout = !ranTimeoutTest ? TimeSpan.FromMilliseconds(1) : TimeSpan.FromMinutes(5), }, cancellationToken); JobResponse compileJobJob; @@ -1366,29 +1381,33 @@ namespace Tgstation.Server.Tests.Live.Instance compileJobJob = await instanceClient.DreamMaker.Compile(cancellationToken); await WaitForJob(compileJobJob, 90, true, ErrorCode.DeploymentTimeout, cancellationToken); + + await instanceClient.DreamMaker.Update(new DreamMakerRequest + { + Timeout = TimeSpan.FromMinutes(5), + }, cancellationToken); ranTimeoutTest = true; } - await instanceClient.DreamMaker.Update(new DreamMakerRequest - { - Timeout = TimeSpan.FromMinutes(5), - }, cancellationToken); - compileJobJob = await instanceClient.DreamMaker.Compile(cancellationToken); await WaitForJob(compileJobJob, 90, false, null, cancellationToken); + // annoying but, with signalR instant job updates, this running task can get queued before the task that processes the watchdog's monitor activation + for (var i = 0; i < 10; ++i) + await Task.Yield(); + var ddInfo = await instanceClient.DreamDaemon.Read(cancellationToken); + var targetJob = ddInfo.StagedCompileJob ?? ddInfo.ActiveCompileJob; + Assert.IsNotNull(targetJob); if (requireApi) - { - var targetJob = ddInfo.StagedCompileJob ?? ddInfo.ActiveCompileJob; - Assert.IsNotNull(targetJob); Assert.IsNotNull(targetJob.DMApiVersion); - } + else + Assert.IsNull(targetJob.DMApiVersion); return ddInfo; } - async Task GracefulWatchdogShutdown(uint timeout, CancellationToken cancellationToken) + async Task GracefulWatchdogShutdown(CancellationToken cancellationToken) { await instanceClient.DreamDaemon.Update(new DreamDaemonRequest { @@ -1398,9 +1417,10 @@ namespace Tgstation.Server.Tests.Live.Instance var newStatus = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.IsTrue(newStatus.SoftShutdown.Value || newStatus.Status.Value == WatchdogStatus.Offline); + var timeout = 20; do { - await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken); + await Task.Delay(TimeSpan.FromSeconds(3), cancellationToken); var ddStatus = await instanceClient.DreamDaemon.Read(cancellationToken); if (ddStatus.Status.Value == WatchdogStatus.Offline) break; @@ -1446,9 +1466,10 @@ namespace Tgstation.Server.Tests.Live.Instance async ValueTask TestLegacyBridgeEndpoint(CancellationToken cancellationToken) { + System.Console.WriteLine("TEST: TestLegacyBridgeEndpoint"); var result = await topicClient.SendTopic(IPAddress.Loopback, "im_out_of_memes=1", FindTopicPort(), cancellationToken); - Assert.AreEqual("yeah gimmie a sec", result.StringData); - await Task.Delay(TimeSpan.FromSeconds(10), cancellationToken); + Assert.IsNotNull(result); + Assert.AreEqual("all gucci", result.StringData); await CheckDMApiFail((await instanceClient.DreamDaemon.Read(cancellationToken)).ActiveCompileJob, cancellationToken); } } diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index 14b26d8ec8..a419fc648c 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -1049,7 +1049,7 @@ namespace Tgstation.Server.Tests.Live var ioManager = new Host.IO.DefaultIOManager(); var repoPath = ioManager.ConcatPath(instance.Path, "Repository"); - var jobsTest = new JobsRequiredTest(instanceClient.Jobs); + await using var jobsTest = new JobsRequiredTest(instanceClient.Jobs); var postWriteHandler = (Host.IO.IPostWriteHandler)(new PlatformIdentifier().IsWindows ? new Host.IO.WindowsPostWriteHandler() : new Host.IO.PosixPostWriteHandler(loggerFactory.CreateLogger())); @@ -1590,7 +1590,7 @@ namespace Tgstation.Server.Tests.Live .ToList(); } - var jrt = new JobsRequiredTest(instanceClient.Jobs); + await using var jrt = new JobsRequiredTest(instanceClient.Jobs); foreach (var job in jobs) { Assert.IsTrue(job.StartedAt.Value >= preStartupTime); @@ -1660,11 +1660,11 @@ namespace Tgstation.Server.Tests.Live .ToList(); } - var jrt = new JobsRequiredTest(instanceClient.Jobs); + await using var jrt = new JobsRequiredTest(instanceClient.Jobs); foreach (var job in jobs) { Assert.IsTrue(job.StartedAt.Value >= preStartupTime); - await jrt.WaitForJob(job, 140, job.Description.Contains("Reconnect chat bot") ? null : false, null, cancellationToken); + await jrt.WaitForJob(job, 140, job.JobCode == JobCode.ReconnectChatBot ? null : false, null, cancellationToken); } } @@ -1684,7 +1684,7 @@ namespace Tgstation.Server.Tests.Live Assert.AreEqual(WatchdogStatus.Online, dd.Status.Value); var compileJob = await instanceClient.DreamMaker.Compile(cancellationToken); - var wdt = new WatchdogTest(edgeVersion, instanceClient, GetInstanceManager(), (ushort)server.ApiUrl.Port, server.HighPriorityDreamDaemon, mainDDPort, server.UsingBasicWatchdog); + await using var wdt = new WatchdogTest(edgeVersion, instanceClient, GetInstanceManager(), (ushort)server.ApiUrl.Port, server.HighPriorityDreamDaemon, mainDDPort, server.UsingBasicWatchdog); await wdt.WaitForJob(compileJob, 30, false, null, cancellationToken); dd = await instanceClient.DreamDaemon.Read(cancellationToken); @@ -1727,13 +1727,15 @@ namespace Tgstation.Server.Tests.Live Assert.AreEqual(WatchdogStatus.Online, currentDD.Status); Assert.AreEqual(expectedStaged, currentDD.StagedCompileJob.Job.Id.Value); - var wdt = new WatchdogTest(edgeVersion, instanceClient, GetInstanceManager(), (ushort)server.ApiUrl.Port, server.HighPriorityDreamDaemon, mainDDPort, server.UsingBasicWatchdog); - currentDD = await wdt.TellWorldToReboot(true, cancellationToken); + await using var wdt = new WatchdogTest(edgeVersion, instanceClient, GetInstanceManager(), (ushort)server.ApiUrl.Port, server.HighPriorityDreamDaemon, mainDDPort, server.UsingBasicWatchdog); + currentDD = await wdt.TellWorldToReboot(false, cancellationToken); Assert.AreEqual(expectedStaged, currentDD.ActiveCompileJob.Job.Id.Value); Assert.IsNull(currentDD.StagedCompileJob); - var repoTest = new RepositoryTest(instanceClient.Repository, instanceClient.Jobs).RunPostTest(cancellationToken); - await new ChatTest(instanceClient.ChatBots, adminClient.Instances, instanceClient.Jobs, instance).RunPostTest(cancellationToken); + await using var repoTestObj = new RepositoryTest(instanceClient.Repository, instanceClient.Jobs); + var repoTest = repoTestObj.RunPostTest(cancellationToken); + await using var chatTestObj = new ChatTest(instanceClient.ChatBots, adminClient.Instances, instanceClient.Jobs, instance); + await chatTestObj.RunPostTest(cancellationToken); await repoTest; await DummyChatProvider.RandomDisconnections(false, cancellationToken); From 02c27eb52da9e65bd42f079f4cf36345d32b8c66 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Thu, 23 Nov 2023 18:01:40 -0500 Subject: [PATCH 04/82] Probably fix shutdown delay condition with sessions --- .../Components/Chat/ChatTrackingContext.cs | 2 +- .../Components/Session/SessionController.cs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Chat/ChatTrackingContext.cs b/src/Tgstation.Server.Host/Components/Chat/ChatTrackingContext.cs index 369ccdeeeb..03c9ee2e54 100644 --- a/src/Tgstation.Server.Host/Components/Chat/ChatTrackingContext.cs +++ b/src/Tgstation.Server.Host/Components/Chat/ChatTrackingContext.cs @@ -16,7 +16,7 @@ namespace Tgstation.Server.Host.Components.Chat /// public bool Active { - get => active; + get => active && onDispose != null; set { if (active == value) diff --git a/src/Tgstation.Server.Host/Components/Session/SessionController.cs b/src/Tgstation.Server.Host/Components/Session/SessionController.cs index d3feab14a6..9094cdf527 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionController.cs @@ -337,9 +337,8 @@ namespace Tgstation.Server.Host.Components.Session Logger.LogTrace("Disposing..."); - // yield then acquire the topic semaphore to prevent new calls from starting - await Task.Yield(); - (await topicSendSemaphore.Lock(CancellationToken.None)).Dispose(); // DCT: None available + reattachTopicCts.Cancel(); + var semaphoreLockTask = topicSendSemaphore.Lock(CancellationToken.None); // DCT: None available if (!released) { @@ -363,6 +362,7 @@ namespace Tgstation.Server.Host.Components.Session if (!released) await Lifetime; // finish the async callback + (await semaphoreLockTask).Dispose(); topicSendSemaphore.Dispose(); } From e6db8713eb38753e1f9098bb0792f472a015d77f Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 26 Nov 2023 22:19:26 -0500 Subject: [PATCH 05/82] Increase test timeout for health check tests --- tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index a5e6f12187..3f1f52866a 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -759,7 +759,7 @@ namespace Tgstation.Server.Tests.Live.Instance ourProcessHandler.Suspend(); - await Task.WhenAny(ourProcessHandler.Lifetime, Task.Delay(TimeSpan.FromMinutes(1), cancellationToken)); + await Task.WhenAny(ourProcessHandler.Lifetime, Task.Delay(TimeSpan.FromMinutes(2), cancellationToken)); Assert.IsTrue(ourProcessHandler.Lifetime.IsCompleted); var timeout = 20; From a979f04e03a2dcc5130cff90ed7a55fdacfc1762 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 27 Nov 2023 12:19:26 -0500 Subject: [PATCH 06/82] Fix race condition in `JobsRequiredTest` cleanup --- .../Live/Instance/JobsRequiredTest.cs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/JobsRequiredTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/JobsRequiredTest.cs index e7c7cb610c..c6501e6766 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/JobsRequiredTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/JobsRequiredTest.cs @@ -50,7 +50,14 @@ namespace Tgstation.Server.Tests.Live.Instance { cancellationTokenSource.Cancel(); cancellationTokenSource.Dispose(); - await hubConnectionTask; + try + { + await hubConnectionTask; + } + catch (OperationCanceledException) + { + } + if (hubConnection != null) await hubConnection.DisposeAsync(); } From e4d6c5d4876d2df89b6da192d69c0152c4a597f7 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 27 Nov 2023 12:49:55 -0500 Subject: [PATCH 07/82] Fix Process dispose handling Fix read timeout workaround unnecessarily disposing too much --- src/Tgstation.Server.Host/System/Process.cs | 30 ++++++++++++++++----- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/src/Tgstation.Server.Host/System/Process.cs b/src/Tgstation.Server.Host/System/Process.cs index da67c8f0e8..210a86db8c 100644 --- a/src/Tgstation.Server.Host/System/Process.cs +++ b/src/Tgstation.Server.Host/System/Process.cs @@ -128,7 +128,7 @@ namespace Tgstation.Server.Host.System /// public async ValueTask DisposeAsync() { - if (Interlocked.Exchange(ref disposed, 1) == 1) + if (Interlocked.Exchange(ref disposed, 1) != 0) return; logger.LogTrace("Disposing PID {pid}...", Id); @@ -146,6 +146,7 @@ namespace Tgstation.Server.Host.System /// public async ValueTask GetCombinedOutput(CancellationToken cancellationToken) { + CheckDisposed(); if (readTask == null) throw new InvalidOperationException("Output/Error stream reading was not enabled!"); @@ -158,7 +159,7 @@ namespace Tgstation.Server.Host.System if (!readTask.IsCompleted) { logger.LogWarning("Detected process output read hang on PID {pid}! Closing handle as a workaround...", Id); - await DisposeAsync(); + cancellationTokenSource.Cancel(); } } @@ -168,6 +169,7 @@ namespace Tgstation.Server.Host.System /// public void Terminate() { + CheckDisposed(); if (handle.HasExited) { logger.LogTrace("PID {pid} already exited", Id); @@ -190,6 +192,7 @@ namespace Tgstation.Server.Host.System /// public void AdjustPriority(bool higher) { + CheckDisposed(); var targetPriority = higher ? ProcessPriorityClass.AboveNormal : ProcessPriorityClass.BelowNormal; try { @@ -205,6 +208,7 @@ namespace Tgstation.Server.Host.System /// public void Suspend() { + CheckDisposed(); try { processFeatures.SuspendProcess(handle); @@ -220,6 +224,7 @@ namespace Tgstation.Server.Host.System /// public void Resume() { + CheckDisposed(); try { processFeatures.ResumeProcess(handle); @@ -235,6 +240,7 @@ namespace Tgstation.Server.Host.System /// public string GetExecutingUsername() { + CheckDisposed(); var result = processFeatures.GetExecutingUsername(handle); logger.LogTrace("PID {pid} Username: {username}", Id, result); return result; @@ -244,6 +250,7 @@ namespace Tgstation.Server.Host.System public ValueTask CreateDump(string outputFile, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(outputFile); + CheckDisposed(); logger.LogTrace("Dumping PID {pid} to {dumpFilePath}...", Id, outputFile); return processFeatures.CreateDump(handle, outputFile, cancellationToken); @@ -255,18 +262,29 @@ namespace Tgstation.Server.Host.System /// A resulting in the or if the process was detached. async Task WrapLifetimeTask() { + bool hasExited; try { await handle.WaitForExitAsync(cancellationTokenSource.Token); - var exitCode = handle.ExitCode; - logger.LogTrace("PID {pid} exited with code {exitCode}", Id, exitCode); - return exitCode; + hasExited = true; } catch (OperationCanceledException ex) { logger.LogTrace(ex, "Process lifetime task cancelled!"); - return null; + hasExited = handle.HasExited; } + + if (!hasExited) + return null; + + var exitCode = handle.ExitCode; + logger.LogTrace("PID {pid} exited with code {exitCode}", Id, exitCode); + return exitCode; } + + /// + /// Throws an if a method of the was called after . + /// + void CheckDisposed() => ObjectDisposedException.ThrowIf(disposed != 0, this); } } From 2ee1ff4665876f6a36cc35b39288eaa1fe621384 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 27 Nov 2023 14:18:10 -0500 Subject: [PATCH 08/82] Add a yield here to potentially fix a CI error We HAD the handle to the process open and running, yet somehow GetProcessesByName failed to find it. --- .../Live/Instance/WatchdogTest.cs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index 3f1f52866a..affa17266e 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -547,7 +547,7 @@ namespace Tgstation.Server.Tests.Live.Instance Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value); ValidateSessionId(daemonStatus, true); - CheckDDPriority(); + await CheckDDPriority(); Assert.AreEqual(false, daemonStatus.SoftRestart); Assert.AreEqual(false, daemonStatus.SoftShutdown); Assert.AreEqual(string.Empty, daemonStatus.AdditionalParameters); @@ -618,7 +618,7 @@ namespace Tgstation.Server.Tests.Live.Instance Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value); ValidateSessionId(daemonStatus, true); - CheckDDPriority(); + await CheckDDPriority(); Assert.AreEqual(false, daemonStatus.SoftRestart); Assert.AreEqual(false, daemonStatus.SoftShutdown); @@ -715,7 +715,7 @@ namespace Tgstation.Server.Tests.Live.Instance await WaitForJob(startJob, 40, false, null, cancellationToken); - CheckDDPriority(); + await CheckDDPriority(); // lock on to DD and pause it so it can't health check var ddProcs = TestLiveServer.GetEngineServerProcessesOnPort(testVersion.Engine.Value, ddPort).Where(x => !x.HasExited).ToList(); @@ -1074,8 +1074,9 @@ namespace Tgstation.Server.Tests.Live.Instance Assert.AreEqual("Footer text", embedsResponse.Embed.Footer?.Text); } - void CheckDDPriority() + async ValueTask CheckDDPriority() { + await Task.Yield(); var allProcesses = TestLiveServer.GetEngineServerProcessesOnPort(testVersion.Engine.Value, ddPort).Where(x => !x.HasExited).ToList(); if (allProcesses.Count == 0) Assert.Fail("Expected engine server to be running here"); @@ -1115,7 +1116,7 @@ namespace Tgstation.Server.Tests.Live.Instance Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value); ValidateSessionId(daemonStatus, true); - CheckDDPriority(); + await CheckDDPriority(); Assert.AreEqual(initialCompileJob.Id, daemonStatus.ActiveCompileJob.Id); var newerCompileJob = daemonStatus.StagedCompileJob; @@ -1163,7 +1164,7 @@ namespace Tgstation.Server.Tests.Live.Instance ValidateSessionId(daemonStatus, true); Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value); Assert.AreEqual(true, daemonStatus.SoftRestart); - CheckDDPriority(); + await CheckDDPriority(); Assert.AreEqual(initialCompileJob.Id, daemonStatus.ActiveCompileJob.Id); var newerCompileJob = daemonStatus.StagedCompileJob; @@ -1201,7 +1202,7 @@ namespace Tgstation.Server.Tests.Live.Instance await WaitForJob(startJob, 70, false, null, cancellationToken); - CheckDDPriority(); + await CheckDDPriority(); var byondInstallJobTask = instanceClient.Engine.SetActiveVersion( new EngineVersionRequest @@ -1267,7 +1268,7 @@ namespace Tgstation.Server.Tests.Live.Instance Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status); Assert.IsTrue(daemonStatus.SoftRestart); - CheckDDPriority(); + await CheckDDPriority(); Assert.AreEqual(ddPort, daemonStatus.CurrentPort); // Try killing the DD process to ensure it gets set to the restoring state From 9ba8952809a8a6f3c11ac05ed7c2f00d84010a33 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Tue, 28 Nov 2023 08:39:12 -0500 Subject: [PATCH 09/82] Fold Code Scanning into CI Pipeline --- .github/workflows/ci-pipeline.yml | 34 +++++++++++++++++- .github/workflows/code-scanning.yml | 55 ----------------------------- 2 files changed, 33 insertions(+), 56 deletions(-) delete mode 100644 .github/workflows/code-scanning.yml diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 5551c047e3..f93709905e 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -82,6 +82,38 @@ jobs: - name: GitHub Requires at Least One Step for a Job run: exit 0 + analyze: + name: Code Scanning + needs: start-ci-run-gate + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + if: (!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success' && ${{ vars.TGS_ENABLE_CODE_QL }} == 'true') + steps: + - name: Setup dotnet + uses: actions/setup-dotnet@v3 + with: + dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' + dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} + + - name: Checkout + uses: actions/checkout@v3 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: csharp + + - name: Build + run: dotnet build -c ReleaseNoWindows -p:TGS_HOST_NO_WEBPANEL=true + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 + with: + category: "/language:csharp" + dmapi-build: name: Build DMAPI needs: start-ci-run-gate @@ -1314,7 +1346,7 @@ jobs: ci-completion-gate: # This job exists so there isn't a moving target for branch protections name: CI Completion Gate - needs: [ pages-build, docker-build, build-deb, build-msi, validate-openapi-spec, upload-code-coverage, check-winget-pr-template ] + needs: [ pages-build, docker-build, build-deb, build-msi, validate-openapi-spec, upload-code-coverage, check-winget-pr-template, analyze ] runs-on: ubuntu-latest if: (!(cancelled() || failure()) && needs.pages-build.result == 'success' && needs.docker-build.result == 'success' && needs.build-deb.result == 'success' && needs.build-msi.result == 'success' && needs.validate-openapi-spec.result == 'success' && needs.upload-code-coverage.result == 'success' && needs.check-winget-pr-template.result == 'success') steps: diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml deleted file mode 100644 index 9442ce5530..0000000000 --- a/.github/workflows/code-scanning.yml +++ /dev/null @@ -1,55 +0,0 @@ -name: 'Code Scanning' - -on: - schedule: - - cron: 0 23 * * 1 - push: - branches: - - dev - - master - - V6 - pull_request: - branches: - - dev - - master - - V6 - -env: - TGS_DOTNET_VERSION: 8 - TGS_DOTNET_QUALITY: ga - -concurrency: - group: "code-scanning-${{ github.head_ref || github.run_id }}-${{ github.event_name }}" - cancel-in-progress: true - -jobs: - analyze: - name: Code Scanning - runs-on: ubuntu-latest - permissions: - actions: read - contents: read - security-events: write - if: ${{ vars.TGS_ENABLE_CODE_QL }} == 'true' - steps: - - name: Setup dotnet - uses: actions/setup-dotnet@v3 - with: - dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' - dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - - - name: Checkout - uses: actions/checkout@v3 - - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 - with: - languages: csharp - - - name: Build - run: dotnet build -c ReleaseNoWindows -p:TGS_HOST_NO_WEBPANEL=true - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 - with: - category: "/language:csharp" From 3b8cbd8e733ecc2a186674376192c5d10033019e Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Tue, 28 Nov 2023 08:39:55 -0500 Subject: [PATCH 10/82] Change CI cron to 4AM EST --- .github/workflows/ci-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index f93709905e..2b51152c16 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -19,7 +19,7 @@ name: 'CI Pipeline' on: schedule: - - cron: 0 23 * * * + - cron: 0 9 * * * push: branches: - dev From 6b37808ec5bd70074e7a13de8e8a84eb928faeea Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Tue, 28 Nov 2023 09:02:28 -0500 Subject: [PATCH 11/82] Revert "Suppress the NU5104 while we're in preview" This reverts commit 2f9fb5df0c66a20960c3472c2dcb45bdbc2980c6. --- src/Tgstation.Server.Client/Tgstation.Server.Client.csproj | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj b/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj index 21eff94c4b..876f1097d6 100644 --- a/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj +++ b/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj @@ -7,7 +7,6 @@ Client library for tgstation-server. json web api tgstation-server tgstation ss13 byond client http $(TGS_NUGET_RELEASE_NOTES_CLIENT) - NU5104 From 0b5a36fea2bfc5e74f2935da962dddf23502cc19 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Wed, 29 Nov 2023 15:06:57 -0500 Subject: [PATCH 12/82] Do not start database services we're not using See https://github.com/actions/runner/issues/822#issuecomment-1524826092 --- .github/workflows/ci-pipeline.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 2b51152c16..796cf6797a 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -625,14 +625,14 @@ jobs: if: (!(cancelled() || failure()) && needs.dmapi-build.result == 'success' && needs.opendream-build.result == 'success') services: # We start all dbs here so we can just code the stuff once mssql: - image: mcr.microsoft.com/mssql/server:2019-latest + image: ${{ (matrix.database-type == 'SqlServer') && 'mcr.microsoft.com/mssql/server:2019-latest' || '' }} env: SA_PASSWORD: myPassword ACCEPT_EULA: 'Y' ports: - 1433:1433 postgres: - image: cyberboss/postgres-max-connections # Fork of _/postgres:latest with max_connections=500 becuase GitHub actions service containers have no way to set command lines. Rebuilds with updates. + image: ${{ (matrix.database-type == 'PostgresSql') && 'cyberboss/postgres-max-connections' || '' }} # Fork of _/postgres:latest with max_connections=500 becuase GitHub actions service containers have no way to set command lines. Rebuilds with updates. ports: - 5432:5432 env: @@ -643,7 +643,7 @@ jobs: --health-timeout 5s --health-retries 5 mariadb: - image: mariadb + image: ${{ (matrix.database-type == 'MariaDB') && 'mariadb' || '' }} ports: - 3306:3306 env: @@ -654,7 +654,7 @@ jobs: --health-timeout=2s --health-retries=3 mysql: - image: mysql:5.7.31 + image: ${{ (matrix.database-type == 'MySql') && 'mysql:5.7.31' || '' }} ports: - 3307:3306 env: From 1016596e5d18e6397b28e9c6221fc3d489340b7c Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Wed, 29 Nov 2023 15:11:11 -0500 Subject: [PATCH 13/82] Add a missing comment to CI --- .github/workflows/ci-pipeline.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 796cf6797a..79bc776719 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -2,6 +2,7 @@ # Does CI on push/PR/cron. Deployments on push when triggered # - Validates Documentation # - Builds C# and DMAPI +# - Runs CodeQL Anaylsis # - Tests everything on massive matrix # - Packages # - Tests package installs/services/uninstalls From b5f7a7f06959a5446afcb5c1e7b0a0b64958f1c3 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Wed, 29 Nov 2023 16:43:30 -0500 Subject: [PATCH 14/82] Fix main page logo not appearing if webpanel was disabled --- .../Controllers/ControlPanelController.cs | 21 ++------ .../Controllers/RootController.cs | 49 ++++++++++++++++--- .../Extensions/ControllerBaseExtensions.cs | 39 +++++++++++++++ .../Views/Root/Index.cshtml | 3 +- tests/Tgstation.Server.Tests/TestVersions.cs | 10 ++-- 5 files changed, 93 insertions(+), 29 deletions(-) diff --git a/src/Tgstation.Server.Host/Controllers/ControlPanelController.cs b/src/Tgstation.Server.Host/Controllers/ControlPanelController.cs index 08d8f74dad..d69947d4ba 100644 --- a/src/Tgstation.Server.Host/Controllers/ControlPanelController.cs +++ b/src/Tgstation.Server.Host/Controllers/ControlPanelController.cs @@ -7,7 +7,6 @@ using System.Threading.Tasks; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.Mvc.Filters; -using Microsoft.AspNetCore.StaticFiles; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Microsoft.Extensions.Primitives; @@ -15,6 +14,7 @@ using Microsoft.Net.Http.Headers; using Tgstation.Server.Api; using Tgstation.Server.Host.Configuration; +using Tgstation.Server.Host.Extensions; namespace Tgstation.Server.Host.Controllers { @@ -136,22 +136,11 @@ namespace Tgstation.Server.Host.Controllers if (Request.Headers.ContainsKey(FetchChannelVaryHeader)) return GetChannelJson(); - var fileInfo = hostEnvironment.WebRootFileProvider.GetFileInfo(appRoute); - if (fileInfo.Exists) - { - logger.LogTrace("Serving static file \"{filename}\"...", appRoute); - var contentTypeProvider = new FileExtensionContentTypeProvider(); - if (!contentTypeProvider.TryGetContentType(fileInfo.Name, out var contentType)) - contentType = MediaTypeNames.Application.Octet; - else if (contentType == MediaTypeNames.Application.Json) - Response.Headers.Add( - HeaderNames.CacheControl, - new StringValues(new[] { "public", "max-age=31536000", "immutable" })); + var foundFile = this.TryServeFile(hostEnvironment, logger, appRoute); + if (foundFile != null) + return foundFile; - return File(appRoute, contentType); - } - else - logger.LogTrace("Requested static file \"{filename}\" does not exist! Redirecting to index...", appRoute); + logger.LogTrace("Requested static file \"{filename}\" does not exist! Redirecting to index...", appRoute); return File("index.html", MediaTypeNames.Text.Html); } diff --git a/src/Tgstation.Server.Host/Controllers/RootController.cs b/src/Tgstation.Server.Host/Controllers/RootController.cs index 850785d612..5de25b1451 100644 --- a/src/Tgstation.Server.Host/Controllers/RootController.cs +++ b/src/Tgstation.Server.Host/Controllers/RootController.cs @@ -2,10 +2,13 @@ using System.Collections.Generic; using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Tgstation.Server.Host.Configuration; +using Tgstation.Server.Host.Extensions; using Tgstation.Server.Host.System; using Tgstation.Server.Host.Utils; @@ -19,14 +22,14 @@ namespace Tgstation.Server.Host.Controllers public sealed class RootController : Controller { /// - /// The route to the TGS logo .svg in the on Windows. + /// The name of the TGS logo .svg in the on Windows. /// - public const string ProjectLogoSvgRouteWindows = "/0176d5d8b7d307f158e0.svg"; + const string LogoSvgWindowsName = "0176d5d8b7d307f158e0"; /// - /// The route to the TGS logo .svg in the on Linux. + /// The name of the TGS logo .svg in the on Linux. /// - public const string ProjectLogoSvgRouteLinux = "/b5616c99bf2052a6bbd7.svg"; + const string LogoSvgLinuxName = "b5616c99bf2052a6bbd7"; /// /// The for the . @@ -38,6 +41,16 @@ namespace Tgstation.Server.Host.Controllers /// readonly IPlatformIdentifier platformIdentifier; + /// + /// THe for the . + /// + readonly IWebHostEnvironment hostEnvironment; + + /// + /// The for the . + /// + readonly ILogger logger; + /// /// The for the . /// @@ -53,16 +66,22 @@ namespace Tgstation.Server.Host.Controllers /// /// The value of . /// The value of . + /// The value of . + /// The value of . /// The containing the value of . /// The containing the value of . public RootController( IAssemblyInformationProvider assemblyInformationProvider, IPlatformIdentifier platformIdentifier, + IWebHostEnvironment hostEnvironment, + ILogger logger, IOptions generalConfigurationOptions, IOptions controlPanelConfigurationOptions) { this.assemblyInformationProvider = assemblyInformationProvider ?? throw new ArgumentNullException(nameof(assemblyInformationProvider)); this.platformIdentifier = platformIdentifier ?? throw new ArgumentNullException(nameof(platformIdentifier)); + this.hostEnvironment = hostEnvironment ?? throw new ArgumentNullException(nameof(hostEnvironment)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); generalConfiguration = generalConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(generalConfigurationOptions)); controlPanelConfiguration = controlPanelConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(controlPanelConfigurationOptions)); } @@ -98,13 +117,29 @@ namespace Tgstation.Server.Host.Controllers var model = new { Links = links, - Svg = platformIdentifier.IsWindows // these are different because of motherfucking line endings -_- - ? ProjectLogoSvgRouteWindows - : ProjectLogoSvgRouteLinux, Title = assemblyInformationProvider.VersionString, }; return View(model); } + + /// + /// Retrieve the logo .svg for the webpanel. + /// + /// The appropriate . + [HttpGet("logo.svg")] + public IActionResult GetLogo() + { +#if NO_WEBPANEL + logger.LogTrace("Cannot serve project logo as TGS was built without the webpanel!"); + return NotFound(); +#else + var logoFileName = platformIdentifier.IsWindows // these are different because of motherfucking line endings -_- + ? LogoSvgWindowsName + : LogoSvgLinuxName; + + return (IActionResult)this.TryServeFile(hostEnvironment, logger, $"{logoFileName}.svg"); +#endif + } } } diff --git a/src/Tgstation.Server.Host/Extensions/ControllerBaseExtensions.cs b/src/Tgstation.Server.Host/Extensions/ControllerBaseExtensions.cs index 9ab3b1cc87..2d1d8119a0 100644 --- a/src/Tgstation.Server.Host/Extensions/ControllerBaseExtensions.cs +++ b/src/Tgstation.Server.Host/Extensions/ControllerBaseExtensions.cs @@ -1,7 +1,13 @@ using System; using System.Net; +using System.Net.Mime; +using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.StaticFiles; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Primitives; +using Microsoft.Net.Http.Headers; using Tgstation.Server.Api.Models; using Tgstation.Server.Api.Models.Response; @@ -30,5 +36,38 @@ namespace Tgstation.Server.Host.Extensions /// A with the given . public static ObjectResult StatusCode(this ControllerBase controller, HttpStatusCode statusCode, object errorMessage) => controller?.StatusCode((int)statusCode, errorMessage) ?? throw new ArgumentNullException(nameof(controller)); + + /// + /// Try to serve a given file . + /// + /// The . + /// The . + /// The . + /// The path to the file in the 'wwwroot'. + /// A if the file was found. otherwise. + public static VirtualFileResult TryServeFile(this ControllerBase controller, IWebHostEnvironment hostEnvironment, ILogger logger, string path) + { + ArgumentNullException.ThrowIfNull(controller); + ArgumentNullException.ThrowIfNull(hostEnvironment); + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(path); + + var fileInfo = hostEnvironment.WebRootFileProvider.GetFileInfo(path); + if (fileInfo.Exists) + { + logger.LogTrace("Serving static file \"{filename}\"...", path); + var contentTypeProvider = new FileExtensionContentTypeProvider(); + if (!contentTypeProvider.TryGetContentType(fileInfo.Name, out var contentType)) + contentType = MediaTypeNames.Application.Octet; + else if (contentType == MediaTypeNames.Application.Json) + controller.Response.Headers.Add( + HeaderNames.CacheControl, + new StringValues(new[] { "public", "max-age=31536000", "immutable" })); + + return controller.File(path, contentType); + } + + return null; + } } } diff --git a/src/Tgstation.Server.Host/Views/Root/Index.cshtml b/src/Tgstation.Server.Host/Views/Root/Index.cshtml index 209a6ad5ac..bc23093e87 100644 --- a/src/Tgstation.Server.Host/Views/Root/Index.cshtml +++ b/src/Tgstation.Server.Host/Views/Root/Index.cshtml @@ -1,5 +1,4 @@ @{ - var svgPath = Model.Svg; var title = Model.Title; @@ -13,7 +12,7 @@ - + @{ if (Model.Links != null) foreach (KeyValuePair kvp in Model.Links) diff --git a/tests/Tgstation.Server.Tests/TestVersions.cs b/tests/Tgstation.Server.Tests/TestVersions.cs index 03b998d438..3dd20d8e7b 100644 --- a/tests/Tgstation.Server.Tests/TestVersions.cs +++ b/tests/Tgstation.Server.Tests/TestVersions.cs @@ -416,11 +416,13 @@ namespace Tgstation.Server.Tests if (!Directory.Exists(directory)) Assert.Inconclusive("Webpanel not built?"); - var logo = new PlatformIdentifier().IsWindows - ? RootController.ProjectLogoSvgRouteWindows - : RootController.ProjectLogoSvgRouteLinux; + static string GetConstField(string name) => (string)typeof(RootController).GetField(name, BindingFlags.NonPublic | BindingFlags.Static).GetValue(null); - var path = $"../../../../../src/Tgstation.Server.Host/wwwroot{logo}"; + var logo = new PlatformIdentifier().IsWindows + ? GetConstField("LogoSvgWindowsName") + : GetConstField("LogoSvgLinuxName"); + + var path = $"../../../../../src/Tgstation.Server.Host/wwwroot/{logo}.svg"; Assert.IsTrue(File.Exists(path)); var content = await File.ReadAllBytesAsync(path); From 2702a2fc2b3568fb149fcad2668287689747d829 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Wed, 29 Nov 2023 17:15:39 -0500 Subject: [PATCH 15/82] Log path repositories are cloned to --- .../Components/Repository/RepositoryManager.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs b/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs index 2940e0d134..5852a58f97 100644 --- a/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs +++ b/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs @@ -127,8 +127,6 @@ namespace Tgstation.Server.Host.Components.Repository CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(url); - - logger.LogInformation("Begin clone {url} (Branch: {initialBranch})", url, initialBranch); lock (semaphore) { if (CloneInProgress) @@ -136,12 +134,14 @@ namespace Tgstation.Server.Host.Components.Repository CloneInProgress = true; } + var repositoryPath = ioManager.ResolvePath(); + logger.LogInformation("Begin clone {url} to {path} (Branch: {initialBranch})", url, repositoryPath, initialBranch); + try { using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken)) { logger.LogTrace("Semaphore acquired for clone"); - var repositoryPath = ioManager.ResolvePath(); if (!await ioManager.DirectoryExists(repositoryPath, cancellationToken)) try { From c40197fcb58b24e9914bec5231b81b7cb2a19eed Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Wed, 29 Nov 2023 17:32:26 -0500 Subject: [PATCH 16/82] Specify `Environment.SpecialFolderOption` where possible Also fix a typo --- src/Tgstation.Server.Host.Service/Program.cs | 4 +++- .../Components/Engine/ByondInstallerBase.cs | 8 +------- .../Components/Engine/PosixByondInstaller.cs | 3 ++- .../Components/Engine/WindowsByondInstaller.cs | 11 ++++++----- .../Configuration/FileLoggingConfiguration.cs | 4 +++- src/Tgstation.Server.Host/Core/Application.cs | 4 +++- tests/Tgstation.Server.Tests/CachingFileDownloader.cs | 4 +++- .../Live/Instance/InstanceTest.cs | 4 +++- 8 files changed, 24 insertions(+), 18 deletions(-) diff --git a/src/Tgstation.Server.Host.Service/Program.cs b/src/Tgstation.Server.Host.Service/Program.cs index c2f72b0901..4a9d39fcd2 100644 --- a/src/Tgstation.Server.Host.Service/Program.cs +++ b/src/Tgstation.Server.Host.Service/Program.cs @@ -179,7 +179,9 @@ namespace Tgstation.Server.Host.Service var exePath = Path.Combine(assemblyDirectory, $"{assemblyNameWithoutExtension}.exe"); var programDataDirectory = Path.Combine( - Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData), + Environment.GetFolderPath( + Environment.SpecialFolder.CommonApplicationData, + Environment.SpecialFolderOption.DoNotVerify), Server.Common.Constants.CanonicalPackageName); using var processInstaller = new ServiceProcessInstaller(); diff --git a/src/Tgstation.Server.Host/Components/Engine/ByondInstallerBase.cs b/src/Tgstation.Server.Host/Components/Engine/ByondInstallerBase.cs index cf20d498a9..f02239969f 100644 --- a/src/Tgstation.Server.Host/Components/Engine/ByondInstallerBase.cs +++ b/src/Tgstation.Server.Host/Components/Engine/ByondInstallerBase.cs @@ -52,7 +52,7 @@ namespace Tgstation.Server.Host.Components.Engine protected override EngineType TargetEngineType => EngineType.Byond; /// - /// Bath to the system user's local BYOND folder. + /// Path to the system user's local BYOND folder. /// protected abstract string PathToUserFolder { get; } @@ -157,12 +157,6 @@ namespace Tgstation.Server.Host.Components.Engine ArgumentNullException.ThrowIfNull(fullDmbPath); var byondDir = PathToUserFolder; - if (String.IsNullOrWhiteSpace(byondDir)) - { - Logger.LogTrace("No relevant user BYOND directory to install a \"{fileName}\" in", TrustedDmbFileName); - return; - } - var cfgDir = IOManager.ConcatPath( byondDir, CfgDirectoryName); diff --git a/src/Tgstation.Server.Host/Components/Engine/PosixByondInstaller.cs b/src/Tgstation.Server.Host/Components/Engine/PosixByondInstaller.cs index cddace53c8..f3c9c561b1 100644 --- a/src/Tgstation.Server.Host/Components/Engine/PosixByondInstaller.cs +++ b/src/Tgstation.Server.Host/Components/Engine/PosixByondInstaller.cs @@ -65,7 +65,8 @@ namespace Tgstation.Server.Host.Components.Engine PathToUserFolder = IOManager.ResolvePath( IOManager.ConcatPath( Environment.GetFolderPath( - Environment.SpecialFolder.UserProfile), + Environment.SpecialFolder.UserProfile, + Environment.SpecialFolderOption.DoNotVerify), "./byond/cache")); } diff --git a/src/Tgstation.Server.Host/Components/Engine/WindowsByondInstaller.cs b/src/Tgstation.Server.Host/Components/Engine/WindowsByondInstaller.cs index 0ebf63ff9f..0f40d3f538 100644 --- a/src/Tgstation.Server.Host/Components/Engine/WindowsByondInstaller.cs +++ b/src/Tgstation.Server.Host/Components/Engine/WindowsByondInstaller.cs @@ -100,11 +100,12 @@ namespace Tgstation.Server.Host.Components.Engine this.processExecutor = processExecutor ?? throw new ArgumentNullException(nameof(processExecutor)); generalConfiguration = generalConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(generalConfigurationOptions)); - var documentsDirectory = Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments); - if (String.IsNullOrWhiteSpace(documentsDirectory)) - PathToUserFolder = null; // happens with the service account - else - PathToUserFolder = IOManager.ResolvePath(IOManager.ConcatPath(documentsDirectory, "BYOND")); + var documentsDirectory = Environment.GetFolderPath( + Environment.SpecialFolder.MyDocuments, + Environment.SpecialFolderOption.DoNotVerify); + + PathToUserFolder = IOManager.ResolvePath( + IOManager.ConcatPath(documentsDirectory, "BYOND")); semaphore = new SemaphoreSlim(1); installedDirectX = false; diff --git a/src/Tgstation.Server.Host/Configuration/FileLoggingConfiguration.cs b/src/Tgstation.Server.Host/Configuration/FileLoggingConfiguration.cs index aa798cfed3..c317fc1b97 100644 --- a/src/Tgstation.Server.Host/Configuration/FileLoggingConfiguration.cs +++ b/src/Tgstation.Server.Host/Configuration/FileLoggingConfiguration.cs @@ -62,7 +62,9 @@ namespace Tgstation.Server.Host.Configuration return platformIdentifier.IsWindows ? ioManager.ConcatPath( - Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData), + Environment.GetFolderPath( + Environment.SpecialFolder.CommonApplicationData, + Environment.SpecialFolderOption.DoNotVerify), assemblyInformationProvider.VersionPrefix, "logs") : ioManager.ConcatPath( diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index 3ce52f39be..10f2728465 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -367,7 +367,9 @@ namespace Tgstation.Server.Host.Core // only global repo manager should be for the OD repo var openDreamRepositoryDirectory = ioManager.ConcatPath( - Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), + Environment.GetFolderPath( + Environment.SpecialFolder.LocalApplicationData, + Environment.SpecialFolderOption.DoNotVerify), assemblyInformationProvider.VersionPrefix, "OpenDreamRepository"); services.AddSingleton( diff --git a/tests/Tgstation.Server.Tests/CachingFileDownloader.cs b/tests/Tgstation.Server.Tests/CachingFileDownloader.cs index bd7d01bbbd..de540b606d 100644 --- a/tests/Tgstation.Server.Tests/CachingFileDownloader.cs +++ b/tests/Tgstation.Server.Tests/CachingFileDownloader.cs @@ -94,7 +94,9 @@ namespace Tgstation.Server.Tests // actions is supposed to cache BYOND for us var dir = Path.Combine( - Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), + Environment.GetFolderPath( + Environment.SpecialFolder.UserProfile, + Environment.SpecialFolderOption.DoNotVerify), "byond-zips-cache", windows ? "windows" : "linux"); path = Path.Combine( diff --git a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs index fc5177e355..611ee9ed6f 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs @@ -85,7 +85,9 @@ namespace Tgstation.Server.Tests.Live.Instance { var ioManager = new DefaultIOManager(); var odRepoDir = ioManager.ConcatPath( - Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), + Environment.GetFolderPath( + Environment.SpecialFolder.LocalApplicationData, + Environment.SpecialFolderOption.DoNotVerify), new AssemblyInformationProvider().VersionPrefix, "OpenDreamRepository"); var odRepoIoManager = new ResolvingIOManager(ioManager, odRepoDir); From 864572b252401fcecfa60bcba46b6196a09ddfc4 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Wed, 29 Nov 2023 18:04:16 -0500 Subject: [PATCH 17/82] Documentation update for OD/Linux --- README.md | 49 ++++++++++++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 8ca58e0560..191499f053 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ This is a toolset to manage production BYOND servers. It includes the ability to ### Pre-Requisites -_Note: If you opt to use the Windows installer, all pre-requisites (including MariaDB) are provided out of the box._ +_Note: If you opt to use the Windows installer, all pre-requisites for running BYOND servers (including MariaDB) are provided out of the box. If you wish to use OpenDream you will need to install the required dotnet SDK manually._ tgstation-server needs a relational database to store it's data. @@ -101,9 +101,19 @@ If using the console version, run `./tgs.bat` in the root of the installation di Installing natively is the recommended way to run tgstation-server on Linux. -##### Ubuntu +##### Ubuntu/Debian Package -Install TGS and all it's dependencies via our apt repository, interactively configure it, and start the service with this one-liner: +You first need to add the appropriate Microsoft package repository for your distribution + +Refer to the Microsoft website for steps for + +- [Ubuntu](https://learn.microsoft.com/en-us/dotnet/core/install/linux-ubuntu#register-the-microsoft-package-repository) +- [Debian 12](https://learn.microsoft.com/en-us/dotnet/core/install/linux-debian#debian-12) +- [Debian 11](https://learn.microsoft.com/en-us/dotnet/core/install/linux-debian#debian-11) +- [Debian 10](https://learn.microsoft.com/en-us/dotnet/core/install/linux-debian#debian-10) +- [Other Distros](https://learn.microsoft.com/en-us/dotnet/core/install/linux-scripted-manual#manual-install) + +After that, install TGS and all it's dependencies via our apt repository, interactively configure it, and start the service with this one-liner: ```sh sudo dpkg --add-architecture i386 \ @@ -119,25 +129,11 @@ sudo dpkg --add-architecture i386 \ The service will execute as the newly created user: `tgstation-server`. -##### Debian - -The `aspnetcore-runtime-8.0` package isn't yet available on mainline Debian and must be [installed from Microsoft](https://learn.microsoft.com/en-us/dotnet/core/install/linux-debian) first. Use the following one-liner to add their packages repository. - -```sh -curl -L https://packages.microsoft.com/config/debian/11/packages-microsoft-prod.deb -o packages-microsoft-prod.deb \ -&& sudo dpkg -i packages-microsoft-prod.deb \ -&& rm packages-microsoft-prod.deb -``` - -After that, run the same command as the Ubuntu installation. - -_Support for more distros coming soon_ - -##### Manual +##### Manual Setup The following dependencies are required. -- aspnetcore-runtime-8.0 (Note, not all supported distros have this package, see the links above for official Microsoft installation instructions) +- aspnetcore-runtime-8.0 (See Prerequisites under the `Ubuntu/Debian Package` section) - libc6-i386 - libstdc++6:i386 - gcc-multilib (Only on 64-bit systems) @@ -185,6 +181,21 @@ Note that automatic configuration reloading is currently not supported in the co If using manual configuration, before starting your container make sure the aforementioned `appsettings.Production.yml` is setup properly. See below +#### OpenDream + +In order for TGS to use [OpenDream](https://github.com/OpenDreamProject/OpenDream), it requires the full .NET SDK to build whichever version your servers target. Whatever that is, it must be available using the `dotnet` command for whichever user runs TGS. + +OpenDream currently requires [.NET SDK 7.0](https://dotnet.microsoft.com/en-us/download/dotnet/7.0) at the time of this writing. You must install this manually. + +On Linux, as long as OpenDream and TGS do not use the same .NET major version, you cannot achieve this with the package manager as they will conflict. The 7.0 SDK can be added to an 8.0 runtime installation via the following steps. + +1. Install `tgstation-server` using any of the above methods. +1. [Download the Linux SDK binaries](https://dotnet.microsoft.com/en-us/download/dotnet/7.0) for your selected architecture. +1. Extract ONLY the contents of the `sdk` directory in the `.tar.gz` to `/usr/share/dotnet/sdk/` +1. Run `sudo chown -R root /usr/share/dotnet/sdk` + +You should now be able to run the `dotnet --list-sdks` command and see an entry for `7.0.XXX [/usr/share/dotnet/sdk]`. + ### Configuring The first time you run TGS you should be prompted with a configuration wizard which will guide you through setting up your `appsettings.Production.yml` From 647d13f041d9b15eb34a51b04c84ce538fd2160a Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Wed, 29 Nov 2023 18:25:51 -0500 Subject: [PATCH 18/82] Fix `NO_WEBPANEL` builds --- src/Tgstation.Server.Host/Controllers/RootController.cs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/Tgstation.Server.Host/Controllers/RootController.cs b/src/Tgstation.Server.Host/Controllers/RootController.cs index 5de25b1451..3ccca028fe 100644 --- a/src/Tgstation.Server.Host/Controllers/RootController.cs +++ b/src/Tgstation.Server.Host/Controllers/RootController.cs @@ -130,16 +130,11 @@ namespace Tgstation.Server.Host.Controllers [HttpGet("logo.svg")] public IActionResult GetLogo() { -#if NO_WEBPANEL - logger.LogTrace("Cannot serve project logo as TGS was built without the webpanel!"); - return NotFound(); -#else var logoFileName = platformIdentifier.IsWindows // these are different because of motherfucking line endings -_- ? LogoSvgWindowsName : LogoSvgLinuxName; - return (IActionResult)this.TryServeFile(hostEnvironment, logger, $"{logoFileName}.svg"); -#endif + return (IActionResult)this.TryServeFile(hostEnvironment, logger, $"{logoFileName}.svg") ?? NotFound(); } } } From 5b42ad0f8f6d8f7ed8e51efc4c4dfc7fb8966fd4 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 2 Dec 2023 11:36:12 -0500 Subject: [PATCH 19/82] Weeding out JobsHubTest errors --- .../Live/Instance/JobsHubTests.cs | 39 ++++++++++++++----- .../Live/TestLiveServer.cs | 2 +- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs b/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs index d9456c22bc..e5f6bd0225 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs @@ -84,7 +84,7 @@ namespace Tgstation.Server.Tests.Live.Instance } } - public async Task Run(CancellationToken cancellationToken) + public async Task Run(CancellationToken cancellationToken) { var neverReceiver = new ShouldNeverReceiveUpdates() { @@ -98,16 +98,32 @@ namespace Tgstation.Server.Tests.Live.Instance }, }; - await using (permedConn = (HubConnection)await permedUser.SubscribeToJobUpdates( + permedConn = (HubConnection)await permedUser.SubscribeToJobUpdates( this, null, null, - cancellationToken)) - await using (permlessConn = (HubConnection)await permlessUser.SubscribeToJobUpdates( - neverReceiver, - null, - null, - cancellationToken)) + cancellationToken); + + try + { + permlessConn = (HubConnection)await permlessUser.SubscribeToJobUpdates( + neverReceiver, + null, + null, + cancellationToken); + } + catch + { + await permedConn.DisposeAsync(); + } + + return FinishAsync(cancellationToken); + } + + async Task FinishAsync(CancellationToken cancellationToken) + { + await using (permedConn) + await using (permlessConn) { Console.WriteLine($"Initial conn1: {permedConn.ConnectionId}"); Console.WriteLine($"Initial conn2: {permlessConn.ConnectionId}"); @@ -193,10 +209,15 @@ namespace Tgstation.Server.Tests.Live.Instance } } + static string JobListFormatter(IEnumerable jobs) => String.Join(Environment.NewLine, jobs.Select(x => $"- I:{x.InstanceId}|JID:{x.Id}|JC:{x.JobCode}|Desc:{x.Description}")); + // some instances may be detached, but our cache remains var accountedJobs = allJobs.Count - missableMissedJobs; var accountedSeenJobs = seenJobs.Where(x => allInstances.Any(i => i.Id.Value == x.Value.InstanceId)).ToList(); - Assert.AreEqual(accountedJobs, accountedSeenJobs.Count, $"Mismatch in seen jobs:{Environment.NewLine}{String.Join(Environment.NewLine, allJobs.Where(x => !seenJobs.Any(y => y.Key == x.Id.Value)).Select(x => $"- I:{x.InstanceId}|JID:{x.Id}|JC:{x.JobCode}|Desc:{x.Description}"))}"); + Assert.AreEqual( + accountedJobs, + accountedSeenJobs.Count, + $"Mismatch in seen jobs:{Environment.NewLine}Not seen in seen:{Environment.NewLine}{JobListFormatter(allJobs.Where(x => !seenJobs.Any(y => y.Key == x.Id.Value)))}{Environment.NewLine}Seen not in all:{Environment.NewLine}{JobListFormatter(seenJobs.Values.Where(x => !allJobs.Any(y => y.Id.Value == x.Id.Value)))}"); Assert.IsTrue(accountedJobs <= seenJobs.Count); Assert.AreNotEqual(0, permlessSeenJobs.Count); Assert.IsTrue(permlessSeenJobs.Count < seenJobs.Count); diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index a419fc648c..8aea9eb1ea 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -1395,11 +1395,11 @@ namespace Tgstation.Server.Tests.Live InstanceResponse odInstance, compatInstance; if (!openDreamOnly) { + jobsHubTestTask = FailFast(await jobsHubTest.Run(cancellationToken)); // returns Task var rootTest = FailFast(RawRequestTests.Run(clientFactory, firstAdminClient, cancellationToken)); var adminTest = FailFast(new AdministrationTest(firstAdminClient.Administration).Run(cancellationToken)); var usersTest = FailFast(new UsersTest(firstAdminClient).Run(cancellationToken)); - jobsHubTestTask = FailFast(jobsHubTest.Run(cancellationToken)); var instanceManagerTest = new InstanceManagerTest(firstAdminClient, server.Directory); var compatInstanceTask = instanceManagerTest.CreateTestInstance("CompatTestsInstance", cancellationToken); var odInstanceTask = instanceManagerTest.CreateTestInstance("OdTestsInstance", cancellationToken); From c1edbed40c5064cb7dd971437cca34a589d2e5cc Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 2 Dec 2023 17:13:20 -0500 Subject: [PATCH 20/82] Fix Linux .NET 7 SDK side-by-side install guide --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 191499f053..d2830e93d9 100644 --- a/README.md +++ b/README.md @@ -191,8 +191,8 @@ On Linux, as long as OpenDream and TGS do not use the same .NET major version, y 1. Install `tgstation-server` using any of the above methods. 1. [Download the Linux SDK binaries](https://dotnet.microsoft.com/en-us/download/dotnet/7.0) for your selected architecture. -1. Extract ONLY the contents of the `sdk` directory in the `.tar.gz` to `/usr/share/dotnet/sdk/` -1. Run `sudo chown -R root /usr/share/dotnet/sdk` +1. Extract everything EXCEPT the `dotnet` executable, `LICENSE.txt``, and `ThirdPartyNotices.txt` in the `.tar.gz` on top of the existing installation directory `/usr/share/dotnet/` +1. Run `sudo chown -R root /usr/share/dotnet` You should now be able to run the `dotnet --list-sdks` command and see an entry for `7.0.XXX [/usr/share/dotnet/sdk]`. From f406a897a20e79bb3e6f9b3674a29b46075952e9 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 3 Dec 2023 13:49:48 -0500 Subject: [PATCH 21/82] Fix a typo in `Chunker` --- .../Components/Interop/Chunker.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Interop/Chunker.cs b/src/Tgstation.Server.Host/Components/Interop/Chunker.cs index d6a0c7c024..85f6de0d6c 100644 --- a/src/Tgstation.Server.Host/Components/Interop/Chunker.cs +++ b/src/Tgstation.Server.Host/Components/Interop/Chunker.cs @@ -57,15 +57,15 @@ namespace Tgstation.Server.Host.Components.Interop /// /// Process a given . /// - /// The of communication that was chunked. + /// The of communication that was chunked. /// The of expected. - /// The callback that receives the completed . + /// The callback that receives the completed . /// The callback that generates a for a given error. /// The . /// The for the operation. /// A resulting in the for the chunked request. - protected async ValueTask ProcessChunk( - Func> completionCallback, + protected async ValueTask ProcessChunk( + Func> completionCallback, Func chunkErrorCallback, ChunkData chunk, CancellationToken cancellationToken) @@ -128,11 +128,11 @@ namespace Tgstation.Server.Host.Components.Interop chunkSets.Remove(requestInfo.PayloadId.Value); } - TCommnication completedCommunication; + TCommunication completedCommunication; var fullCommunicationJson = String.Concat(payloads); try { - completedCommunication = JsonConvert.DeserializeObject(fullCommunicationJson, DMApiConstants.SerializerSettings); + completedCommunication = JsonConvert.DeserializeObject(fullCommunicationJson, DMApiConstants.SerializerSettings); } catch (Exception ex) { From b29d93a66a6b7f21800def23cbf90132400569e6 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 3 Dec 2023 13:59:28 -0500 Subject: [PATCH 22/82] Migrate priority topic sending to an extension --- .../Components/Session/SessionController.cs | 38 +++-------- .../Extensions/TopicClientExtensions.cs | 68 +++++++++++++++++++ 2 files changed, 77 insertions(+), 29 deletions(-) create mode 100644 src/Tgstation.Server.Host/Extensions/TopicClientExtensions.cs diff --git a/src/Tgstation.Server.Host/Components/Session/SessionController.cs b/src/Tgstation.Server.Host/Components/Session/SessionController.cs index 9094cdf527..cefdcff76d 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionController.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.Globalization; using System.Linq; -using System.Net; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -22,6 +21,7 @@ using Tgstation.Server.Host.Components.Engine; using Tgstation.Server.Host.Components.Interop; using Tgstation.Server.Host.Components.Interop.Bridge; using Tgstation.Server.Host.Components.Interop.Topic; +using Tgstation.Server.Host.Extensions; using Tgstation.Server.Host.System; using Tgstation.Server.Host.Utils; @@ -884,35 +884,15 @@ namespace Tgstation.Server.Host.Components.Session } var targetPort = ReattachInformation.Port; - Byond.TopicSender.TopicResponse byondResponse = null; - var firstSend = true; - + Byond.TopicSender.TopicResponse byondResponse; using (await topicSendSemaphore.Lock(cancellationToken)) - { - const int PrioritySendAttempts = 5; - var endpoint = new IPEndPoint(IPAddress.Loopback, targetPort); - for (var i = PrioritySendAttempts - 1; i >= 0 && (priority || firstSend); --i) - try - { - firstSend = false; - - Logger.LogTrace("Begin topic request"); - byondResponse = await byondTopicSender.SendTopic( - endpoint, - queryString, - cancellationToken); - - Logger.LogTrace("End topic request"); - break; - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - Logger.LogWarning(ex, "SendTopic exception!{retryDetails}", priority ? $" {i} attempts remaining." : String.Empty); - - if (priority && i > 0) - await asyncDelayer.Delay(TimeSpan.FromSeconds(2), cancellationToken); - } - } + byondResponse = await byondTopicSender.SendWithOptionalPriority( + asyncDelayer, + Logger, + queryString, + targetPort, + priority, + cancellationToken); if (byondResponse == null) { diff --git a/src/Tgstation.Server.Host/Extensions/TopicClientExtensions.cs b/src/Tgstation.Server.Host/Extensions/TopicClientExtensions.cs new file mode 100644 index 0000000000..07cd38bd1f --- /dev/null +++ b/src/Tgstation.Server.Host/Extensions/TopicClientExtensions.cs @@ -0,0 +1,68 @@ +using System; +using System.Net; +using System.Threading; +using System.Threading.Tasks; + +using Byond.TopicSender; + +using Microsoft.Extensions.Logging; + +using Tgstation.Server.Host.Utils; + +namespace Tgstation.Server.Host.Extensions +{ + /// + /// Extension methods for . + /// + static class TopicClientExtensions + { + /// + /// Send a with optional repeated priority. + /// + /// The to send with. + /// The to use for delayed retries if an error occurs. + /// The to write to. + /// The to send. + /// The local port to send the topic to. + /// If priority retries should be used. + /// The for the operation. + /// A resulting in the on success, on failure. + public static async ValueTask SendWithOptionalPriority( + this ITopicClient topicClient, + IAsyncDelayer delayer, + ILogger logger, + string queryString, + ushort port, + bool priority, + CancellationToken cancellationToken) + { + const int PrioritySendAttempts = 5; + var endpoint = new IPEndPoint(IPAddress.Loopback, port); + var firstSend = true; + + for (var i = PrioritySendAttempts - 1; i >= 0 && (priority || firstSend); --i) + try + { + firstSend = false; + + logger.LogTrace("Begin topic request"); + var byondResponse = await topicClient.SendTopic( + endpoint, + queryString, + cancellationToken); + + logger.LogTrace("End topic request"); + return byondResponse; + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + logger.LogWarning(ex, "SendTopic exception!{retryDetails}", priority ? $" {i} attempts remaining." : String.Empty); + + if (priority && i > 0) + await delayer.Delay(TimeSpan.FromSeconds(2), cancellationToken); + } + + return null; + } + } +} From 060bd6126db8bd64e2d9086d7bc232c747619826 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 3 Dec 2023 14:09:29 -0500 Subject: [PATCH 23/82] Switch to using topic extension method in tests --- .../Live/Instance/WatchdogTest.cs | 74 +++++++++++-------- .../Live/TestLiveServer.cs | 12 ++- 2 files changed, 51 insertions(+), 35 deletions(-) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index affa17266e..e011fcc178 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -208,7 +208,9 @@ namespace Tgstation.Server.Tests.Live.Instance // reimplement TellWorldToReboot because it expects a new deployment and we don't care System.Console.WriteLine("TEST: Hack world reboot topic..."); - var result = await topicClient.SendTopic(IPAddress.Loopback, "tgs_integration_test_special_tactics=1", FindTopicPort(), cancellationToken); + var result = await SendTestTopic( + "tgs_integration_test_special_tactics=1", + cancellationToken); Assert.AreEqual("ack", result.StringData); using var tempCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); @@ -234,13 +236,25 @@ namespace Tgstation.Server.Tests.Live.Instance await RunTest(false); } + async ValueTask SendTestTopic(string queryString, CancellationToken cancellationToken) + { + using var loggerFactory = LoggerFactory.Create(builder => + { + builder.AddConsole(); + builder.SetMinimumLevel(LogLevel.Trace); + }); + return await topicClient.SendWithOptionalPriority( + new AsyncDelayer(), + loggerFactory.CreateLogger(), + queryString, + FindTopicPort(), + true, + cancellationToken); + } + async ValueTask BroadcastTest(CancellationToken cancellationToken) { - var topicRequestResult = await topicClient.SendTopic( - IPAddress.Loopback, - $"tgs_integration_test_tactics_broadcast=1", - FindTopicPort(), - cancellationToken); + var topicRequestResult = await SendTestTopic("tgs_integration_test_tactics_broadcast=1", cancellationToken); Assert.IsNotNull(topicRequestResult); Assert.AreEqual("!!NULL!!", topicRequestResult.StringData); @@ -251,10 +265,8 @@ namespace Tgstation.Server.Tests.Live.Instance BroadcastMessage = TestBroadcastMessage, }, cancellationToken); - topicRequestResult = await topicClient.SendTopic( - IPAddress.Loopback, - $"tgs_integration_test_tactics_broadcast=1", - FindTopicPort(), + topicRequestResult = await SendTestTopic( + "tgs_integration_test_tactics_broadcast=1", cancellationToken); Assert.IsNotNull(topicRequestResult); @@ -324,10 +336,8 @@ namespace Tgstation.Server.Tests.Live.Instance ValidateSessionId(currentStatus, true); Assert.AreEqual(expectedStaged.Id, currentStatus.ActiveCompileJob.Id); - var topicRequestResult = await topicClient.SendTopic( - IPAddress.Loopback, - $"shadow_wizard_money_gang=1", - FindTopicPort(), + var topicRequestResult = await SendTestTopic( + "shadow_wizard_money_gang=1", cancellationToken); Assert.IsNotNull(topicRequestResult); @@ -456,10 +466,8 @@ namespace Tgstation.Server.Tests.Live.Instance async Task SendChatOverloadCommand(CancellationToken cancellationToken) { // for the code coverage really... - var topicRequestResult = await topicClient.SendTopic( - IPAddress.Loopback, - $"tgs_integration_test_tactics5=1", - FindTopicPort(), + var topicRequestResult = await SendTestTopic( + "tgs_integration_test_tactics5=1", cancellationToken); Assert.IsNotNull(topicRequestResult); @@ -740,10 +748,8 @@ namespace Tgstation.Server.Tests.Live.Instance Assert.IsFalse(ddProc.HasExited); // check DD agrees - var topicRequestResult = await topicClient.SendTopic( - IPAddress.Loopback, - $"tgs_integration_test_tactics8=1", - FindTopicPort(), + var topicRequestResult = await SendTestTopic( + "tgs_integration_test_tactics8=1", cancellationToken); Assert.IsNotNull(topicRequestResult); @@ -852,7 +858,9 @@ namespace Tgstation.Server.Tests.Live.Instance System.Console.WriteLine("TEST: Sending Bridge tests topic..."); - var bridgeTestTopicResult = await topicClient.SendTopic(IPAddress.Loopback, $"tgs_integration_test_tactics2={accessIdentifier}", FindTopicPort(), cancellationToken); + var bridgeTestTopicResult = await SendTestTopic( + $"tgs_integration_test_tactics2={accessIdentifier}", + cancellationToken); Assert.AreEqual("ack2", bridgeTestTopicResult.StringData); await bridgeTestsTcs.Task.WaitAsync(cancellationToken); @@ -902,10 +910,8 @@ namespace Tgstation.Server.Tests.Live.Instance try { System.Console.WriteLine($"Topic send limit test S:{currentSize}..."); - topicRequestResult = await topicClient.SendTopic( - IPAddress.Loopback, + topicRequestResult = await SendTestTopic( $"tgs_integration_test_tactics3={topicClient.SanitizeString(JsonConvert.SerializeObject(topic, DMApiConstants.SerializerSettings))}", - FindTopicPort(), cancellationToken); } catch (ArgumentOutOfRangeException) @@ -943,10 +949,8 @@ namespace Tgstation.Server.Tests.Live.Instance { var currentSize = baseSize + (int)Math.Pow(2, nextPow); System.Console.WriteLine($"Topic recieve limit test S:{currentSize}..."); - var topicRequestResult = await topicClient.SendTopic( - IPAddress.Loopback, + var topicRequestResult = await SendTestTopic( $"tgs_integration_test_tactics4={topicClient.SanitizeString(currentSize.ToString())}", - FindTopicPort(), cancellationToken); if (topicRequestResult.ResponseType != TopicResponseType.StringResponse @@ -1332,7 +1336,13 @@ namespace Tgstation.Server.Tests.Live.Instance System.Console.WriteLine("TEST: Sending world reboot topic..."); - var result = await topicClient.SendTopic(IPAddress.Loopback, "tgs_integration_test_special_tactics=1", topicPort, cancellationToken); + var result = await topicClient.SendWithOptionalPriority( + new AsyncDelayer(), + Mock.Of(), + $"tgs_integration_test_special_tactics=1", + topicPort, + true, + cancellationToken); Assert.AreEqual("ack", result.StringData); using var tempCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); @@ -1468,7 +1478,9 @@ namespace Tgstation.Server.Tests.Live.Instance async ValueTask TestLegacyBridgeEndpoint(CancellationToken cancellationToken) { System.Console.WriteLine("TEST: TestLegacyBridgeEndpoint"); - var result = await topicClient.SendTopic(IPAddress.Loopback, "im_out_of_memes=1", FindTopicPort(), cancellationToken); + var result = await SendTestTopic( + "im_out_of_memes=1", + cancellationToken); Assert.IsNotNull(result); Assert.AreEqual("all gucci", result.StringData); await CheckDMApiFail((await instanceClient.DreamDaemon.Read(cancellationToken)).ActiveCompileJob, cancellationToken); diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index 8aea9eb1ea..a968d57219 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -1532,10 +1532,12 @@ namespace Tgstation.Server.Tests.Live // test the reattach message queueing // for the code coverage really... - var topicRequestResult = await WatchdogTest.StaticTopicClient.SendTopic( - IPAddress.Loopback, + var topicRequestResult = await WatchdogTest.StaticTopicClient.SendWithOptionalPriority( + new AsyncDelayer(), + Mock.Of(), $"tgs_integration_test_tactics6=1", mainDDPort, + true, cancellationToken); Assert.IsNotNull(topicRequestResult); @@ -1608,10 +1610,12 @@ namespace Tgstation.Server.Tests.Live var chatReadTask = instanceClient.ChatBots.List(null, cancellationToken); // Check the DMAPI got the channels again https://github.com/tgstation/tgstation-server/issues/1490 - topicRequestResult = await WatchdogTest.StaticTopicClient.SendTopic( - IPAddress.Loopback, + topicRequestResult = await WatchdogTest.StaticTopicClient.SendWithOptionalPriority( + new AsyncDelayer(), + Mock.Of(), $"tgs_integration_test_tactics7=1", mainDDPort, + true, cancellationToken); Assert.IsNotNull(topicRequestResult); From 15aa09597012625c94675cd41ae06adde761c8d4 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 3 Dec 2023 14:11:48 -0500 Subject: [PATCH 24/82] Prominent errored windows tests artifact names --- .github/workflows/ci-pipeline.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 79bc776719..41d156de2a 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -564,11 +564,19 @@ jobs: fi - name: Store Live Tests Output + if: ${{ steps.live-tests.outputs.succeeded == 'YES' }} uses: actions/upload-artifact@v3 with: name: windows-integration-test-logs-${{ matrix.configuration }}-${{ matrix.watchdog-type }}-${{ matrix.database-type }} path: ./test_output.txt + - name: Store Errored Live Tests Output + if: ${{ steps.live-tests.outputs.succeeded != 'YES' }} + uses: actions/upload-artifact@v3 + with: + name: errored-windows-test-logs-${{ matrix.configuration }}-${{ matrix.watchdog-type }}-${{ matrix.database-type }} + path: ./test_output.txt + - name: Fail if Live Tests Failed if: ${{ steps.live-tests.outputs.succeeded != 'YES' }} run: exit 1 From d1e26bc37429b94a0f376374d49f73807b970bb4 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 3 Dec 2023 14:18:18 -0500 Subject: [PATCH 25/82] Fix dumping multiple times in the same second --- .../Components/Watchdog/WatchdogBase.cs | 29 ++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs index 33961aa1f6..59aebe0c57 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs @@ -447,19 +447,28 @@ namespace Tgstation.Server.Host.Components.Watchdog public async ValueTask CreateDump(CancellationToken cancellationToken) { const string DumpDirectory = "ProcessDumps"; - await diagnosticsIOManager.CreateDirectory(DumpDirectory, cancellationToken); + using (await SemaphoreSlimContext.Lock(synchronizationSemaphore, cancellationToken)) + { + var dumpFileNameTemplate = diagnosticsIOManager.ResolvePath( + diagnosticsIOManager.ConcatPath( + DumpDirectory, + $"DreamDaemon-{DateTimeOffset.UtcNow.ToFileStamp()}.dmp")); - var dumpFileName = diagnosticsIOManager.ResolvePath( - diagnosticsIOManager.ConcatPath( - DumpDirectory, - $"DreamDaemon-{DateTimeOffset.UtcNow.ToFileStamp()}.dmp")); + var dumpFileName = dumpFileNameTemplate; + var iteration = 0; + while (await diagnosticsIOManager.FileExists(dumpFileName, cancellationToken)) + dumpFileName = $"{dumpFileNameTemplate} ({++iteration})"; - var session = GetActiveController(); - if (session?.Lifetime.IsCompleted != false) - throw new JobException(ErrorCode.GameServerOffline); + if (iteration == 0) + await diagnosticsIOManager.CreateDirectory(DumpDirectory, cancellationToken); - Logger.LogInformation("Dumping session to {dumpFileName}...", dumpFileName); - await session.CreateDump(dumpFileName, cancellationToken); + var session = GetActiveController(); + if (session?.Lifetime.IsCompleted != false) + throw new JobException(ErrorCode.GameServerOffline); + + Logger.LogInformation("Dumping session to {dumpFileName}...", dumpFileName); + await session.CreateDump(dumpFileName, cancellationToken); + } } /// From c6ac59529046e4070933266d6cf2624dea21d304 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 3 Dec 2023 14:21:53 -0500 Subject: [PATCH 26/82] Improve an assert --- tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index e011fcc178..0668eb4d0a 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -923,7 +923,11 @@ namespace Tgstation.Server.Tests.Live.Instance || topicRequestResult.StringData != "pass") { if (topicRequestResult != null) + { + Assert.AreEqual(TopicResponseType.StringResponse, topicRequestResult.ResponseType, $"String data is: {topicRequestResult.StringData ?? "<>"}"); Assert.AreEqual("fail", topicRequestResult.StringData); + } + if (currentSize == lastSize + 1) break; baseSize = lastSize; From 10188c782fe3ff7504c51c71b0dd50fe425fd4e1 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 3 Dec 2023 14:29:38 -0500 Subject: [PATCH 27/82] Hopefully fix bug in JobsHubTest --- .../Live/Instance/JobsHubTests.cs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs b/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs index e5f6bd0225..099877690f 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs @@ -211,13 +211,18 @@ namespace Tgstation.Server.Tests.Live.Instance static string JobListFormatter(IEnumerable jobs) => String.Join(Environment.NewLine, jobs.Select(x => $"- I:{x.InstanceId}|JID:{x.Id}|JC:{x.JobCode}|Desc:{x.Description}")); + var jobsSeenByHubButNotInAllJobs = seenJobs.Values.Where(x => !allJobs.Any(y => y.Id.Value == x.Id.Value)).ToList(); + // some instances may be detached, but our cache remains var accountedJobs = allJobs.Count - missableMissedJobs; - var accountedSeenJobs = seenJobs.Where(x => allInstances.Any(i => i.Id.Value == x.Value.InstanceId)).ToList(); + var errorMessage = $"Mismatch in seen jobs:{Environment.NewLine}Not seen in seen:{Environment.NewLine}{JobListFormatter(allJobs.Where(x => !seenJobs.Any(y => y.Key == x.Id.Value)))}{Environment.NewLine}Seen not in all:{Environment.NewLine}{JobListFormatter(jobsSeenByHubButNotInAllJobs)}"; Assert.AreEqual( accountedJobs, - accountedSeenJobs.Count, - $"Mismatch in seen jobs:{Environment.NewLine}Not seen in seen:{Environment.NewLine}{JobListFormatter(allJobs.Where(x => !seenJobs.Any(y => y.Key == x.Id.Value)))}{Environment.NewLine}Seen not in all:{Environment.NewLine}{JobListFormatter(seenJobs.Values.Where(x => !allJobs.Any(y => y.Id.Value == x.Id.Value)))}"); + seenJobs.Count - jobsSeenByHubButNotInAllJobs.Count, + errorMessage); + Assert.IsTrue( + jobsSeenByHubButNotInAllJobs.All(job => job.JobCode.Value == JobCode.Move), + errorMessage); Assert.IsTrue(accountedJobs <= seenJobs.Count); Assert.AreNotEqual(0, permlessSeenJobs.Count); Assert.IsTrue(permlessSeenJobs.Count < seenJobs.Count); From 69f4c7e0b69500baf4e94f133864a92aa76ed699 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 3 Dec 2023 14:38:45 -0500 Subject: [PATCH 28/82] Add some diagnostic messages for tests --- tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index 0668eb4d0a..bdd0180e9c 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -763,7 +763,9 @@ namespace Tgstation.Server.Tests.Live.Instance }, cancellationToken); ValidateSessionId(ddStatus, true); + global::System.Console.WriteLine($"WATCHDOG TEST {instanceClient.Metadata.Id}: COMMENCE PROCESS SUSPEND FOR HEALTH CHECK DEATH PID {ourProcessHandler.Id}."); ourProcessHandler.Suspend(); + global::System.Console.WriteLine($"WATCHDOG TEST {instanceClient.Metadata.Id}: FINISH PROCESS SUSPEND FOR HEALTH CHECK DEATH. WAITING FOR LIFETIME {ourProcessHandler.Id}."); await Task.WhenAny(ourProcessHandler.Lifetime, Task.Delay(TimeSpan.FromMinutes(2), cancellationToken)); Assert.IsTrue(ourProcessHandler.Lifetime.IsCompleted); From da78b8e8a15b3cb059bab11f6012e1cebfebf5c8 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 3 Dec 2023 14:53:45 -0500 Subject: [PATCH 29/82] Fix deadlock with `DumpOnHealthCheckRestart` --- .../Components/Watchdog/WatchdogBase.cs | 54 +++++++++++-------- 1 file changed, 31 insertions(+), 23 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs index 59aebe0c57..60b3deb05f 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs @@ -446,29 +446,8 @@ namespace Tgstation.Server.Host.Components.Watchdog /// public async ValueTask CreateDump(CancellationToken cancellationToken) { - const string DumpDirectory = "ProcessDumps"; using (await SemaphoreSlimContext.Lock(synchronizationSemaphore, cancellationToken)) - { - var dumpFileNameTemplate = diagnosticsIOManager.ResolvePath( - diagnosticsIOManager.ConcatPath( - DumpDirectory, - $"DreamDaemon-{DateTimeOffset.UtcNow.ToFileStamp()}.dmp")); - - var dumpFileName = dumpFileNameTemplate; - var iteration = 0; - while (await diagnosticsIOManager.FileExists(dumpFileName, cancellationToken)) - dumpFileName = $"{dumpFileNameTemplate} ({++iteration})"; - - if (iteration == 0) - await diagnosticsIOManager.CreateDirectory(DumpDirectory, cancellationToken); - - var session = GetActiveController(); - if (session?.Lifetime.IsCompleted != false) - throw new JobException(ErrorCode.GameServerOffline); - - Logger.LogInformation("Dumping session to {dumpFileName}...", dumpFileName); - await session.CreateDump(dumpFileName, cancellationToken); - } + await CreateDumpNoLock(cancellationToken); } /// @@ -1153,7 +1132,7 @@ namespace Tgstation.Server.Host.Components.Watchdog Logger.LogDebug("DumpOnHealthCheckRestart enabled."); try { - await CreateDump(cancellationToken); + await CreateDumpNoLock(cancellationToken); } catch (JobException ex) { @@ -1203,5 +1182,34 @@ namespace Tgstation.Server.Host.Components.Watchdog .Where(nullableChannelId => nullableChannelId.HasValue) .Select(nullableChannelId => nullableChannelId.Value)); } + + /// + /// Attempt to create a process dump for the game server. Requires a lock on . + /// + /// The for the operation. + /// A representing the running operation. + async ValueTask CreateDumpNoLock(CancellationToken cancellationToken) + { + const string DumpDirectory = "ProcessDumps"; + var dumpFileNameTemplate = diagnosticsIOManager.ResolvePath( + diagnosticsIOManager.ConcatPath( + DumpDirectory, + $"DreamDaemon-{DateTimeOffset.UtcNow.ToFileStamp()}.dmp")); + + var dumpFileName = dumpFileNameTemplate; + var iteration = 0; + while (await diagnosticsIOManager.FileExists(dumpFileName, cancellationToken)) + dumpFileName = $"{dumpFileNameTemplate} ({++iteration})"; + + if (iteration == 0) + await diagnosticsIOManager.CreateDirectory(DumpDirectory, cancellationToken); + + var session = GetActiveController(); + if (session?.Lifetime.IsCompleted != false) + throw new JobException(ErrorCode.GameServerOffline); + + Logger.LogInformation("Dumping session to {dumpFileName}...", dumpFileName); + await session.CreateDump(dumpFileName, cancellationToken); + } } } From 6d6645e9a20697d13d3dfa1b5dcf1d0fa3a357b4 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 4 Dec 2023 22:25:06 -0500 Subject: [PATCH 30/82] Add a diagnostic message to an `Assert` --- tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs b/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs index 099877690f..17fb4c0d89 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs @@ -204,7 +204,7 @@ namespace Tgstation.Server.Tests.Live.Instance var wasMissableJob = job.JobCode == JobCode.ReconnectChatBot || job.JobCode == JobCode.StartupWatchdogLaunch || job.JobCode == JobCode.StartupWatchdogReattach; - Assert.IsTrue(wasMissableJob); + Assert.IsTrue(wasMissableJob, $"Found unexpected missed job: {job.Description}"); ++missableMissedJobs; } } From 081b8d19a6e2ea5d7dfdce03a3c5a90ed5dd9849 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 4 Dec 2023 22:26:59 -0500 Subject: [PATCH 31/82] Assert that the process is running before the Linux links test --- tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index bdd0180e9c..576d025fef 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -680,7 +680,9 @@ namespace Tgstation.Server.Tests.Live.Instance var pid = proc.Id; var foundLivePath = false; var allPaths = new List(); - foreach (var fd in Directory.EnumerateFiles($"/proc/{pid}/fd")) + + Assert.IsFalse(proc.HasExited); + foreach (var fd in Directory.GetFiles($"/proc/{pid}/fd")) { var sb = new StringBuilder(UInt16.MaxValue); if (Syscall.readlink(fd, sb) == -1) From 711c04bc03c5a251ea84b2eb256291443c7ed5b7 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Tue, 12 Dec 2023 10:55:04 -0500 Subject: [PATCH 32/82] Update Nuget packages --- build/NugetCommon.props | 2 +- src/Tgstation.Server.Host/Tgstation.Server.Host.csproj | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build/NugetCommon.props b/build/NugetCommon.props index d1c341efd0..f07a00a656 100644 --- a/build/NugetCommon.props +++ b/build/NugetCommon.props @@ -25,7 +25,7 @@ - + diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index 5be73fa4ee..04f82f6b2d 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -107,7 +107,7 @@ - + @@ -125,7 +125,7 @@ - + From 5231bd8a816a91595c5d50a91f33d80f914fa296 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Wed, 13 Dec 2023 21:49:22 -0500 Subject: [PATCH 33/82] Update to LibGit2Sharp 0.29.0 --- .../Components/Repository/Repository.cs | 76 ++++------------ .../Repository/RepositoryManager.cs | 21 ++--- .../Extensions/FetchOptionsExtensions.cs | 91 +++++++++++++++++++ .../Tgstation.Server.Host.csproj | 2 +- .../Repository/TestRepositoryFactory.cs | 9 +- 5 files changed, 122 insertions(+), 77 deletions(-) create mode 100644 src/Tgstation.Server.Host/Extensions/FetchOptionsExtensions.cs diff --git a/src/Tgstation.Server.Host/Components/Repository/Repository.cs b/src/Tgstation.Server.Host/Components/Repository/Repository.cs index a3b81d2bad..125ceded30 100644 --- a/src/Tgstation.Server.Host/Components/Repository/Repository.cs +++ b/src/Tgstation.Server.Host/Components/Repository/Repository.cs @@ -247,20 +247,16 @@ namespace Tgstation.Server.Host.Components.Repository libGitRepo, refSpecList, remote, - new FetchOptions - { - Prune = true, - OnProgress = (a) => !cancellationToken.IsCancellationRequested, - OnTransferProgress = TransferProgressHandler( - progressReporter.CreateSection($"Fetch {refSpec}", progressFactor), - cancellationToken), - OnUpdateTips = (a, b, c) => !cancellationToken.IsCancellationRequested, - CredentialsProvider = credentialsProvider.GenerateCredentialsHandler(username, password), - }, + new FetchOptions().Hydrate( + logger, + progressReporter.CreateSection($"Fetch {refSpec}", progressFactor), + credentialsProvider.GenerateCredentialsHandler(username, password), + cancellationToken), logMessage); } - catch (UserCancelledException) + catch (UserCancelledException ex) { + logger.LogTrace(ex, "Suppressing fetch cancel exception"); } catch (LibGit2SharpException ex) { @@ -439,14 +435,12 @@ namespace Tgstation.Server.Host.Components.Repository var fetchOptions = new FetchOptions { Prune = true, - OnProgress = (a) => !cancellationToken.IsCancellationRequested, - OnUpdateTips = (a, b, c) => !cancellationToken.IsCancellationRequested, - CredentialsProvider = credentialsProvider.GenerateCredentialsHandler(username, password), TagFetchMode = TagFetchMode.All, - }; - - if (progressReporter != null) - fetchOptions.OnTransferProgress = TransferProgressHandler(progressReporter.CreateSection("Fetch Origin", 1.0), cancellationToken); + }.Hydrate( + logger, + progressReporter?.CreateSection("Fetch Origin", 1.0), + credentialsProvider.GenerateCredentialsHandler(username, password), + cancellationToken); commands.Fetch( libGitRepo, @@ -1041,19 +1035,18 @@ namespace Tgstation.Server.Host.Components.Repository var submoduleUpdateOptions = new SubmoduleUpdateOptions { Init = true, - OnProgress = output => !cancellationToken.IsCancellationRequested, - OnUpdateTips = (a, b, c) => !cancellationToken.IsCancellationRequested, - CredentialsProvider = credentialsProvider.GenerateCredentialsHandler(username, password), + OnCheckoutNotify = (_, _) => !cancellationToken.IsCancellationRequested, }; + submoduleUpdateOptions.FetchOptions.Hydrate( + logger, + progressReporter?.CreateSection($"Fetch submodule {submodule.Name}", factor), + credentialsProvider.GenerateCredentialsHandler(username, password), + cancellationToken); + if (progressReporter != null) - { - submoduleUpdateOptions.OnTransferProgress = TransferProgressHandler( - progressReporter.CreateSection($"Fetch submodule {submodule.Name}", factor), - cancellationToken); submoduleUpdateOptions.OnCheckoutProgress = CheckoutProgressHandler( progressReporter.CreateSection($"Checkout submodule {submodule.Name}", factor)); - } logger.LogDebug("Updating submodule {submoduleName}...", submodule.Name); Task RawSubModuleUpdate() => Task.Factory.StartNew( @@ -1131,37 +1124,6 @@ namespace Tgstation.Server.Host.Components.Repository progressReporter.ReportProgress(percentage); }; - - /// - /// Generate a from a given and . - /// - /// The of the operation. - /// The for the operation. - /// A new based on . - TransferProgressHandler TransferProgressHandler(JobProgressReporter progressReporter, CancellationToken cancellationToken) => (transferProgress) => - { - double? percentage; - var totalObjectsToProcess = transferProgress.TotalObjects * 2; - var processedObjects = transferProgress.IndexedObjects + transferProgress.ReceivedObjects; - if (totalObjectsToProcess < processedObjects || totalObjectsToProcess == 0) - percentage = null; - else - { - percentage = (double)processedObjects / totalObjectsToProcess; - if (percentage < 0) - percentage = null; - } - - if (percentage == null) - logger.LogDebug( - "Bad transfer progress values (Please tell Cyberboss)! Indexed: {indexed}, Received: {received}, Total: {total}", - transferProgress.IndexedObjects, - transferProgress.ReceivedObjects, - transferProgress.TotalObjects); - - progressReporter.ReportProgress(percentage); - return !cancellationToken.IsCancellationRequested; - }; } #pragma warning restore CA1506 } diff --git a/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs b/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs index 5852a58f97..8ce9b60318 100644 --- a/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs +++ b/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs @@ -9,6 +9,7 @@ using Microsoft.Extensions.Logging; using Tgstation.Server.Api.Models; using Tgstation.Server.Host.Components.Events; using Tgstation.Server.Host.Configuration; +using Tgstation.Server.Host.Extensions; using Tgstation.Server.Host.IO; using Tgstation.Server.Host.Jobs; using Tgstation.Server.Host.Utils; @@ -149,20 +150,7 @@ namespace Tgstation.Server.Host.Components.Repository var checkoutProgressReporter = progressReporter?.CreateSection(null, 0.25f); var cloneOptions = new CloneOptions { - OnProgress = (a) => !cancellationToken.IsCancellationRequested, - OnTransferProgress = (a) => - { - if (cloneProgressReporter != null) - { - var percentage = ((double)a.IndexedObjects + a.ReceivedObjects) / (a.TotalObjects * 2); - cloneProgressReporter.ReportProgress(percentage); - } - - return !cancellationToken.IsCancellationRequested; - }, RecurseSubmodules = recurseSubmodules, - OnUpdateTips = (a, b, c) => !cancellationToken.IsCancellationRequested, - RepositoryOperationStarting = (a) => !cancellationToken.IsCancellationRequested, OnCheckoutProgress = (path, completed, remaining) => { if (checkoutProgressReporter == null) @@ -172,9 +160,14 @@ namespace Tgstation.Server.Host.Components.Repository checkoutProgressReporter.ReportProgress(percentage); }, BranchName = initialBranch, - CredentialsProvider = repositoryFactory.GenerateCredentialsHandler(username, password), }; + cloneOptions.FetchOptions.Hydrate( + logger, + cloneProgressReporter, + repositoryFactory.GenerateCredentialsHandler(username, password), + cancellationToken); + await repositoryFactory.Clone( url, cloneOptions, diff --git a/src/Tgstation.Server.Host/Extensions/FetchOptionsExtensions.cs b/src/Tgstation.Server.Host/Extensions/FetchOptionsExtensions.cs new file mode 100644 index 0000000000..e1b47d644a --- /dev/null +++ b/src/Tgstation.Server.Host/Extensions/FetchOptionsExtensions.cs @@ -0,0 +1,91 @@ +using System; +using System.Threading; + +using LibGit2Sharp; +using LibGit2Sharp.Handlers; + +using Microsoft.Extensions.Logging; + +using Tgstation.Server.Host.Jobs; + +namespace Tgstation.Server.Host.Extensions +{ + /// + /// Extension methods for . + /// + static class FetchOptionsExtensions + { + /// + /// Hydrate a given set of . + /// + /// The to hydrate. + /// The for the operation. + /// The optional . + /// The optional . + /// The for the operation. + /// The hydrated . + public static FetchOptions Hydrate( + this FetchOptions fetchOptions, + ILogger logger, + JobProgressReporter progressReporter, + CredentialsHandler credentialsHandler, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(fetchOptions); + ArgumentNullException.ThrowIfNull(logger); + + fetchOptions.OnProgress = _ => !cancellationToken.IsCancellationRequested; + fetchOptions.OnTransferProgress = transferProgress => + { + if (progressReporter != null) + { + var percentage = ((double)transferProgress.IndexedObjects + transferProgress.ReceivedObjects) / (transferProgress.TotalObjects * 2); + progressReporter.ReportProgress(percentage); + } + + return !cancellationToken.IsCancellationRequested; + }; + fetchOptions.OnUpdateTips = (_, _, _) => !cancellationToken.IsCancellationRequested; + fetchOptions.CredentialsProvider = credentialsHandler; + fetchOptions.RepositoryOperationStarting = _ => !cancellationToken.IsCancellationRequested; + fetchOptions.OnTransferProgress = TransferProgressHandler( + logger, + progressReporter, + cancellationToken); + + return fetchOptions; + } + + /// + /// Generate a from a given and . + /// + /// The for the operation. + /// The optional of the operation. + /// The for the operation. + /// A new based on . + static TransferProgressHandler TransferProgressHandler(ILogger logger, JobProgressReporter progressReporter, CancellationToken cancellationToken) => transferProgress => + { + double? percentage; + var totalObjectsToProcess = transferProgress.TotalObjects * 2; + var processedObjects = transferProgress.IndexedObjects + transferProgress.ReceivedObjects; + if (totalObjectsToProcess < processedObjects || totalObjectsToProcess == 0) + percentage = null; + else + { + percentage = (double)processedObjects / totalObjectsToProcess; + if (percentage < 0) + percentage = null; + } + + if (percentage == null) + logger.LogDebug( + "Bad transfer progress values (Please tell Cyberboss)! Indexed: {indexed}, Received: {received}, Total: {total}", + transferProgress.IndexedObjects, + transferProgress.ReceivedObjects, + transferProgress.TotalObjects); + + progressReporter?.ReportProgress(percentage); + return !cancellationToken.IsCancellationRequested; + }; + } +} diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index 04f82f6b2d..af8a133ea0 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -73,7 +73,7 @@ - + diff --git a/tests/Tgstation.Server.Host.Tests/Components/Repository/TestRepositoryFactory.cs b/tests/Tgstation.Server.Host.Tests/Components/Repository/TestRepositoryFactory.cs index bbf369efce..04e7cdb66b 100644 --- a/tests/Tgstation.Server.Host.Tests/Components/Repository/TestRepositoryFactory.cs +++ b/tests/Tgstation.Server.Host.Tests/Components/Repository/TestRepositoryFactory.cs @@ -1,4 +1,4 @@ -using LibGit2Sharp; +using LibGit2Sharp; using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; @@ -39,12 +39,11 @@ namespace Tgstation.Server.Host.Components.Repository.Tests try { var factory = CreateFactory(); + var cloneOpts = new CloneOptions(); + cloneOpts.FetchOptions.CredentialsProvider = factory.GenerateCredentialsHandler(null, null); await factory.Clone( new Uri("https://github.com/Cyberboss/Test"), - new CloneOptions - { - CredentialsProvider = factory.GenerateCredentialsHandler(null, null) - }, + cloneOpts, tempDir, default); From ef0bf6c0cf54356d90485fbdb5154fe7a9f0b0c2 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Wed, 13 Dec 2023 22:59:50 -0500 Subject: [PATCH 34/82] Force this variable to check for `true` --- tests/Tgstation.Server.Tests/Live/TestLiveServer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index a968d57219..52b0b3f90e 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -1213,7 +1213,7 @@ namespace Tgstation.Server.Tests.Live [TestMethod] public Task TestOpenDreamExclusiveTgsOperation() { - if (String.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("TGS_TEST_OD_EXCLUSIVE"))) + if (Environment.GetEnvironmentVariable("TGS_TEST_OD_EXCLUSIVE") != "true") Assert.Inconclusive("This test is covered by TestStandardTgsOperation"); return TestStandardTgsOperation(true); From 7af656d3d53d1814b9402aadb472416ab8f432a6 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Wed, 13 Dec 2023 23:00:14 -0500 Subject: [PATCH 35/82] Some better test crash diagnostics --- tests/Tgstation.Server.Tests/TestSystemInteraction.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/Tgstation.Server.Tests/TestSystemInteraction.cs b/tests/Tgstation.Server.Tests/TestSystemInteraction.cs index 1075b698d3..4570c58b94 100644 --- a/tests/Tgstation.Server.Tests/TestSystemInteraction.cs +++ b/tests/Tgstation.Server.Tests/TestSystemInteraction.cs @@ -69,12 +69,12 @@ namespace Tgstation.Server.Tests Assert.AreEqual(0, exitCode); } - Assert.IsTrue(File.Exists(tempFile)); + Assert.IsTrue(File.Exists(tempFile), $"Could not find temp file: {tempFile}"); var result = File.ReadAllText(tempFile).Trim(); // no guarantees about order - Assert.IsTrue(result.Contains("Hello World!")); - Assert.IsTrue(result.Contains("Hello Error!")); + Assert.IsTrue(result.Contains("Hello World!"), $"Result: {result}"); + Assert.IsTrue(result.Contains("Hello Error!"), $"Result: {result}"); } finally { From 4f63b9619b4b2f2f4103e3325a35803c09f7707c Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Thu, 14 Dec 2023 17:48:16 -0500 Subject: [PATCH 36/82] Add some whitespace --- .../Components/Session/TopicClientFactory.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Tgstation.Server.Host/Components/Session/TopicClientFactory.cs b/src/Tgstation.Server.Host/Components/Session/TopicClientFactory.cs index 40605e5de8..29768e7604 100644 --- a/src/Tgstation.Server.Host/Components/Session/TopicClientFactory.cs +++ b/src/Tgstation.Server.Host/Components/Session/TopicClientFactory.cs @@ -1,6 +1,7 @@ using System; using Byond.TopicSender; + using Microsoft.Extensions.Logging; namespace Tgstation.Server.Host.Components.Session From 3921e60dc70f4921dcdafacc5045989da9135202 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Thu, 14 Dec 2023 22:42:14 -0500 Subject: [PATCH 37/82] Add a log line --- .../Components/Watchdog/AdvancedWatchdog.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Tgstation.Server.Host/Components/Watchdog/AdvancedWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/AdvancedWatchdog.cs index 33140b9e57..4495a99fe3 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/AdvancedWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/AdvancedWatchdog.cs @@ -395,6 +395,7 @@ namespace Tgstation.Server.Host.Components.Watchdog try { + Logger.LogTrace("Making new provider {id} active...", newProvider.CompileJob.Id); await newProvider.MakeActive(cancellationToken); } finally From 7b3fa4a53cf9dab6e2c73779ce23d62ecefef452 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Thu, 14 Dec 2023 23:10:12 -0500 Subject: [PATCH 38/82] Some additional test diagnostics --- .../Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index 576d025fef..8fcccf0589 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -18,6 +18,7 @@ using System.Linq; using System.Net; using System.Net.Sockets; using System.Reflection; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Text; using System.Threading; @@ -1334,15 +1335,15 @@ namespace Tgstation.Server.Tests.Live.Instance return ddProc != null; } - public Task TellWorldToReboot(bool waitForOnlineIfRestoring, CancellationToken cancellationToken) - => TellWorldToReboot2(instanceClient, topicClient, FindTopicPort(), waitForOnlineIfRestoring || testVersion.Engine.Value == EngineType.OpenDream, cancellationToken); - public static async Task TellWorldToReboot2(IInstanceClient instanceClient, ITopicClient topicClient, ushort topicPort, bool waitForOnlineIfRestoring, CancellationToken cancellationToken) + public Task TellWorldToReboot(bool waitForOnlineIfRestoring, CancellationToken cancellationToken, [CallerLineNumber]int source = 0) + => TellWorldToReboot2(instanceClient, topicClient, FindTopicPort(), waitForOnlineIfRestoring || testVersion.Engine.Value == EngineType.OpenDream, cancellationToken, source); + public static async Task TellWorldToReboot2(IInstanceClient instanceClient, ITopicClient topicClient, ushort topicPort, bool waitForOnlineIfRestoring, CancellationToken cancellationToken, [CallerLineNumber]int source = 0, [CallerFilePath]string path = null) { var daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.IsNotNull(daemonStatus.StagedCompileJob); var initialSession = daemonStatus.ActiveCompileJob; - System.Console.WriteLine("TEST: Sending world reboot topic..."); + System.Console.WriteLine($"TEST: Sending world reboot topic @ {path}#L{source}"); var result = await topicClient.SendWithOptionalPriority( new AsyncDelayer(), From ea1fecb4fd547c1be9e3b622427e19534e34e0bd Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Thu, 14 Dec 2023 23:11:25 -0500 Subject: [PATCH 39/82] Update `Byond.TopicSender` to v7.1.0 --- src/Tgstation.Server.Host/Tgstation.Server.Host.csproj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index af8a133ea0..448d040017 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -63,7 +63,7 @@ - + @@ -129,6 +129,7 @@ + From dc08e87be0eed7d84a4971acc18025a27915bb54 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Thu, 14 Dec 2023 23:21:20 -0500 Subject: [PATCH 40/82] Minor diagnostics for this flaky test --- tests/Tgstation.Server.Tests/TestSystemInteraction.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/Tgstation.Server.Tests/TestSystemInteraction.cs b/tests/Tgstation.Server.Tests/TestSystemInteraction.cs index 4570c58b94..45a017a968 100644 --- a/tests/Tgstation.Server.Tests/TestSystemInteraction.cs +++ b/tests/Tgstation.Server.Tests/TestSystemInteraction.cs @@ -45,13 +45,17 @@ namespace Tgstation.Server.Tests [TestMethod] public async Task TestScriptExecutionWithFileOutput() { - using var loggerFactory = LoggerFactory.Create(x => { }); + using var loggerFactory = LoggerFactory.Create(builder => + { + builder.AddConsole(); + builder.SetMinimumLevel(LogLevel.Trace); + }); var platformIdentifier = new PlatformIdentifier(); var processExecutor = new ProcessExecutor( Mock.Of(), Mock.Of(), new DefaultIOManager(), - Mock.Of>(), + loggerFactory.CreateLogger(), loggerFactory); var tempFile = Path.GetTempFileName(); From 7304cf1a8d2f273a22521684f2e9c51874b8b0fe Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 15 Dec 2023 16:18:02 -0500 Subject: [PATCH 41/82] Clean up the JobsHubTest assertions again --- tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs b/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs index 17fb4c0d89..2a7587d43d 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs @@ -215,7 +215,7 @@ namespace Tgstation.Server.Tests.Live.Instance // some instances may be detached, but our cache remains var accountedJobs = allJobs.Count - missableMissedJobs; - var errorMessage = $"Mismatch in seen jobs:{Environment.NewLine}Not seen in seen:{Environment.NewLine}{JobListFormatter(allJobs.Where(x => !seenJobs.Any(y => y.Key == x.Id.Value)))}{Environment.NewLine}Seen not in all:{Environment.NewLine}{JobListFormatter(jobsSeenByHubButNotInAllJobs)}"; + var errorMessage = $"Mismatch in seen jobs:{Environment.NewLine}Not seen in seen:{Environment.NewLine}{JobListFormatter(allJobs.Where(x => !seenJobs.Any(y => y.Key == x.Id.Value)))}{Environment.NewLine}Seen not in all:{Environment.NewLine}{JobListFormatter(jobsSeenByHubButNotInAllJobs)}{Environment.NewLine}Current Instances: {String.Join(", ", allInstances.Select(i => i.Id.Value))}"; Assert.AreEqual( accountedJobs, seenJobs.Count - jobsSeenByHubButNotInAllJobs.Count, From d932b7e943974a79d334098f20ffe7f0426c7cac Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 15 Dec 2023 19:40:27 -0500 Subject: [PATCH 42/82] Revert "Fold Code Scanning into CI Pipeline" This reverts commit 9ba8952809a8a6f3c11ac05ed7c2f00d84010a33. --- .github/workflows/ci-pipeline.yml | 34 +----------------- .github/workflows/code-scanning.yml | 55 +++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 33 deletions(-) create mode 100644 .github/workflows/code-scanning.yml diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 41d156de2a..4f5a0f95d2 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -83,38 +83,6 @@ jobs: - name: GitHub Requires at Least One Step for a Job run: exit 0 - analyze: - name: Code Scanning - needs: start-ci-run-gate - runs-on: ubuntu-latest - permissions: - actions: read - contents: read - security-events: write - if: (!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success' && ${{ vars.TGS_ENABLE_CODE_QL }} == 'true') - steps: - - name: Setup dotnet - uses: actions/setup-dotnet@v3 - with: - dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' - dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - - - name: Checkout - uses: actions/checkout@v3 - - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 - with: - languages: csharp - - - name: Build - run: dotnet build -c ReleaseNoWindows -p:TGS_HOST_NO_WEBPANEL=true - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 - with: - category: "/language:csharp" - dmapi-build: name: Build DMAPI needs: start-ci-run-gate @@ -1355,7 +1323,7 @@ jobs: ci-completion-gate: # This job exists so there isn't a moving target for branch protections name: CI Completion Gate - needs: [ pages-build, docker-build, build-deb, build-msi, validate-openapi-spec, upload-code-coverage, check-winget-pr-template, analyze ] + needs: [ pages-build, docker-build, build-deb, build-msi, validate-openapi-spec, upload-code-coverage, check-winget-pr-template ] runs-on: ubuntu-latest if: (!(cancelled() || failure()) && needs.pages-build.result == 'success' && needs.docker-build.result == 'success' && needs.build-deb.result == 'success' && needs.build-msi.result == 'success' && needs.validate-openapi-spec.result == 'success' && needs.upload-code-coverage.result == 'success' && needs.check-winget-pr-template.result == 'success') steps: diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml new file mode 100644 index 0000000000..9442ce5530 --- /dev/null +++ b/.github/workflows/code-scanning.yml @@ -0,0 +1,55 @@ +name: 'Code Scanning' + +on: + schedule: + - cron: 0 23 * * 1 + push: + branches: + - dev + - master + - V6 + pull_request: + branches: + - dev + - master + - V6 + +env: + TGS_DOTNET_VERSION: 8 + TGS_DOTNET_QUALITY: ga + +concurrency: + group: "code-scanning-${{ github.head_ref || github.run_id }}-${{ github.event_name }}" + cancel-in-progress: true + +jobs: + analyze: + name: Code Scanning + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + if: ${{ vars.TGS_ENABLE_CODE_QL }} == 'true' + steps: + - name: Setup dotnet + uses: actions/setup-dotnet@v3 + with: + dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x' + dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} + + - name: Checkout + uses: actions/checkout@v3 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: csharp + + - name: Build + run: dotnet build -c ReleaseNoWindows -p:TGS_HOST_NO_WEBPANEL=true + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 + with: + category: "/language:csharp" From 198f0f714ffa71c98bc9e672ce5bee94b024d054 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 15 Dec 2023 19:41:22 -0500 Subject: [PATCH 43/82] Remove code scanning cron --- .github/workflows/code-scanning.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index 9442ce5530..a0e8069a33 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -1,8 +1,6 @@ name: 'Code Scanning' on: - schedule: - - cron: 0 23 * * 1 push: branches: - dev From e0f67bb6dff432ea213b4bf572c5e4611bce00f7 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 15 Dec 2023 19:50:05 -0500 Subject: [PATCH 44/82] More `JobsHubTests` diagnostics --- tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs b/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs index 2a7587d43d..8b8de95ffe 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs @@ -169,6 +169,11 @@ namespace Tgstation.Server.Tests.Live.Instance .Select(CheckInstance); var allJobs = (await ValueTaskExtensions.WhenAll(allJobsTask, allInstances.Count)).SelectMany(x => x).ToList(); + + var uniqueAllJobs = allJobs.GroupBy(x => x.Id.Value).Select(x => x.First()).ToList(); + + Assert.AreEqual(allJobs.Count, uniqueAllJobs.Count); + var missableMissedJobs = 0; foreach (var job in allJobs) { @@ -204,7 +209,7 @@ namespace Tgstation.Server.Tests.Live.Instance var wasMissableJob = job.JobCode == JobCode.ReconnectChatBot || job.JobCode == JobCode.StartupWatchdogLaunch || job.JobCode == JobCode.StartupWatchdogReattach; - Assert.IsTrue(wasMissableJob, $"Found unexpected missed job: {job.Description}"); + Assert.IsTrue(wasMissableJob, $"Found unexpected missed job: #{job.Id.Value} - {job.JobCode} - {job.Description}"); ++missableMissedJobs; } } From 92cc9e7f6a3633a9763ade683bf0955cd12d2ee6 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 15 Dec 2023 22:09:12 -0500 Subject: [PATCH 45/82] More diagnostic asserts --- .../Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs b/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs index 8b8de95ffe..013de8ea71 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs @@ -170,9 +170,11 @@ namespace Tgstation.Server.Tests.Live.Instance var allJobs = (await ValueTaskExtensions.WhenAll(allJobsTask, allInstances.Count)).SelectMany(x => x).ToList(); - var uniqueAllJobs = allJobs.GroupBy(x => x.Id.Value).Select(x => x.First()).ToList(); + var groups = allJobs.GroupBy(x => x.Id.Value).ToList(); + var uniqueAllJobs = groups.Select(x => x.First()).ToList(); - Assert.AreEqual(allJobs.Count, uniqueAllJobs.Count); + static string JobListFormatter(IEnumerable jobs) => String.Join(Environment.NewLine, jobs.Select(x => $"- I:{x.InstanceId}|JID:{x.Id}|JC:{x.JobCode}|Desc:{x.Description}")); + Assert.AreEqual(allJobs.Count, uniqueAllJobs.Count, $"Duplicated Jobs:{Environment.NewLine}{JobListFormatter(groups.Where(x => x.Count() > 1).Select(x => x.First()))}"); var missableMissedJobs = 0; foreach (var job in allJobs) @@ -214,8 +216,6 @@ namespace Tgstation.Server.Tests.Live.Instance } } - static string JobListFormatter(IEnumerable jobs) => String.Join(Environment.NewLine, jobs.Select(x => $"- I:{x.InstanceId}|JID:{x.Id}|JC:{x.JobCode}|Desc:{x.Description}")); - var jobsSeenByHubButNotInAllJobs = seenJobs.Values.Where(x => !allJobs.Any(y => y.Id.Value == x.Id.Value)).ToList(); // some instances may be detached, but our cache remains From 1665d564d245050ecc5fa5211234755ff79e63e2 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 16 Dec 2023 09:21:33 -0500 Subject: [PATCH 46/82] Use proper TCS method --- tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs b/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs index 013de8ea71..6f578ed6d7 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; @@ -66,7 +66,7 @@ namespace Tgstation.Server.Tests.Live.Instance } catch(Exception ex) { - finishTcs.SetException(ex); + finishTcs.TrySetException(ex); } return Task.CompletedTask; From ba454f0ac05dd3b80429596a1c7e563610c0e98c Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 16 Dec 2023 09:22:12 -0500 Subject: [PATCH 47/82] Add a missing throw --- tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs b/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs index 6f578ed6d7..8f42d165d4 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs @@ -115,6 +115,7 @@ namespace Tgstation.Server.Tests.Live.Instance catch { await permedConn.DisposeAsync(); + throw; } return FinishAsync(cancellationToken); From aa51b67ea1f767acced1958022d4891e4ede8648 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 16 Dec 2023 10:07:18 -0500 Subject: [PATCH 48/82] Handle startup jobs a bit better --- .../Extensions/JobCodeExtensions.cs | 25 +++++++++++++++++++ src/Tgstation.Server.Host/Jobs/JobService.cs | 8 +++++- .../Live/Instance/JobsHubTests.cs | 7 +++--- .../Live/TestLiveServer.cs | 18 ++++++------- 4 files changed, 42 insertions(+), 16 deletions(-) create mode 100644 src/Tgstation.Server.Api/Extensions/JobCodeExtensions.cs diff --git a/src/Tgstation.Server.Api/Extensions/JobCodeExtensions.cs b/src/Tgstation.Server.Api/Extensions/JobCodeExtensions.cs new file mode 100644 index 0000000000..90c379f824 --- /dev/null +++ b/src/Tgstation.Server.Api/Extensions/JobCodeExtensions.cs @@ -0,0 +1,25 @@ +using System; + +using Tgstation.Server.Api.Models; + +namespace Tgstation.Server.Api.Extensions +{ + /// + /// Extension methods for the . + /// + public static class JobCodeExtensions + { + /// + /// If a given can be triggered by TGS startup. + /// + /// The . + /// if the can trigger before startup, otherwise. + public static bool IsServerStartupJob(this JobCode jobCode) + => jobCode switch + { + JobCode.Unknown or JobCode.Move or JobCode.RepositoryClone or JobCode.RepositoryUpdate or JobCode.RepositoryAutoUpdate or JobCode.RepositoryDelete or JobCode.EngineOfficialInstall or JobCode.EngineCustomInstall or JobCode.EngineDelete or JobCode.Deployment or JobCode.AutomaticDeployment or JobCode.WatchdogLaunch or JobCode.WatchdogRestart or JobCode.WatchdogDump => false, + JobCode.StartupWatchdogLaunch or JobCode.StartupWatchdogReattach or JobCode.ReconnectChatBot => true, + _ => throw new InvalidOperationException($"Invalid JobCode: {jobCode}"), + }; + } +} diff --git a/src/Tgstation.Server.Host/Jobs/JobService.cs b/src/Tgstation.Server.Host/Jobs/JobService.cs index 87387c60a9..abdb26170c 100644 --- a/src/Tgstation.Server.Host/Jobs/JobService.cs +++ b/src/Tgstation.Server.Host/Jobs/JobService.cs @@ -11,6 +11,7 @@ using Microsoft.Extensions.Logging; using Serilog.Context; +using Tgstation.Server.Api.Extensions; using Tgstation.Server.Api.Hubs; using Tgstation.Server.Api.Models.Response; using Tgstation.Server.Common.Extensions; @@ -452,7 +453,12 @@ namespace Tgstation.Server.Host.Jobs } } - var instanceCoreProvider = await activationTcs.Task.WaitAsync(cancellationToken); + var activationTask = activationTcs.Task; + + Debug.Assert(activationTask.IsCompleted || job.JobCode.Value.IsServerStartupJob(), "Non-server startup job registered before activation!"); + + var instanceCoreProvider = await activationTask.WaitAsync(cancellationToken); + QueueHubUpdate(job.ToApi(), false); logger.LogTrace("Starting job..."); diff --git a/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs b/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs index 8f42d165d4..19be3fe30b 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; @@ -8,6 +8,7 @@ using System.Threading.Tasks; using Microsoft.AspNetCore.SignalR.Client; using Microsoft.VisualStudio.TestTools.UnitTesting; +using Tgstation.Server.Api.Extensions; using Tgstation.Server.Api.Hubs; using Tgstation.Server.Api.Models; using Tgstation.Server.Api.Models.Request; @@ -209,9 +210,7 @@ namespace Tgstation.Server.Tests.Live.Instance } else { - var wasMissableJob = job.JobCode == JobCode.ReconnectChatBot - || job.JobCode == JobCode.StartupWatchdogLaunch - || job.JobCode == JobCode.StartupWatchdogReattach; + var wasMissableJob = job.JobCode.Value.IsServerStartupJob(); Assert.IsTrue(wasMissableJob, $"Found unexpected missed job: #{job.Id.Value} - {job.JobCode} - {job.Description}"); ++missableMissedJobs; } diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index 52b0b3f90e..5707027a92 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -28,6 +28,7 @@ using Newtonsoft.Json; using Npgsql; using Tgstation.Server.Api; +using Tgstation.Server.Api.Extensions; using Tgstation.Server.Api.Models; using Tgstation.Server.Api.Models.Request; using Tgstation.Server.Api.Models.Response; @@ -1653,16 +1654,11 @@ namespace Tgstation.Server.Tests.Live { var jobs = await instanceClient.Jobs.ListActive(null, cancellationToken); if (jobs.Count == 0) - { - var entities = await instanceClient.Jobs.List(null, cancellationToken); - var getTasks = entities - .Select(e => instanceClient.Jobs.GetId(e, cancellationToken)) - .ToList(); - - jobs = (await ValueTaskExtensions.WhenAll(getTasks)) + jobs = (await instanceClient.Jobs.List(null, cancellationToken)) .Where(x => x.StartedAt.Value > preStartupTime) - .ToList(); - } + .ToList(); + else + jobs = jobs.Where(x => x.JobCode.Value.IsServerStartupJob()).ToList(); await using var jrt = new JobsRequiredTest(instanceClient.Jobs); foreach (var job in jobs) @@ -1679,9 +1675,9 @@ namespace Tgstation.Server.Tests.Live var edgeVersion = await EngineTest.GetEdgeVersion(EngineType.Byond, fileDownloader, cancellationToken); await using (var adminClient = await CreateAdminClient(server.ApiUrl, cancellationToken)) { + await jobsHubTest.WaitForReconnect(cancellationToken); var instanceClient = adminClient.Instances.CreateClient(instance); await WaitForInitialJobs(instanceClient); - await jobsHubTest.WaitForReconnect(cancellationToken); var dd = await instanceClient.DreamDaemon.Read(cancellationToken); @@ -1722,9 +1718,9 @@ namespace Tgstation.Server.Tests.Live serverTask = server.Run(cancellationToken).AsTask(); await using (var adminClient = await CreateAdminClient(server.ApiUrl, cancellationToken)) { + await jobsHubTest.WaitForReconnect(cancellationToken); var instanceClient = adminClient.Instances.CreateClient(instance); await WaitForInitialJobs(instanceClient); - await jobsHubTest.WaitForReconnect(cancellationToken); var currentDD = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.AreEqual(expectedCompileJobId, currentDD.ActiveCompileJob.Id.Value); From e6bc0c9105ad43d97194e1684ce9f74c74089953 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 16 Dec 2023 10:07:56 -0500 Subject: [PATCH 49/82] Disable primary constructor recommendation --- build/analyzers.ruleset | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/build/analyzers.ruleset b/build/analyzers.ruleset index 034747c138..237aa86fe4 100644 --- a/build/analyzers.ruleset +++ b/build/analyzers.ruleset @@ -1,4 +1,4 @@ - + @@ -88,7 +88,6 @@ - @@ -666,6 +665,10 @@ + + + + @@ -751,6 +754,7 @@ + @@ -1026,8 +1030,6 @@ - - @@ -1043,4 +1045,4 @@ - + \ No newline at end of file From feca232b933962a55c43099c02fdcab1f9499aad Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 16 Dec 2023 10:14:22 -0500 Subject: [PATCH 50/82] Update `Byond.TopicSender` to `8.0.0` --- src/Tgstation.Server.Host/Tgstation.Server.Host.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index 448d040017..6d50031a8d 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -63,7 +63,7 @@ - + From f4d33747b5174928e3f66e889363a13d69344e7c Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 16 Dec 2023 10:15:10 -0500 Subject: [PATCH 51/82] Update Z.EntityFramework.Plus to `8.101.1.2` --- src/Tgstation.Server.Host/Tgstation.Server.Host.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index 6d50031a8d..6a2262dfa4 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -125,7 +125,7 @@ - + From 8ab50abef61c8f03b96045fea6d6a73fbba73b07 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 16 Dec 2023 10:27:53 -0500 Subject: [PATCH 52/82] Fix a message --- .../System/ProcessExecutor.cs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/Tgstation.Server.Host/System/ProcessExecutor.cs b/src/Tgstation.Server.Host/System/ProcessExecutor.cs index f6218f5cd9..acea2604ff 100644 --- a/src/Tgstation.Server.Host/System/ProcessExecutor.cs +++ b/src/Tgstation.Server.Host/System/ProcessExecutor.cs @@ -125,13 +125,19 @@ namespace Tgstation.Server.Host.System if (!noShellExecute && readStandardHandles) throw new InvalidOperationException("Requesting output/error reading requires noShellExecute to be true!"); - logger.LogDebug( - noShellExecute - ? "Launching process in {workingDirectory}: {exe} {arguments}" - : "Shell launching process in {workingDirectory}: {exe} {arguments}", + if (noShellExecute) + logger.LogDebug( + "Launching process in {workingDirectory}: {exe} {arguments}", workingDirectory, fileName, arguments); + else + logger.LogDebug( + "Shell launching process in {workingDirectory}: {exe} {arguments}", + workingDirectory, + fileName, + arguments); + var handle = new global::System.Diagnostics.Process(); try { From 6ce9695533e23c857329950098c540c27ca796cd Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 16 Dec 2023 11:22:01 -0500 Subject: [PATCH 53/82] Redo Process output reading to better workaround https://github.com/dotnet/runtime/issues/28583 Also test harder --- src/Tgstation.Server.Host/System/IProcess.cs | 5 +- src/Tgstation.Server.Host/System/Process.cs | 27 +---- .../System/ProcessExecutor.cs | 102 ++++++++++-------- .../System/TestPosixSignalHandler.cs | 3 +- .../Live/Instance/WatchdogTest.cs | 1 - .../Live/TestLiveServer.cs | 1 - .../TestSystemInteraction.cs | 52 ++++----- tests/Tgstation.Server.Tests/TestVersions.cs | 3 +- 8 files changed, 89 insertions(+), 105 deletions(-) diff --git a/src/Tgstation.Server.Host/System/IProcess.cs b/src/Tgstation.Server.Host/System/IProcess.cs index 8df102b2b2..dc9a1d5eac 100644 --- a/src/Tgstation.Server.Host/System/IProcess.cs +++ b/src/Tgstation.Server.Host/System/IProcess.cs @@ -23,13 +23,12 @@ namespace Tgstation.Server.Host.System /// Get the stderr and stdout output of the . /// /// The for the operation. - /// A resulting in the stderr and stdout output of the . + /// A resulting in the stderr and stdout output of the . /// /// To guarantee that all data is received from the when redirecting streams to a file /// the result of this function must be ed before is called. - /// May call internally if the process has exited. /// - ValueTask GetCombinedOutput(CancellationToken cancellationToken); + Task GetCombinedOutput(CancellationToken cancellationToken); /// /// Asycnhronously terminates the process. diff --git a/src/Tgstation.Server.Host/System/Process.cs b/src/Tgstation.Server.Host/System/Process.cs index 210a86db8c..266e220708 100644 --- a/src/Tgstation.Server.Host/System/Process.cs +++ b/src/Tgstation.Server.Host/System/Process.cs @@ -7,7 +7,6 @@ using Microsoft.Extensions.Logging; using Microsoft.Win32.SafeHandles; using Tgstation.Server.Host.IO; -using Tgstation.Server.Host.Utils; namespace Tgstation.Server.Host.System { @@ -28,11 +27,6 @@ namespace Tgstation.Server.Host.System /// readonly IProcessFeatures processFeatures; - /// - /// The for the . - /// - readonly IAsyncDelayer asyncDelayer; - /// /// The for the . /// @@ -68,7 +62,6 @@ namespace Tgstation.Server.Host.System /// Initializes a new instance of the class. /// /// The value of . - /// The value of . /// The value of . /// The override value of . /// The value of . @@ -76,7 +69,6 @@ namespace Tgstation.Server.Host.System /// If was NOT just created. public Process( IProcessFeatures processFeatures, - IAsyncDelayer asyncDelayer, global::System.Diagnostics.Process handle, CancellationTokenSource readerCts, Task readTask, @@ -92,7 +84,6 @@ namespace Tgstation.Server.Host.System cancellationTokenSource = readerCts ?? new CancellationTokenSource(); this.processFeatures = processFeatures ?? throw new ArgumentNullException(nameof(processFeatures)); - this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer)); this.readTask = readTask; @@ -144,26 +135,12 @@ namespace Tgstation.Server.Host.System } /// - public async ValueTask GetCombinedOutput(CancellationToken cancellationToken) + public Task GetCombinedOutput(CancellationToken cancellationToken) { - CheckDisposed(); if (readTask == null) throw new InvalidOperationException("Output/Error stream reading was not enabled!"); - // workaround for https://github.com/dotnet/runtime/issues/28583 (?) - if (handle.HasExited) - { - handle.WaitForExit(); - await Task.WhenAny(readTask, asyncDelayer.Delay(TimeSpan.FromSeconds(30), cancellationToken)); - - if (!readTask.IsCompleted) - { - logger.LogWarning("Detected process output read hang on PID {pid}! Closing handle as a workaround...", Id); - cancellationTokenSource.Cancel(); - } - } - - return await readTask.WaitAsync(cancellationToken); + return readTask.WaitAsync(cancellationToken); } /// diff --git a/src/Tgstation.Server.Host/System/ProcessExecutor.cs b/src/Tgstation.Server.Host/System/ProcessExecutor.cs index acea2604ff..4f54ce62b3 100644 --- a/src/Tgstation.Server.Host/System/ProcessExecutor.cs +++ b/src/Tgstation.Server.Host/System/ProcessExecutor.cs @@ -7,7 +7,6 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Tgstation.Server.Host.IO; -using Tgstation.Server.Host.Utils; namespace Tgstation.Server.Host.System { @@ -24,11 +23,6 @@ namespace Tgstation.Server.Host.System /// readonly IProcessFeatures processFeatures; - /// - /// The for the . - /// - readonly IAsyncDelayer asyncDelayer; - /// /// The for the . /// @@ -65,19 +59,16 @@ namespace Tgstation.Server.Host.System /// Initializes a new instance of the class. /// /// The value of . - /// The value of . /// The value of . /// The value of . /// The value of . public ProcessExecutor( IProcessFeatures processFeatures, - IAsyncDelayer asyncDelayer, IIOManager ioManager, ILogger logger, ILoggerFactory loggerFactory) { this.processFeatures = processFeatures ?? throw new ArgumentNullException(nameof(processFeatures)); - this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer)); this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); this.loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory)); @@ -122,9 +113,6 @@ namespace Tgstation.Server.Host.System ArgumentNullException.ThrowIfNull(workingDirectory); ArgumentNullException.ThrowIfNull(arguments); - if (!noShellExecute && readStandardHandles) - throw new InvalidOperationException("Requesting output/error reading requires noShellExecute to be true!"); - if (noShellExecute) logger.LogDebug( "Launching process in {workingDirectory}: {exe} {arguments}", @@ -151,10 +139,10 @@ namespace Tgstation.Server.Host.System CancellationTokenSource disposeCts = null; try { - TaskCompletionSource processStartTcs = null; + TaskCompletionSource processStartTcs = null; if (readStandardHandles) { - processStartTcs = new TaskCompletionSource(); + processStartTcs = new TaskCompletionSource(); handle.StartInfo.RedirectStandardOutput = true; handle.StartInfo.RedirectStandardError = true; @@ -162,6 +150,7 @@ namespace Tgstation.Server.Host.System readTask = ConsumeReaders(handle, processStartTcs.Task, fileRedirect, disposeCts.Token); } + int pid; try { ExclusiveProcessLaunchLock.EnterReadLock(); @@ -174,7 +163,8 @@ namespace Tgstation.Server.Host.System ExclusiveProcessLaunchLock.ExitReadLock(); } - processStartTcs?.SetResult(); + pid = handle.Id; + processStartTcs?.SetResult(pid); } catch (Exception ex) { @@ -184,7 +174,6 @@ namespace Tgstation.Server.Host.System var process = new Process( processFeatures, - asyncDelayer, handle, disposeCts, readTask, @@ -231,43 +220,53 @@ namespace Tgstation.Server.Host.System /// Consume the stdout/stderr streams into a . /// /// The . - /// The that completes when starts. + /// The resulting in the of the started process. /// The optional path to redirect the streams to. /// The that triggers when the is disposed. /// A resulting in the program's output/error text if is , otherwise. - async Task ConsumeReaders(global::System.Diagnostics.Process handle, Task startTask, string fileRedirect, CancellationToken disposeToken) + async Task ConsumeReaders(global::System.Diagnostics.Process handle, Task startupAndPid, string fileRedirect, CancellationToken disposeToken) { - await startTask; - - var pid = handle.Id; + var pid = await startupAndPid; logger.LogTrace("Starting read for PID {pid}...", pid); - // once we obtain these handles we're responsible for them - using var stdOutHandle = handle.StandardOutput; - using var stdErrHandle = handle.StandardError; - Task outputReadTask = null, errorReadTask = null; + var stdOutHandle = handle.StandardOutput; + var stdErrHandle = handle.StandardError; bool outputOpen = true, errorOpen = true; - async Task GetNextLine() + var outputReadTask = stdOutHandle.ReadLineAsync(disposeToken).AsTask(); + var errorReadTask = stdErrHandle.ReadLineAsync(disposeToken).AsTask(); + + async ValueTask GetNextLine() { - if (outputOpen && outputReadTask == null) - outputReadTask = stdOutHandle.ReadLineAsync(disposeToken).AsTask(); + var nextLineTask = outputOpen + ? errorOpen + ? await Task.WhenAny(outputReadTask, errorReadTask) + : outputReadTask + : errorReadTask; - if (errorOpen && errorReadTask == null) - errorReadTask = stdErrHandle.ReadLineAsync(disposeToken).AsTask(); + var line = await nextLineTask; - var completedTask = await Task.WhenAny(outputReadTask ?? errorReadTask, errorReadTask ?? outputReadTask); - var line = await completedTask.WaitAsync(disposeToken); - if (completedTask == outputReadTask) + // Important to retrigger the reading block asap after pushing + if (nextLineTask == outputReadTask) { - outputReadTask = null; if (line == null) + { outputOpen = false; + outputReadTask = null; + } + else + outputReadTask = stdOutHandle.ReadLineAsync(disposeToken).AsTask(); } else { - errorReadTask = null; + global::System.Diagnostics.Debug.Assert(nextLineTask == errorReadTask, "How is this incorrect?"); + if (line == null) + { errorOpen = false; + errorReadTask = null; + } + else + errorReadTask = stdErrHandle.ReadLineAsync(disposeToken).AsTask(); } if (line == null && (errorOpen || outputOpen)) @@ -279,20 +278,32 @@ namespace Tgstation.Server.Host.System await using var fileStream = fileRedirect != null ? ioManager.CreateAsyncSequentialWriteStream(fileRedirect) : null; await using var writer = fileStream != null ? new StreamWriter(fileStream) : null; - string text; var stringBuilder = fileStream == null ? new StringBuilder() : null; + ulong fileFlushToken = 0; + async ValueTask QueueWrite(ValueTask previous, string text) + { + if (fileStream != null) + { + var startFlushToken = Interlocked.Increment(ref fileFlushToken); + await previous; + await writer.WriteLineAsync(text.AsMemory(), disposeToken); + + // only flush if there isn't anything rapidly coming in + if (fileFlushToken == startFlushToken) + await writer.FlushAsync(disposeToken); + } + else + stringBuilder.AppendLine(text); + } + + ValueTask writeQueue = ValueTask.CompletedTask; try { + string text; while ((text = await GetNextLine()) != null) - { - if (fileStream != null) - { - await writer.WriteLineAsync(text.AsMemory(), disposeToken); - await writer.FlushAsync(disposeToken); - } - else - stringBuilder.AppendLine(text); - } + writeQueue = QueueWrite(writeQueue, text); + + await writeQueue; logger.LogTrace("Finished read for PID {pid}", pid); } @@ -318,7 +329,6 @@ namespace Tgstation.Server.Host.System var pid = handle.Id; return new Process( processFeatures, - asyncDelayer, handle, null, null, diff --git a/tests/Tgstation.Server.Host.Tests/System/TestPosixSignalHandler.cs b/tests/Tgstation.Server.Host.Tests/System/TestPosixSignalHandler.cs index 407145730c..6728fe09b6 100644 --- a/tests/Tgstation.Server.Host.Tests/System/TestPosixSignalHandler.cs +++ b/tests/Tgstation.Server.Host.Tests/System/TestPosixSignalHandler.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.IO; using System.Reflection; using System.Threading; @@ -60,7 +60,6 @@ namespace Tgstation.Server.Host.System.Tests new Lazy(() => processExecutor), new DefaultIOManager(), loggerFactory.CreateLogger()), - new AsyncDelayer(), Mock.Of(), loggerFactory.CreateLogger(), loggerFactory); diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index 8fcccf0589..6071dade63 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -739,7 +739,6 @@ namespace Tgstation.Server.Tests.Live.Instance RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? new WindowsProcessFeatures(Mock.Of>()) : new PosixProcessFeatures(new Lazy(() => executor), Mock.Of(), Mock.Of>()), - Mock.Of(), Mock.Of(), Mock.Of>(), LoggerFactory.Create(x => { })); diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index 5707027a92..bc0a8c5e5a 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -1081,7 +1081,6 @@ namespace Tgstation.Server.Tests.Live new PlatformIdentifier().IsWindows ? new WindowsProcessFeatures(loggerFactory.CreateLogger()) : new PosixProcessFeatures(new Lazy(() => processExecutor), ioManager, loggerFactory.CreateLogger()), - new AsyncDelayer(), ioManager, loggerFactory.CreateLogger(), loggerFactory); diff --git a/tests/Tgstation.Server.Tests/TestSystemInteraction.cs b/tests/Tgstation.Server.Tests/TestSystemInteraction.cs index 45a017a968..98b7c82f44 100644 --- a/tests/Tgstation.Server.Tests/TestSystemInteraction.cs +++ b/tests/Tgstation.Server.Tests/TestSystemInteraction.cs @@ -24,7 +24,6 @@ namespace Tgstation.Server.Tests var platformIdentifier = new PlatformIdentifier(); var processExecutor = new ProcessExecutor( Mock.Of(), - Mock.Of(), new DefaultIOManager(), Mock.Of>(), loggerFactory); @@ -53,36 +52,39 @@ namespace Tgstation.Server.Tests var platformIdentifier = new PlatformIdentifier(); var processExecutor = new ProcessExecutor( Mock.Of(), - Mock.Of(), new DefaultIOManager(), loggerFactory.CreateLogger(), loggerFactory); - var tempFile = Path.GetTempFileName(); - File.Delete(tempFile); - try - { - await using (var process = processExecutor.LaunchProcess("test." + platformIdentifier.ScriptFileExtension, ".", string.Empty, tempFile, true, true)) - { - using var cts = new CancellationTokenSource(); - cts.CancelAfter(3000); - var exitCode = await process.Lifetime.WaitAsync(cts.Token); - - await process.GetCombinedOutput(cts.Token); - - Assert.AreEqual(0, exitCode); - } - - Assert.IsTrue(File.Exists(tempFile), $"Could not find temp file: {tempFile}"); - var result = File.ReadAllText(tempFile).Trim(); - - // no guarantees about order - Assert.IsTrue(result.Contains("Hello World!"), $"Result: {result}"); - Assert.IsTrue(result.Contains("Hello Error!"), $"Result: {result}"); - } - finally + // run on a loop to spot the hang + for (var i = 0; i < 1000; ++i) { + var tempFile = Path.GetTempFileName(); File.Delete(tempFile); + try + { + await using (var process = processExecutor.LaunchProcess("test." + platformIdentifier.ScriptFileExtension, ".", string.Empty, tempFile, true, true)) + { + using var cts = new CancellationTokenSource(); + cts.CancelAfter(3000); + var exitCode = await process.Lifetime.WaitAsync(cts.Token); + + await process.GetCombinedOutput(cts.Token); + + Assert.AreEqual(0, exitCode); + } + + Assert.IsTrue(File.Exists(tempFile), $"Could not find temp file: {tempFile}"); + var result = File.ReadAllText(tempFile).Trim(); + + // no guarantees about order + Assert.IsTrue(result.Contains("Hello World!"), $"Result: {result}"); + Assert.IsTrue(result.Contains("Hello Error!"), $"Result: {result}"); + } + finally + { + File.Delete(tempFile); + } } } } diff --git a/tests/Tgstation.Server.Tests/TestVersions.cs b/tests/Tgstation.Server.Tests/TestVersions.cs index 3dd20d8e7b..1ef0da6fd7 100644 --- a/tests/Tgstation.Server.Tests/TestVersions.cs +++ b/tests/Tgstation.Server.Tests/TestVersions.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.IO; using System.IO.Compression; using System.Globalization; @@ -208,7 +208,6 @@ namespace Tgstation.Server.Tests new Lazy(() => null), Mock.Of(), loggerFactory.CreateLogger()), - Mock.Of(), Mock.Of(), loggerFactory.CreateLogger(), loggerFactory); From c06af410415a3d7afeef01504e21adc7dc172532 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 16 Dec 2023 11:22:08 -0500 Subject: [PATCH 54/82] Cleanup some messages --- .../System/TestPosixSignalHandler.cs | 4 ++-- tests/Tgstation.Server.Tests/TestVersions.cs | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/Tgstation.Server.Host.Tests/System/TestPosixSignalHandler.cs b/tests/Tgstation.Server.Host.Tests/System/TestPosixSignalHandler.cs index 6728fe09b6..083657a075 100644 --- a/tests/Tgstation.Server.Host.Tests/System/TestPosixSignalHandler.cs +++ b/tests/Tgstation.Server.Host.Tests/System/TestPosixSignalHandler.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.IO; using System.Reflection; using System.Threading; @@ -54,7 +54,7 @@ namespace Tgstation.Server.Host.System.Tests builder.SetMinimumLevel(LogLevel.Trace); }); - IProcessExecutor processExecutor = null; + ProcessExecutor processExecutor = null; processExecutor = new ProcessExecutor( new PosixProcessFeatures( new Lazy(() => processExecutor), diff --git a/tests/Tgstation.Server.Tests/TestVersions.cs b/tests/Tgstation.Server.Tests/TestVersions.cs index 1ef0da6fd7..34dad9e006 100644 --- a/tests/Tgstation.Server.Tests/TestVersions.cs +++ b/tests/Tgstation.Server.Tests/TestVersions.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.IO; using System.IO.Compression; using System.Globalization; @@ -463,8 +463,8 @@ namespace Tgstation.Server.Tests EngineVersion engineVersion, Stream byondBytes, ByondInstallerBase byondInstaller, - IIOManager ioManager, - IProcessExecutor processExecutor, + DefaultIOManager ioManager, + ProcessExecutor processExecutor, string tempPath) { using (byondBytes) From 37c54c1fbdb28b2b1139231d5f3bf4aaece55a63 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 16 Dec 2023 11:26:31 -0500 Subject: [PATCH 55/82] Increase some timeouts that were too low --- tests/Tgstation.Server.Tests/Live/Instance/DeploymentTest.cs | 2 +- tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/DeploymentTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/DeploymentTest.cs index d0d074e6e8..a0cf0d9af1 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/DeploymentTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/DeploymentTest.cs @@ -139,7 +139,7 @@ namespace Tgstation.Server.Tests.Live.Instance var updatedDD = await dreamDaemonClient.Update(new DreamDaemonRequest { - StartupTimeout = 15, + StartupTimeout = 30, Port = ddPort }, cancellationToken); Assert.AreEqual(15U, updatedDD.StartupTimeout); diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index 6071dade63..acb1a095e0 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -133,7 +133,7 @@ namespace Tgstation.Server.Tests.Live.Instance // Increase startup timeout, disable heartbeats, enable map threads because we've tested without for years instanceClient.DreamDaemon.Update(new DreamDaemonRequest { - StartupTimeout = 15, + StartupTimeout = 30, HealthCheckSeconds = 0, Port = ddPort, MapThreads = 2, From 0e209b933fc9a078902293a47edf530fc2e092bf Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 16 Dec 2023 11:34:04 -0500 Subject: [PATCH 56/82] Essentially remove live tests token expiry --- tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs b/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs index 6fce2c452c..61a7a36176 100644 --- a/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs +++ b/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs @@ -153,6 +153,7 @@ namespace Tgstation.Server.Tests.Live $"Session:LowPriorityDeploymentProcesses={LowPriorityDeployments}", $"General:SkipAddingByondFirewallException={!TestingUtils.RunningInGitHubActions}", $"General:OpenDreamGitUrl={OpenDreamUrl}", + $"Security:TokenExpiryMinutes=120", // timeouts are useless for us }; swarmArgs = new List(); From f870aa0b178baf03a08bd1edc3d5d6dafa29f562 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 16 Dec 2023 11:37:39 -0500 Subject: [PATCH 57/82] Remove some whitespace --- tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs index 611ee9ed6f..a50d4a6504 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs @@ -221,7 +221,6 @@ namespace Tgstation.Server.Tests.Live.Instance await chatRequest; await Task.Yield(); - await Task.WhenAll( jrt.WaitForJob(installJob2.InstallJob, EngineTest.EngineInstallationTimeout(compatVersion) + 30, false, null, cancellationToken), jrt.WaitForJob(cloneRequest.Result.ActiveJob, 60, false, null, cancellationToken), From a3689ac4a9d7aed7c09dc675a2cf0efa25b04443 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 16 Dec 2023 13:14:11 -0500 Subject: [PATCH 58/82] Remove bad reference --- src/Tgstation.Server.Host/Tgstation.Server.Host.csproj | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index 6a2262dfa4..a47c87fbd2 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -129,7 +129,6 @@ - From 670fe5084760236f2f4e96c806e6cb565b46a1fd Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 16 Dec 2023 13:16:57 -0500 Subject: [PATCH 59/82] Fix assert --- tests/Tgstation.Server.Tests/Live/Instance/DeploymentTest.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/DeploymentTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/DeploymentTest.cs index a0cf0d9af1..8cc7d0b6fb 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/DeploymentTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/DeploymentTest.cs @@ -142,7 +142,7 @@ namespace Tgstation.Server.Tests.Live.Instance StartupTimeout = 30, Port = ddPort }, cancellationToken); - Assert.AreEqual(15U, updatedDD.StartupTimeout); + Assert.AreEqual(30U, updatedDD.StartupTimeout); Assert.AreEqual(ddPort, updatedDD.Port); async Task CompileAfterByondInstall() From 2d5def6fdb4fc33b8df430a17c8ad3ead559369c Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 16 Dec 2023 15:06:42 -0500 Subject: [PATCH 60/82] Correct firewall rule name for OpenDream --- .../Components/Engine/WindowsOpenDreamInstaller.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Components/Engine/WindowsOpenDreamInstaller.cs b/src/Tgstation.Server.Host/Components/Engine/WindowsOpenDreamInstaller.cs index 4f42884c7f..4062884604 100644 --- a/src/Tgstation.Server.Host/Components/Engine/WindowsOpenDreamInstaller.cs +++ b/src/Tgstation.Server.Host/Components/Engine/WindowsOpenDreamInstaller.cs @@ -102,7 +102,7 @@ namespace Tgstation.Server.Host.Components.Engine // I really wish we could add the instance name here but // 1. It'd make IByondInstaller need to be transient per-instance and WindowsByondInstaller relys on being a singleton for its DX installer call // 2. The instance could be renamed, so it'd have to be an unfriendly ID anyway. - var ruleName = $"TGS DreamDaemon {version}"; + var ruleName = $"TGS OpenDream {version}"; exitCode = await WindowsFirewallHelper.AddFirewallException( ProcessExecutor, From f12ecfd73832bd9437562e9b33542b5ef8aa2792 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 16 Dec 2023 15:45:17 -0500 Subject: [PATCH 61/82] Fix connecting to an existing SQLite database --- src/Tgstation.Server.Host/Setup/SetupWizard.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Tgstation.Server.Host/Setup/SetupWizard.cs b/src/Tgstation.Server.Host/Setup/SetupWizard.cs index b9671572b2..79f30c22b9 100644 --- a/src/Tgstation.Server.Host/Setup/SetupWizard.cs +++ b/src/Tgstation.Server.Host/Setup/SetupWizard.cs @@ -600,6 +600,8 @@ namespace Tgstation.Server.Host.Setup }; CreateTestConnection(csb.ConnectionString); + + csb.Mode = SqliteOpenMode.ReadWriteCreate; databaseConfiguration.ConnectionString = csb.ConnectionString; } From e28db583daa86a96a9f1aac1b106b89483cb6031 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 16 Dec 2023 16:14:11 -0500 Subject: [PATCH 62/82] Remove superfluous assignments --- src/Tgstation.Server.Host/System/ProcessExecutor.cs | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/Tgstation.Server.Host/System/ProcessExecutor.cs b/src/Tgstation.Server.Host/System/ProcessExecutor.cs index 4f54ce62b3..52b2c8d222 100644 --- a/src/Tgstation.Server.Host/System/ProcessExecutor.cs +++ b/src/Tgstation.Server.Host/System/ProcessExecutor.cs @@ -1,4 +1,5 @@ using System; +using System.Diagnostics; using System.IO; using System.Text; using System.Threading; @@ -247,24 +248,16 @@ namespace Tgstation.Server.Host.System // Important to retrigger the reading block asap after pushing if (nextLineTask == outputReadTask) - { if (line == null) - { outputOpen = false; - outputReadTask = null; - } else outputReadTask = stdOutHandle.ReadLineAsync(disposeToken).AsTask(); - } else { - global::System.Diagnostics.Debug.Assert(nextLineTask == errorReadTask, "How is this incorrect?"); + Debug.Assert(nextLineTask == errorReadTask, "How is this incorrect?"); if (line == null) - { errorOpen = false; - errorReadTask = null; - } else errorReadTask = stdErrHandle.ReadLineAsync(disposeToken).AsTask(); } From fba969b2f1ab26faf06c24ccfb8345236d585539 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 16 Dec 2023 16:14:39 -0500 Subject: [PATCH 63/82] Setup basic WSL debug environment --- .../Properties/launchSettings.json | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/Tgstation.Server.Host/Properties/launchSettings.json b/src/Tgstation.Server.Host/Properties/launchSettings.json index 52ba379ea3..b93f29050d 100644 --- a/src/Tgstation.Server.Host/Properties/launchSettings.json +++ b/src/Tgstation.Server.Host/Properties/launchSettings.json @@ -7,9 +7,15 @@ "ASPNETCORE_ENVIRONMENT": "Development" } }, - "Docker": { - "commandName": "Docker", - "publishAllPorts": true + "WSL": { + "commandName": "WSL2", + "distributionName": "Ubuntu", + "environmentVariables": { + "DOTNET_ENVIRONMENT": "Development", + "ASPNETCORE_ENVIRONMENT": "Development", + "Database__ConnectionString": "Data Source=192.168.2.16,1433;Initial Catalog=TGS_Linux;User Id=tgs_debug;Password=asdf;Encrypt=False;Application Name=tgstation-server", + "General__ValidInstancePaths__0": "/home/dominion/tgs_debug_pen" + } } } -} \ No newline at end of file +} From 50c2fd20ad6458461a29850a58c604d63fbbb0ec Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 16 Dec 2023 16:16:15 -0500 Subject: [PATCH 64/82] Increase OpenDream test installation timeout again --- tests/Tgstation.Server.Tests/Live/Instance/EngineTest.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/EngineTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/EngineTest.cs index 83e7f322c9..57c4255130 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/EngineTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/EngineTest.cs @@ -140,7 +140,7 @@ namespace Tgstation.Server.Tests.Live.Instance case EngineType.Byond: return 30; case EngineType.OpenDream: - return 300; + return 500; default: throw new InvalidOperationException($"Unknown engine type: {testVersion.Engine.Value}"); } From 35e8c33cd2918f0ad8135b5c5eb3e68685d9314e Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 16 Dec 2023 16:31:15 -0500 Subject: [PATCH 65/82] Add more assertions because something is fucky --- src/Tgstation.Server.Host/System/ProcessExecutor.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Tgstation.Server.Host/System/ProcessExecutor.cs b/src/Tgstation.Server.Host/System/ProcessExecutor.cs index 52b2c8d222..d6782d4512 100644 --- a/src/Tgstation.Server.Host/System/ProcessExecutor.cs +++ b/src/Tgstation.Server.Host/System/ProcessExecutor.cs @@ -238,6 +238,10 @@ namespace Tgstation.Server.Host.System async ValueTask GetNextLine() { + Debug.Assert(outputOpen || errorOpen, "We shouldn't be here if neither stream is open"); + Debug.Assert(outputOpen || outputReadTask.IsCompleted, "Output open flag mismatch"); + Debug.Assert(errorOpen || errorReadTask.IsCompleted, "Error open flag mismatch"); + var nextLineTask = outputOpen ? errorOpen ? await Task.WhenAny(outputReadTask, errorReadTask) From 0810db2c8eb2ab09a2e12511bd9d87827e4ab26c Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 16 Dec 2023 16:49:54 -0500 Subject: [PATCH 66/82] Clean up some Stylecop 'L's --- .../Components/Engine/OpenDreamInstaller.cs | 4 +++- .../Engine/WindowsOpenDreamInstaller.cs | 21 +++++++++++-------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs b/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs index bbd09ed1c1..6b356ba539 100644 --- a/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs +++ b/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs @@ -276,7 +276,9 @@ namespace Tgstation.Server.Host.Components.Engine cancellationToken))); } - await ValueTaskExtensions.WhenAll(MoveDirs(), MoveFiles()); + var dirsMoveTask = MoveDirs(); + var outputFilesMoveTask = MoveFiles(); + await ValueTaskExtensions.WhenAll(dirsMoveTask, outputFilesMoveTask); await IOManager.DeleteDirectory(sourcePath, cancellationToken); } diff --git a/src/Tgstation.Server.Host/Components/Engine/WindowsOpenDreamInstaller.cs b/src/Tgstation.Server.Host/Components/Engine/WindowsOpenDreamInstaller.cs index 4062884604..c97ca10418 100644 --- a/src/Tgstation.Server.Host/Components/Engine/WindowsOpenDreamInstaller.cs +++ b/src/Tgstation.Server.Host/Components/Engine/WindowsOpenDreamInstaller.cs @@ -56,15 +56,18 @@ namespace Tgstation.Server.Host.Components.Engine /// public override ValueTask Install(EngineVersion version, string installPath, CancellationToken cancellationToken) - => ValueTaskExtensions.WhenAll( - base.Install( - version, - installPath, - cancellationToken), - AddServerFirewallException( - version, - installPath, - cancellationToken)); + { + var installTask = base.Install( + version, + installPath, + cancellationToken); + var firewallTask = AddServerFirewallException( + version, + installPath, + cancellationToken); + + return ValueTaskExtensions.WhenAll(installTask, firewallTask); + } /// protected override async ValueTask HandleExtremelyLongPathOperation(Func shortenedPathOperation, string originalPath, CancellationToken cancellationToken) From 441985ff83ea22fdf13c683700ae0d835130046e Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 16 Dec 2023 16:50:11 -0500 Subject: [PATCH 67/82] Further readdress process output handling --- .../System/ProcessExecutor.cs | 65 +++++++++++++------ 1 file changed, 45 insertions(+), 20 deletions(-) diff --git a/src/Tgstation.Server.Host/System/ProcessExecutor.cs b/src/Tgstation.Server.Host/System/ProcessExecutor.cs index d6782d4512..29ea83129a 100644 --- a/src/Tgstation.Server.Host/System/ProcessExecutor.cs +++ b/src/Tgstation.Server.Host/System/ProcessExecutor.cs @@ -3,6 +3,7 @@ using System.Diagnostics; using System.IO; using System.Text; using System.Threading; +using System.Threading.Channels; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -223,9 +224,9 @@ namespace Tgstation.Server.Host.System /// The . /// The resulting in the of the started process. /// The optional path to redirect the streams to. - /// The that triggers when the is disposed. + /// The for the operation. /// A resulting in the program's output/error text if is , otherwise. - async Task ConsumeReaders(global::System.Diagnostics.Process handle, Task startupAndPid, string fileRedirect, CancellationToken disposeToken) + async Task ConsumeReaders(global::System.Diagnostics.Process handle, Task startupAndPid, string fileRedirect, CancellationToken cancellationToken) { var pid = await startupAndPid; logger.LogTrace("Starting read for PID {pid}...", pid); @@ -233,8 +234,11 @@ namespace Tgstation.Server.Host.System var stdOutHandle = handle.StandardOutput; var stdErrHandle = handle.StandardError; bool outputOpen = true, errorOpen = true; - var outputReadTask = stdOutHandle.ReadLineAsync(disposeToken).AsTask(); - var errorReadTask = stdErrHandle.ReadLineAsync(disposeToken).AsTask(); + + Task ReadHandle(StreamReader handle) => handle.ReadLineAsync(cancellationToken).AsTask(); + + var outputReadTask = ReadHandle(stdOutHandle); + var errorReadTask = ReadHandle(stdErrHandle); async ValueTask GetNextLine() { @@ -255,7 +259,7 @@ namespace Tgstation.Server.Host.System if (line == null) outputOpen = false; else - outputReadTask = stdOutHandle.ReadLineAsync(disposeToken).AsTask(); + outputReadTask = ReadHandle(stdOutHandle); else { Debug.Assert(nextLineTask == errorReadTask, "How is this incorrect?"); @@ -263,7 +267,7 @@ namespace Tgstation.Server.Host.System if (line == null) errorOpen = false; else - errorReadTask = stdErrHandle.ReadLineAsync(disposeToken).AsTask(); + errorReadTask = ReadHandle(stdErrHandle); } if (line == null && (errorOpen || outputOpen)) @@ -276,31 +280,52 @@ namespace Tgstation.Server.Host.System await using var writer = fileStream != null ? new StreamWriter(fileStream) : null; var stringBuilder = fileStream == null ? new StringBuilder() : null; - ulong fileFlushToken = 0; - async ValueTask QueueWrite(ValueTask previous, string text) + + var dataChannel = Channel.CreateUnbounded( + new UnboundedChannelOptions + { + AllowSynchronousContinuations = false, + SingleReader = true, + SingleWriter = true, + }); + + async ValueTask OutputWriter() { + var enumerable = dataChannel.Reader.ReadAllAsync(cancellationToken); if (fileStream != null) { - var startFlushToken = Interlocked.Increment(ref fileFlushToken); - await previous; - await writer.WriteLineAsync(text.AsMemory(), disposeToken); + var enumerator = enumerable.GetAsyncEnumerator(cancellationToken); + var nextEnumeration = enumerator.MoveNextAsync(); + while (await nextEnumeration) + { + var text = enumerator.Current; + nextEnumeration = enumerator.MoveNextAsync(); + await writer.WriteLineAsync(text.AsMemory(), cancellationToken); - // only flush if there isn't anything rapidly coming in - if (fileFlushToken == startFlushToken) - await writer.FlushAsync(disposeToken); + if (!nextEnumeration.IsCompleted) + await writer.FlushAsync(cancellationToken); + } } else - stringBuilder.AppendLine(text); + await foreach (var text in enumerable) + stringBuilder.AppendLine(text); } - ValueTask writeQueue = ValueTask.CompletedTask; try { - string text; - while ((text = await GetNextLine()) != null) - writeQueue = QueueWrite(writeQueue, text); + var outputWriterTask = OutputWriter(); + try + { + string text; + while ((text = await GetNextLine()) != null) + await dataChannel.Writer.WriteAsync(text, cancellationToken); - await writeQueue; + dataChannel.Writer.Complete(); + } + finally + { + await outputWriterTask; + } logger.LogTrace("Finished read for PID {pid}", pid); } From 40c5ba97b4c280793e371db46a2f5f8a1a62e39a Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 16 Dec 2023 16:51:44 -0500 Subject: [PATCH 68/82] Update to `actions/checkout@v4` --- .github/workflows/ci-pipeline.yml | 70 ++++++++++++++--------------- .github/workflows/code-scanning.yml | 2 +- 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 4f5a0f95d2..1efe65423b 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -137,11 +137,11 @@ jobs: exit 0 - name: Checkout (Branch) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name == 'push' || github.event_name == 'schedule' - name: Checkout (PR Merge) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: ref: "refs/pull/${{ github.event.number }}/merge" @@ -183,11 +183,11 @@ jobs: dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - name: Checkout (Branch) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name == 'push' || github.event_name == 'schedule' - name: Checkout (PR Merge) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: ref: "refs/pull/${{ github.event.number }}/merge" @@ -223,11 +223,11 @@ jobs: dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - name: Checkout (Branch) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name == 'push' || github.event_name == 'schedule' - name: Checkout (PR Merge) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: ref: "refs/pull/${{ github.event.number }}/merge" @@ -287,11 +287,11 @@ jobs: if: (!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success') steps: - name: Checkout (Branch) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name == 'push' || github.event_name == 'schedule' - name: Checkout (PR Merge) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: ref: "refs/pull/${{ github.event.number }}/merge" @@ -325,11 +325,11 @@ jobs: dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - name: Checkout (Branch) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name == 'push' || github.event_name == 'schedule' - name: Checkout (PR Merge) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: ref: "refs/pull/${{ github.event.number }}/merge" @@ -376,11 +376,11 @@ jobs: dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - name: Checkout (Branch) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name == 'push' || github.event_name == 'schedule' - name: Checkout (PR Merge) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: ref: "refs/pull/${{ github.event.number }}/merge" @@ -491,11 +491,11 @@ jobs: echo "TGS_TEST_DATABASE_TYPE=SqlServer" >> $GITHUB_ENV - name: Checkout (Branch) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name == 'push' || github.event_name == 'schedule' - name: Checkout (PR Merge) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: ref: "refs/pull/${{ github.event.number }}/merge" @@ -696,11 +696,11 @@ jobs: run: echo "General__UseBasicWatchdog=true" >> $GITHUB_ENV - name: Checkout (Branch) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name == 'push' || github.event_name == 'schedule' - name: Checkout (PR Merge) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: ref: "refs/pull/${{ github.event.number }}/merge" @@ -775,11 +775,11 @@ jobs: run: npm i -g ibm-openapi-validator@0.51.3 - name: Checkout (Branch) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name == 'push' || github.event_name == 'schedule' - name: Checkout (PR Merge) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: ref: "refs/pull/${{ github.event.number }}/merge" @@ -800,11 +800,11 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout (Branch) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name == 'push' || github.event_name == 'schedule' - name: Checkout (PR Merge) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: ref: "refs/pull/${{ github.event.number }}/merge" @@ -1088,11 +1088,11 @@ jobs: echo "New dotnet path should be $DOTNET_PATH" - name: Checkout (Branch) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name == 'push' || github.event_name == 'schedule' - name: Checkout (PR Merge) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: ref: "refs/pull/${{ github.event.number }}/merge" @@ -1171,11 +1171,11 @@ jobs: dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - name: Checkout (Branch) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name == 'push' || github.event_name == 'schedule' - name: Checkout (PR Merge) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: ref: "refs/pull/${{ github.event.number }}/merge" @@ -1303,11 +1303,11 @@ jobs: echo "pr_template_sha=$(cat commits.json | jq '.[0].sha')" >> $GITHUB_OUTPUT - name: Checkout (Branch) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name == 'push' || github.event_name == 'schedule' - name: Checkout (PR Merge) - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: github.event_name != 'push' && github.event_name != 'schedule' with: ref: "refs/pull/${{ github.event.number }}/merge" @@ -1352,7 +1352,7 @@ jobs: dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Restore run: dotnet restore @@ -1416,7 +1416,7 @@ jobs: dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Restore run: dotnet restore @@ -1479,7 +1479,7 @@ jobs: dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Restore run: dotnet restore @@ -1530,7 +1530,7 @@ jobs: dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Restore run: dotnet restore @@ -1554,7 +1554,7 @@ jobs: dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Restore run: dotnet restore @@ -1766,7 +1766,7 @@ jobs: dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Restore run: dotnet restore @@ -1805,7 +1805,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Parse TGS version run: | @@ -1829,7 +1829,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Parse TGS version run: | @@ -1862,7 +1862,7 @@ jobs: run: winget install wingetcreate --version 1.2.8.0 --disable-interactivity --accept-source-agreements # Pinned due to breaking every other version - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Build ReleaseNotes run: dotnet build -c Release -p:TGS_HOST_NO_WEBPANEL=true tools/Tgstation.Server.ReleaseNotes diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index a0e8069a33..a7baad0e0e 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -37,7 +37,7 @@ jobs: dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Initialize CodeQL uses: github/codeql-action/init@v2 From b86b4765eecc8a0006ece1c5b2a9636cf6d7491d Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 16 Dec 2023 17:06:23 -0500 Subject: [PATCH 69/82] Another `ProcessExecutor.ConsumeReaders` rewrite --- .../System/ProcessExecutor.cs | 79 +++++++------------ 1 file changed, 28 insertions(+), 51 deletions(-) diff --git a/src/Tgstation.Server.Host/System/ProcessExecutor.cs b/src/Tgstation.Server.Host/System/ProcessExecutor.cs index 29ea83129a..0deae3507a 100644 --- a/src/Tgstation.Server.Host/System/ProcessExecutor.cs +++ b/src/Tgstation.Server.Host/System/ProcessExecutor.cs @@ -1,5 +1,4 @@ using System; -using System.Diagnostics; using System.IO; using System.Text; using System.Threading; @@ -8,6 +7,7 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using Tgstation.Server.Common.Extensions; using Tgstation.Server.Host.IO; namespace Tgstation.Server.Host.System @@ -233,50 +233,9 @@ namespace Tgstation.Server.Host.System var stdOutHandle = handle.StandardOutput; var stdErrHandle = handle.StandardError; - bool outputOpen = true, errorOpen = true; - Task ReadHandle(StreamReader handle) => handle.ReadLineAsync(cancellationToken).AsTask(); - - var outputReadTask = ReadHandle(stdOutHandle); - var errorReadTask = ReadHandle(stdErrHandle); - - async ValueTask GetNextLine() - { - Debug.Assert(outputOpen || errorOpen, "We shouldn't be here if neither stream is open"); - Debug.Assert(outputOpen || outputReadTask.IsCompleted, "Output open flag mismatch"); - Debug.Assert(errorOpen || errorReadTask.IsCompleted, "Error open flag mismatch"); - - var nextLineTask = outputOpen - ? errorOpen - ? await Task.WhenAny(outputReadTask, errorReadTask) - : outputReadTask - : errorReadTask; - - var line = await nextLineTask; - - // Important to retrigger the reading block asap after pushing - if (nextLineTask == outputReadTask) - if (line == null) - outputOpen = false; - else - outputReadTask = ReadHandle(stdOutHandle); - else - { - Debug.Assert(nextLineTask == errorReadTask, "How is this incorrect?"); - - if (line == null) - errorOpen = false; - else - errorReadTask = ReadHandle(stdErrHandle); - } - - if (line == null && (errorOpen || outputOpen)) - return await GetNextLine(); - - return line; - } - - await using var fileStream = fileRedirect != null ? ioManager.CreateAsyncSequentialWriteStream(fileRedirect) : null; + bool writingToFile; + await using var fileStream = (writingToFile = fileRedirect != null) ? ioManager.CreateAsyncSequentialWriteStream(fileRedirect) : null; await using var writer = fileStream != null ? new StreamWriter(fileStream) : null; var stringBuilder = fileStream == null ? new StringBuilder() : null; @@ -284,15 +243,34 @@ namespace Tgstation.Server.Host.System var dataChannel = Channel.CreateUnbounded( new UnboundedChannelOptions { - AllowSynchronousContinuations = false, + AllowSynchronousContinuations = !writingToFile, SingleReader = true, - SingleWriter = true, + SingleWriter = false, }); + var handlesOpen = 2; + async ValueTask DrainHandle(StreamReader reader) + { + while (true) + { + var line = await reader.ReadLineAsync(cancellationToken); + if (line == null) + { + var handlesRemaining = Interlocked.Decrement(ref handlesOpen); + if (handlesRemaining == 0) + dataChannel.Writer.Complete(); + + break; + } + + await dataChannel.Writer.WriteAsync(line, cancellationToken); + } + } + async ValueTask OutputWriter() { var enumerable = dataChannel.Reader.ReadAllAsync(cancellationToken); - if (fileStream != null) + if (writingToFile) { var enumerator = enumerable.GetAsyncEnumerator(cancellationToken); var nextEnumeration = enumerator.MoveNextAsync(); @@ -316,11 +294,10 @@ namespace Tgstation.Server.Host.System var outputWriterTask = OutputWriter(); try { - string text; - while ((text = await GetNextLine()) != null) - await dataChannel.Writer.WriteAsync(text, cancellationToken); + var outputReadTask = DrainHandle(stdOutHandle); + var errorReadTask = DrainHandle(stdErrHandle); - dataChannel.Writer.Complete(); + await ValueTaskExtensions.WhenAll(outputReadTask, errorReadTask); } finally { From 1c14cf60e495d70682d8b7ee88de2b52f889fbf5 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 16 Dec 2023 17:06:28 -0500 Subject: [PATCH 70/82] Simplify a null check --- src/Tgstation.Server.Host/Database/DatabaseContext.cs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/Tgstation.Server.Host/Database/DatabaseContext.cs b/src/Tgstation.Server.Host/Database/DatabaseContext.cs index cca6313b53..3dd14560f6 100644 --- a/src/Tgstation.Server.Host/Database/DatabaseContext.cs +++ b/src/Tgstation.Server.Host/Database/DatabaseContext.cs @@ -245,11 +245,8 @@ namespace Tgstation.Server.Host.Database const string ConfigureMethodName = nameof(SqlServerDatabaseContext.ConfigureWith); var configureFunction = typeof(TDatabaseContext).GetMethod( ConfigureMethodName, - BindingFlags.Public | BindingFlags.Static); - - if (configureFunction == null) - throw new InvalidOperationException($"Context type {typeof(TDatabaseContext).FullName} missing static {ConfigureMethodName} function!"); - + BindingFlags.Public | BindingFlags.Static) + ?? throw new InvalidOperationException($"Context type {typeof(TDatabaseContext).FullName} missing static {ConfigureMethodName} function!"); return (optionsBuilder, config) => configureFunction.Invoke(null, new object[] { optionsBuilder, config }); } From 343b8c69ba8de71d003f481c3c28e34b4f78027e Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 16 Dec 2023 22:33:42 -0500 Subject: [PATCH 71/82] Sync topics sent in tests with `SessionController` --- .../Components/Session/SessionController.cs | 18 +++---- .../Live/Instance/WatchdogTest.cs | 49 +++++++++++-------- .../Live/TestLiveServer.cs | 26 +++++----- 3 files changed, 52 insertions(+), 41 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Session/SessionController.cs b/src/Tgstation.Server.Host/Components/Session/SessionController.cs index cefdcff76d..2e57410eab 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionController.cs @@ -108,6 +108,11 @@ namespace Tgstation.Server.Host.Components.Session /// public ReattachInformation ReattachInformation { get; } + /// + /// The used to prevent concurrent calls into /world/Topic(). + /// + public FifoSemaphore TopicSendSemaphore { get; } + /// /// The for the . /// @@ -148,11 +153,6 @@ namespace Tgstation.Server.Host.Components.Session /// readonly TaskCompletionSource initialBridgeRequestTcs; - /// - /// The used to prevent concurrent calls into /world/Topic(). - /// - readonly FifoSemaphore topicSendSemaphore; - /// /// The metadata. /// @@ -285,7 +285,7 @@ namespace Tgstation.Server.Host.Components.Session initialBridgeRequestTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); reattachTopicCts = new CancellationTokenSource(); - topicSendSemaphore = new FifoSemaphore(); + TopicSendSemaphore = new FifoSemaphore(); synchronizationLock = new object(); if (apiValidationSession || DMApiAvailable) @@ -338,7 +338,7 @@ namespace Tgstation.Server.Host.Components.Session Logger.LogTrace("Disposing..."); reattachTopicCts.Cancel(); - var semaphoreLockTask = topicSendSemaphore.Lock(CancellationToken.None); // DCT: None available + var semaphoreLockTask = TopicSendSemaphore.Lock(CancellationToken.None); // DCT: None available if (!released) { @@ -363,7 +363,7 @@ namespace Tgstation.Server.Host.Components.Session await Lifetime; // finish the async callback (await semaphoreLockTask).Dispose(); - topicSendSemaphore.Dispose(); + TopicSendSemaphore.Dispose(); } /// @@ -885,7 +885,7 @@ namespace Tgstation.Server.Host.Components.Session var targetPort = ReattachInformation.Port; Byond.TopicSender.TopicResponse byondResponse; - using (await topicSendSemaphore.Lock(cancellationToken)) + using (await TopicSendSemaphore.Lock(cancellationToken)) byondResponse = await byondTopicSender.SendWithOptionalPriority( asyncDelayer, Logger, diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index acb1a095e0..c62fbd67b5 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -237,20 +237,33 @@ namespace Tgstation.Server.Tests.Live.Instance await RunTest(false); } - async ValueTask SendTestTopic(string queryString, CancellationToken cancellationToken) + ValueTask SendTestTopic(string queryString, CancellationToken cancellationToken) + => SendTestTopic(queryString, topicClient, instanceManager.GetInstanceReference(instanceClient.Metadata), FindTopicPort(), cancellationToken); + + public static async ValueTask SendTestTopic(string queryString, ITopicClient topicClient, IInstanceReference instanceReference, ushort topicPort, CancellationToken cancellationToken) { - using var loggerFactory = LoggerFactory.Create(builder => + using (instanceReference) { - builder.AddConsole(); - builder.SetMinimumLevel(LogLevel.Trace); - }); - return await topicClient.SendWithOptionalPriority( - new AsyncDelayer(), - loggerFactory.CreateLogger(), - queryString, - FindTopicPort(), - true, - cancellationToken); + using var loggerFactory = LoggerFactory.Create(builder => + { + builder.AddConsole(); + builder.SetMinimumLevel(LogLevel.Trace); + }); + + var watchdog = instanceReference?.Watchdog; + var session = (SessionController)watchdog?.GetType().GetMethod("GetActiveController", BindingFlags.Instance | BindingFlags.NonPublic)?.Invoke(watchdog, null); + + using (session != null + ? await session.TopicSendSemaphore.Lock(cancellationToken) + : null) + return await topicClient.SendWithOptionalPriority( + new AsyncDelayer(), + loggerFactory.CreateLogger(), + queryString, + topicPort, + true, + cancellationToken); + } } async ValueTask BroadcastTest(CancellationToken cancellationToken) @@ -1335,8 +1348,8 @@ namespace Tgstation.Server.Tests.Live.Instance } public Task TellWorldToReboot(bool waitForOnlineIfRestoring, CancellationToken cancellationToken, [CallerLineNumber]int source = 0) - => TellWorldToReboot2(instanceClient, topicClient, FindTopicPort(), waitForOnlineIfRestoring || testVersion.Engine.Value == EngineType.OpenDream, cancellationToken, source); - public static async Task TellWorldToReboot2(IInstanceClient instanceClient, ITopicClient topicClient, ushort topicPort, bool waitForOnlineIfRestoring, CancellationToken cancellationToken, [CallerLineNumber]int source = 0, [CallerFilePath]string path = null) + => TellWorldToReboot2(instanceClient, instanceManager, topicClient, FindTopicPort(), waitForOnlineIfRestoring || testVersion.Engine.Value == EngineType.OpenDream, cancellationToken, source); + public static async Task TellWorldToReboot2(IInstanceClient instanceClient, IInstanceManager instanceManager, ITopicClient topicClient, ushort topicPort, bool waitForOnlineIfRestoring, CancellationToken cancellationToken, [CallerLineNumber]int source = 0, [CallerFilePath]string path = null) { var daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.IsNotNull(daemonStatus.StagedCompileJob); @@ -1344,13 +1357,7 @@ namespace Tgstation.Server.Tests.Live.Instance System.Console.WriteLine($"TEST: Sending world reboot topic @ {path}#L{source}"); - var result = await topicClient.SendWithOptionalPriority( - new AsyncDelayer(), - Mock.Of(), - $"tgs_integration_test_special_tactics=1", - topicPort, - true, - cancellationToken); + var result = await SendTestTopic("tgs_integration_test_special_tactics=1", topicClient, instanceManager.GetInstanceReference(instanceClient.Metadata), topicPort, cancellationToken); Assert.AreEqual("ack", result.StringData); using var tempCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index bc0a8c5e5a..b027ca4b4f 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -1532,12 +1532,11 @@ namespace Tgstation.Server.Tests.Live // test the reattach message queueing // for the code coverage really... - var topicRequestResult = await WatchdogTest.StaticTopicClient.SendWithOptionalPriority( - new AsyncDelayer(), - Mock.Of(), - $"tgs_integration_test_tactics6=1", + var topicRequestResult = await WatchdogTest.SendTestTopic( + "tgs_integration_test_tactics6=1", + WatchdogTest.StaticTopicClient, + null, mainDDPort, - true, cancellationToken); Assert.IsNotNull(topicRequestResult); @@ -1610,12 +1609,11 @@ namespace Tgstation.Server.Tests.Live var chatReadTask = instanceClient.ChatBots.List(null, cancellationToken); // Check the DMAPI got the channels again https://github.com/tgstation/tgstation-server/issues/1490 - topicRequestResult = await WatchdogTest.StaticTopicClient.SendWithOptionalPriority( - new AsyncDelayer(), - Mock.Of(), - $"tgs_integration_test_tactics7=1", + topicRequestResult = await WatchdogTest.SendTestTopic( + "tgs_integration_test_tactics7=1", + WatchdogTest.StaticTopicClient, + GetInstanceManager().GetInstanceReference(instanceClient.Metadata), mainDDPort, - true, cancellationToken); Assert.IsNotNull(topicRequestResult); @@ -1626,7 +1624,13 @@ namespace Tgstation.Server.Tests.Live Assert.AreEqual(connectedChannelCount, topicRequestResult.FloatData.Value); - dd = await WatchdogTest.TellWorldToReboot2(instanceClient, WatchdogTest.StaticTopicClient, mainDDPort, true, cancellationToken); + dd = await WatchdogTest.TellWorldToReboot2( + instanceClient, + GetInstanceManager(), + WatchdogTest.StaticTopicClient, + mainDDPort, + true, + cancellationToken); Assert.AreEqual(WatchdogStatus.Online, dd.Status.Value); // if this assert fails, you likely have to crack open the debugger and read test_fail_reason.txt manually Assert.IsNull(dd.StagedCompileJob); From 768a91eabc674ac4ec674c0ca08be556c663b323 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 16 Dec 2023 22:44:22 -0500 Subject: [PATCH 72/82] Cleanup a bunch of messages --- .../Components/StaticFiles/Configuration.cs | 47 +++++++++---------- .../Components/Watchdog/WatchdogBase.cs | 3 +- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs index f42b4cc7e2..eecb350e43 100644 --- a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs +++ b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs @@ -298,8 +298,7 @@ namespace Tgstation.Server.Host.Components.StaticFiles string GetFileSha() { var content = synchronousIOManager.ReadFile(path); - using var sha1 = SHA1.Create(); - return String.Join(String.Empty, sha1.ComputeHash(content).Select(b => b.ToString("x2", CultureInfo.InvariantCulture))); + return String.Join(String.Empty, SHA1.HashData(content).Select(b => b.ToString("x2", CultureInfo.InvariantCulture))); } var originalSha = GetFileSha(); @@ -406,12 +405,10 @@ namespace Tgstation.Server.Host.Components.StaticFiles var fileName = ioManager.GetFileName(file); // need to normalize - bool ignored; - if (platformIdentifier.IsWindows) - ignored = ignoreFiles.Any(y => fileName.ToUpperInvariant() == y.ToUpperInvariant()); - else - ignored = ignoreFiles.Any(y => fileName == y); - + var fileComparison = platformIdentifier.IsWindows + ? StringComparison.OrdinalIgnoreCase + : StringComparison.Ordinal; + var ignored = ignoreFiles.Any(y => fileName.Equals(y, fileComparison)); if (ignored) { logger.LogTrace("Ignoring static file {fileName}...", fileName); @@ -442,12 +439,14 @@ namespace Tgstation.Server.Host.Components.StaticFiles using (var reader = new StringReader(ignoreFileText)) { cancellationToken.ThrowIfCancellationRequested(); - var line = await reader.ReadLineAsync(); + var line = await reader.ReadLineAsync(cancellationToken); if (!String.IsNullOrEmpty(line)) ignoreFiles.Add(line); } - await ValueTaskExtensions.WhenAll(SymlinkBase(true), SymlinkBase(false)); + var filesSymlinkTask = SymlinkBase(true); + var dirsSymlinkTask = SymlinkBase(false); + await ValueTaskExtensions.WhenAll(filesSymlinkTask, dirsSymlinkTask); } } @@ -615,7 +614,7 @@ namespace Tgstation.Server.Host.Components.StaticFiles scriptName => x.StartsWith(scriptName, StringComparison.Ordinal))) .ToList(); - if (!scriptFiles.Any()) + if (scriptFiles.Count == 0) { logger.LogTrace("No event scripts starting with \"{scriptName}\" detected", String.Join("\" or \"", scriptNames)); return; @@ -710,19 +709,19 @@ namespace Tgstation.Server.Host.Components.StaticFiles return; await ioManager.CreateDirectory(CodeModificationsSubdirectory, cancellationToken); - await ValueTaskExtensions.WhenAll( - ioManager.WriteAllBytes( - ioManager.ConcatPath( - CodeModificationsSubdirectory, - CodeModificationsHeadFile), - Encoding.UTF8.GetBytes(DefaultHeadInclude), - cancellationToken), - ioManager.WriteAllBytes( - ioManager.ConcatPath( - CodeModificationsSubdirectory, - CodeModificationsTailFile), - Encoding.UTF8.GetBytes(DefaultTailInclude), - cancellationToken)); + var headWriteTask = ioManager.WriteAllBytes( + ioManager.ConcatPath( + CodeModificationsSubdirectory, + CodeModificationsHeadFile), + Encoding.UTF8.GetBytes(DefaultHeadInclude), + cancellationToken); + var tailWriteTask = ioManager.WriteAllBytes( + ioManager.ConcatPath( + CodeModificationsSubdirectory, + CodeModificationsTailFile), + Encoding.UTF8.GetBytes(DefaultTailInclude), + cancellationToken); + await ValueTaskExtensions.WhenAll(headWriteTask, tailWriteTask); } return Task.WhenAll( diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs index 60b3deb05f..4fb3e1677f 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs @@ -748,8 +748,9 @@ namespace Tgstation.Server.Host.Components.Watchdog try { var sessionEventTask = relayToSession ? ((IEventConsumer)this).HandleEvent(eventType, parameters, false, cancellationToken) : ValueTask.CompletedTask; + var eventConsumerTask = eventConsumer.HandleEvent(eventType, parameters, false, cancellationToken); await ValueTaskExtensions.WhenAll( - eventConsumer.HandleEvent(eventType, parameters, false, cancellationToken), + eventConsumerTask, sessionEventTask); } catch (JobException ex) From 612ec04d763f50215babbdbd27e1362328145bb1 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 16 Dec 2023 22:45:12 -0500 Subject: [PATCH 73/82] Disable `IDE0028` and `IDE0301` --- build/analyzers.ruleset | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/build/analyzers.ruleset b/build/analyzers.ruleset index 237aa86fe4..e61e800d00 100644 --- a/build/analyzers.ruleset +++ b/build/analyzers.ruleset @@ -668,7 +668,9 @@ + + @@ -718,8 +720,8 @@ - - + + @@ -755,6 +757,7 @@ + @@ -956,8 +959,8 @@ - - + + From a9d21eccf746139b63787a932b4b201b2f2bc642 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 16 Dec 2023 22:47:56 -0500 Subject: [PATCH 74/82] Fix another style message --- .../Components/StaticFiles/Configuration.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs index eecb350e43..5e871c2551 100644 --- a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs +++ b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs @@ -71,7 +71,7 @@ namespace Tgstation.Server.Host.Components.StaticFiles /// /// Map of s to the filename of the event scripts they trigger. /// - static readonly IReadOnlyDictionary> EventTypeScriptFileNameMap = new Dictionary>( + public static IReadOnlyDictionary> EventTypeScriptFileNameMap { get; } = new Dictionary>( Enum.GetValues(typeof(EventType)) .Cast() .Select( From 2cff33c5b39bc734db219911e05d2dc8045283ec Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 17 Dec 2023 09:16:35 -0500 Subject: [PATCH 75/82] I am once again asking you to fix process output reading --- .../System/ProcessExecutor.cs | 82 ++++++++++--------- 1 file changed, 43 insertions(+), 39 deletions(-) diff --git a/src/Tgstation.Server.Host/System/ProcessExecutor.cs b/src/Tgstation.Server.Host/System/ProcessExecutor.cs index 0deae3507a..8df021f795 100644 --- a/src/Tgstation.Server.Host/System/ProcessExecutor.cs +++ b/src/Tgstation.Server.Host/System/ProcessExecutor.cs @@ -1,4 +1,5 @@ using System; +using System.Diagnostics; using System.IO; using System.Text; using System.Threading; @@ -7,7 +8,6 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using Tgstation.Server.Common.Extensions; using Tgstation.Server.Host.IO; namespace Tgstation.Server.Host.System @@ -145,9 +145,6 @@ namespace Tgstation.Server.Host.System if (readStandardHandles) { processStartTcs = new TaskCompletionSource(); - handle.StartInfo.RedirectStandardOutput = true; - handle.StartInfo.RedirectStandardError = true; - disposeCts = new CancellationTokenSource(); readTask = ConsumeReaders(handle, processStartTcs.Task, fileRedirect, disposeCts.Token); } @@ -228,11 +225,8 @@ namespace Tgstation.Server.Host.System /// A resulting in the program's output/error text if is , otherwise. async Task ConsumeReaders(global::System.Diagnostics.Process handle, Task startupAndPid, string fileRedirect, CancellationToken cancellationToken) { - var pid = await startupAndPid; - logger.LogTrace("Starting read for PID {pid}...", pid); - - var stdOutHandle = handle.StandardOutput; - var stdErrHandle = handle.StandardError; + handle.StartInfo.RedirectStandardOutput = true; + handle.StartInfo.RedirectStandardError = true; bool writingToFile; await using var fileStream = (writingToFile = fileRedirect != null) ? ioManager.CreateAsyncSequentialWriteStream(fileRedirect) : null; @@ -249,24 +243,31 @@ namespace Tgstation.Server.Host.System }); var handlesOpen = 2; - async ValueTask DrainHandle(StreamReader reader) + async void DataReceivedHandler(object sender, DataReceivedEventArgs eventArgs) { - while (true) + var line = eventArgs.Data; + if (line == null) { - var line = await reader.ReadLineAsync(cancellationToken); - if (line == null) - { - var handlesRemaining = Interlocked.Decrement(ref handlesOpen); - if (handlesRemaining == 0) - dataChannel.Writer.Complete(); + var handlesRemaining = Interlocked.Decrement(ref handlesOpen); + if (handlesRemaining == 0) + dataChannel.Writer.Complete(); - break; - } + return; + } + try + { await dataChannel.Writer.WriteAsync(line, cancellationToken); } + catch (OperationCanceledException ex) + { + logger.LogWarning(ex, "Handle channel write interrupted!"); + } } + handle.OutputDataReceived += DataReceivedHandler; + handle.ErrorDataReceived += DataReceivedHandler; + async ValueTask OutputWriter() { var enumerable = dataChannel.Reader.ReadAllAsync(cancellationToken); @@ -289,28 +290,31 @@ namespace Tgstation.Server.Host.System stringBuilder.AppendLine(text); } - try - { - var outputWriterTask = OutputWriter(); - try - { - var outputReadTask = DrainHandle(stdOutHandle); - var errorReadTask = DrainHandle(stdErrHandle); + var pid = await startupAndPid; + logger.LogTrace("Starting read for PID {pid}...", pid); - await ValueTaskExtensions.WhenAll(outputReadTask, errorReadTask); - } - finally - { - await outputWriterTask; - } - - logger.LogTrace("Finished read for PID {pid}", pid); - } - catch (OperationCanceledException ex) + using (cancellationToken.Register(() => dataChannel.Writer.TryComplete())) { - logger.LogWarning(ex, "PID {pid} stream reading interrupted!", pid); - if (fileStream != null) - await writer.WriteLineAsync("-- Process detached, log truncated. This is likely due a to TGS restart --"); + handle.BeginOutputReadLine(); + using (cancellationToken.Register(handle.CancelOutputRead)) + { + handle.BeginErrorReadLine(); + using (cancellationToken.Register(handle.CancelErrorRead)) + { + try + { + await OutputWriter(); + + logger.LogTrace("Finished read for PID {pid}", pid); + } + catch (OperationCanceledException ex) + { + logger.LogWarning(ex, "PID {pid} stream reading interrupted!", pid); + if (fileStream != null) + await writer.WriteLineAsync("-- Process detached, log truncated. This is likely due a to TGS restart --"); + } + } + } } return stringBuilder?.ToString(); From 5eef0905116649934ad2f9c4ecb29da3e8bfc826 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 17 Dec 2023 09:53:50 -0500 Subject: [PATCH 76/82] Suppress OD build output in CI --- .../Components/Engine/OpenDreamInstaller.cs | 18 ++++++++++++++---- .../Configuration/GeneralConfiguration.cs | 5 +++++ src/Tgstation.Server.Host/appsettings.yml | 1 + .../Live/LiveTestingServer.cs | 1 + 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs b/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs index 6b356ba539..b8ffc98334 100644 --- a/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs +++ b/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs @@ -230,18 +230,28 @@ namespace Tgstation.Server.Host.Components.Engine shortenedPath, $"run -c Release --project OpenDreamPackageTool -- --tgs -o {shortenedDeployPath}", null, - true, - true); + !GeneralConfiguration.OpenDreamSuppressInstallOutput, + !GeneralConfiguration.OpenDreamSuppressInstallOutput); using (cancellationToken.Register(() => buildProcess.Terminate())) buildExitCode = await buildProcess.Lifetime; - Logger.LogTrace("OD build complete, waiting for output..."); + string output; + + if (!GeneralConfiguration.OpenDreamSuppressInstallOutput) + { + var buildOutputTask = buildProcess.GetCombinedOutput(cancellationToken); + if (!buildOutputTask.IsCompleted) + Logger.LogTrace("OD build complete, waiting for output..."); + output = await buildOutputTask; + } + else + output = ""; Logger.LogDebug( "OpenDream build exited with code {exitCode}:{newLine}{output}", buildExitCode, Environment.NewLine, - await buildProcess.GetCombinedOutput(cancellationToken)); + output); }, sourcePath, cancellationToken); diff --git a/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs b/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs index f33ef79b11..8e2fe59579 100644 --- a/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs +++ b/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs @@ -145,6 +145,11 @@ namespace Tgstation.Server.Host.Configuration /// public string OpenDreamGitTagPrefix { get; set; } = DefaultOpenDreamGitTagPrefix; + /// + /// If the dotnet output of creating an OpenDream installation should be suppressed. Known to cause issues in CI. + /// + public bool OpenDreamSuppressInstallOutput { get; set; } + /// /// Initializes a new instance of the class. /// diff --git a/src/Tgstation.Server.Host/appsettings.yml b/src/Tgstation.Server.Host/appsettings.yml index 60624d3617..56f87f2169 100644 --- a/src/Tgstation.Server.Host/appsettings.yml +++ b/src/Tgstation.Server.Host/appsettings.yml @@ -18,6 +18,7 @@ General: DeploymentDirectoryCopyTasksPerCore: 100 # Maximum number of concurrent file copy operations PER available CPU core OpenDreamGitUrl: https://github.com/OpenDreamProject/OpenDream # The repository to retrieve OpenDream from OpenDreamGitTagPrefix: v # The prefix to the OpenDream semver as tags appear in the git repository + OpenDreamSuppressInstallOutput: false # Suppress the dotnet output of creating an OpenDream installation. Known to cause hangs in CI. Session: HighPriorityLiveDreamDaemon: false # If DreamDaemon instances should run as higher priority processes LowPriorityDeploymentProcesses: true # If TGS Deployments should run as lower priority processes diff --git a/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs b/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs index 61a7a36176..645edac147 100644 --- a/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs +++ b/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs @@ -154,6 +154,7 @@ namespace Tgstation.Server.Tests.Live $"General:SkipAddingByondFirewallException={!TestingUtils.RunningInGitHubActions}", $"General:OpenDreamGitUrl={OpenDreamUrl}", $"Security:TokenExpiryMinutes=120", // timeouts are useless for us + $"General:OpenDreamSuppressInstallOutput={TestingUtils.RunningInGitHubActions}", }; swarmArgs = new List(); From da10d0f1519418fdca04b7c3d4e6676c78fb1067 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 17 Dec 2023 16:23:32 -0500 Subject: [PATCH 77/82] Double a test timeout --- tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index c62fbd67b5..7924638e4f 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -782,7 +782,7 @@ namespace Tgstation.Server.Tests.Live.Instance ourProcessHandler.Suspend(); global::System.Console.WriteLine($"WATCHDOG TEST {instanceClient.Metadata.Id}: FINISH PROCESS SUSPEND FOR HEALTH CHECK DEATH. WAITING FOR LIFETIME {ourProcessHandler.Id}."); - await Task.WhenAny(ourProcessHandler.Lifetime, Task.Delay(TimeSpan.FromMinutes(2), cancellationToken)); + await Task.WhenAny(ourProcessHandler.Lifetime, Task.Delay(TimeSpan.FromMinutes(4), cancellationToken)); Assert.IsTrue(ourProcessHandler.Lifetime.IsCompleted); var timeout = 20; From 4b4bdcf25afe7ba7490769bfcf141816156852a0 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 17 Dec 2023 16:27:36 -0500 Subject: [PATCH 78/82] Use larger page size here --- tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs b/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs index 19be3fe30b..f5edc04ea1 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; @@ -156,7 +156,10 @@ namespace Tgstation.Server.Tests.Live.Instance Online = true, }, cancellationToken); - var jobs = await permedUser.Instances.CreateClient(instance).Jobs.List(null, cancellationToken); + var jobs = await permedUser.Instances.CreateClient(instance).Jobs.List(new PaginationSettings + { + PageSize = 100 + }, cancellationToken); if (wasOffline) await permedUser.Instances.Update(new InstanceUpdateRequest { From 577ecb0d0243f836933c7998cd9a96696baa4356 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 17 Dec 2023 16:27:43 -0500 Subject: [PATCH 79/82] Show all duplicated jobs --- tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs b/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs index f5edc04ea1..84985b6a9b 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; @@ -179,7 +179,7 @@ namespace Tgstation.Server.Tests.Live.Instance var uniqueAllJobs = groups.Select(x => x.First()).ToList(); static string JobListFormatter(IEnumerable jobs) => String.Join(Environment.NewLine, jobs.Select(x => $"- I:{x.InstanceId}|JID:{x.Id}|JC:{x.JobCode}|Desc:{x.Description}")); - Assert.AreEqual(allJobs.Count, uniqueAllJobs.Count, $"Duplicated Jobs:{Environment.NewLine}{JobListFormatter(groups.Where(x => x.Count() > 1).Select(x => x.First()))}"); + Assert.AreEqual(allJobs.Count, uniqueAllJobs.Count, $"Duplicated Jobs:{Environment.NewLine}{JobListFormatter(groups.Where(x => x.Count() > 1).SelectMany(x => x))}"); var missableMissedJobs = 0; foreach (var job in allJobs) From 93da4d87664ea1cb16a69b20713bf221a51d1dc1 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 17 Dec 2023 16:32:51 -0500 Subject: [PATCH 80/82] More robust test port allocation --- .../Live/TestLiveServer.cs | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index b027ca4b4f..8c81e5451d 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -54,12 +54,12 @@ namespace Tgstation.Server.Tests.Live { public static readonly Version TestUpdateVersion = new(5, 11, 0); - static readonly ushort mainDDPort = FreeTcpPort(); - static readonly ushort mainDMPort = FreeTcpPort(mainDDPort); - static readonly ushort compatDMPort = FreeTcpPort(mainDDPort, mainDMPort); - static readonly ushort compatDDPort = FreeTcpPort(mainDDPort, mainDMPort, compatDMPort); - static readonly ushort odDMPort = FreeTcpPort(mainDDPort, mainDMPort, compatDMPort, compatDDPort); - static readonly ushort odDDPort = FreeTcpPort(mainDDPort, mainDMPort, compatDMPort, compatDDPort, odDMPort); + static readonly Lazy odDMPort = new Lazy(() => FreeTcpPort()); + static readonly Lazy odDDPort = new Lazy(() => FreeTcpPort(odDMPort.Value)); + static readonly Lazy compatDMPort = new Lazy(() => FreeTcpPort(odDDPort.Value, odDMPort.Value)); + static readonly Lazy compatDDPort = new Lazy(() => FreeTcpPort(odDDPort.Value, odDMPort.Value, compatDMPort.Value)); + static readonly Lazy mainDDPort = new Lazy(() => FreeTcpPort(odDDPort.Value, odDMPort.Value, compatDMPort.Value, compatDDPort.Value)); + static readonly Lazy mainDMPort = new Lazy(() => FreeTcpPort(odDDPort.Value, odDMPort.Value, compatDMPort.Value, compatDDPort.Value, mainDDPort.Value)); readonly ServerClientFactory clientFactory = new (new ProductHeaderValue(Assembly.GetExecutingAssembly().GetName().Name, Assembly.GetExecutingAssembly().GetName().Version.ToString())); @@ -167,7 +167,7 @@ namespace Tgstation.Server.Tests.Live result = (ushort)((IPEndPoint)l.LocalEndpoint).Port; } - while (usedPorts.Contains(result)); + while (usedPorts.Contains(result) || result < 10000); } finally { @@ -1457,8 +1457,8 @@ namespace Tgstation.Server.Tests.Live await edgeODVersionTask, server.OpenDreamUrl, firstAdminClient.Instances.CreateClient(odInstance), - odDMPort, - odDDPort, + odDMPort.Value, + odDDPort.Value, server.HighPriorityDreamDaemon, server.UsingBasicWatchdog, cancellationToken); @@ -1484,8 +1484,8 @@ namespace Tgstation.Server.Tests.Live }, server.OpenDreamUrl, firstAdminClient.Instances.CreateClient(compatInstance), - compatDMPort, - compatDDPort, + compatDMPort.Value, + compatDDPort.Value, server.HighPriorityDreamDaemon, server.UsingBasicWatchdog, cancellationToken)); @@ -1497,8 +1497,8 @@ namespace Tgstation.Server.Tests.Live instanceTest .RunTests( instanceClient, - mainDMPort, - mainDDPort, + mainDMPort.Value, + mainDDPort.Value, server.HighPriorityDreamDaemon, server.LowPriorityDeployments, server.UsingBasicWatchdog, @@ -1536,7 +1536,7 @@ namespace Tgstation.Server.Tests.Live "tgs_integration_test_tactics6=1", WatchdogTest.StaticTopicClient, null, - mainDDPort, + mainDDPort.Value, cancellationToken); Assert.IsNotNull(topicRequestResult); @@ -1613,7 +1613,7 @@ namespace Tgstation.Server.Tests.Live "tgs_integration_test_tactics7=1", WatchdogTest.StaticTopicClient, GetInstanceManager().GetInstanceReference(instanceClient.Metadata), - mainDDPort, + mainDDPort.Value, cancellationToken); Assert.IsNotNull(topicRequestResult); @@ -1628,7 +1628,7 @@ namespace Tgstation.Server.Tests.Live instanceClient, GetInstanceManager(), WatchdogTest.StaticTopicClient, - mainDDPort, + mainDDPort.Value, true, cancellationToken); @@ -1687,7 +1687,7 @@ namespace Tgstation.Server.Tests.Live Assert.AreEqual(WatchdogStatus.Online, dd.Status.Value); var compileJob = await instanceClient.DreamMaker.Compile(cancellationToken); - await using var wdt = new WatchdogTest(edgeVersion, instanceClient, GetInstanceManager(), (ushort)server.ApiUrl.Port, server.HighPriorityDreamDaemon, mainDDPort, server.UsingBasicWatchdog); + await using var wdt = new WatchdogTest(edgeVersion, instanceClient, GetInstanceManager(), (ushort)server.ApiUrl.Port, server.HighPriorityDreamDaemon, mainDDPort.Value, server.UsingBasicWatchdog); await wdt.WaitForJob(compileJob, 30, false, null, cancellationToken); dd = await instanceClient.DreamDaemon.Read(cancellationToken); @@ -1730,7 +1730,7 @@ namespace Tgstation.Server.Tests.Live Assert.AreEqual(WatchdogStatus.Online, currentDD.Status); Assert.AreEqual(expectedStaged, currentDD.StagedCompileJob.Job.Id.Value); - await using var wdt = new WatchdogTest(edgeVersion, instanceClient, GetInstanceManager(), (ushort)server.ApiUrl.Port, server.HighPriorityDreamDaemon, mainDDPort, server.UsingBasicWatchdog); + await using var wdt = new WatchdogTest(edgeVersion, instanceClient, GetInstanceManager(), (ushort)server.ApiUrl.Port, server.HighPriorityDreamDaemon, mainDDPort.Value, server.UsingBasicWatchdog); currentDD = await wdt.TellWorldToReboot(false, cancellationToken); Assert.AreEqual(expectedStaged, currentDD.ActiveCompileJob.Job.Id.Value); Assert.IsNull(currentDD.StagedCompileJob); From 638418534e15bf3364ef2cde45f84b3078017bd5 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 17 Dec 2023 16:41:26 -0500 Subject: [PATCH 81/82] Fix `DummyChatProvider` interference with health check tests --- .../Tgstation.Server.Tests/Live/DummyChatProvider.cs | 4 ++++ .../Live/Instance/WatchdogTest.cs | 12 ++++++++++++ 2 files changed, 16 insertions(+) diff --git a/tests/Tgstation.Server.Tests/Live/DummyChatProvider.cs b/tests/Tgstation.Server.Tests/Live/DummyChatProvider.cs index 2edb4686ed..14cc51dd4d 100644 --- a/tests/Tgstation.Server.Tests/Live/DummyChatProvider.cs +++ b/tests/Tgstation.Server.Tests/Live/DummyChatProvider.cs @@ -44,6 +44,8 @@ namespace Tgstation.Server.Tests.Live ulong channelIdAllocator; + public static Task MessageGuard = Task.CompletedTask; + static IAsyncDelayer CreateMockDelayer() { // at time of writing, this is used exclusively for the reconnection interval which works in minutes @@ -219,6 +221,8 @@ namespace Tgstation.Server.Tests.Live var delay = random.Next(0, 10000); await Task.Delay(delay, cancellationToken); + await MessageGuard; + // %5 chance to disconnect randomly if (enableRandomDisconnections != 0 && random.Next(0, 100) > 95) connected = false; diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index 7924638e4f..c2fa31ba31 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -165,9 +165,21 @@ namespace Tgstation.Server.Tests.Live.Instance await RunLongRunningTestThenUpdateWithNewDme(cancellationToken); await RunLongRunningTestThenUpdateWithByondVersionSwitch(cancellationToken); + // no chatty bullshit while we test health checks + var tcs = new TaskCompletionSource(); + var oldTask = Interlocked.Exchange(ref DummyChatProvider.MessageGuard, tcs.Task); + await RunHealthCheckTest(true, cancellationToken); await RunHealthCheckTest(false, cancellationToken); + async void Cleanup() + { + await oldTask; + tcs.SetResult(); + } + + Cleanup(); + await InteropTestsForLongRunningDme(cancellationToken); await instanceClient.DreamDaemon.Update(new DreamDaemonRequest From 2bb85b6aeaea0074770287259dae5385236338d1 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 17 Dec 2023 18:17:21 -0500 Subject: [PATCH 82/82] Simpler test port allocation --- tests/Tgstation.Server.Tests/Live/TestLiveServer.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index 8c81e5451d..6edee102ea 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -147,8 +147,12 @@ namespace Tgstation.Server.Tests.Live static ushort FreeTcpPort(params ushort[] usedPorts) { + var portList = new ushort[] { 42069, 42070, 42071, 42072, 42073, 42074 }; + return portList.First(x => !usedPorts.Contains(x)); + /* ushort result; var listeners = new List(); + try { do @@ -177,6 +181,7 @@ namespace Tgstation.Server.Tests.Live } } return result; + */ } [ClassInitialize]