From 4c846014350bb0d3ecc0be4dc2ea7b455faa446c Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 16 Oct 2023 21:31:26 -0400 Subject: [PATCH 01/40] Fix DMAPI not returning a value for world.TgsSecurityLevel() --- build/Version.props | 2 +- src/DMAPI/tgs.dm | 2 +- src/DMAPI/tgs/core/core.dm | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build/Version.props b/build/Version.props index 6a908f98fd..ef2f652396 100644 --- a/build/Version.props +++ b/build/Version.props @@ -9,7 +9,7 @@ 6.0.1 11.1.2 12.1.2 - 6.5.3 + 6.5.4 5.6.1 1.4.0 1.2.1 diff --git a/src/DMAPI/tgs.dm b/src/DMAPI/tgs.dm index 6187a67825..d0466b806f 100644 --- a/src/DMAPI/tgs.dm +++ b/src/DMAPI/tgs.dm @@ -1,6 +1,6 @@ // tgstation-server DMAPI -#define TGS_DMAPI_VERSION "6.5.3" +#define TGS_DMAPI_VERSION "6.5.4" // All functions and datums outside this document are subject to change with any version and should not be relied on. diff --git a/src/DMAPI/tgs/core/core.dm b/src/DMAPI/tgs/core/core.dm index 41a0473394..aa4084904b 100644 --- a/src/DMAPI/tgs/core/core.dm +++ b/src/DMAPI/tgs/core/core.dm @@ -153,4 +153,4 @@ /world/TgsSecurityLevel() var/datum/tgs_api/api = TGS_READ_GLOBAL(tgs) if(api) - api.SecurityLevel() + return api.SecurityLevel() From c8984cc109134062e7e5546d7ea5eb450d420cd2 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 16 Oct 2023 21:59:41 -0400 Subject: [PATCH 02/40] Fix a log message typo --- src/Tgstation.Server.Host/Utils/GitHub/GitHubClientFactory.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Utils/GitHub/GitHubClientFactory.cs b/src/Tgstation.Server.Host/Utils/GitHub/GitHubClientFactory.cs index 47a67728ab..7595a70a43 100644 --- a/src/Tgstation.Server.Host/Utils/GitHub/GitHubClientFactory.cs +++ b/src/Tgstation.Server.Host/Utils/GitHub/GitHubClientFactory.cs @@ -153,7 +153,7 @@ namespace Tgstation.Server.Host.Utils.GitHub rateLimitInfo.Reset.ToString("o")); else logger.LogDebug( - "Requested GitHub client has {remainingRequests} requests remaining after the usage {lastUse}. Limit resets at {resetTime}", + "Requested GitHub client has {remainingRequests} requests remaining after the usage at {lastUse}. Limit resets at {resetTime}", rateLimitInfo.Remaining, lastUsed, rateLimitInfo.Reset.ToString("o")); From 8a5d0d40e585a3d19a2d51979cab76fa14b24816 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 16 Oct 2023 21:59:44 -0400 Subject: [PATCH 03/40] Test DMAPI security level and CWD is Live --- tests/DMAPI/LongRunning/Test.dm | 26 +++++++ .../LongRunning/long_running_test_rooted.dme | 17 +++++ .../Live/Instance/ConfigurationTest.cs | 10 ++- .../Live/Instance/InstanceTest.cs | 3 +- .../Live/Instance/WatchdogTest.cs | 68 ++++++++++++++++++- tgstation-server.sln | 1 + 6 files changed, 121 insertions(+), 4 deletions(-) create mode 100644 tests/DMAPI/LongRunning/long_running_test_rooted.dme diff --git a/tests/DMAPI/LongRunning/Test.dm b/tests/DMAPI/LongRunning/Test.dm index bed428eceb..d3515034ee 100644 --- a/tests/DMAPI/LongRunning/Test.dm +++ b/tests/DMAPI/LongRunning/Test.dm @@ -11,6 +11,12 @@ dab() TgsNew(new /datum/tgs_event_handler/impl, TGS_SECURITY_SAFE) + var/sec = TgsSecurityLevel() + if(isnull(sec)) + FailTest("TGS Security level was null!") + + log << "Running in security level: [sec]" + if(params["expect_chat_channels"]) var/list/channels = TgsChatChannelInfo() if(!length(channels)) @@ -140,6 +146,26 @@ var/run_bridge_test kajigger_test = TRUE return "we love casting spells" + var/expected_path = data["vaporeon"] + if(expected_path) + var/command + if(world.system_type == MS_WINDOWS) + command = "C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe -ExecutionPolicy Bypass -Command \"if('[expected_path]' -ne $(Get-Location)){ (Get-Location).Path | Out-File \"fuck_up.txt\"; exit 1; }\"" + else + command = "if \[\[ \"[expected_path]\" != \"$(pwd)\" \]\]; then echo $(pwd) > fuck_up.txt; exit 1; fi" + + world.log << "shell: [command]" + var/exitCode = shell(command) + + if(isnull(exitCode) || exitCode != 0) + var/fuck_up_reason = "DIDN'T READ" + if(fexists("fuck_up.txt")) + fuck_up_reason = "COULDN'T READ" + fuck_up_reason = file2text("fuck_up.txt") + return "Dir check shell command failed with code [exitCode || "null"]: [fuck_up_reason]" + + return "is the most pokemon of all time" + TgsChatBroadcast(new /datum/tgs_message_content("Recieved non-tgs topic: `[T]`")) return "feck" diff --git a/tests/DMAPI/LongRunning/long_running_test_rooted.dme b/tests/DMAPI/LongRunning/long_running_test_rooted.dme new file mode 100644 index 0000000000..d8e9d61c96 --- /dev/null +++ b/tests/DMAPI/LongRunning/long_running_test_rooted.dme @@ -0,0 +1,17 @@ +// Hand crafted DME, will not work if saved with DreamMaker + +// BEGIN_INTERNALS +// END_INTERNALS + +// BEGIN_FILE_DIR +#define FILE_DIR . +// END_FILE_DIR + +// BEGIN_PREFERENCES +// END_PREFERENCES + +// BEGIN_INCLUDE +#include "tests/DMAPI/LongRunning/Config.dm" +#include "tests/DMAPI/test_prelude.dm" +#include "tests/DMAPI/LongRunning/Test.dm" +// END_INCLUDE diff --git a/tests/Tgstation.Server.Tests/Live/Instance/ConfigurationTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/ConfigurationTest.cs index 1008406cd7..2dea3e5b12 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/ConfigurationTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/ConfigurationTest.cs @@ -92,7 +92,7 @@ namespace Tgstation.Server.Tests.Live.Instance await configurationClient.CreateDirectory(staticDir, cancellationToken); } - public Task SetupDMApiTests(CancellationToken cancellationToken) + public Task SetupDMApiTests(bool includingRoot, CancellationToken cancellationToken) { // just use an I/O manager here var ioManager = new DefaultIOManager(); @@ -104,6 +104,12 @@ namespace Tgstation.Server.Tests.Live.Instance ioManager.ConcatPath(instance.Path, "Repository", "tests", "DMAPI"), null, cancellationToken), + includingRoot + ? ioManager.CopyFile( + "../../../../DMAPI/LongRunning/long_running_test_rooted.dme", + ioManager.ConcatPath(instance.Path, "Repository", "long_running_test_rooted.dme"), + cancellationToken) + : Task.CompletedTask, ioManager.CopyDirectory( Enumerable.Empty(), null, @@ -175,8 +181,8 @@ namespace Tgstation.Server.Tests.Live.Instance } public Task RunPreWatchdog(CancellationToken cancellationToken) => Task.WhenAll( + SetupDMApiTests(false, cancellationToken), SequencedApiTests(cancellationToken), - SetupDMApiTests(cancellationToken), TestPregeneratedFilesExist(cancellationToken)); } } diff --git a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs index 9265e0e7f6..4457994d79 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs @@ -63,6 +63,7 @@ namespace Tgstation.Server.Tests.Live.Instance await chatTask; await dmTask; + await configTest.SetupDMApiTests(true, cancellationToken); await byondTask; await new WatchdogTest( @@ -172,7 +173,7 @@ namespace Tgstation.Server.Tests.Live.Instance dmUpdateRequest, cloneRequest); - var configSetupTask = new ConfigurationTest(instanceClient.Configuration, instanceClient.Metadata).SetupDMApiTests(cancellationToken); + var configSetupTask = new ConfigurationTest(instanceClient.Configuration, instanceClient.Metadata).SetupDMApiTests(true, cancellationToken); if (TestingUtils.RunningInGitHubActions || String.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("TGS_TEST_GITHUB_TOKEN")) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index 641085529c..bd1671e05e 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -131,6 +131,8 @@ namespace Tgstation.Server.Tests.Live.Instance await RunLongRunningTestThenUpdate(cancellationToken); + await TestDDIsntResolvingSymlink(cancellationToken); + await RunLongRunningTestThenUpdateWithNewDme(cancellationToken); await RunLongRunningTestThenUpdateWithByondVersionSwitch(cancellationToken); @@ -150,6 +152,70 @@ namespace Tgstation.Server.Tests.Live.Instance System.Console.WriteLine($"TEST: END WATCHDOG TESTS {instanceClient.Metadata.Name}"); } + async ValueTask TestDDIsntResolvingSymlink(CancellationToken cancellationToken) + { + System.Console.WriteLine("STARTING SYMLINK RESOLVE TEST"); + var deployTask = DeployTestDme("long_running_test_rooted", DreamDaemonSecurity.Trusted, true, cancellationToken); + + var previous = await instanceClient.DreamDaemon.Read(cancellationToken); + if (previous.SecurityLevel.Value != DreamDaemonSecurity.Trusted) + { + var updated = await instanceClient.DreamDaemon.Update(new DreamDaemonRequest + { + SecurityLevel = DreamDaemonSecurity.Trusted + }, cancellationToken); + + Assert.AreEqual(DreamDaemonSecurity.Trusted, updated.SecurityLevel); + } + + var startState = await deployTask; + + await WaitForJob(await StartDD(cancellationToken), 30, false, null, cancellationToken); + + var command = topicClient.SanitizeString( + Path.GetFullPath( + Path.Combine( + instanceClient.Metadata.Path, + "Game", + "Live"))); + command = $"vaporeon={command}"; + + var topicRequestResult = await topicClient.SendTopic( + IPAddress.Loopback, + command, + ddPort, + cancellationToken); + + Assert.IsNotNull(topicRequestResult); + Assert.AreEqual("is the most pokemon of all time", topicRequestResult.StringData); + + await DeployTestDme("long_running_test_rooted", DreamDaemonSecurity.Trusted, true, cancellationToken); + var newState = await TellWorldToReboot(cancellationToken); + + Assert.AreNotEqual(startState.ActiveCompileJob.Id, newState.ActiveCompileJob.Id); + topicRequestResult = await topicClient.SendTopic( + IPAddress.Loopback, + command, + ddPort, + cancellationToken); + + Assert.IsNotNull(topicRequestResult); + Assert.AreEqual("is the most pokemon of all time", topicRequestResult.StringData); + + await instanceClient.DreamDaemon.Shutdown(cancellationToken); + if (previous.SecurityLevel.Value != DreamDaemonSecurity.Trusted) + { + var updated = await instanceClient.DreamDaemon.Update(new DreamDaemonRequest + { + SecurityLevel = previous.SecurityLevel + }, cancellationToken); + + Assert.AreEqual(previous.SecurityLevel, updated.SecurityLevel); + } + + System.Console.WriteLine("END SYMLINK RESOLVE TEST"); + } + async Task InteropTestsForLongRunningDme(CancellationToken cancellationToken) { await StartAndLeaveRunning(cancellationToken); @@ -1101,7 +1167,7 @@ namespace Tgstation.Server.Tests.Live.Instance var refreshed = await instanceClient.DreamMaker.Update(new DreamMakerRequest { ApiValidationSecurityLevel = deploymentSecurity, - ProjectName = $"tests/DMAPI/{dmeName}", + ProjectName = dmeName.Contains("rooted") ? dmeName : $"tests/DMAPI/{dmeName}", RequireDMApiValidation = requireApi, Timeout = TimeSpan.FromMilliseconds(1), }, cancellationToken); diff --git a/tgstation-server.sln b/tgstation-server.sln index 6d183435b8..20eef245e1 100644 --- a/tgstation-server.sln +++ b/tgstation-server.sln @@ -157,6 +157,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "LongRunning", "LongRunning" tests\DMAPI\LongRunning\Config.dm = tests\DMAPI\LongRunning\Config.dm tests\DMAPI\LongRunning\long_running_test.dme = tests\DMAPI\LongRunning\long_running_test.dme tests\DMAPI\LongRunning\long_running_test_copy.dme = tests\DMAPI\LongRunning\long_running_test_copy.dme + tests\DMAPI\LongRunning\long_running_test_rooted.dme = tests\DMAPI\LongRunning\long_running_test_rooted.dme tests\DMAPI\LongRunning\Test.dm = tests\DMAPI\LongRunning\Test.dm EndProjectSection EndProject From 2f5e62c9a011b591d59860613e93fc50cb9ff4f4 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 20 Oct 2023 23:30:39 -0400 Subject: [PATCH 04/40] A note to help future errors on this assert --- tests/Tgstation.Server.Tests/Live/TestLiveServer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index 5a7da4471a..f058d9795f 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -1471,7 +1471,7 @@ namespace Tgstation.Server.Tests.Live await WatchdogTest.TellWorldToReboot2(instanceClient, WatchdogTest.StaticTopicClient, mainDDPort, cancellationToken); dd = await instanceClient.DreamDaemon.Read(cancellationToken); - Assert.AreEqual(WatchdogStatus.Online, dd.Status.Value); + Assert.AreEqual(WatchdogStatus.Online, dd.Status.Value); // if this assert fails, you likely have to crack open the debugger and read test_fail_reason.txt manually Assert.IsNull(dd.StagedCompileJob); Assert.AreEqual(initialStaged, dd.ActiveCompileJob.Id); From a6cb81902fe075f0f7361bc433db3c1a0be8c2d0 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 20 Oct 2023 23:44:44 -0400 Subject: [PATCH 05/40] Fix RuntimeInformation after reattach having null SecurityLevel and Visibility --- build/Version.props | 2 +- src/DMAPI/tgs/v5/__interop_version.dm | 2 +- .../Components/Interop/Bridge/RuntimeInformation.cs | 12 ++++++++---- .../Components/Session/ReattachInformation.cs | 6 ++---- .../Components/Session/SessionControllerFactory.cs | 13 ++++++++----- .../Models/ReattachInformationBase.cs | 4 ++-- 6 files changed, 22 insertions(+), 17 deletions(-) diff --git a/build/Version.props b/build/Version.props index ef2f652396..4a4c556f24 100644 --- a/build/Version.props +++ b/build/Version.props @@ -10,7 +10,7 @@ 11.1.2 12.1.2 6.5.4 - 5.6.1 + 5.6.2 1.4.0 1.2.1 1.0.2 diff --git a/src/DMAPI/tgs/v5/__interop_version.dm b/src/DMAPI/tgs/v5/__interop_version.dm index 5d3d491a73..1b52b31d6a 100644 --- a/src/DMAPI/tgs/v5/__interop_version.dm +++ b/src/DMAPI/tgs/v5/__interop_version.dm @@ -1 +1 @@ -"5.6.1" +"5.6.2" diff --git a/src/Tgstation.Server.Host/Components/Interop/Bridge/RuntimeInformation.cs b/src/Tgstation.Server.Host/Components/Interop/Bridge/RuntimeInformation.cs index 5d0d13b7fa..b7d05b67d7 100644 --- a/src/Tgstation.Server.Host/Components/Interop/Bridge/RuntimeInformation.cs +++ b/src/Tgstation.Server.Host/Components/Interop/Bridge/RuntimeInformation.cs @@ -2,6 +2,8 @@ using System.Collections.Generic; using System.Linq; +using Newtonsoft.Json; + using Tgstation.Server.Api.Models; using Tgstation.Server.Host.Components.Chat; using Tgstation.Server.Host.Components.Deployment; @@ -42,12 +44,14 @@ namespace Tgstation.Server.Host.Components.Interop.Bridge /// /// The level of the launch. /// - public DreamDaemonSecurity? SecurityLevel { get; } + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Include)] + public DreamDaemonSecurity SecurityLevel { get; } /// /// The level of the launch. /// - public DreamDaemonVisibility? Visibility { get; } + [JsonProperty(DefaultValueHandling = DefaultValueHandling.Include)] + public DreamDaemonVisibility Visibility { get; } /// /// The s in the launch. @@ -70,8 +74,8 @@ namespace Tgstation.Server.Host.Components.Interop.Bridge IDmbProvider dmbProvider, Version serverVersion, string instanceName, - DreamDaemonSecurity? securityLevel, - DreamDaemonVisibility? visibility, + DreamDaemonSecurity securityLevel, + DreamDaemonVisibility visibility, ushort serverPort, bool apiValidateOnly) : base(chatTrackingContext?.Channels ?? throw new ArgumentNullException(nameof(chatTrackingContext))) diff --git a/src/Tgstation.Server.Host/Components/Session/ReattachInformation.cs b/src/Tgstation.Server.Host/Components/Session/ReattachInformation.cs index 2c9007f4f3..6c429d2d0a 100644 --- a/src/Tgstation.Server.Host/Components/Session/ReattachInformation.cs +++ b/src/Tgstation.Server.Host/Components/Session/ReattachInformation.cs @@ -76,13 +76,11 @@ namespace Tgstation.Server.Host.Components.Session Dmb = dmb ?? throw new ArgumentNullException(nameof(dmb)); ProcessId = process?.Id ?? throw new ArgumentNullException(nameof(process)); RuntimeInformation = runtimeInformation ?? throw new ArgumentNullException(nameof(runtimeInformation)); - if (!runtimeInformation.SecurityLevel.HasValue) - throw new ArgumentException("runtimeInformation must have a valid SecurityLevel!", nameof(runtimeInformation)); AccessIdentifier = accessIdentifier ?? throw new ArgumentNullException(nameof(accessIdentifier)); - LaunchSecurityLevel = runtimeInformation.SecurityLevel.Value; - LaunchVisibility = runtimeInformation.Visibility.Value; + LaunchSecurityLevel = runtimeInformation.SecurityLevel; + LaunchVisibility = runtimeInformation.Visibility; Port = port; runtimeInformationLock = new object(); diff --git a/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs b/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs index 96228c3c32..939632524a 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs @@ -336,7 +336,8 @@ namespace Tgstation.Server.Host.Components.Session var runtimeInformation = CreateRuntimeInformation( dmbProvider, chatTrackingContext, - launchParameters, + launchParameters.SecurityLevel.Value, + launchParameters.Visibility.Value, apiValidate); var reattachInformation = new ReattachInformation( @@ -430,7 +431,8 @@ namespace Tgstation.Server.Host.Components.Session var runtimeInformation = CreateRuntimeInformation( reattachInformation.Dmb, chatTrackingContext, - null, + reattachInformation.LaunchSecurityLevel, + reattachInformation.LaunchVisibility, false); reattachInformation.SetRuntimeInformation(runtimeInformation); @@ -632,15 +634,16 @@ namespace Tgstation.Server.Host.Components.Session RuntimeInformation CreateRuntimeInformation( IDmbProvider dmbProvider, IChatTrackingContext chatTrackingContext, - DreamDaemonLaunchParameters launchParameters, + DreamDaemonSecurity securityLevel, + DreamDaemonVisibility visibility, bool apiValidateOnly) => new ( chatTrackingContext, dmbProvider, assemblyInformationProvider.Version, instance.Name, - launchParameters?.SecurityLevel, - launchParameters?.Visibility, + securityLevel, + visibility, serverPortProvider.HttpApiPort, apiValidateOnly); diff --git a/src/Tgstation.Server.Host/Models/ReattachInformationBase.cs b/src/Tgstation.Server.Host/Models/ReattachInformationBase.cs index 76dfbfda36..5cad9efbce 100644 --- a/src/Tgstation.Server.Host/Models/ReattachInformationBase.cs +++ b/src/Tgstation.Server.Host/Models/ReattachInformationBase.cs @@ -32,13 +32,13 @@ namespace Tgstation.Server.Host.Models /// The level DreamDaemon was launched with. /// [Required] - public DreamDaemonSecurity? LaunchSecurityLevel { get; set; } + public DreamDaemonSecurity LaunchSecurityLevel { get; set; } /// /// The DreamDaemon was launched with. /// [Required] - public DreamDaemonVisibility? LaunchVisibility { get; set; } + public DreamDaemonVisibility LaunchVisibility { get; set; } /// /// Initializes a new instance of the class. From 6ab9fc51fcd68b908eabd945d8695ad4e8c5f813 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 21 Oct 2023 00:16:36 -0400 Subject: [PATCH 06/40] Remove unnecessary `RequiredAttribute`s --- src/Tgstation.Server.Host/Models/ReattachInformationBase.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/Tgstation.Server.Host/Models/ReattachInformationBase.cs b/src/Tgstation.Server.Host/Models/ReattachInformationBase.cs index 5cad9efbce..241fae5141 100644 --- a/src/Tgstation.Server.Host/Models/ReattachInformationBase.cs +++ b/src/Tgstation.Server.Host/Models/ReattachInformationBase.cs @@ -1,5 +1,4 @@ using System; -using System.ComponentModel.DataAnnotations; using System.Globalization; using Tgstation.Server.Api.Models; @@ -31,13 +30,11 @@ namespace Tgstation.Server.Host.Models /// /// The level DreamDaemon was launched with. /// - [Required] public DreamDaemonSecurity LaunchSecurityLevel { get; set; } /// /// The DreamDaemon was launched with. /// - [Required] public DreamDaemonVisibility LaunchVisibility { get; set; } /// From 42b09376d2d90f783452814333dc6c2bec08ef2a Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 21 Oct 2023 00:18:30 -0400 Subject: [PATCH 07/40] Fix documentation comment errors --- .../Components/Session/SessionControllerFactory.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs b/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs index 939632524a..cf0dea0468 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs @@ -628,7 +628,8 @@ namespace Tgstation.Server.Host.Components.Session /// /// The . /// The . - /// The if any. + /// The the server was launched with. + /// The the server was launched with. /// The value of . /// A new class. RuntimeInformation CreateRuntimeInformation( From 279671c7cbd8d2e5ea184e2461c42af14b2b11df Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 21 Oct 2023 03:49:28 -0400 Subject: [PATCH 08/40] Remove failing and badly conceived test --- tests/DMAPI/LongRunning/Test.dm | 20 ------ .../Live/Instance/WatchdogTest.cs | 66 ------------------- 2 files changed, 86 deletions(-) diff --git a/tests/DMAPI/LongRunning/Test.dm b/tests/DMAPI/LongRunning/Test.dm index d3515034ee..a77c3377b5 100644 --- a/tests/DMAPI/LongRunning/Test.dm +++ b/tests/DMAPI/LongRunning/Test.dm @@ -146,26 +146,6 @@ var/run_bridge_test kajigger_test = TRUE return "we love casting spells" - var/expected_path = data["vaporeon"] - if(expected_path) - var/command - if(world.system_type == MS_WINDOWS) - command = "C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe -ExecutionPolicy Bypass -Command \"if('[expected_path]' -ne $(Get-Location)){ (Get-Location).Path | Out-File \"fuck_up.txt\"; exit 1; }\"" - else - command = "if \[\[ \"[expected_path]\" != \"$(pwd)\" \]\]; then echo $(pwd) > fuck_up.txt; exit 1; fi" - - world.log << "shell: [command]" - var/exitCode = shell(command) - - if(isnull(exitCode) || exitCode != 0) - var/fuck_up_reason = "DIDN'T READ" - if(fexists("fuck_up.txt")) - fuck_up_reason = "COULDN'T READ" - fuck_up_reason = file2text("fuck_up.txt") - return "Dir check shell command failed with code [exitCode || "null"]: [fuck_up_reason]" - - return "is the most pokemon of all time" - TgsChatBroadcast(new /datum/tgs_message_content("Recieved non-tgs topic: `[T]`")) return "feck" diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index bd1671e05e..329fab6aa1 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -131,8 +131,6 @@ namespace Tgstation.Server.Tests.Live.Instance await RunLongRunningTestThenUpdate(cancellationToken); - await TestDDIsntResolvingSymlink(cancellationToken); - await RunLongRunningTestThenUpdateWithNewDme(cancellationToken); await RunLongRunningTestThenUpdateWithByondVersionSwitch(cancellationToken); @@ -152,70 +150,6 @@ namespace Tgstation.Server.Tests.Live.Instance System.Console.WriteLine($"TEST: END WATCHDOG TESTS {instanceClient.Metadata.Name}"); } - async ValueTask TestDDIsntResolvingSymlink(CancellationToken cancellationToken) - { - System.Console.WriteLine("STARTING SYMLINK RESOLVE TEST"); - var deployTask = DeployTestDme("long_running_test_rooted", DreamDaemonSecurity.Trusted, true, cancellationToken); - - var previous = await instanceClient.DreamDaemon.Read(cancellationToken); - if (previous.SecurityLevel.Value != DreamDaemonSecurity.Trusted) - { - var updated = await instanceClient.DreamDaemon.Update(new DreamDaemonRequest - { - SecurityLevel = DreamDaemonSecurity.Trusted - }, cancellationToken); - - Assert.AreEqual(DreamDaemonSecurity.Trusted, updated.SecurityLevel); - } - - var startState = await deployTask; - - await WaitForJob(await StartDD(cancellationToken), 30, false, null, cancellationToken); - - var command = topicClient.SanitizeString( - Path.GetFullPath( - Path.Combine( - instanceClient.Metadata.Path, - "Game", - "Live"))); - command = $"vaporeon={command}"; - - var topicRequestResult = await topicClient.SendTopic( - IPAddress.Loopback, - command, - ddPort, - cancellationToken); - - Assert.IsNotNull(topicRequestResult); - Assert.AreEqual("is the most pokemon of all time", topicRequestResult.StringData); - - await DeployTestDme("long_running_test_rooted", DreamDaemonSecurity.Trusted, true, cancellationToken); - var newState = await TellWorldToReboot(cancellationToken); - - Assert.AreNotEqual(startState.ActiveCompileJob.Id, newState.ActiveCompileJob.Id); - topicRequestResult = await topicClient.SendTopic( - IPAddress.Loopback, - command, - ddPort, - cancellationToken); - - Assert.IsNotNull(topicRequestResult); - Assert.AreEqual("is the most pokemon of all time", topicRequestResult.StringData); - - await instanceClient.DreamDaemon.Shutdown(cancellationToken); - if (previous.SecurityLevel.Value != DreamDaemonSecurity.Trusted) - { - var updated = await instanceClient.DreamDaemon.Update(new DreamDaemonRequest - { - SecurityLevel = previous.SecurityLevel - }, cancellationToken); - - Assert.AreEqual(previous.SecurityLevel, updated.SecurityLevel); - } - - System.Console.WriteLine("END SYMLINK RESOLVE TEST"); - } - async Task InteropTestsForLongRunningDme(CancellationToken cancellationToken) { await StartAndLeaveRunning(cancellationToken); From c48ce8deed0103bab9365871ccf6ad56a817f0d6 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 21 Oct 2023 12:00:55 -0400 Subject: [PATCH 09/40] Fix deployments always timing out if DMAPI validation fails --- .../Components/Session/SessionController.cs | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/Tgstation.Server.Host/Components/Session/SessionController.cs b/src/Tgstation.Server.Host/Components/Session/SessionController.cs index 2da8c03b58..c792054b83 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionController.cs @@ -192,6 +192,11 @@ namespace Tgstation.Server.Host.Components.Session /// volatile Task rebootGate; + /// + /// for shutting down the server if it is taking too long after validation. + /// + volatile Task postValidationShutdownTask; + /// /// The number of currently active calls to from TgsReboot(). /// @@ -307,6 +312,9 @@ namespace Tgstation.Server.Host.Components.Session { var exitCode = await process.Lifetime; await postLifetimeCallback(); + if (postValidationShutdownTask != null) + await postValidationShutdownTask; + return exitCode; } @@ -655,12 +663,40 @@ namespace Tgstation.Server.Host.Components.Session throw new ObjectDisposedException(nameof(SessionController)); } + /// + /// Terminates the server after ten seconds if it does not exit. + /// + /// A that this method s before executing. If the is , this method will return immediately. + /// A representing the running operation. + async Task PostValidationShutdown(Task proceedTask) + { + Logger.LogTrace("Entered post validation terminate task."); + if (!await proceedTask) + { + Logger.LogTrace("Not running post validation terminate task for repeated bridge request."); + return; + } + + Logger.LogDebug("Server will terminated in 10s if it does not exit..."); + var delayTask = asyncDelayer.Delay(TimeSpan.FromSeconds(10), CancellationToken.None); // DCT: None available + var completedTask = await Task.WhenAny(process.Lifetime, delayTask); + if (completedTask == delayTask) + { + Logger.LogWarning("DMAPI took too long to shutdown server after validation request!"); + process.Terminate(); + apiValidationStatus = ApiValidationStatus.BadValidationRequest; + } + else + Logger.LogTrace("Server exited properly post validation."); + } + /// /// Handle a set of bridge . /// /// The to handle. /// The for the operation. /// A resulting in the for the request or if the request could not be dispatched. +#pragma warning disable CA1502 // TODO: Decomplexify async Task ProcessBridgeCommand(BridgeParameters parameters, CancellationToken cancellationToken) { var response = new BridgeResponse(); @@ -734,7 +770,14 @@ namespace Tgstation.Server.Host.Components.Session break; case BridgeCommandType.Startup: + var proceedTcs = new TaskCompletionSource(); + var firstValidationRequest = Interlocked.CompareExchange(ref postValidationShutdownTask, PostValidationShutdown(proceedTcs.Task), null) == null; + proceedTcs.SetResult(firstValidationRequest); apiValidationStatus = ApiValidationStatus.BadValidationRequest; + + if (!firstValidationRequest) + return BridgeError("Startup bridge request was repeated!"); + if (parameters.Version == null) return BridgeError("Missing dmApiVersion field!"); @@ -808,6 +851,7 @@ namespace Tgstation.Server.Host.Components.Session return response; } +#pragma warning restore CA1502 /// /// Log and return a for a given . From f124c2aaec13351521b14f2288d8e0c40ef855e8 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 21 Oct 2023 12:01:04 -0400 Subject: [PATCH 10/40] Add a decomplexify TODO --- src/Tgstation.Server.Api/ApiHeaders.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Api/ApiHeaders.cs b/src/Tgstation.Server.Api/ApiHeaders.cs index 2d27908acd..7fe6edafc3 100644 --- a/src/Tgstation.Server.Api/ApiHeaders.cs +++ b/src/Tgstation.Server.Api/ApiHeaders.cs @@ -164,7 +164,7 @@ namespace Tgstation.Server.Api /// The containing the serialized . /// If a missing should be ignored. /// Thrown if the constitue invalid . -#pragma warning disable CA1502 +#pragma warning disable CA1502 // TODO: Decomplexify public ApiHeaders(RequestHeaders requestHeaders, bool ignoreMissingAuth = false) { if (requestHeaders == null) From 571c555360aa3b9fb06d9d0b1592fbd976121408 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Wed, 18 Oct 2023 00:19:16 -0400 Subject: [PATCH 11/40] Finally find a way to test if Linux is being shitty to us about links --- tests/DMAPI/LongRunning/Test.dm | 9 ++++ tests/DMAPI/LongRunning/long_running_test.dme | 3 ++ .../LongRunning/long_running_test_copy.dme | 1 + tests/DMAPI/LongRunning/resource.txt | 8 ++++ .../Live/Instance/WatchdogTest.cs | 48 +++++++++++++++++++ tgstation-server.sln | 1 + 6 files changed, 70 insertions(+) create mode 100644 tests/DMAPI/LongRunning/resource.txt diff --git a/tests/DMAPI/LongRunning/Test.dm b/tests/DMAPI/LongRunning/Test.dm index a77c3377b5..657fadbfd0 100644 --- a/tests/DMAPI/LongRunning/Test.dm +++ b/tests/DMAPI/LongRunning/Test.dm @@ -22,6 +22,15 @@ if(!length(channels)) FailTest("Expected some chat channels!") + var/test_str = "aljsdhfjahsfkjnsalkjdfhskljdackmcnvxkljhvkjsdanv,jdshlkufhklasjeFDhfjkalhdkjlfhalksfdjh" + var/res_contents = file2text('resource.txt') // we need a .rsc to be generated + + if(!findtext(res_contents, test_str)) + FailTest("Failed to resource? Did not find magic: [res_contents]") + + if(!fexists("[DME_NAME].rsc")) + FailTest("Failed to create .rsc!") + StartAsync() /proc/dab() diff --git a/tests/DMAPI/LongRunning/long_running_test.dme b/tests/DMAPI/LongRunning/long_running_test.dme index b927014455..322587647d 100644 --- a/tests/DMAPI/LongRunning/long_running_test.dme +++ b/tests/DMAPI/LongRunning/long_running_test.dme @@ -13,5 +13,8 @@ // BEGIN_INCLUDE #include "Config.dm" #include "../test_prelude.dm" +#ifndef DME_NAME +#define DME_NAME "long_running_test" +#endif #include "Test.dm" // END_INCLUDE diff --git a/tests/DMAPI/LongRunning/long_running_test_copy.dme b/tests/DMAPI/LongRunning/long_running_test_copy.dme index b4da8e0d96..e443c4b331 100644 --- a/tests/DMAPI/LongRunning/long_running_test_copy.dme +++ b/tests/DMAPI/LongRunning/long_running_test_copy.dme @@ -11,5 +11,6 @@ // END_PREFERENCES // BEGIN_INCLUDE +#define DME_NAME "long_running_test_copy" #include "long_running_test.dme" // END_INCLUDE diff --git a/tests/DMAPI/LongRunning/resource.txt b/tests/DMAPI/LongRunning/resource.txt new file mode 100644 index 0000000000..94b848fe2a --- /dev/null +++ b/tests/DMAPI/LongRunning/resource.txt @@ -0,0 +1,8 @@ + + if(!length(channels)) + FailTest("Expected some chat channels!") + + var/test_str = "aljsdhfjahsfkjnsalkjdfhskljdackmcnvxkljhvkjsdanv,jdshlkufhklasjeFDhfjkalhdkjlfhalksfdjh" + var/self_contents = file2text(file('Test.dm')) // we need a .rsc to be generated + + if(!(test_str in self_contents)) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index 329fab6aa1..30b2ef3656 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -3,17 +3,22 @@ using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; +using Mono.Unix; +using Mono.Unix.Native; + using Moq; using Newtonsoft.Json; using System; +using System.Collections.Generic; using System.Globalization; using System.IO; using System.Linq; using System.Net; using System.Net.Sockets; using System.Runtime.InteropServices; +using System.Text; using System.Threading; using System.Threading.Tasks; @@ -471,6 +476,47 @@ namespace Tgstation.Server.Tests.Live.Instance Assert.AreEqual(string.Empty, daemonStatus.AdditionalParameters); } + void TestLinuxIsntBeingFuckingCheekyAboutFilePaths(DreamDaemonResponse currentStatus, CompileJobResponse previousStatus, CancellationToken cancellationToken) + { + if (new PlatformIdentifier().IsWindows) + return; + + Assert.IsNotNull(currentStatus.ActiveCompileJob); + Assert.IsTrue(currentStatus.ActiveCompileJob.DmeName.Contains("long_running_test")); + Assert.AreEqual(WatchdogStatus.Online, currentStatus.Status); + + var procs = TestLiveServer.GetDDProcessesOnPort(currentStatus.Port.Value); + Assert.AreEqual(1, procs.Count); + var failingLinks = new List(); + using var proc = procs[0]; + var pid = proc.Id; + var foundLivePath = false; + var allPaths = new List(); + foreach (var fd in Directory.EnumerateFiles($"/proc/{pid}/fd")) + { + var sb = new StringBuilder(); + if (Syscall.readlink(fd, sb) == -1) + throw new UnixIOException(Stdlib.GetLastError()); + + var path = sb.ToString(); + + allPaths.Add(path); + if (path.Contains($"Game/{previousStatus.DirectoryName}")) + failingLinks.Add($"Found fd {fd} resolving to previous absolute path game dir path: {path}"); + + if (path.Contains($"Game/{currentStatus.ActiveCompileJob.DirectoryName}")) + failingLinks.Add($"Found fd {fd} resolving to current absolute path game dir path: {path}"); + + if (path.Contains($"Game/Live")) + foundLivePath = true; + } + + if (!foundLivePath) + failingLinks.Add($"Failed to find a path containing the 'Live' directory! Found {allPaths.Count}: \"{String.Join("\", \"", allPaths)}\""); + + Assert.IsTrue(failingLinks.Count == 0, String.Join(Environment.NewLine, failingLinks)); + } + async Task RunHealthCheckTest(bool checkDump, CancellationToken cancellationToken) { System.Console.WriteLine("TEST: WATCHDOG HEALTH CHECK TEST"); @@ -901,6 +947,8 @@ namespace Tgstation.Server.Tests.Live.Instance Assert.AreNotEqual(initialCompileJob.Id, daemonStatus.ActiveCompileJob.Id); Assert.IsNull(daemonStatus.StagedCompileJob); + TestLinuxIsntBeingFuckingCheekyAboutFilePaths(daemonStatus, initialCompileJob, cancellationToken); + await instanceClient.DreamDaemon.Shutdown(cancellationToken); daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); diff --git a/tgstation-server.sln b/tgstation-server.sln index 20eef245e1..c1cde46821 100644 --- a/tgstation-server.sln +++ b/tgstation-server.sln @@ -158,6 +158,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "LongRunning", "LongRunning" tests\DMAPI\LongRunning\long_running_test.dme = tests\DMAPI\LongRunning\long_running_test.dme tests\DMAPI\LongRunning\long_running_test_copy.dme = tests\DMAPI\LongRunning\long_running_test_copy.dme tests\DMAPI\LongRunning\long_running_test_rooted.dme = tests\DMAPI\LongRunning\long_running_test_rooted.dme + tests\DMAPI\LongRunning\resource.txt = tests\DMAPI\LongRunning\resource.txt tests\DMAPI\LongRunning\Test.dm = tests\DMAPI\LongRunning\Test.dm EndProjectSection EndProject From 44729820eddd656f55c2090eb31c83e203eaa324 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 21 Oct 2023 12:27:53 -0400 Subject: [PATCH 12/40] Remove unused variable and fix test buffer size --- tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs | 4 ++-- tests/Tgstation.Server.Tests/Live/TestLiveServer.cs | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index 30b2ef3656..96ef4d3c20 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -494,7 +494,7 @@ namespace Tgstation.Server.Tests.Live.Instance var allPaths = new List(); foreach (var fd in Directory.EnumerateFiles($"/proc/{pid}/fd")) { - var sb = new StringBuilder(); + var sb = new StringBuilder(UInt16.MaxValue); if (Syscall.readlink(fd, sb) == -1) throw new UnixIOException(Stdlib.GetLastError()); @@ -512,7 +512,7 @@ namespace Tgstation.Server.Tests.Live.Instance } if (!foundLivePath) - failingLinks.Add($"Failed to find a path containing the 'Live' directory! Found {allPaths.Count}: \"{String.Join("\", \"", allPaths)}\""); + failingLinks.Add($"Failed to find a path containing the 'Live' directory!"); Assert.IsTrue(failingLinks.Count == 0, String.Join(Environment.NewLine, failingLinks)); } diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index f058d9795f..a78b984fc2 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -1326,8 +1326,6 @@ namespace Tgstation.Server.Tests.Live async Task RunInstanceTests() { - // Some earlier linux BYOND versions have a critical bug where replacing the directory in non-basic watchdogs causes the DreamDaemon cwd to change - var canRunCompatTests = new PlatformIdentifier().IsWindows; var compatTests = FailFast( instanceTest .RunCompatTests( From a0d583b5961d353d24024e7e200283cdae799a8a Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 21 Oct 2023 18:01:41 -0400 Subject: [PATCH 13/40] New hard link system for PosixWatchdog - IDmbProvider is now `AsyncDisposable` as opposed to `IDisposable`. - Add hard link support to `ISymlinkFactory`. - `PosixWatchdog` now has to mirror deployment structure as hard links before swapping. Because of this deployments may not immediately be applied if the reboots immediately after they were completed. --- .../Components/Deployment/DmbFactory.cs | 7 +- .../Components/Deployment/DmbProvider.cs | 9 +- .../Components/Deployment/DreamMaker.cs | 2 +- .../Deployment/HardLinkDmbProvider.cs | 222 ++++++++++++++++++ .../Components/Deployment/IDmbProvider.cs | 2 +- .../Deployment/SwappableDmbProvider.cs | 62 +++-- .../Deployment/TemporaryDmbProvider.cs | 5 +- .../Components/Session/ISessionController.cs | 4 +- .../Components/Session/SessionController.cs | 11 +- .../Components/Watchdog/BasicWatchdog.cs | 2 +- .../Components/Watchdog/PosixWatchdog.cs | 21 +- .../Watchdog/PosixWatchdogFactory.cs | 1 + .../Components/Watchdog/WindowsWatchdog.cs | 67 ++++-- .../IO/ISymlinkFactory.cs | 9 + .../IO/PosixSymlinkFactory.cs | 16 ++ .../IO/WindowsSymlinkFactory.cs | 4 + 16 files changed, 380 insertions(+), 64 deletions(-) create mode 100644 src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs diff --git a/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs b/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs index 09567ba79c..df587cab11 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs @@ -148,15 +148,18 @@ namespace Tgstation.Server.Host.Components.Deployment cancellationToken); } + ValueTask dmbDisposeTask; lock (jobLockCounts) { - nextDmbProvider?.Dispose(); + dmbDisposeTask = nextDmbProvider?.DisposeAsync() ?? ValueTask.CompletedTask; nextDmbProvider = newProvider; // Oh god dammit var temp = Interlocked.Exchange(ref newerDmbTcs, new TaskCompletionSource()); temp.SetResult(); } + + await dmbDisposeTask; } /// @@ -320,7 +323,7 @@ namespace Tgstation.Server.Host.Components.Deployment finally { if (!providerSubmitted) - newProvider.Dispose(); + await newProvider.DisposeAsync(); } } #pragma warning restore CA1506 diff --git a/src/Tgstation.Server.Host/Components/Deployment/DmbProvider.cs b/src/Tgstation.Server.Host/Components/Deployment/DmbProvider.cs index f3711ede4f..770409f790 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/DmbProvider.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/DmbProvider.cs @@ -1,4 +1,5 @@ using System; +using System.Threading.Tasks; using Tgstation.Server.Host.IO; using Tgstation.Server.Host.Models; @@ -30,7 +31,7 @@ namespace Tgstation.Server.Host.Components.Deployment readonly string directoryAppend; /// - /// The to run when is called. + /// The to run when is called. /// Action onDispose; @@ -50,7 +51,11 @@ namespace Tgstation.Server.Host.Components.Deployment } /// - public void Dispose() => onDispose?.Invoke(); + public ValueTask DisposeAsync() + { + onDispose?.Invoke(); + return ValueTask.CompletedTask; + } /// public void KeepAlive() => onDispose = null; diff --git a/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs b/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs index 665efba795..307a1bb107 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs @@ -802,7 +802,7 @@ namespace Tgstation.Server.Host.Components.Deployment job.MinimumSecurityLevel = securityLevel; // needed for the TempDmbProvider ApiValidationStatus validationStatus; - using (var provider = new TemporaryDmbProvider(ioManager.ResolvePath(job.DirectoryName.ToString()), String.Concat(job.DmeName, DmbExtension), job)) + await using (var provider = new TemporaryDmbProvider(ioManager.ResolvePath(job.DirectoryName.ToString()), String.Concat(job.DmeName, DmbExtension), job)) await using (var controller = await sessionControllerFactory.LaunchNew(provider, byondLock, launchParameters, true, cancellationToken)) { var launchResult = await controller.LaunchResult.WaitAsync(cancellationToken); diff --git a/src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs b/src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs new file mode 100644 index 0000000000..e5ddede026 --- /dev/null +++ b/src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs @@ -0,0 +1,222 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +using Microsoft.Extensions.Logging; + +using Tgstation.Server.Host.Configuration; +using Tgstation.Server.Host.Extensions; +using Tgstation.Server.Host.IO; +using Tgstation.Server.Host.Utils; + +namespace Tgstation.Server.Host.Components.Deployment +{ + /// + /// A that uses hard links. + /// + sealed class HardLinkDmbProvider : SwappableDmbProvider + { + /// + /// The for . + /// + readonly CancellationTokenSource cancellationTokenSource; + + /// + /// The representing the base provider mirroring operation. + /// + readonly Task mirroringTask; + + /// + /// The for the . + /// + readonly ILogger logger; + + /// + /// Initializes a new instance of the class. + /// + /// The for the . + /// The for the . + /// The for the . + /// The value of . + /// The for the . + public HardLinkDmbProvider( + IDmbProvider baseProvider, + IIOManager ioManager, + ISymlinkFactory symlinkFactory, + ILogger logger, + GeneralConfiguration generalConfiguration) + : base( + baseProvider, + ioManager, + symlinkFactory) + { + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + cancellationTokenSource = new CancellationTokenSource(); + try + { + mirroringTask = MirrorSourceDirectory(generalConfiguration.GetCopyDirectoryTaskThrottle(), cancellationTokenSource.Token); + } + catch + { + cancellationTokenSource.Dispose(); + throw; + } + } + + /// + public override async ValueTask DisposeAsync() + { + cancellationTokenSource.Cancel(); + cancellationTokenSource.Dispose(); + try + { + await mirroringTask; + } + catch (OperationCanceledException ex) + { + logger.LogDebug(ex, "Mirroring task cancelled!"); + } + + await base.DisposeAsync(); + } + + /// + public override Task FinishActivationPreparation(CancellationToken cancellationToken) + { + if (!mirroringTask.IsCompleted) + logger.LogTrace("Waiting for mirroring to complete..."); + + return mirroringTask.WaitAsync(cancellationToken); + } + + /// + protected override async Task DoSwap(CancellationToken cancellationToken) + { + var mirroredDir = await mirroringTask.WaitAsync(cancellationToken); + var goAheadTcs = new TaskCompletionSource(); + + // I feel dirty... + async void DisposeOfOldDirectory() + { + var directoryMoved = false; + var disposePath = Guid.NewGuid().ToString(); + try + { + await IOManager.MoveDirectory(LiveGameDirectory, disposePath, cancellationToken); + directoryMoved = true; + goAheadTcs.SetResult(); + await IOManager.DeleteDirectory(disposePath, CancellationToken.None); // DCT: We're detached at this point + } + catch (Exception ex) + { + if (directoryMoved) + logger.LogWarning(ex, "Failed to delete hard linked directory: {disposePath}", disposePath); + else + { + logger.LogDebug(ex, "Live directory appears to not exist"); + goAheadTcs.SetResult(); + } + } + } + + DisposeOfOldDirectory(); + await goAheadTcs.Task; + await IOManager.MoveDirectory(mirroredDir, LiveGameDirectory, cancellationToken); + } + + /// + /// Mirror the . + /// + /// The optional maximum number of simultaneous tasks allowed to execute. + /// The for the operation. + /// A resulting in the full path to the mirrored directory. + async Task MirrorSourceDirectory(int? taskThrottle, CancellationToken cancellationToken) + { + var stopwatch = Stopwatch.StartNew(); + var mirrorGuid = Guid.NewGuid(); + logger.LogDebug("Starting to mirror {sourceDir} as hard links to {mirrorGuid}...", CompileJob.DirectoryName, mirrorGuid); + if (taskThrottle.HasValue && taskThrottle < 1) + throw new ArgumentOutOfRangeException(nameof(taskThrottle), taskThrottle, "taskThrottle must be at least 1!"); + + var src = IOManager.ResolvePath(CompileJob.DirectoryName.ToString()); + var dest = IOManager.ResolvePath(mirrorGuid.ToString()); + + using var semaphore = taskThrottle.HasValue ? new SemaphoreSlim(taskThrottle.Value) : null; + await Task.WhenAll(MirrorDirectoryImpl(src, dest, semaphore, cancellationToken)); + stopwatch.Stop(); + + logger.LogDebug( + "Finished mirror of {sourceDir} to {mirrorGuid} in {seconds}s...", + CompileJob.DirectoryName, + mirrorGuid, + stopwatch.Elapsed.TotalSeconds.ToString("0.##", CultureInfo.InvariantCulture)); + + return dest; + } + + /// + /// Recursively create tasks to create a hard link directory mirror of to . + /// + /// The source directory path. + /// The destination directory path. + /// Optional used to limit degree of parallelism. + /// The for the operation. + /// A of s representing the running operations. The first returned is always the necessary call to . + /// I genuinely don't know how this will work with symlinked files. Waiting for the issue report I guess. + IEnumerable MirrorDirectoryImpl(string src, string dest, SemaphoreSlim semaphore, CancellationToken cancellationToken) + { + var dir = new DirectoryInfo(src); + Task subdirCreationTask = null; + foreach (var subDirectory in dir.EnumerateDirectories()) + { + // check if we are a symbolic link + if (!subDirectory.Attributes.HasFlag(FileAttributes.Directory) || subDirectory.Attributes.HasFlag(FileAttributes.ReparsePoint)) + { + logger.LogTrace("Skipping symlink to {subdir}", subDirectory.Name); + continue; + } + + var checkingSubdirCreationTask = true; + foreach (var copyTask in MirrorDirectoryImpl(subDirectory.FullName, Path.Combine(dest, subDirectory.Name), semaphore, cancellationToken)) + { + if (subdirCreationTask == null) + { + subdirCreationTask = copyTask; + yield return subdirCreationTask; + } + else if (!checkingSubdirCreationTask) + yield return copyTask; + + checkingSubdirCreationTask = false; + } + } + + foreach (var fileInfo in dir.EnumerateFiles()) + { + if (subdirCreationTask == null) + { + subdirCreationTask = IOManager.CreateDirectory(dest, cancellationToken); + yield return subdirCreationTask; + } + + var sourceFile = fileInfo.FullName; + var destFile = IOManager.ConcatPath(dest, fileInfo.Name); + + async Task LinkThisFile() + { + await subdirCreationTask.WaitAsync(cancellationToken); + using var lockContext = semaphore != null + ? await SemaphoreSlimContext.Lock(semaphore, cancellationToken) + : null; + await SymlinkFactory.CreateHardLink(sourceFile, destFile, cancellationToken); + } + + yield return LinkThisFile(); + } + } + } +} diff --git a/src/Tgstation.Server.Host/Components/Deployment/IDmbProvider.cs b/src/Tgstation.Server.Host/Components/Deployment/IDmbProvider.cs index efc773a919..c1c477ffff 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/IDmbProvider.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/IDmbProvider.cs @@ -7,7 +7,7 @@ namespace Tgstation.Server.Host.Components.Deployment /// /// Provides absolute paths to the latest compiled .dmbs. /// - public interface IDmbProvider : IDisposable + public interface IDmbProvider : IAsyncDisposable { /// /// The file name of the .dmb. diff --git a/src/Tgstation.Server.Host/Components/Deployment/SwappableDmbProvider.cs b/src/Tgstation.Server.Host/Components/Deployment/SwappableDmbProvider.cs index e496453222..14cdc268e2 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/SwappableDmbProvider.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/SwappableDmbProvider.cs @@ -10,7 +10,7 @@ namespace Tgstation.Server.Host.Components.Deployment /// /// A that uses symlinks. /// - sealed class SwappableDmbProvider : IDmbProvider + class SwappableDmbProvider : IDmbProvider { /// /// The directory where the is symlinked to. @@ -21,7 +21,7 @@ namespace Tgstation.Server.Host.Components.Deployment public string DmbName => baseProvider.DmbName; /// - public string Directory => ioManager.ResolvePath(LiveGameDirectory); + public string Directory => IOManager.ResolvePath(LiveGameDirectory); /// public CompileJob CompileJob => baseProvider.CompileJob; @@ -31,20 +31,20 @@ namespace Tgstation.Server.Host.Components.Deployment /// public bool Swapped => swapped != 0; - /// - /// The we are swapping for. - /// - readonly IDmbProvider baseProvider; - /// /// The to use. /// - readonly IIOManager ioManager; + protected IIOManager IOManager { get; } /// /// The to use. /// - readonly ISymlinkFactory symlinkFactory; + protected ISymlinkFactory SymlinkFactory { get; } + + /// + /// The we are swapping for. + /// + readonly IDmbProvider baseProvider; /// /// Backing field for . @@ -55,17 +55,17 @@ namespace Tgstation.Server.Host.Components.Deployment /// Initializes a new instance of the class. /// /// The value of . - /// The value of . - /// The value of . + /// The value of . + /// The value of . public SwappableDmbProvider(IDmbProvider baseProvider, IIOManager ioManager, ISymlinkFactory symlinkFactory) { this.baseProvider = baseProvider ?? throw new ArgumentNullException(nameof(baseProvider)); - this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); - this.symlinkFactory = symlinkFactory ?? throw new ArgumentNullException(nameof(symlinkFactory)); + IOManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); + SymlinkFactory = symlinkFactory ?? throw new ArgumentNullException(nameof(symlinkFactory)); } /// - public void Dispose() => baseProvider.Dispose(); + public virtual ValueTask DisposeAsync() => baseProvider.DisposeAsync(); /// public void KeepAlive() => baseProvider.KeepAlive(); @@ -75,19 +75,37 @@ namespace Tgstation.Server.Host.Components.Deployment /// /// The for the operation. /// A representing the running operation. - public async Task MakeActive(CancellationToken cancellationToken) + public Task MakeActive(CancellationToken cancellationToken) { if (Interlocked.Exchange(ref swapped, 1) != 0) throw new InvalidOperationException("Already swapped!"); - if (symlinkFactory.SymlinkedDirectoriesAreDeletedAsFiles) - await ioManager.DeleteFile(LiveGameDirectory, cancellationToken); - else - await ioManager.DeleteDirectory(LiveGameDirectory, cancellationToken); + return DoSwap(cancellationToken); + } - await symlinkFactory.CreateSymbolicLink( - ioManager.ResolvePath(baseProvider.Directory), - ioManager.ResolvePath(LiveGameDirectory), + /// + /// Should be . before calling to ensure the is ready to instantly swap. Can be called multiple times. + /// + /// The for the operation. + /// A representing the preparation process. + public virtual Task FinishActivationPreparation(CancellationToken cancellationToken) + => Task.CompletedTask; + + /// + /// Perform the swapping action. + /// + /// The for the operation. + /// A representing the running operation. + protected virtual async Task DoSwap(CancellationToken cancellationToken) + { + if (SymlinkFactory.SymlinkedDirectoriesAreDeletedAsFiles) + await IOManager.DeleteFile(LiveGameDirectory, cancellationToken); + else + await IOManager.DeleteDirectory(LiveGameDirectory, cancellationToken); + + await SymlinkFactory.CreateSymbolicLink( + IOManager.ResolvePath(baseProvider.Directory), + IOManager.ResolvePath(LiveGameDirectory), cancellationToken); } } diff --git a/src/Tgstation.Server.Host/Components/Deployment/TemporaryDmbProvider.cs b/src/Tgstation.Server.Host/Components/Deployment/TemporaryDmbProvider.cs index 0bf982cfa3..4476d9d2c3 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/TemporaryDmbProvider.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/TemporaryDmbProvider.cs @@ -1,4 +1,5 @@ using System; +using System.Threading.Tasks; using Tgstation.Server.Host.Models; @@ -32,9 +33,7 @@ namespace Tgstation.Server.Host.Components.Deployment } /// - public void Dispose() - { - } + public ValueTask DisposeAsync() => ValueTask.CompletedTask; /// public void KeepAlive() => throw new NotSupportedException(); diff --git a/src/Tgstation.Server.Host/Components/Session/ISessionController.cs b/src/Tgstation.Server.Host/Components/Session/ISessionController.cs index 3ded84429a..93f10eb17c 100644 --- a/src/Tgstation.Server.Host/Components/Session/ISessionController.cs +++ b/src/Tgstation.Server.Host/Components/Session/ISessionController.cs @@ -128,7 +128,7 @@ namespace Tgstation.Server.Host.Components.Session /// Replace the in use with a given , disposing the old one. /// /// The new . - /// An to be disposed once certain that the original is no longer in use. - IDisposable ReplaceDmbProvider(IDmbProvider newProvider); + /// An to be disposed once certain that the original is no longer in use. + IAsyncDisposable ReplaceDmbProvider(IDmbProvider newProvider); } } diff --git a/src/Tgstation.Server.Host/Components/Session/SessionController.cs b/src/Tgstation.Server.Host/Components/Session/SessionController.cs index 2da8c03b58..eab32bdce1 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionController.cs @@ -350,8 +350,13 @@ namespace Tgstation.Server.Host.Components.Session await process.DisposeAsync(); byondLock.Dispose(); bridgeRegistration?.Dispose(); - ReattachInformation.Dmb.Dispose(); - ReattachInformation.InitialDmb?.Dispose(); + var regularDmbDisposeTask = ReattachInformation.Dmb.DisposeAsync(); + var initialDmb = ReattachInformation.InitialDmb; + if (initialDmb != null) + await initialDmb.DisposeAsync(); + + await regularDmbDisposeTask; + chatTrackingContext.Dispose(); reattachTopicCts.Dispose(); @@ -552,7 +557,7 @@ namespace Tgstation.Server.Host.Components.Session public void Resume() => process.Resume(); /// - public IDisposable ReplaceDmbProvider(IDmbProvider dmbProvider) + public IAsyncDisposable ReplaceDmbProvider(IDmbProvider dmbProvider) { var oldDmb = ReattachInformation.Dmb; ReattachInformation.Dmb = dmbProvider ?? throw new ArgumentNullException(nameof(dmbProvider)); diff --git a/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs index 8396ce7c99..fc5c3461b0 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs @@ -268,7 +268,7 @@ namespace Tgstation.Server.Host.Components.Watchdog // server didn't get control of this dmb if (dmbToUse != null && !serverWasActive) - dmbToUse.Dispose(); + await dmbToUse.DisposeAsync(); throw; } diff --git a/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs index 044ef3f1ef..3bf2927e57 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs @@ -1,4 +1,5 @@ -using System.Threading; +using System; +using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -9,6 +10,7 @@ using Tgstation.Server.Host.Components.Deployment; using Tgstation.Server.Host.Components.Deployment.Remote; using Tgstation.Server.Host.Components.Events; using Tgstation.Server.Host.Components.Session; +using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.Core; using Tgstation.Server.Host.IO; using Tgstation.Server.Host.Jobs; @@ -21,6 +23,11 @@ namespace Tgstation.Server.Host.Components.Watchdog /// sealed class PosixWatchdog : WindowsWatchdog { + /// + /// The for the . + /// + readonly GeneralConfiguration generalConfiguration; + /// /// Initializes a new instance of the class. /// @@ -39,6 +46,7 @@ namespace Tgstation.Server.Host.Components.Watchdog /// The for the . /// The for the . /// The for the . + /// The value of . /// The autostart value for the . public PosixWatchdog( IChatManager chat, @@ -56,6 +64,7 @@ namespace Tgstation.Server.Host.Components.Watchdog ILogger logger, DreamDaemonLaunchParameters initialLaunchParameters, Api.Models.Instance instance, + GeneralConfiguration generalConfiguration, bool autoStart) : base( chat, @@ -75,13 +84,15 @@ namespace Tgstation.Server.Host.Components.Watchdog instance, autoStart) { + this.generalConfiguration = generalConfiguration ?? throw new ArgumentNullException(nameof(generalConfiguration)); } /// protected override Task ApplyInitialDmb(CancellationToken cancellationToken) - { - // not necessary to hold initial .dmb on Linux because of based inode deletes - return Task.CompletedTask; - } + => Task.CompletedTask; // not necessary to hold initial .dmb on Linux because of based inode deletes + + /// + protected override SwappableDmbProvider CreateSwappableDmbProvider(IDmbProvider dmbProvider) + => new HardLinkDmbProvider(dmbProvider, GameIOManager, SymlinkFactory, Logger, generalConfiguration); } } diff --git a/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdogFactory.cs b/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdogFactory.cs index 6774c73cb5..787127a05c 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdogFactory.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdogFactory.cs @@ -76,6 +76,7 @@ namespace Tgstation.Server.Host.Components.Watchdog LoggerFactory.CreateLogger(), settings, instance, + GeneralConfiguration, settings.AutoStart ?? throw new ArgumentNullException(nameof(settings))); } } diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs index c8d6c7eb44..0e42c3b660 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs @@ -37,7 +37,7 @@ namespace Tgstation.Server.Host.Components.Watchdog /// /// The for the . /// - readonly ISymlinkFactory symlinkFactory; + protected ISymlinkFactory SymlinkFactory { get; } /// /// of s that are waiting to clean up old deployments. @@ -68,7 +68,7 @@ namespace Tgstation.Server.Host.Components.Watchdog /// The for the . /// The for the . /// The value of . - /// The value of . + /// The value of . /// The for the . /// The for the . /// The for the . @@ -109,7 +109,7 @@ namespace Tgstation.Server.Host.Components.Watchdog try { GameIOManager = gameIOManager ?? throw new ArgumentNullException(nameof(gameIOManager)); - this.symlinkFactory = symlinkFactory ?? throw new ArgumentNullException(nameof(symlinkFactory)); + SymlinkFactory = symlinkFactory ?? throw new ArgumentNullException(nameof(symlinkFactory)); deploymentCleanupTasks = new List(); } @@ -131,7 +131,7 @@ namespace Tgstation.Server.Host.Components.Watchdog // If we reach this point, we can guarantee PrepServerForLaunch will be called before starting again. ActiveSwappable = null; - pendingSwappable?.Dispose(); + await (pendingSwappable?.DisposeAsync() ?? ValueTask.CompletedTask); pendingSwappable = null; await DrainDeploymentCleanupTasks(true); @@ -142,9 +142,10 @@ namespace Tgstation.Server.Host.Components.Watchdog { if (pendingSwappable != null) { - var updateTask = BeforeApplyDmb(pendingSwappable.CompileJob, cancellationToken); + Task RunPrequel() => BeforeApplyDmb(pendingSwappable.CompileJob, cancellationToken); - if (!pendingSwappable.Swapped) + var needToSwap = !pendingSwappable.Swapped; + if (needToSwap) { // IMPORTANT: THE SESSIONCONTROLLER SHOULD STILL BE PROCESSING THE BRIDGE REQUEST SO WE KNOW DD IS SLEEPING // OTHERWISE, IT COULD RETURN TO /world/Reboot() TOO EARLY AND LOAD THE WRONG .DMB @@ -153,18 +154,29 @@ namespace Tgstation.Server.Host.Components.Watchdog // integration test logging will catch this Logger.LogError( "The reboot bridge request completed before the watchdog could suspend the server! This can lead to buggy DreamDaemon behaviour and should be reported! To ensure stability, we will need to hard reboot the server"); - await updateTask; + await RunPrequel(); return MonitorAction.Restart; } - await PerformDmbSwap(pendingSwappable, cancellationToken); + // DCT: Not necessary + if (!pendingSwappable.FinishActivationPreparation(CancellationToken.None).IsCompleted) + { + // rare pokemon + Logger.LogInformation("Deployed .dme is not ready to swap, delaying until next reboot!"); + Chat.QueueWatchdogMessage("The pending deployment was not ready to be activated this reboot. It will be applied at the next one."); + return MonitorAction.Continue; + } } + var updateTask = RunPrequel(); + if (needToSwap) + await PerformDmbSwap(pendingSwappable, cancellationToken); + var currentCompileJobId = Server.ReattachInformation.Dmb.CompileJob.Id; await DrainDeploymentCleanupTasks(false); - IDisposable lingeringDeployment; + IAsyncDisposable lingeringDeployment; var localDeploymentCleanupGate = new TaskCompletionSource(); async Task CleanupLingeringDeployment() { @@ -191,7 +203,7 @@ namespace Tgstation.Server.Host.Components.Watchdog ? " due to timeout!" : "..."); - lingeringDeployment.Dispose(); + await lingeringDeployment.DisposeAsync(); } var oldDeploymentCleanupGate = Interlocked.Exchange(ref deploymentCleanupGate, localDeploymentCleanupGate); @@ -247,31 +259,31 @@ namespace Tgstation.Server.Host.Components.Watchdog if (!canSeamlesslySwap) { - compileJobProvider.Dispose(); + await compileJobProvider.DisposeAsync(); await base.HandleNewDmbAvailable(cancellationToken); return; } - SwappableDmbProvider windowsProvider = null; + SwappableDmbProvider swappableProvider = null; try { - windowsProvider = new SwappableDmbProvider(compileJobProvider, GameIOManager, symlinkFactory); + swappableProvider = CreateSwappableDmbProvider(compileJobProvider); if (ActiveCompileJob.DMApiVersion == null) { Logger.LogWarning("Active compile job has no DMAPI! Commencing immediate .dmb swap. Note this behavior is known to be buggy in some DM code contexts. See https://github.com/tgstation/tgstation-server/issues/1550"); - await PerformDmbSwap(windowsProvider, cancellationToken); + await PerformDmbSwap(swappableProvider, cancellationToken); } } catch (Exception ex) { Logger.LogError(ex, "Exception while swapping"); - IDmbProvider providerToDispose = windowsProvider ?? compileJobProvider; - providerToDispose.Dispose(); + IDmbProvider providerToDispose = swappableProvider ?? compileJobProvider; + await providerToDispose.DisposeAsync(); throw; } - pendingSwappable?.Dispose(); - pendingSwappable = windowsProvider; + await (pendingSwappable?.DisposeAsync() ?? ValueTask.CompletedTask); + pendingSwappable = swappableProvider; } /// @@ -284,7 +296,7 @@ namespace Tgstation.Server.Host.Components.Watchdog Logger.LogTrace("Prep for server launch"); - ActiveSwappable = new SwappableDmbProvider(dmbToUse, GameIOManager, symlinkFactory); + ActiveSwappable = CreateSwappableDmbProvider(dmbToUse); try { await InitialLink(cancellationToken); @@ -310,6 +322,14 @@ namespace Tgstation.Server.Host.Components.Watchdog Server.ReattachInformation.InitialDmb = await DmbFactory.FromCompileJob(Server.CompileJob, cancellationToken); } + /// + /// Create a for a given . + /// + /// The to create a for. + /// A new . + protected virtual SwappableDmbProvider CreateSwappableDmbProvider(IDmbProvider dmbProvider) + => new SwappableDmbProvider(dmbProvider, GameIOManager, SymlinkFactory); + /// protected override async Task SessionStartupPersist(CancellationToken cancellationToken) { @@ -332,10 +352,11 @@ namespace Tgstation.Server.Host.Components.Watchdog /// /// The for the operation. /// A representing the running operation. - Task InitialLink(CancellationToken cancellationToken) + async ValueTask InitialLink(CancellationToken cancellationToken) { - Logger.LogTrace("Symlinking compile job..."); - return ActiveSwappable.MakeActive(cancellationToken); + await ActiveSwappable.FinishActivationPreparation(cancellationToken); + Logger.LogTrace("Linking compile job..."); + await ActiveSwappable.MakeActive(cancellationToken); } /// @@ -348,6 +369,8 @@ namespace Tgstation.Server.Host.Components.Watchdog { Logger.LogDebug("Swapping to compile job {id}...", newProvider.CompileJob.Id); + await newProvider.FinishActivationPreparation(cancellationToken); + var suspended = false; var server = Server; try diff --git a/src/Tgstation.Server.Host/IO/ISymlinkFactory.cs b/src/Tgstation.Server.Host/IO/ISymlinkFactory.cs index 1adbd99d97..08320a3d15 100644 --- a/src/Tgstation.Server.Host/IO/ISymlinkFactory.cs +++ b/src/Tgstation.Server.Host/IO/ISymlinkFactory.cs @@ -22,5 +22,14 @@ namespace Tgstation.Server.Host.IO /// The for the operation. /// A representing the running operation. Task CreateSymbolicLink(string targetPath, string linkPath, CancellationToken cancellationToken); + + /// + /// Creates a hard link. + /// + /// The path to the hard target. + /// The path to the link. + /// The for the operation. + /// A representing the running operation. + Task CreateHardLink(string targetPath, string linkPath, CancellationToken cancellationToken); } } diff --git a/src/Tgstation.Server.Host/IO/PosixSymlinkFactory.cs b/src/Tgstation.Server.Host/IO/PosixSymlinkFactory.cs index ddbcbb86fb..05d64a3bf0 100644 --- a/src/Tgstation.Server.Host/IO/PosixSymlinkFactory.cs +++ b/src/Tgstation.Server.Host/IO/PosixSymlinkFactory.cs @@ -15,6 +15,22 @@ namespace Tgstation.Server.Host.IO /// public bool SymlinkedDirectoriesAreDeletedAsFiles => true; + /// + public Task CreateHardLink(string targetPath, string linkPath, CancellationToken cancellationToken) => Task.Factory.StartNew( + () => + { + ArgumentNullException.ThrowIfNull(targetPath); + ArgumentNullException.ThrowIfNull(linkPath); + + cancellationToken.ThrowIfCancellationRequested(); + var fsInfo = new UnixFileInfo(targetPath); + cancellationToken.ThrowIfCancellationRequested(); + fsInfo.CreateLink(linkPath); + }, + cancellationToken, + DefaultIOManager.BlockingTaskCreationOptions, + TaskScheduler.Current); + /// public Task CreateSymbolicLink(string targetPath, string linkPath, CancellationToken cancellationToken) => Task.Factory.StartNew( () => diff --git a/src/Tgstation.Server.Host/IO/WindowsSymlinkFactory.cs b/src/Tgstation.Server.Host/IO/WindowsSymlinkFactory.cs index 18abde6b8f..864242d992 100644 --- a/src/Tgstation.Server.Host/IO/WindowsSymlinkFactory.cs +++ b/src/Tgstation.Server.Host/IO/WindowsSymlinkFactory.cs @@ -16,6 +16,10 @@ namespace Tgstation.Server.Host.IO /// public bool SymlinkedDirectoriesAreDeletedAsFiles => false; + /// + public Task CreateHardLink(string targetPath, string linkPath, CancellationToken cancellationToken) + => throw new NotSupportedException(); + /// public Task CreateSymbolicLink(string targetPath, string linkPath, CancellationToken cancellationToken) => Task.Factory.StartNew( () => From f6a843e651942907221f2ed5a6cf54749f9d0825 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 21 Oct 2023 18:04:14 -0400 Subject: [PATCH 14/40] Rename `WindowsWatchdog` to `AdvancedWatchdog` --- .../{WindowsWatchdog.cs => AdvancedWatchdog.cs} | 14 +++++++------- .../Components/Watchdog/PosixWatchdog.cs | 8 ++++---- .../Components/Watchdog/WindowsWatchdogFactory.cs | 4 ++-- 3 files changed, 13 insertions(+), 13 deletions(-) rename src/Tgstation.Server.Host/Components/Watchdog/{WindowsWatchdog.cs => AdvancedWatchdog.cs} (97%) diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/AdvancedWatchdog.cs similarity index 97% rename from src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs rename to src/Tgstation.Server.Host/Components/Watchdog/AdvancedWatchdog.cs index 0e42c3b660..3552149c3c 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/AdvancedWatchdog.cs @@ -20,9 +20,9 @@ using Tgstation.Server.Host.Utils; namespace Tgstation.Server.Host.Components.Watchdog { /// - /// A that, instead of killing servers for updates, uses the wonders of symlinks to swap out changes without killing DreamDaemon. + /// A that, instead of killing servers for updates, uses the wonders of filesystem links to swap out changes without killing the server process. /// - class WindowsWatchdog : BasicWatchdog + class AdvancedWatchdog : BasicWatchdog { /// /// The for . @@ -30,12 +30,12 @@ namespace Tgstation.Server.Host.Components.Watchdog protected SwappableDmbProvider ActiveSwappable { get; private set; } /// - /// The for the pointing to the Game directory. + /// The for the pointing to the Game directory. /// protected IIOManager GameIOManager { get; } /// - /// The for the . + /// The for the . /// protected ISymlinkFactory SymlinkFactory { get; } @@ -55,7 +55,7 @@ namespace Tgstation.Server.Host.Components.Watchdog volatile TaskCompletionSource deploymentCleanupGate; /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// /// The for the . /// The for the . @@ -73,7 +73,7 @@ namespace Tgstation.Server.Host.Components.Watchdog /// The for the . /// The for the . /// The autostart value for the . - public WindowsWatchdog( + public AdvancedWatchdog( IChatManager chat, ISessionControllerFactory sessionControllerFactory, IDmbFactory dmbFactory, @@ -86,7 +86,7 @@ namespace Tgstation.Server.Host.Components.Watchdog IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory, IIOManager gameIOManager, ISymlinkFactory symlinkFactory, - ILogger logger, + ILogger logger, DreamDaemonLaunchParameters initialLaunchParameters, Api.Models.Instance instance, bool autoStart) diff --git a/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs index 3bf2927e57..961fee9988 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs @@ -19,9 +19,9 @@ using Tgstation.Server.Host.Utils; namespace Tgstation.Server.Host.Components.Watchdog { /// - /// A variant of the that works on POSIX systems. + /// A variant of the that works on POSIX systems. /// - sealed class PosixWatchdog : WindowsWatchdog + sealed class PosixWatchdog : AdvancedWatchdog { /// /// The for the . @@ -41,8 +41,8 @@ namespace Tgstation.Server.Host.Components.Watchdog /// The for the . /// The for the . /// The for the . - /// The pointing to the game directory for the .. - /// The for the . + /// The pointing to the game directory for the .. + /// The for the . /// The for the . /// The for the . /// The for the . diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs index 7909be59af..fd7bdfe9b2 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs @@ -18,7 +18,7 @@ using Tgstation.Server.Host.Utils; namespace Tgstation.Server.Host.Components.Watchdog { /// - /// for creating s. + /// for creating s. /// class WindowsWatchdogFactory : WatchdogFactory { @@ -78,7 +78,7 @@ namespace Tgstation.Server.Host.Components.Watchdog remoteDeploymentManagerFactory, gameIOManager, SymlinkFactory, - LoggerFactory.CreateLogger(), + LoggerFactory.CreateLogger(), settings, instance, settings.AutoStart ?? throw new ArgumentNullException(nameof(settings))); From 4b02e43c3841c9f1a869a3cc2a8ed74334e0be3c Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 21 Oct 2023 18:10:35 -0400 Subject: [PATCH 15/40] Abstract `AdvancedWatchdog` Move Windows specific functionality out to `WindowsWatchdog`. --- .../Components/Watchdog/AdvancedWatchdog.cs | 16 ++-- .../Components/Watchdog/WindowsWatchdog.cs | 90 +++++++++++++++++++ .../Watchdog/WindowsWatchdogFactory.cs | 2 +- 3 files changed, 97 insertions(+), 11 deletions(-) create mode 100644 src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs diff --git a/src/Tgstation.Server.Host/Components/Watchdog/AdvancedWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/AdvancedWatchdog.cs index 3552149c3c..07954e5a88 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/AdvancedWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/AdvancedWatchdog.cs @@ -22,7 +22,7 @@ namespace Tgstation.Server.Host.Components.Watchdog /// /// A that, instead of killing servers for updates, uses the wonders of filesystem links to swap out changes without killing the server process. /// - class AdvancedWatchdog : BasicWatchdog + abstract class AdvancedWatchdog : BasicWatchdog { /// /// The for . @@ -125,7 +125,7 @@ namespace Tgstation.Server.Host.Components.Watchdog } /// - protected override async Task DisposeAndNullControllersImpl() + protected sealed override async Task DisposeAndNullControllersImpl() { await base.DisposeAndNullControllersImpl(); @@ -138,7 +138,7 @@ namespace Tgstation.Server.Host.Components.Watchdog } /// - protected override async Task HandleNormalReboot(CancellationToken cancellationToken) + protected sealed override async Task HandleNormalReboot(CancellationToken cancellationToken) { if (pendingSwappable != null) { @@ -231,7 +231,7 @@ namespace Tgstation.Server.Host.Components.Watchdog } /// - protected override async Task HandleNewDmbAvailable(CancellationToken cancellationToken) + protected sealed override async Task HandleNewDmbAvailable(CancellationToken cancellationToken) { IDmbProvider compileJobProvider = DmbFactory.LockNextDmb(1); bool canSeamlesslySwap = true; @@ -317,18 +317,14 @@ namespace Tgstation.Server.Host.Components.Watchdog /// /// The for the operation. /// A representing the running operation. - protected virtual async Task ApplyInitialDmb(CancellationToken cancellationToken) - { - Server.ReattachInformation.InitialDmb = await DmbFactory.FromCompileJob(Server.CompileJob, cancellationToken); - } + protected abstract Task ApplyInitialDmb(CancellationToken cancellationToken); /// /// Create a for a given . /// /// The to create a for. /// A new . - protected virtual SwappableDmbProvider CreateSwappableDmbProvider(IDmbProvider dmbProvider) - => new SwappableDmbProvider(dmbProvider, GameIOManager, SymlinkFactory); + protected abstract SwappableDmbProvider CreateSwappableDmbProvider(IDmbProvider dmbProvider); /// protected override async Task SessionStartupPersist(CancellationToken cancellationToken) diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs new file mode 100644 index 0000000000..917bafa134 --- /dev/null +++ b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs @@ -0,0 +1,90 @@ +using System.Threading; +using System.Threading.Tasks; + +using Microsoft.Extensions.Logging; + +using Tgstation.Server.Api.Models.Internal; +using Tgstation.Server.Host.Components.Chat; +using Tgstation.Server.Host.Components.Deployment; +using Tgstation.Server.Host.Components.Deployment.Remote; +using Tgstation.Server.Host.Components.Events; +using Tgstation.Server.Host.Components.Session; +using Tgstation.Server.Host.Core; +using Tgstation.Server.Host.IO; +using Tgstation.Server.Host.Jobs; +using Tgstation.Server.Host.Utils; + +namespace Tgstation.Server.Host.Components.Watchdog +{ + /// + /// A variant of the that works on Windows systems. + /// + sealed class WindowsWatchdog : AdvancedWatchdog + { + /// + /// Initializes a new instance of the class. + /// + /// The for the . + /// The for the . + /// The for the . + /// The for the . + /// The for the . + /// The for the . + /// The for the . + /// The for the . + /// The for the . + /// The for the . + /// The pointing to the game directory for the .. + /// The for the . + /// The for the . + /// The for the . + /// The for the . + /// The autostart value for the . + public WindowsWatchdog( + IChatManager chat, + ISessionControllerFactory sessionControllerFactory, + IDmbFactory dmbFactory, + ISessionPersistor sessionPersistor, + IJobManager jobManager, + IServerControl serverControl, + IAsyncDelayer asyncDelayer, + IIOManager diagnosticsIOManager, + IEventConsumer eventConsumer, + IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory, + IIOManager gameIOManager, + ISymlinkFactory symlinkFactory, + ILogger logger, + DreamDaemonLaunchParameters initialLaunchParameters, + Api.Models.Instance instance, + bool autoStart) + : base( + chat, + sessionControllerFactory, + dmbFactory, + sessionPersistor, + jobManager, + serverControl, + asyncDelayer, + diagnosticsIOManager, + eventConsumer, + remoteDeploymentManagerFactory, + gameIOManager, + symlinkFactory, + logger, + initialLaunchParameters, + instance, + autoStart) + { + } + + /// + protected override async Task ApplyInitialDmb(CancellationToken cancellationToken) + { + Server.ReattachInformation.InitialDmb = await DmbFactory.FromCompileJob(Server.CompileJob, cancellationToken); + } + + /// + protected override SwappableDmbProvider CreateSwappableDmbProvider(IDmbProvider dmbProvider) + => new SwappableDmbProvider(dmbProvider, GameIOManager, SymlinkFactory); + } +} diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs index fd7bdfe9b2..4cbe20577e 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs @@ -78,7 +78,7 @@ namespace Tgstation.Server.Host.Components.Watchdog remoteDeploymentManagerFactory, gameIOManager, SymlinkFactory, - LoggerFactory.CreateLogger(), + LoggerFactory.CreateLogger(), settings, instance, settings.AutoStart ?? throw new ArgumentNullException(nameof(settings))); From afceee3a96a4aa979046f4344901f284f647ffcd Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 21 Oct 2023 18:24:54 -0400 Subject: [PATCH 16/40] I was unsure about `DefaultIOManager.DeleteDirectory`, so here's another unit test. --- .../IO/TestIOManager.cs | 58 +++++++++++++++++-- 1 file changed, 53 insertions(+), 5 deletions(-) diff --git a/tests/Tgstation.Server.Host.Tests/IO/TestIOManager.cs b/tests/Tgstation.Server.Host.Tests/IO/TestIOManager.cs index f92ffaa04d..5e34e809dc 100644 --- a/tests/Tgstation.Server.Host.Tests/IO/TestIOManager.cs +++ b/tests/Tgstation.Server.Host.Tests/IO/TestIOManager.cs @@ -1,11 +1,12 @@ -using Microsoft.VisualStudio.TestTools.UnitTesting; - -using Remora.Discord.API.Objects; - -using System; +using System; using System.IO; +using System.Linq; +using System.Text; +using System.Threading; using System.Threading.Tasks; +using Microsoft.VisualStudio.TestTools.UnitTesting; + using Tgstation.Server.Host.System; namespace Tgstation.Server.Host.IO.Tests @@ -38,6 +39,53 @@ namespace Tgstation.Server.Host.IO.Tests } } + [TestMethod] + public async Task TestDeleteDirectoryWithSymlinkInsideDoesntRecurse() + { + var linkFactory = (ISymlinkFactory)(new PlatformIdentifier().IsWindows + ? new WindowsSymlinkFactory() + : new PosixSymlinkFactory()); + + var tempPath = Path.GetTempFileName(); + File.Delete(tempPath); + Directory.CreateDirectory(tempPath); + try + { + var targetDir = ioManager.ConcatPath(tempPath, "targetdir"); + await ioManager.CreateDirectory(targetDir, CancellationToken.None); + var fileInTargetDir = ioManager.ConcatPath(targetDir, "test1.txt"); + + var expectedBytes = Encoding.UTF8.GetBytes("I want to live"); + await ioManager.WriteAllBytes(fileInTargetDir, expectedBytes, CancellationToken.None); + + var testDir = ioManager.ConcatPath(tempPath, "testdir"); + await ioManager.CreateDirectory(testDir, CancellationToken.None); + var symlinkedFile = ioManager.ConcatPath(testDir, "test1.txt"); + var symlinkedDir = ioManager.ConcatPath(testDir, "linkedDir"); + + await linkFactory.CreateSymbolicLink(targetDir, symlinkedDir, CancellationToken.None); + await linkFactory.CreateSymbolicLink(fileInTargetDir, symlinkedFile, CancellationToken.None); + + Assert.IsTrue(await ioManager.DirectoryExists(symlinkedDir, CancellationToken.None)); + Assert.IsTrue(await ioManager.FileExists(symlinkedFile, CancellationToken.None)); + Assert.IsTrue(await ioManager.FileExists(ioManager.ConcatPath(symlinkedDir, "test1.txt"), CancellationToken.None)); + Assert.IsTrue(await ioManager.FileExists(fileInTargetDir, CancellationToken.None)); + + await ioManager.DeleteDirectory(testDir, CancellationToken.None); + + Assert.IsFalse(await ioManager.DirectoryExists(symlinkedDir, CancellationToken.None)); + Assert.IsFalse(await ioManager.FileExists(symlinkedFile, CancellationToken.None)); + Assert.IsFalse(await ioManager.FileExists(ioManager.ConcatPath(symlinkedDir, "test1.txt"), CancellationToken.None)); + Assert.IsTrue(await ioManager.FileExists(fileInTargetDir, CancellationToken.None)); + Assert.IsTrue(expectedBytes.SequenceEqual(await ioManager.ReadAllBytes(fileInTargetDir, CancellationToken.None))); + } + catch + { + Directory.Delete(tempPath, true); + throw; + } + } + [TestMethod] public async Task TestFileExists() { From 8129773d69d7e8d4c27c3783bf8cf5f962da30e1 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 21 Oct 2023 18:37:01 -0400 Subject: [PATCH 17/40] Extract symlinking behavior to `SymlinkDmbProvider` --- .../Deployment/HardLinkDmbProvider.cs | 16 ++++--- .../Deployment/SwappableDmbProvider.cs | 44 +++++++------------ .../Deployment/SymlinkDmbProvider.cs | 44 +++++++++++++++++++ .../Components/Watchdog/WindowsWatchdog.cs | 2 +- 4 files changed, 70 insertions(+), 36 deletions(-) create mode 100644 src/Tgstation.Server.Host/Components/Deployment/SymlinkDmbProvider.cs diff --git a/src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs b/src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs index e5ddede026..634af753f4 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs @@ -111,15 +111,17 @@ namespace Tgstation.Server.Host.Components.Deployment goAheadTcs.SetResult(); await IOManager.DeleteDirectory(disposePath, CancellationToken.None); // DCT: We're detached at this point } + catch (DirectoryNotFoundException ex) + { + logger.LogDebug(ex, "Live directory appears to not exist"); + if (!directoryMoved) + goAheadTcs.SetResult(); + } catch (Exception ex) { - if (directoryMoved) - logger.LogWarning(ex, "Failed to delete hard linked directory: {disposePath}", disposePath); - else - { - logger.LogDebug(ex, "Live directory appears to not exist"); - goAheadTcs.SetResult(); - } + logger.LogWarning(ex, "Failed to delete hard linked directory: {disposePath}", disposePath); + if (!directoryMoved) + goAheadTcs.SetException(ex); } } diff --git a/src/Tgstation.Server.Host/Components/Deployment/SwappableDmbProvider.cs b/src/Tgstation.Server.Host/Components/Deployment/SwappableDmbProvider.cs index 14cdc268e2..b82f5cc676 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/SwappableDmbProvider.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/SwappableDmbProvider.cs @@ -8,29 +8,34 @@ using Tgstation.Server.Host.Models; namespace Tgstation.Server.Host.Components.Deployment { /// - /// A that uses symlinks. + /// A that uses filesystem links to change directory structure underneath the server process. /// - class SwappableDmbProvider : IDmbProvider + abstract class SwappableDmbProvider : IDmbProvider { /// - /// The directory where the is symlinked to. + /// The directory where the is symlinked to. /// public const string LiveGameDirectory = "Live"; /// - public string DmbName => baseProvider.DmbName; + public string DmbName => BaseProvider.DmbName; /// public string Directory => IOManager.ResolvePath(LiveGameDirectory); /// - public CompileJob CompileJob => baseProvider.CompileJob; + public CompileJob CompileJob => BaseProvider.CompileJob; /// /// If has been run. /// public bool Swapped => swapped != 0; + /// + /// The we are swapping for. + /// + protected IDmbProvider BaseProvider { get; } + /// /// The to use. /// @@ -41,11 +46,6 @@ namespace Tgstation.Server.Host.Components.Deployment /// protected ISymlinkFactory SymlinkFactory { get; } - /// - /// The we are swapping for. - /// - readonly IDmbProvider baseProvider; - /// /// Backing field for . /// @@ -54,21 +54,21 @@ namespace Tgstation.Server.Host.Components.Deployment /// /// Initializes a new instance of the class. /// - /// The value of . + /// The value of . /// The value of . /// The value of . public SwappableDmbProvider(IDmbProvider baseProvider, IIOManager ioManager, ISymlinkFactory symlinkFactory) { - this.baseProvider = baseProvider ?? throw new ArgumentNullException(nameof(baseProvider)); + BaseProvider = baseProvider ?? throw new ArgumentNullException(nameof(baseProvider)); IOManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); SymlinkFactory = symlinkFactory ?? throw new ArgumentNullException(nameof(symlinkFactory)); } /// - public virtual ValueTask DisposeAsync() => baseProvider.DisposeAsync(); + public virtual ValueTask DisposeAsync() => BaseProvider.DisposeAsync(); /// - public void KeepAlive() => baseProvider.KeepAlive(); + public void KeepAlive() => BaseProvider.KeepAlive(); /// /// Make the active by replacing the live link with our . @@ -88,25 +88,13 @@ namespace Tgstation.Server.Host.Components.Deployment /// /// The for the operation. /// A representing the preparation process. - public virtual Task FinishActivationPreparation(CancellationToken cancellationToken) - => Task.CompletedTask; + public abstract Task FinishActivationPreparation(CancellationToken cancellationToken); /// /// Perform the swapping action. /// /// The for the operation. /// A representing the running operation. - protected virtual async Task DoSwap(CancellationToken cancellationToken) - { - if (SymlinkFactory.SymlinkedDirectoriesAreDeletedAsFiles) - await IOManager.DeleteFile(LiveGameDirectory, cancellationToken); - else - await IOManager.DeleteDirectory(LiveGameDirectory, cancellationToken); - - await SymlinkFactory.CreateSymbolicLink( - IOManager.ResolvePath(baseProvider.Directory), - IOManager.ResolvePath(LiveGameDirectory), - cancellationToken); - } + protected abstract Task DoSwap(CancellationToken cancellationToken); } } diff --git a/src/Tgstation.Server.Host/Components/Deployment/SymlinkDmbProvider.cs b/src/Tgstation.Server.Host/Components/Deployment/SymlinkDmbProvider.cs new file mode 100644 index 0000000000..7e0845b95c --- /dev/null +++ b/src/Tgstation.Server.Host/Components/Deployment/SymlinkDmbProvider.cs @@ -0,0 +1,44 @@ +using System.Threading; +using System.Threading.Tasks; + +using Tgstation.Server.Host.IO; + +namespace Tgstation.Server.Host.Components.Deployment +{ + /// + /// A that uses symlinks. + /// + sealed class SymlinkDmbProvider : SwappableDmbProvider + { + /// + /// Initializes a new instance of the class. + /// + /// The for the . + /// The for the . + /// The for the . + public SymlinkDmbProvider( + IDmbProvider baseProvider, + IIOManager ioManager, + ISymlinkFactory symlinkFactory) + : base(baseProvider, ioManager, symlinkFactory) + { + } + + /// + public override Task FinishActivationPreparation(CancellationToken cancellationToken) => Task.CompletedTask; + + /// + protected override async Task DoSwap(CancellationToken cancellationToken) + { + if (SymlinkFactory.SymlinkedDirectoriesAreDeletedAsFiles) + await IOManager.DeleteFile(LiveGameDirectory, cancellationToken); + else + await IOManager.DeleteDirectory(LiveGameDirectory, cancellationToken); + + await SymlinkFactory.CreateSymbolicLink( + IOManager.ResolvePath(BaseProvider.Directory), + IOManager.ResolvePath(LiveGameDirectory), + cancellationToken); + } + } +} diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs index 917bafa134..d882296b06 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs @@ -85,6 +85,6 @@ namespace Tgstation.Server.Host.Components.Watchdog /// protected override SwappableDmbProvider CreateSwappableDmbProvider(IDmbProvider dmbProvider) - => new SwappableDmbProvider(dmbProvider, GameIOManager, SymlinkFactory); + => new SymlinkDmbProvider(dmbProvider, GameIOManager, SymlinkFactory); } } From fd6de0fee2d8ab0b2e950de59a94001335952a24 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 21 Oct 2023 18:42:19 -0400 Subject: [PATCH 18/40] Rename *`SymlinkFactory` to *`FilesystemLinkFactory` to better reflect new behavior --- .../Deployment/HardLinkDmbProvider.cs | 8 +++---- .../Deployment/SwappableDmbProvider.cs | 10 ++++----- .../Deployment/SymlinkDmbProvider.cs | 10 ++++----- .../Components/InstanceFactory.cs | 12 +++++----- .../Components/StaticFiles/Configuration.cs | 12 +++++----- .../Components/Watchdog/AdvancedWatchdog.cs | 10 ++++----- .../Components/Watchdog/PosixWatchdog.cs | 8 +++---- .../Watchdog/PosixWatchdogFactory.cs | 8 +++---- .../Components/Watchdog/WindowsWatchdog.cs | 8 +++---- .../Watchdog/WindowsWatchdogFactory.cs | 12 +++++----- src/Tgstation.Server.Host/Core/Application.cs | 4 ++-- ...nkFactory.cs => IFilesystemLinkFactory.cs} | 2 +- ...ctory.cs => PosixFilesystemLinkFactory.cs} | 4 ++-- ...ory.cs => WindowsFilesystemLinkFactory.cs} | 4 ++-- .../StaticFiles/TestConfiguration.cs | 2 +- ...actory.cs => TestFilesystemLinkFactory.cs} | 22 +++++++++---------- .../IO/TestIOManager.cs | 6 ++--- .../System/TestSymlinkFactory.cs | 6 ++--- 18 files changed, 74 insertions(+), 74 deletions(-) rename src/Tgstation.Server.Host/IO/{ISymlinkFactory.cs => IFilesystemLinkFactory.cs} (97%) rename src/Tgstation.Server.Host/IO/{PosixSymlinkFactory.cs => PosixFilesystemLinkFactory.cs} (92%) rename src/Tgstation.Server.Host/IO/{WindowsSymlinkFactory.cs => WindowsFilesystemLinkFactory.cs} (92%) rename tests/Tgstation.Server.Host.Tests/IO/{TestSymlinkFactory.cs => TestFilesystemLinkFactory.cs} (80%) diff --git a/src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs b/src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs index 634af753f4..cd9a3f1185 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs @@ -40,19 +40,19 @@ namespace Tgstation.Server.Host.Components.Deployment /// /// The for the . /// The for the . - /// The for the . + /// The for the . /// The value of . /// The for the . public HardLinkDmbProvider( IDmbProvider baseProvider, IIOManager ioManager, - ISymlinkFactory symlinkFactory, + IFilesystemLinkFactory linkFactory, ILogger logger, GeneralConfiguration generalConfiguration) : base( baseProvider, ioManager, - symlinkFactory) + linkFactory) { this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); cancellationTokenSource = new CancellationTokenSource(); @@ -214,7 +214,7 @@ namespace Tgstation.Server.Host.Components.Deployment using var lockContext = semaphore != null ? await SemaphoreSlimContext.Lock(semaphore, cancellationToken) : null; - await SymlinkFactory.CreateHardLink(sourceFile, destFile, cancellationToken); + await LinkFactory.CreateHardLink(sourceFile, destFile, cancellationToken); } yield return LinkThisFile(); diff --git a/src/Tgstation.Server.Host/Components/Deployment/SwappableDmbProvider.cs b/src/Tgstation.Server.Host/Components/Deployment/SwappableDmbProvider.cs index b82f5cc676..b00ba3d518 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/SwappableDmbProvider.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/SwappableDmbProvider.cs @@ -42,9 +42,9 @@ namespace Tgstation.Server.Host.Components.Deployment protected IIOManager IOManager { get; } /// - /// The to use. + /// The to use. /// - protected ISymlinkFactory SymlinkFactory { get; } + protected IFilesystemLinkFactory LinkFactory { get; } /// /// Backing field for . @@ -56,12 +56,12 @@ namespace Tgstation.Server.Host.Components.Deployment /// /// The value of . /// The value of . - /// The value of . - public SwappableDmbProvider(IDmbProvider baseProvider, IIOManager ioManager, ISymlinkFactory symlinkFactory) + /// The value of . + public SwappableDmbProvider(IDmbProvider baseProvider, IIOManager ioManager, IFilesystemLinkFactory symlinkFactory) { BaseProvider = baseProvider ?? throw new ArgumentNullException(nameof(baseProvider)); IOManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); - SymlinkFactory = symlinkFactory ?? throw new ArgumentNullException(nameof(symlinkFactory)); + LinkFactory = symlinkFactory ?? throw new ArgumentNullException(nameof(symlinkFactory)); } /// diff --git a/src/Tgstation.Server.Host/Components/Deployment/SymlinkDmbProvider.cs b/src/Tgstation.Server.Host/Components/Deployment/SymlinkDmbProvider.cs index 7e0845b95c..2edc58d37e 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/SymlinkDmbProvider.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/SymlinkDmbProvider.cs @@ -15,12 +15,12 @@ namespace Tgstation.Server.Host.Components.Deployment /// /// The for the . /// The for the . - /// The for the . + /// The for the . public SymlinkDmbProvider( IDmbProvider baseProvider, IIOManager ioManager, - ISymlinkFactory symlinkFactory) - : base(baseProvider, ioManager, symlinkFactory) + IFilesystemLinkFactory linkFactory) + : base(baseProvider, ioManager, linkFactory) { } @@ -30,12 +30,12 @@ namespace Tgstation.Server.Host.Components.Deployment /// protected override async Task DoSwap(CancellationToken cancellationToken) { - if (SymlinkFactory.SymlinkedDirectoriesAreDeletedAsFiles) + if (LinkFactory.SymlinkedDirectoriesAreDeletedAsFiles) await IOManager.DeleteFile(LiveGameDirectory, cancellationToken); else await IOManager.DeleteDirectory(LiveGameDirectory, cancellationToken); - await SymlinkFactory.CreateSymbolicLink( + await LinkFactory.CreateSymbolicLink( IOManager.ResolvePath(BaseProvider.Directory), IOManager.ResolvePath(LiveGameDirectory), cancellationToken); diff --git a/src/Tgstation.Server.Host/Components/InstanceFactory.cs b/src/Tgstation.Server.Host/Components/InstanceFactory.cs index 8c7a8df85e..a3c7ba2192 100644 --- a/src/Tgstation.Server.Host/Components/InstanceFactory.cs +++ b/src/Tgstation.Server.Host/Components/InstanceFactory.cs @@ -66,9 +66,9 @@ namespace Tgstation.Server.Host.Components readonly ISynchronousIOManager synchronousIOManager; /// - /// The for the . + /// The for the . /// - readonly ISymlinkFactory symlinkFactory; + readonly IFilesystemLinkFactory linkFactory; /// /// The for the . @@ -173,7 +173,7 @@ namespace Tgstation.Server.Host.Components /// The value of . /// The value of . /// The value of . - /// The value of . + /// The value of . /// The value of . /// The value of . /// The value of . @@ -199,7 +199,7 @@ namespace Tgstation.Server.Host.Components ITopicClientFactory topicClientFactory, ICryptographySuite cryptographySuite, ISynchronousIOManager synchronousIOManager, - ISymlinkFactory symlinkFactory, + IFilesystemLinkFactory linkFactory, IByondInstaller byondInstaller, IChatManagerFactory chatFactory, IProcessExecutor processExecutor, @@ -225,7 +225,7 @@ namespace Tgstation.Server.Host.Components this.topicClientFactory = topicClientFactory ?? throw new ArgumentNullException(nameof(topicClientFactory)); this.cryptographySuite = cryptographySuite ?? throw new ArgumentNullException(nameof(cryptographySuite)); this.synchronousIOManager = synchronousIOManager ?? throw new ArgumentNullException(nameof(synchronousIOManager)); - this.symlinkFactory = symlinkFactory ?? throw new ArgumentNullException(nameof(symlinkFactory)); + this.linkFactory = linkFactory ?? throw new ArgumentNullException(nameof(linkFactory)); this.byondInstaller = byondInstaller ?? throw new ArgumentNullException(nameof(byondInstaller)); this.chatFactory = chatFactory ?? throw new ArgumentNullException(nameof(chatFactory)); this.processExecutor = processExecutor ?? throw new ArgumentNullException(nameof(processExecutor)); @@ -275,7 +275,7 @@ namespace Tgstation.Server.Host.Components var configuration = new StaticFiles.Configuration( configurationIoManager, synchronousIOManager, - symlinkFactory, + linkFactory, processExecutor, postWriteHandler, platformIdentifier, diff --git a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs index 087503d68c..eeb5cd71c6 100644 --- a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs +++ b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs @@ -94,9 +94,9 @@ namespace Tgstation.Server.Host.Components.StaticFiles readonly ISynchronousIOManager synchronousIOManager; /// - /// The for . + /// The for . /// - readonly ISymlinkFactory symlinkFactory; + readonly IFilesystemLinkFactory linkFactory; /// /// The for . @@ -153,7 +153,7 @@ namespace Tgstation.Server.Host.Components.StaticFiles /// /// The value of . /// The value of . - /// The value of . + /// The value of . /// The value of . /// The value of . /// The value of . @@ -164,7 +164,7 @@ namespace Tgstation.Server.Host.Components.StaticFiles public Configuration( IIOManager ioManager, ISynchronousIOManager synchronousIOManager, - ISymlinkFactory symlinkFactory, + IFilesystemLinkFactory linkFactory, IProcessExecutor processExecutor, IPostWriteHandler postWriteHandler, IPlatformIdentifier platformIdentifier, @@ -175,7 +175,7 @@ namespace Tgstation.Server.Host.Components.StaticFiles { this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); this.synchronousIOManager = synchronousIOManager ?? throw new ArgumentNullException(nameof(synchronousIOManager)); - this.symlinkFactory = symlinkFactory ?? throw new ArgumentNullException(nameof(symlinkFactory)); + this.linkFactory = linkFactory ?? throw new ArgumentNullException(nameof(linkFactory)); this.processExecutor = processExecutor ?? throw new ArgumentNullException(nameof(processExecutor)); this.postWriteHandler = postWriteHandler ?? throw new ArgumentNullException(nameof(postWriteHandler)); this.platformIdentifier = platformIdentifier ?? throw new ArgumentNullException(nameof(platformIdentifier)); @@ -450,7 +450,7 @@ namespace Tgstation.Server.Host.Components.StaticFiles var fileExists = await fileExistsTask; if (fileExists) await ioManager.DeleteFile(destPath, cancellationToken); - await symlinkFactory.CreateSymbolicLink(ioManager.ResolvePath(file), ioManager.ResolvePath(destPath), cancellationToken); + await linkFactory.CreateSymbolicLink(ioManager.ResolvePath(file), ioManager.ResolvePath(destPath), cancellationToken); })); } diff --git a/src/Tgstation.Server.Host/Components/Watchdog/AdvancedWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/AdvancedWatchdog.cs index 07954e5a88..48d2b3e254 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/AdvancedWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/AdvancedWatchdog.cs @@ -35,9 +35,9 @@ namespace Tgstation.Server.Host.Components.Watchdog protected IIOManager GameIOManager { get; } /// - /// The for the . + /// The for the . /// - protected ISymlinkFactory SymlinkFactory { get; } + protected IFilesystemLinkFactory LinkFactory { get; } /// /// of s that are waiting to clean up old deployments. @@ -68,7 +68,7 @@ namespace Tgstation.Server.Host.Components.Watchdog /// The for the . /// The for the . /// The value of . - /// The value of . + /// The value of . /// The for the . /// The for the . /// The for the . @@ -85,7 +85,7 @@ namespace Tgstation.Server.Host.Components.Watchdog IEventConsumer eventConsumer, IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory, IIOManager gameIOManager, - ISymlinkFactory symlinkFactory, + IFilesystemLinkFactory linkFactory, ILogger logger, DreamDaemonLaunchParameters initialLaunchParameters, Api.Models.Instance instance, @@ -109,7 +109,7 @@ namespace Tgstation.Server.Host.Components.Watchdog try { GameIOManager = gameIOManager ?? throw new ArgumentNullException(nameof(gameIOManager)); - SymlinkFactory = symlinkFactory ?? throw new ArgumentNullException(nameof(symlinkFactory)); + LinkFactory = linkFactory ?? throw new ArgumentNullException(nameof(linkFactory)); deploymentCleanupTasks = new List(); } diff --git a/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs index 961fee9988..eb563bd1fa 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs @@ -42,7 +42,7 @@ namespace Tgstation.Server.Host.Components.Watchdog /// The for the . /// The for the . /// The pointing to the game directory for the .. - /// The for the . + /// The for the . /// The for the . /// The for the . /// The for the . @@ -60,7 +60,7 @@ namespace Tgstation.Server.Host.Components.Watchdog IEventConsumer eventConsumer, IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory, IIOManager gameIOManager, - ISymlinkFactory symlinkFactory, + IFilesystemLinkFactory linkFactory, ILogger logger, DreamDaemonLaunchParameters initialLaunchParameters, Api.Models.Instance instance, @@ -78,7 +78,7 @@ namespace Tgstation.Server.Host.Components.Watchdog eventConsumer, remoteDeploymentManagerFactory, gameIOManager, - symlinkFactory, + linkFactory, logger, initialLaunchParameters, instance, @@ -93,6 +93,6 @@ namespace Tgstation.Server.Host.Components.Watchdog /// protected override SwappableDmbProvider CreateSwappableDmbProvider(IDmbProvider dmbProvider) - => new HardLinkDmbProvider(dmbProvider, GameIOManager, SymlinkFactory, Logger, generalConfiguration); + => new HardLinkDmbProvider(dmbProvider, GameIOManager, LinkFactory, Logger, generalConfiguration); } } diff --git a/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdogFactory.cs b/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdogFactory.cs index 787127a05c..64c8672262 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdogFactory.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdogFactory.cs @@ -29,21 +29,21 @@ namespace Tgstation.Server.Host.Components.Watchdog /// The for the . /// The for the . /// The for the . - /// The for the . + /// The for the . /// The for for the . public PosixWatchdogFactory( IServerControl serverControl, ILoggerFactory loggerFactory, IJobManager jobManager, IAsyncDelayer asyncDelayer, - ISymlinkFactory symlinkFactory, + IFilesystemLinkFactory linkFactory, IOptions generalConfigurationOptions) : base( serverControl, loggerFactory, jobManager, asyncDelayer, - symlinkFactory, + linkFactory, generalConfigurationOptions) { } @@ -72,7 +72,7 @@ namespace Tgstation.Server.Host.Components.Watchdog eventConsumer, remoteDeploymentManagerFactory, gameIOManager, - SymlinkFactory, + LinkFactory, LoggerFactory.CreateLogger(), settings, instance, diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs index d882296b06..05c1f10088 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs @@ -35,7 +35,7 @@ namespace Tgstation.Server.Host.Components.Watchdog /// The for the . /// The for the . /// The pointing to the game directory for the .. - /// The for the . + /// The for the . /// The for the . /// The for the . /// The for the . @@ -52,7 +52,7 @@ namespace Tgstation.Server.Host.Components.Watchdog IEventConsumer eventConsumer, IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory, IIOManager gameIOManager, - ISymlinkFactory symlinkFactory, + IFilesystemLinkFactory linkFactory, ILogger logger, DreamDaemonLaunchParameters initialLaunchParameters, Api.Models.Instance instance, @@ -69,7 +69,7 @@ namespace Tgstation.Server.Host.Components.Watchdog eventConsumer, remoteDeploymentManagerFactory, gameIOManager, - symlinkFactory, + linkFactory, logger, initialLaunchParameters, instance, @@ -85,6 +85,6 @@ namespace Tgstation.Server.Host.Components.Watchdog /// protected override SwappableDmbProvider CreateSwappableDmbProvider(IDmbProvider dmbProvider) - => new SymlinkDmbProvider(dmbProvider, GameIOManager, SymlinkFactory); + => new SymlinkDmbProvider(dmbProvider, GameIOManager, LinkFactory); } } diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs index 4cbe20577e..89ed6b145d 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs @@ -23,9 +23,9 @@ namespace Tgstation.Server.Host.Components.Watchdog class WindowsWatchdogFactory : WatchdogFactory { /// - /// The for the . + /// The for the . /// - protected ISymlinkFactory SymlinkFactory { get; } + protected IFilesystemLinkFactory LinkFactory { get; } /// /// Initializes a new instance of the class. @@ -34,14 +34,14 @@ namespace Tgstation.Server.Host.Components.Watchdog /// The for the . /// The for the . /// The for the . - /// The value of . + /// The value of . /// The for for the . public WindowsWatchdogFactory( IServerControl serverControl, ILoggerFactory loggerFactory, IJobManager jobManager, IAsyncDelayer asyncDelayer, - ISymlinkFactory symlinkFactory, + IFilesystemLinkFactory symlinkFactory, IOptions generalConfigurationOptions) : base( serverControl, @@ -50,7 +50,7 @@ namespace Tgstation.Server.Host.Components.Watchdog asyncDelayer, generalConfigurationOptions) { - SymlinkFactory = symlinkFactory ?? throw new ArgumentNullException(nameof(symlinkFactory)); + LinkFactory = symlinkFactory ?? throw new ArgumentNullException(nameof(symlinkFactory)); } /// @@ -77,7 +77,7 @@ namespace Tgstation.Server.Host.Components.Watchdog eventConsumer, remoteDeploymentManagerFactory, gameIOManager, - SymlinkFactory, + LinkFactory, LoggerFactory.CreateLogger(), settings, instance, diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index ed1cb97cad..18abdcacf3 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -336,7 +336,7 @@ namespace Tgstation.Server.Host.Core { AddWatchdog(services, postSetupServices); services.AddSingleton(); - services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); @@ -349,7 +349,7 @@ namespace Tgstation.Server.Host.Core { AddWatchdog(services, postSetupServices); services.AddSingleton(); - services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); diff --git a/src/Tgstation.Server.Host/IO/ISymlinkFactory.cs b/src/Tgstation.Server.Host/IO/IFilesystemLinkFactory.cs similarity index 97% rename from src/Tgstation.Server.Host/IO/ISymlinkFactory.cs rename to src/Tgstation.Server.Host/IO/IFilesystemLinkFactory.cs index 08320a3d15..3cf0622801 100644 --- a/src/Tgstation.Server.Host/IO/ISymlinkFactory.cs +++ b/src/Tgstation.Server.Host/IO/IFilesystemLinkFactory.cs @@ -6,7 +6,7 @@ namespace Tgstation.Server.Host.IO /// /// For creating filesystem symbolic links. /// - interface ISymlinkFactory + interface IFilesystemLinkFactory { /// /// If directory symlinks must be deleted as files would in the current environment. diff --git a/src/Tgstation.Server.Host/IO/PosixSymlinkFactory.cs b/src/Tgstation.Server.Host/IO/PosixFilesystemLinkFactory.cs similarity index 92% rename from src/Tgstation.Server.Host/IO/PosixSymlinkFactory.cs rename to src/Tgstation.Server.Host/IO/PosixFilesystemLinkFactory.cs index 05d64a3bf0..b80af9e122 100644 --- a/src/Tgstation.Server.Host/IO/PosixSymlinkFactory.cs +++ b/src/Tgstation.Server.Host/IO/PosixFilesystemLinkFactory.cs @@ -8,9 +8,9 @@ using Mono.Unix; namespace Tgstation.Server.Host.IO { /// - /// for posix systems. + /// for POSIX systems. /// - sealed class PosixSymlinkFactory : ISymlinkFactory + sealed class PosixFilesystemLinkFactory : IFilesystemLinkFactory { /// public bool SymlinkedDirectoriesAreDeletedAsFiles => true; diff --git a/src/Tgstation.Server.Host/IO/WindowsSymlinkFactory.cs b/src/Tgstation.Server.Host/IO/WindowsFilesystemLinkFactory.cs similarity index 92% rename from src/Tgstation.Server.Host/IO/WindowsSymlinkFactory.cs rename to src/Tgstation.Server.Host/IO/WindowsFilesystemLinkFactory.cs index 864242d992..1513f4e166 100644 --- a/src/Tgstation.Server.Host/IO/WindowsSymlinkFactory.cs +++ b/src/Tgstation.Server.Host/IO/WindowsFilesystemLinkFactory.cs @@ -9,9 +9,9 @@ using Tgstation.Server.Host.System; namespace Tgstation.Server.Host.IO { /// - /// for windows systems. + /// for windows systems. /// - sealed class WindowsSymlinkFactory : ISymlinkFactory + sealed class WindowsFilesystemLinkFactory : IFilesystemLinkFactory { /// public bool SymlinkedDirectoriesAreDeletedAsFiles => false; diff --git a/tests/Tgstation.Server.Host.Tests/Components/StaticFiles/TestConfiguration.cs b/tests/Tgstation.Server.Host.Tests/Components/StaticFiles/TestConfiguration.cs index 68241988d1..ff3bd0b5b8 100644 --- a/tests/Tgstation.Server.Host.Tests/Components/StaticFiles/TestConfiguration.cs +++ b/tests/Tgstation.Server.Host.Tests/Components/StaticFiles/TestConfiguration.cs @@ -47,7 +47,7 @@ namespace Tgstation.Server.Host.Components.StaticFiles.Tests var configuration = new Configuration( ioManager, new SynchronousIOManager(), - Mock.Of(), + Mock.Of(), Mock.Of(), Mock.Of(), Mock.Of(), diff --git a/tests/Tgstation.Server.Host.Tests/IO/TestSymlinkFactory.cs b/tests/Tgstation.Server.Host.Tests/IO/TestFilesystemLinkFactory.cs similarity index 80% rename from tests/Tgstation.Server.Host.Tests/IO/TestSymlinkFactory.cs rename to tests/Tgstation.Server.Host.Tests/IO/TestFilesystemLinkFactory.cs index 27646a9dc7..e735c48541 100644 --- a/tests/Tgstation.Server.Host.Tests/IO/TestSymlinkFactory.cs +++ b/tests/Tgstation.Server.Host.Tests/IO/TestFilesystemLinkFactory.cs @@ -8,17 +8,17 @@ using System.Threading.Tasks; namespace Tgstation.Server.Host.IO.Tests { [TestClass] - public sealed class TestSymlinkFactory + public sealed class TestFilesystemLinkFactory { - static ISymlinkFactory symlinkFactory; + static IFilesystemLinkFactory linkFactory; [ClassInitialize] public static void SelectFactory(TestContext _) { if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - symlinkFactory = new WindowsSymlinkFactory(); + linkFactory = new WindowsFilesystemLinkFactory(); else - symlinkFactory = new PosixSymlinkFactory(); + linkFactory = new PosixFilesystemLinkFactory(); } public static bool HasPermissionToMakeSymlinks() @@ -43,10 +43,10 @@ namespace Tgstation.Server.Host.IO.Tests f2 = f1 + ".linked"; File.WriteAllText(f1, Text); - await Assert.ThrowsExceptionAsync(() => symlinkFactory.CreateSymbolicLink(null, null, default)); - await Assert.ThrowsExceptionAsync(() => symlinkFactory.CreateSymbolicLink(f1, null, default)); + await Assert.ThrowsExceptionAsync(() => linkFactory.CreateSymbolicLink(null, null, default)); + await Assert.ThrowsExceptionAsync(() => linkFactory.CreateSymbolicLink(f1, null, default)); - await symlinkFactory.CreateSymbolicLink(f1, f2, default); + await linkFactory.CreateSymbolicLink(f1, f2, default); Assert.IsTrue(File.Exists(f2)); var f2Contents = File.ReadAllText(f2); @@ -76,10 +76,10 @@ namespace Tgstation.Server.Host.IO.Tests var p1 = Path.Combine(f1, FileName); File.WriteAllText(p1, Text); - await Assert.ThrowsExceptionAsync(() => symlinkFactory.CreateSymbolicLink(null, null, default)); - await Assert.ThrowsExceptionAsync(() => symlinkFactory.CreateSymbolicLink(f1, null, default)); + await Assert.ThrowsExceptionAsync(() => linkFactory.CreateSymbolicLink(null, null, default)); + await Assert.ThrowsExceptionAsync(() => linkFactory.CreateSymbolicLink(f1, null, default)); - await symlinkFactory.CreateSymbolicLink(f1, f2, default); + await linkFactory.CreateSymbolicLink(f1, f2, default); var p2 = Path.Combine(f2, FileName); Assert.IsTrue(File.Exists(p2)); @@ -104,7 +104,7 @@ namespace Tgstation.Server.Host.IO.Tests try { - await symlinkFactory.CreateSymbolicLink(BadPath, BadPath, default); + await linkFactory.CreateSymbolicLink(BadPath, BadPath, default); Assert.Fail("No exception thrown!"); } catch { } diff --git a/tests/Tgstation.Server.Host.Tests/IO/TestIOManager.cs b/tests/Tgstation.Server.Host.Tests/IO/TestIOManager.cs index 5e34e809dc..00319f4f30 100644 --- a/tests/Tgstation.Server.Host.Tests/IO/TestIOManager.cs +++ b/tests/Tgstation.Server.Host.Tests/IO/TestIOManager.cs @@ -42,9 +42,9 @@ namespace Tgstation.Server.Host.IO.Tests [TestMethod] public async Task TestDeleteDirectoryWithSymlinkInsideDoesntRecurse() { - var linkFactory = (ISymlinkFactory)(new PlatformIdentifier().IsWindows - ? new WindowsSymlinkFactory() - : new PosixSymlinkFactory()); + var linkFactory = (IFilesystemLinkFactory)(new PlatformIdentifier().IsWindows + ? new WindowsFilesystemLinkFactory() + : new PosixFilesystemLinkFactory()); var tempPath = Path.GetTempFileName(); File.Delete(tempPath); diff --git a/tests/Tgstation.Server.Host.Tests/System/TestSymlinkFactory.cs b/tests/Tgstation.Server.Host.Tests/System/TestSymlinkFactory.cs index 8e488f11b0..4d3c5be749 100644 --- a/tests/Tgstation.Server.Host.Tests/System/TestSymlinkFactory.cs +++ b/tests/Tgstation.Server.Host.Tests/System/TestSymlinkFactory.cs @@ -12,9 +12,9 @@ namespace Tgstation.Server.Host.System.Tests [TestClass] public sealed class TestSymlinkFactory { - readonly ISymlinkFactory factory = new PlatformIdentifier().IsWindows - ? new WindowsSymlinkFactory() - : new PosixSymlinkFactory(); + readonly IFilesystemLinkFactory factory = new PlatformIdentifier().IsWindows + ? new WindowsFilesystemLinkFactory() + : new PosixFilesystemLinkFactory(); [TestMethod] public async Task TestSymlinks() From 5bb655e3f6d3acc6963054f9517c8cd5257c522b Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 21 Oct 2023 18:48:14 -0400 Subject: [PATCH 19/40] Add test for hard links --- .../IO/TestFilesystemLinkFactory.cs | 57 ++++++++++++++++++- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/tests/Tgstation.Server.Host.Tests/IO/TestFilesystemLinkFactory.cs b/tests/Tgstation.Server.Host.Tests/IO/TestFilesystemLinkFactory.cs index e735c48541..dbf2aaf2a8 100644 --- a/tests/Tgstation.Server.Host.Tests/IO/TestFilesystemLinkFactory.cs +++ b/tests/Tgstation.Server.Host.Tests/IO/TestFilesystemLinkFactory.cs @@ -1,10 +1,12 @@ -using Microsoft.VisualStudio.TestTools.UnitTesting; -using System; +using System; using System.IO; using System.Runtime.InteropServices; using System.Security.Principal; +using System.Threading; using System.Threading.Tasks; +using Microsoft.VisualStudio.TestTools.UnitTesting; + namespace Tgstation.Server.Host.IO.Tests { [TestClass] @@ -30,6 +32,57 @@ namespace Tgstation.Server.Host.IO.Tests return principal.IsInRole(WindowsBuiltInRole.Administrator); } + [TestMethod] + public async Task TestHardLinkWorks() + { + string f2 = null; + var f1 = Path.GetTempFileName(); + try + { + f2 = f1 + ".linked"; + const string Text = "Hello world"; + File.WriteAllText(f1, Text); + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + await Assert.ThrowsExceptionAsync(() => linkFactory.CreateHardLink(f1, f2, CancellationToken.None)); + Assert.Inconclusive("Windows does not support hardlinks"); + } + + await Assert.ThrowsExceptionAsync(() => linkFactory.CreateHardLink(null, null, CancellationToken.None)); + await Assert.ThrowsExceptionAsync(() => linkFactory.CreateHardLink(f1, null, CancellationToken.None)); + + await linkFactory.CreateHardLink(f1, f2, default); + Assert.IsTrue(File.Exists(f2)); + + var f2Contents = File.ReadAllText(f2); + Assert.AreEqual(Text, f2Contents); + + const string NewText = "asdf"; + File.WriteAllText(f1, NewText); + + f2Contents = File.ReadAllText(f2); + Assert.AreEqual(NewText, f2Contents); + + const string NewText2 = "fdsa"; + File.WriteAllText(f2, NewText2); + + var f1Contents = File.ReadAllText(f1); + Assert.AreEqual(NewText2, f1Contents); + + File.Delete(f1); + Assert.IsFalse(File.Exists(f1)); + Assert.IsTrue(File.Exists(f2)); + f2Contents = File.ReadAllText(f2); + Assert.AreEqual(NewText2, f2Contents); + } + finally + { + File.Delete(f2); + File.Delete(f1); + } + } + [TestMethod] public async Task TestFileWorks() { From abdf8b6bfd620ebad7b973c8bb9b737dd7792338 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 21 Oct 2023 19:06:41 -0400 Subject: [PATCH 20/40] Fix build --- .../Live/Instance/ConfigurationTest.cs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/ConfigurationTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/ConfigurationTest.cs index 53064c9b7c..dac5b9b53c 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/ConfigurationTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/ConfigurationTest.cs @@ -93,7 +93,7 @@ namespace Tgstation.Server.Tests.Live.Instance await configurationClient.CreateDirectory(staticDir, cancellationToken); } - public Task SetupDMApiTests(bool includingRoot, CancellationToken cancellationToken) + public ValueTask SetupDMApiTests(bool includingRoot, CancellationToken cancellationToken) { // just use an I/O manager here var ioManager = new DefaultIOManager(); @@ -110,7 +110,7 @@ namespace Tgstation.Server.Tests.Live.Instance "../../../../DMAPI/LongRunning/long_running_test_rooted.dme", ioManager.ConcatPath(instance.Path, "Repository", "long_running_test_rooted.dme"), cancellationToken) - : Task.CompletedTask, + : ValueTask.CompletedTask, ioManager.CopyDirectory( Enumerable.Empty(), null, @@ -182,9 +182,8 @@ namespace Tgstation.Server.Tests.Live.Instance } public Task RunPreWatchdog(CancellationToken cancellationToken) => Task.WhenAll( - SetupDMApiTests(false, cancellationToken), + SetupDMApiTests(false, cancellationToken).AsTask(), SequencedApiTests(cancellationToken), - SetupDMApiTests(cancellationToken), TestPregeneratedFilesExist(cancellationToken)); } } From 53eafffbd07371fddcac177403d7e6d9dbf48ecc Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 21 Oct 2023 00:02:59 -0400 Subject: [PATCH 21/40] Add world.TgsVisibility() to DMAPI --- build/Version.props | 2 +- src/DMAPI/tgs.dm | 13 ++++++++++++- src/DMAPI/tgs/core/core.dm | 5 +++++ src/DMAPI/tgs/core/datum.dm | 3 +++ src/DMAPI/tgs/v5/_defines.dm | 1 + src/DMAPI/tgs/v5/api.dm | 6 ++++++ src/DMAPI/tgs/v5/undefs.dm | 1 + tests/DMAPI/LongRunning/Test.dm | 6 ++++++ 8 files changed, 35 insertions(+), 2 deletions(-) diff --git a/build/Version.props b/build/Version.props index 99cef97f0d..5022bac52e 100644 --- a/build/Version.props +++ b/build/Version.props @@ -9,7 +9,7 @@ 7.0.0 11.1.2 13.0.0 - 6.5.4 + 6.6.0 5.6.2 1.4.0 1.2.1 diff --git a/src/DMAPI/tgs.dm b/src/DMAPI/tgs.dm index d0466b806f..9825cd118b 100644 --- a/src/DMAPI/tgs.dm +++ b/src/DMAPI/tgs.dm @@ -1,6 +1,6 @@ // tgstation-server DMAPI -#define TGS_DMAPI_VERSION "6.5.4" +#define TGS_DMAPI_VERSION "6.6.0" // All functions and datums outside this document are subject to change with any version and should not be relied on. @@ -129,6 +129,13 @@ /// DreamDaemon Ultrasafe security level. #define TGS_SECURITY_ULTRASAFE 2 +/// DreamDaemon public visibility level. +#define TGS_VISIBILITY_PUBLIC 0 +/// DreamDaemon private visibility level. +#define TGS_VISIBILITY_PRIVATE 1 +/// DreamDaemon invisible visibility level. +#define TGS_VISIBILITY_INVISIBLE 2 + //REQUIRED HOOKS /** @@ -458,6 +465,10 @@ /world/proc/TgsSecurityLevel() return +/// Returns the current BYOND visibility level as a TGS_VISIBILITY_ define if TGS is present, null otherwise. Requires TGS to be using interop API version 5 or higher otherwise the string "___unimplemented" wil be returned. This function may sleep if the call to [/world/proc/TgsNew] is sleeping! +/world/proc/TgsVisibility() + return + /// Returns a list of active [/datum/tgs_revision_information/test_merge]s if TGS is present, null otherwise. This function may sleep if the call to [/world/proc/TgsNew] is sleeping! /world/proc/TgsTestMerges() return diff --git a/src/DMAPI/tgs/core/core.dm b/src/DMAPI/tgs/core/core.dm index aa4084904b..b9a9f27a28 100644 --- a/src/DMAPI/tgs/core/core.dm +++ b/src/DMAPI/tgs/core/core.dm @@ -154,3 +154,8 @@ var/datum/tgs_api/api = TGS_READ_GLOBAL(tgs) if(api) return api.SecurityLevel() + +/world/TgsVisibility() + var/datum/tgs_api/api = TGS_READ_GLOBAL(tgs) + if(api) + return api.Visibility() diff --git a/src/DMAPI/tgs/core/datum.dm b/src/DMAPI/tgs/core/datum.dm index 68b0330fe8..93377079aa 100644 --- a/src/DMAPI/tgs/core/datum.dm +++ b/src/DMAPI/tgs/core/datum.dm @@ -57,3 +57,6 @@ TGS_PROTECT_DATUM(/datum/tgs_api) /datum/tgs_api/proc/SecurityLevel() return TGS_UNIMPLEMENTED + +/datum/tgs_api/proc/Visibility() + return TGS_UNIMPLEMENTED diff --git a/src/DMAPI/tgs/v5/_defines.dm b/src/DMAPI/tgs/v5/_defines.dm index f973338daa..bdcd4e4dd5 100644 --- a/src/DMAPI/tgs/v5/_defines.dm +++ b/src/DMAPI/tgs/v5/_defines.dm @@ -48,6 +48,7 @@ #define DMAPI5_RUNTIME_INFORMATION_REVISION "revision" #define DMAPI5_RUNTIME_INFORMATION_TEST_MERGES "testMerges" #define DMAPI5_RUNTIME_INFORMATION_SECURITY_LEVEL "securityLevel" +#define DMAPI5_RUNTIME_INFORMATION_VISIBILITY "visibility" #define DMAPI5_CHAT_UPDATE_CHANNELS "channels" diff --git a/src/DMAPI/tgs/v5/api.dm b/src/DMAPI/tgs/v5/api.dm index 34cc43f876..45250efc46 100644 --- a/src/DMAPI/tgs/v5/api.dm +++ b/src/DMAPI/tgs/v5/api.dm @@ -4,6 +4,7 @@ var/instance_name var/security_level + var/visibility var/reboot_mode = TGS_REBOOT_MODE_NORMAL @@ -54,6 +55,7 @@ version = new /datum/tgs_version(runtime_information[DMAPI5_RUNTIME_INFORMATION_SERVER_VERSION]) security_level = runtime_information[DMAPI5_RUNTIME_INFORMATION_SECURITY_LEVEL] + visibility = runtime_information[DMAPI5_RUNTIME_INFORMATION_VISIBILITY] instance_name = runtime_information[DMAPI5_RUNTIME_INFORMATION_INSTANCE_NAME] var/list/revisionData = runtime_information[DMAPI5_RUNTIME_INFORMATION_REVISION] @@ -252,3 +254,7 @@ /datum/tgs_api/v5/SecurityLevel() RequireInitialBridgeResponse() return security_level + +/datum/tgs_api/v5/Visibility() + RequireInitialBridgeResponse() + return visibility diff --git a/src/DMAPI/tgs/v5/undefs.dm b/src/DMAPI/tgs/v5/undefs.dm index c679737dfc..f163adaaaf 100644 --- a/src/DMAPI/tgs/v5/undefs.dm +++ b/src/DMAPI/tgs/v5/undefs.dm @@ -48,6 +48,7 @@ #undef DMAPI5_RUNTIME_INFORMATION_REVISION #undef DMAPI5_RUNTIME_INFORMATION_TEST_MERGES #undef DMAPI5_RUNTIME_INFORMATION_SECURITY_LEVEL +#undef DMAPI5_RUNTIME_INFORMATION_VISIBILITY #undef DMAPI5_CHAT_UPDATE_CHANNELS diff --git a/tests/DMAPI/LongRunning/Test.dm b/tests/DMAPI/LongRunning/Test.dm index a77c3377b5..68597e10d0 100644 --- a/tests/DMAPI/LongRunning/Test.dm +++ b/tests/DMAPI/LongRunning/Test.dm @@ -17,6 +17,12 @@ log << "Running in security level: [sec]" + var/vis = TgsVisibility() + if(isnull(vis)) + FailTest("TGS Visibility was null!") + + log << "Running in visibility: [vis]" + if(params["expect_chat_channels"]) var/list/channels = TgsChatChannelInfo() if(!length(channels)) From cc1c955bc8f2b037f257a3eeec977683533e53c5 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 21 Oct 2023 19:16:47 -0400 Subject: [PATCH 22/40] We call it the Advanced Watchdog now --- .github/workflows/ci-pipeline.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 6ce724ea5d..766e9660b4 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -365,7 +365,7 @@ jobs: fail-fast: false matrix: database-type: [ 'SqlServer', 'Sqlite', 'PostgresSql', 'MariaDB', 'MySql' ] - watchdog-type: [ 'Basic', 'System' ] + watchdog-type: [ 'Basic', 'Advanced' ] configuration: [ 'Debug', 'Release' ] runs-on: windows-latest steps: @@ -735,25 +735,25 @@ jobs: name: linux-unit-test-coverage-Release path: ./code_coverage/unit_tests/linux_unit_tests_release - - name: Retrieve Linux Integration Test Coverage (Release, System, Sqlite) + - name: Retrieve Linux Integration Test Coverage (Release, Advanced, Sqlite) uses: actions/download-artifact@v3 with: name: linux-integration-test-coverage-Release-System-Sqlite path: ./code_coverage/integration_tests/linux_integration_tests_release_system_sqlite - - name: Retrieve Linux Integration Test Coverage (Release, System, PostgresSql) + - name: Retrieve Linux Integration Test Coverage (Release, Advanced, PostgresSql) uses: actions/download-artifact@v3 with: name: linux-integration-test-coverage-Release-System-PostgresSql path: ./code_coverage/integration_tests/linux_integration_tests_release_system_mariadb - - name: Retrieve Linux Integration Test Coverage (Release, System, MariaDB) + - name: Retrieve Linux Integration Test Coverage (Release, Advanced, MariaDB) uses: actions/download-artifact@v3 with: name: linux-integration-test-coverage-Release-System-MariaDB path: ./code_coverage/integration_tests/linux_integration_tests_release_system_mysql - - name: Retrieve Linux Integration Test Coverage (Release, System, MySql) + - name: Retrieve Linux Integration Test Coverage (Release, Advanced, MySql) uses: actions/download-artifact@v3 with: name: linux-integration-test-coverage-Release-System-MySql @@ -789,13 +789,13 @@ jobs: name: linux-integration-test-coverage-Debug-System-Sqlite path: ./code_coverage/integration_tests/linux_integration_tests_debug_system_sqlite - - name: Retrieve Linux Integration Test Coverage (Debug, System, PostgresSql) + - name: Retrieve Linux Integration Test Coverage (Debug, Advanced, PostgresSql) uses: actions/download-artifact@v3 with: name: linux-integration-test-coverage-Debug-System-PostgresSql path: ./code_coverage/integration_tests/linux_integration_tests_debug_system_mariadb - - name: Retrieve Linux Integration Test Coverage (Debug, System, MariaDB) + - name: Retrieve Linux Integration Test Coverage (Debug, Advanced, MariaDB) uses: actions/download-artifact@v3 with: name: linux-integration-test-coverage-Debug-System-MariaDB From 71f5c739d8ba9d68ef691935d3b2e18b2755761f Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 21 Oct 2023 19:21:11 -0400 Subject: [PATCH 23/40] Fix tests using `BasicWatchdog` failing --- .../Live/Instance/InstanceTest.cs | 6 ++++-- .../Live/Instance/WatchdogTest.cs | 12 +++++++----- .../Tgstation.Server.Tests/Live/LiveTestingServer.cs | 4 ++++ tests/Tgstation.Server.Tests/Live/TestLiveServer.cs | 6 ++++-- 4 files changed, 19 insertions(+), 9 deletions(-) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs index 4457994d79..2940500e20 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs @@ -43,6 +43,7 @@ namespace Tgstation.Server.Tests.Live.Instance ushort ddPort, bool highPrioDD, bool lowPrioDeployment, + bool usingBasicWatchdog, CancellationToken cancellationToken) { var byondTest = new ByondTest(instanceClient.Byond, instanceClient.Jobs, fileDownloader, instanceClient.Metadata); @@ -67,7 +68,7 @@ namespace Tgstation.Server.Tests.Live.Instance await byondTask; await new WatchdogTest( - await ByondTest.GetEdgeVersion(fileDownloader, cancellationToken), instanceClient, instanceManager, serverPort, highPrioDD, ddPort).Run(cancellationToken); + await ByondTest.GetEdgeVersion(fileDownloader, cancellationToken), instanceClient, instanceManager, serverPort, highPrioDD, ddPort, usingBasicWatchdog).Run(cancellationToken); } public async Task RunCompatTests( @@ -76,6 +77,7 @@ namespace Tgstation.Server.Tests.Live.Instance ushort dmPort, ushort ddPort, bool highPrioDD, + bool usingBasicWatchdog, CancellationToken cancellationToken) { System.Console.WriteLine($"COMPAT TEST START: {compatVersion}"); @@ -189,7 +191,7 @@ namespace Tgstation.Server.Tests.Live.Instance await configSetupTask; - await new WatchdogTest(compatVersion, instanceClient, instanceManager, serverPort, highPrioDD, ddPort).Run(cancellationToken); + await new WatchdogTest(compatVersion, instanceClient, instanceManager, serverPort, highPrioDD, ddPort, usingBasicWatchdog).Run(cancellationToken); await instanceManagerClient.Update(new InstanceUpdateRequest { diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index 96ef4d3c20..b59e9532fe 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -63,10 +63,11 @@ namespace Tgstation.Server.Tests.Live.Instance readonly bool highPrioDD; readonly TopicClient topicClient; readonly Version testVersion; + readonly bool usingBasicWatchdog; bool ranTimeoutTest = false; - public WatchdogTest(Version testVersion, IInstanceClient instanceClient, InstanceManager instanceManager, ushort serverPort, bool highPrioDD, ushort ddPort) + public WatchdogTest(Version testVersion, IInstanceClient instanceClient, InstanceManager instanceManager, ushort serverPort, bool highPrioDD, ushort ddPort, bool usingBasicWatchdog) : base(instanceClient.Jobs) { this.instanceClient = instanceClient ?? throw new ArgumentNullException(nameof(instanceClient)); @@ -75,8 +76,9 @@ namespace Tgstation.Server.Tests.Live.Instance this.highPrioDD = highPrioDD; this.ddPort = ddPort; this.testVersion = testVersion ?? throw new ArgumentNullException(nameof(testVersion)); + this.usingBasicWatchdog = usingBasicWatchdog; - this.topicClient = new(new SocketParameters + topicClient = new(new SocketParameters { SendTimeout = TimeSpan.FromSeconds(30), ReceiveTimeout = TimeSpan.FromSeconds(30), @@ -476,9 +478,9 @@ namespace Tgstation.Server.Tests.Live.Instance Assert.AreEqual(string.Empty, daemonStatus.AdditionalParameters); } - void TestLinuxIsntBeingFuckingCheekyAboutFilePaths(DreamDaemonResponse currentStatus, CompileJobResponse previousStatus, CancellationToken cancellationToken) + void TestLinuxIsntBeingFuckingCheekyAboutFilePaths(DreamDaemonResponse currentStatus, CompileJobResponse previousStatus) { - if (new PlatformIdentifier().IsWindows) + if (new PlatformIdentifier().IsWindows || usingBasicWatchdog) return; Assert.IsNotNull(currentStatus.ActiveCompileJob); @@ -947,7 +949,7 @@ namespace Tgstation.Server.Tests.Live.Instance Assert.AreNotEqual(initialCompileJob.Id, daemonStatus.ActiveCompileJob.Id); Assert.IsNull(daemonStatus.StagedCompileJob); - TestLinuxIsntBeingFuckingCheekyAboutFilePaths(daemonStatus, initialCompileJob, cancellationToken); + TestLinuxIsntBeingFuckingCheekyAboutFilePaths(daemonStatus, initialCompileJob); await instanceClient.DreamDaemon.Shutdown(cancellationToken); diff --git a/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs b/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs index 56263c1717..33213f6f7c 100644 --- a/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs +++ b/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs @@ -63,6 +63,8 @@ namespace Tgstation.Server.Tests.Live public bool HighPriorityDreamDaemon { get; } public bool LowPriorityDeployments { get; } + public bool UsingBasicWatchdog { get; } + public bool RestartRequested => RealServer.RestartRequested; readonly List args; @@ -89,6 +91,8 @@ namespace Tgstation.Server.Tests.Live var gitHubAccessToken = Environment.GetEnvironmentVariable("TGS_TEST_GITHUB_TOKEN"); var dumpOpenAPISpecPathEnvVar = Environment.GetEnvironmentVariable("TGS_TEST_DUMP_API_SPEC"); + UsingBasicWatchdog = Boolean.TryParse(Environment.GetEnvironmentVariable("General__UseBasicWatchdog"), out var result) && result; + if (String.IsNullOrEmpty(DatabaseType)) Assert.Inconclusive("No database type configured in env var TGS_TEST_DATABASE_TYPE!"); diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index a78b984fc2..21ab8be844 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -1336,6 +1336,7 @@ namespace Tgstation.Server.Tests.Live compatDMPort, compatDDPort, server.HighPriorityDreamDaemon, + server.UsingBasicWatchdog, cancellationToken)); if (TestingUtils.RunningInGitHubActions) // they only have 2 cores, can't handle intense parallelization @@ -1349,6 +1350,7 @@ namespace Tgstation.Server.Tests.Live mainDDPort, server.HighPriorityDreamDaemon, server.LowPriorityDeployments, + server.UsingBasicWatchdog, cancellationToken)); await compatTests; @@ -1529,7 +1531,7 @@ namespace Tgstation.Server.Tests.Live Assert.AreEqual(WatchdogStatus.Online, dd.Status.Value); var compileJob = await instanceClient.DreamMaker.Compile(cancellationToken); - var wdt = new WatchdogTest(edgeByond, instanceClient, GetInstanceManager(), (ushort)server.Url.Port, server.HighPriorityDreamDaemon, mainDDPort); + var wdt = new WatchdogTest(edgeByond, instanceClient, GetInstanceManager(), (ushort)server.Url.Port, server.HighPriorityDreamDaemon, mainDDPort, server.UsingBasicWatchdog); await wdt.WaitForJob(compileJob, 30, false, null, cancellationToken); dd = await instanceClient.DreamDaemon.Read(cancellationToken); @@ -1576,7 +1578,7 @@ namespace Tgstation.Server.Tests.Live Assert.AreEqual(WatchdogStatus.Online, currentDD.Status); Assert.AreEqual(expectedStaged, currentDD.StagedCompileJob.Job.Id.Value); - var wdt = new WatchdogTest(edgeByond, instanceClient, GetInstanceManager(), (ushort)server.Url.Port, server.HighPriorityDreamDaemon, mainDDPort); + var wdt = new WatchdogTest(edgeByond, instanceClient, GetInstanceManager(), (ushort)server.Url.Port, server.HighPriorityDreamDaemon, mainDDPort, server.UsingBasicWatchdog); currentDD = await wdt.TellWorldToReboot(cancellationToken); Assert.AreEqual(expectedStaged, currentDD.ActiveCompileJob.Job.Id.Value); Assert.IsNull(currentDD.StagedCompileJob); From b22368086becd526e59da65aa6dc71a65d3f8d6a Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 21 Oct 2023 20:40:04 -0400 Subject: [PATCH 24/40] Additional test logging --- tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index b59e9532fe..784fce5f67 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -502,7 +502,7 @@ namespace Tgstation.Server.Tests.Live.Instance var path = sb.ToString(); - allPaths.Add(path); + allPaths.Add($"Path: {path}"); if (path.Contains($"Game/{previousStatus.DirectoryName}")) failingLinks.Add($"Found fd {fd} resolving to previous absolute path game dir path: {path}"); @@ -516,7 +516,7 @@ namespace Tgstation.Server.Tests.Live.Instance if (!foundLivePath) failingLinks.Add($"Failed to find a path containing the 'Live' directory!"); - Assert.IsTrue(failingLinks.Count == 0, String.Join(Environment.NewLine, failingLinks)); + Assert.IsTrue(failingLinks.Count == 0, String.Join(Environment.NewLine, failingLinks.Concat(allPaths))); } async Task RunHealthCheckTest(bool checkDump, CancellationToken cancellationToken) From b53b29189b86081b714e2557e2bf298285a10fef Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 21 Oct 2023 23:17:00 -0400 Subject: [PATCH 25/40] Fix naming of CI matrix steps --- .github/workflows/ci-pipeline.yml | 90 +++++++++++++++---------------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 766e9660b4..8e75c03afb 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -382,7 +382,7 @@ jobs: dotnet-version: ${{ env.TGS_DOTNET_VERSION }} - name: Set TGS_TEST_DUMP_API_SPEC - if: ${{ matrix.configuration == 'Release' && matrix.watchdog-type == 'System' && matrix.database-type == 'SqlServer' }} + if: ${{ matrix.configuration == 'Release' && matrix.watchdog-type == 'Advanced' && matrix.database-type == 'SqlServer' }} run: echo "TGS_TEST_DUMP_API_SPEC=yes" >> $Env:GITHUB_ENV - name: Set General__UseBasicWatchdog @@ -473,7 +473,7 @@ jobs: path: ./TestResults/ - name: Store OpenAPI Spec - if: ${{ matrix.configuration == 'Release' && matrix.watchdog-type == 'System' && matrix.database-type == 'SqlServer' }} + if: ${{ matrix.configuration == 'Release' && matrix.watchdog-type == 'Advanced' && matrix.database-type == 'SqlServer' }} uses: actions/upload-artifact@v3 with: name: openapi-spec @@ -562,7 +562,7 @@ jobs: fail-fast: false matrix: database-type: [ 'Sqlite', 'PostgresSql', 'MariaDB', 'MySql' ] - watchdog-type: [ 'Basic', 'System' ] + watchdog-type: [ 'Basic', 'Advanced' ] configuration: [ 'Debug', 'Release' ] runs-on: ubuntu-latest steps: @@ -645,7 +645,7 @@ jobs: path: ./TestResults/ - name: Package Server Console - if: ${{ matrix.configuration == 'Release' && matrix.watchdog-type == 'System' && matrix.database-type == 'MariaDB' }} + if: ${{ matrix.configuration == 'Release' && matrix.watchdog-type == 'Advanced' && matrix.database-type == 'MariaDB' }} run: | cd src/Tgstation.Server.Host.Console dotnet publish -c ${{ matrix.configuration }} -o ../../artifacts/Console @@ -657,7 +657,7 @@ jobs: build/RemoveUnsupportedRuntimes.sh artifacts/Console - name: Package Server Update Package - if: ${{ matrix.configuration == 'Release' && matrix.watchdog-type == 'System' && matrix.database-type == 'PostgresSql' }} + if: ${{ matrix.configuration == 'Release' && matrix.watchdog-type == 'Advanced' && matrix.database-type == 'PostgresSql' }} run: | cd src/Tgstation.Server.Host dotnet publish -c ${{ matrix.configuration }}NoWindows --no-build -o ../../artifacts/ServerUpdate @@ -666,14 +666,14 @@ jobs: build/RemoveUnsupportedRuntimes.sh artifacts/ServerUpdate - name: Store Server Console - if: ${{ matrix.configuration == 'Release' && matrix.watchdog-type == 'System' && matrix.database-type == 'MariaDB' }} + if: ${{ matrix.configuration == 'Release' && matrix.watchdog-type == 'Advanced' && matrix.database-type == 'MariaDB' }} uses: actions/upload-artifact@v3 with: name: ServerConsole path: artifacts/Console/ - name: Store Server Update Package - if: ${{ matrix.configuration == 'Release' && matrix.watchdog-type == 'System' && matrix.database-type == 'PostgresSql' }} + if: ${{ matrix.configuration == 'Release' && matrix.watchdog-type == 'Advanced' && matrix.database-type == 'PostgresSql' }} uses: actions/upload-artifact@v3 with: name: ServerUpdatePackage @@ -738,97 +738,97 @@ jobs: - name: Retrieve Linux Integration Test Coverage (Release, Advanced, Sqlite) uses: actions/download-artifact@v3 with: - name: linux-integration-test-coverage-Release-System-Sqlite + name: linux-integration-test-coverage-Release-Advanced-Sqlite path: ./code_coverage/integration_tests/linux_integration_tests_release_system_sqlite - name: Retrieve Linux Integration Test Coverage (Release, Advanced, PostgresSql) uses: actions/download-artifact@v3 with: - name: linux-integration-test-coverage-Release-System-PostgresSql + name: linux-integration-test-coverage-Release-Advanced-PostgresSql path: ./code_coverage/integration_tests/linux_integration_tests_release_system_mariadb - name: Retrieve Linux Integration Test Coverage (Release, Advanced, MariaDB) uses: actions/download-artifact@v3 with: - name: linux-integration-test-coverage-Release-System-MariaDB + name: linux-integration-test-coverage-Release-Advanced-MariaDB path: ./code_coverage/integration_tests/linux_integration_tests_release_system_mysql - name: Retrieve Linux Integration Test Coverage (Release, Advanced, MySql) uses: actions/download-artifact@v3 with: - name: linux-integration-test-coverage-Release-System-MySql + name: linux-integration-test-coverage-Release-Advanced-MySql path: ./code_coverage/integration_tests/linux_integration_tests_release_system_mysql - name: Retrieve Linux Integration Test Coverage (Release, Basic, Sqlite) uses: actions/download-artifact@v3 with: - name: linux-integration-test-coverage-Release-System-Sqlite + name: linux-integration-test-coverage-Release-Advanced-Sqlite path: ./code_coverage/integration_tests/linux_integration_tests_release_basic_sqlite - name: Retrieve Linux Integration Test Coverage (Release, Basic, PostgresSql) uses: actions/download-artifact@v3 with: - name: linux-integration-test-coverage-Release-System-PostgresSql + name: linux-integration-test-coverage-Release-Advanced-PostgresSql path: ./code_coverage/integration_tests/linux_integration_tests_release_basic_mariadb - name: Retrieve Linux Integration Test Coverage (Release, Basic, MariaDB) uses: actions/download-artifact@v3 with: - name: linux-integration-test-coverage-Release-System-MariaDB + name: linux-integration-test-coverage-Release-Advanced-MariaDB path: ./code_coverage/integration_tests/linux_integration_tests_release_basic_mysql - name: Retrieve Linux Integration Test Coverage (Release, Basic, MySql) uses: actions/download-artifact@v3 with: - name: linux-integration-test-coverage-Release-System-MySql + name: linux-integration-test-coverage-Release-Advanced-MySql path: ./code_coverage/integration_tests/linux_integration_tests_release_basic_mysql - - name: Retrieve Linux Integration Test Coverage (Debug, System, Sqlite) + - name: Retrieve Linux Integration Test Coverage (Debug, Advanced, Sqlite) uses: actions/download-artifact@v3 with: - name: linux-integration-test-coverage-Debug-System-Sqlite + name: linux-integration-test-coverage-Debug-Advanced-Sqlite path: ./code_coverage/integration_tests/linux_integration_tests_debug_system_sqlite - name: Retrieve Linux Integration Test Coverage (Debug, Advanced, PostgresSql) uses: actions/download-artifact@v3 with: - name: linux-integration-test-coverage-Debug-System-PostgresSql + name: linux-integration-test-coverage-Debug-Advanced-PostgresSql path: ./code_coverage/integration_tests/linux_integration_tests_debug_system_mariadb - name: Retrieve Linux Integration Test Coverage (Debug, Advanced, MariaDB) uses: actions/download-artifact@v3 with: - name: linux-integration-test-coverage-Debug-System-MariaDB + name: linux-integration-test-coverage-Debug-Advanced-MariaDB path: ./code_coverage/integration_tests/linux_integration_tests_debug_system_mysql - - name: Retrieve Linux Integration Test Coverage (Debug, System, MySql) + - name: Retrieve Linux Integration Test Coverage (Debug, Advanced, MySql) uses: actions/download-artifact@v3 with: - name: linux-integration-test-coverage-Debug-System-MySql + name: linux-integration-test-coverage-Debug-Advanced-MySql path: ./code_coverage/integration_tests/linux_integration_tests_debug_system_mysql - name: Retrieve Linux Integration Test Coverage (Debug, Basic, Sqlite) uses: actions/download-artifact@v3 with: - name: linux-integration-test-coverage-Debug-System-Sqlite + name: linux-integration-test-coverage-Debug-Advanced-Sqlite path: ./code_coverage/integration_tests/linux_integration_tests_debug_basic_sqlite - name: Retrieve Linux Integration Test Coverage (Debug, Basic, PostgresSql) uses: actions/download-artifact@v3 with: - name: linux-integration-test-coverage-Debug-System-PostgresSql + name: linux-integration-test-coverage-Debug-Advanced-PostgresSql path: ./code_coverage/integration_tests/linux_integration_tests_debug_basic_mariadb - name: Retrieve Linux Integration Test Coverage (Debug, Basic, MariaDB) uses: actions/download-artifact@v3 with: - name: linux-integration-test-coverage-Debug-System-MariaDB + name: linux-integration-test-coverage-Debug-Advanced-MariaDB path: ./code_coverage/integration_tests/linux_integration_tests_debug_basic_mysql - name: Retrieve Linux Integration Test Coverage (Debug, Basic, MySql) uses: actions/download-artifact@v3 with: - name: linux-integration-test-coverage-Debug-System-MySql + name: linux-integration-test-coverage-Debug-Advanced-MySql path: ./code_coverage/integration_tests/linux_integration_tests_debug_basic_mysql - name: Retrieve Windows Unit Test Coverage (Release) @@ -849,16 +849,16 @@ jobs: name: windows-integration-test-coverage-Release-Basic-SqlServer path: ./code_coverage/integration_tests/windows_integration_tests_release_basic_sqlserver - - name: Retrieve Windows Integration Test Coverage (Debug, System, SqlServer) + - name: Retrieve Windows Integration Test Coverage (Debug, Advanced, SqlServer) uses: actions/download-artifact@v3 with: - name: windows-integration-test-coverage-Debug-System-SqlServer + name: windows-integration-test-coverage-Debug-Advanced-SqlServer path: ./code_coverage/integration_tests/windows_integration_tests_debug_system_sqlserver - - name: Retrieve Windows Integration Test Coverage (Release, System, SqlServer) + - name: Retrieve Windows Integration Test Coverage (Release, Advanced, SqlServer) uses: actions/download-artifact@v3 with: - name: windows-integration-test-coverage-Release-System-SqlServer + name: windows-integration-test-coverage-Release-Advanced-SqlServer path: ./code_coverage/integration_tests/windows_integration_tests_release_system_sqlserver - name: Retrieve Windows Integration Test Coverage (Debug, Basic, MariaDB) @@ -873,16 +873,16 @@ jobs: name: windows-integration-test-coverage-Release-Basic-MariaDB path: ./code_coverage/integration_tests/windows_integration_tests_release_basic_mariadb - - name: Retrieve Windows Integration Test Coverage (Debug, System, MariaDB) + - name: Retrieve Windows Integration Test Coverage (Debug, Advanced, MariaDB) uses: actions/download-artifact@v3 with: - name: windows-integration-test-coverage-Debug-System-MariaDB + name: windows-integration-test-coverage-Debug-Advanced-MariaDB path: ./code_coverage/integration_tests/windows_integration_tests_debug_system_mariadb - - name: Retrieve Windows Integration Test Coverage (Release, System, MariaDB) + - name: Retrieve Windows Integration Test Coverage (Release, Advanced, MariaDB) uses: actions/download-artifact@v3 with: - name: windows-integration-test-coverage-Release-System-MariaDB + name: windows-integration-test-coverage-Release-Advanced-MariaDB path: ./code_coverage/integration_tests/windows_integration_tests_release_system_mariadb - name: Retrieve Windows Integration Test Coverage (Debug, Basic, MySql) @@ -897,16 +897,16 @@ jobs: name: windows-integration-test-coverage-Release-Basic-MySql path: ./code_coverage/integration_tests/windows_integration_tests_release_basic_mysql - - name: Retrieve Windows Integration Test Coverage (Debug, System, MySql) + - name: Retrieve Windows Integration Test Coverage (Debug, Advanced, MySql) uses: actions/download-artifact@v3 with: - name: windows-integration-test-coverage-Debug-System-MySql + name: windows-integration-test-coverage-Debug-Advanced-MySql path: ./code_coverage/integration_tests/windows_integration_tests_debug_system_mysql - - name: Retrieve Windows Integration Test Coverage (Release, System, MySql) + - name: Retrieve Windows Integration Test Coverage (Release, Advanced, MySql) uses: actions/download-artifact@v3 with: - name: windows-integration-test-coverage-Release-System-MySql + name: windows-integration-test-coverage-Release-Advanced-MySql path: ./code_coverage/integration_tests/windows_integration_tests_release_system_mysql - name: Retrieve Windows Integration Test Coverage (Debug, Basic, PostgresSql) @@ -921,16 +921,16 @@ jobs: name: windows-integration-test-coverage-Release-Basic-PostgresSql path: ./code_coverage/integration_tests/windows_integration_tests_release_basic_postgressql - - name: Retrieve Windows Integration Test Coverage (Debug, System, PostgresSql) + - name: Retrieve Windows Integration Test Coverage (Debug, Advanced, PostgresSql) uses: actions/download-artifact@v3 with: - name: windows-integration-test-coverage-Debug-System-PostgresSql + name: windows-integration-test-coverage-Debug-Advanced-PostgresSql path: ./code_coverage/integration_tests/windows_integration_tests_debug_system_postgressql - - name: Retrieve Windows Integration Test Coverage (Release, System, PostgresSql) + - name: Retrieve Windows Integration Test Coverage (Release, Advanced, PostgresSql) uses: actions/download-artifact@v3 with: - name: windows-integration-test-coverage-Release-System-PostgresSql + name: windows-integration-test-coverage-Release-Advanced-PostgresSql path: ./code_coverage/integration_tests/windows_integration_tests_release_system_postgressql - name: Retrieve Windows Integration Test Coverage (Debug, Basic, Sqlite) @@ -945,16 +945,16 @@ jobs: name: windows-integration-test-coverage-Release-Basic-Sqlite path: ./code_coverage/integration_tests/windows_integration_tests_release_basic_sqlite - - name: Retrieve Windows Integration Test Coverage (Debug, System, Sqlite) + - name: Retrieve Windows Integration Test Coverage (Debug, Advanced, Sqlite) uses: actions/download-artifact@v3 with: - name: windows-integration-test-coverage-Debug-System-Sqlite + name: windows-integration-test-coverage-Debug-Advanced-Sqlite path: ./code_coverage/integration_tests/windows_integration_tests_debug_system_sqlite - - name: Retrieve Windows Integration Test Coverage (Release, System, Sqlite) + - name: Retrieve Windows Integration Test Coverage (Release, Advanced, Sqlite) uses: actions/download-artifact@v3 with: - name: windows-integration-test-coverage-Release-System-Sqlite + name: windows-integration-test-coverage-Release-Advanced-Sqlite path: ./code_coverage/integration_tests/windows_integration_tests_release_system_sqlite - name: Upload Coverage to CodeCov From cfca47d999b1090dbc57fd418197075ae94759d8 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 21 Oct 2023 23:20:08 -0400 Subject: [PATCH 26/40] Version bump to 5.16.3 --- build/Version.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/Version.props b/build/Version.props index 4a4c556f24..c9c9e39ca0 100644 --- a/build/Version.props +++ b/build/Version.props @@ -3,7 +3,7 @@ - 5.16.2 + 5.16.3 4.7.1 9.12.0 6.0.1 From 103adf13d1c580256414e06b415cea1a100e1d1b Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 22 Oct 2023 09:12:28 -0400 Subject: [PATCH 27/40] Increase delay before running `--link-winget` Prevents PR content from being overwritten by the MS bot --- .github/workflows/ci-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 8e75c03afb..02945d00e4 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -1771,5 +1771,5 @@ jobs: - name: Run ReleaseNotes with --link-winget shell: powershell run: | - Sleep 15 + Sleep 600 dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --link-winget ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} From bf23f03b9118a1eb895e017ac3f9cc072f74f0e9 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 22 Oct 2023 09:17:34 -0400 Subject: [PATCH 28/40] Fix DMAPI post validate timeout applying to all sessions --- .../Components/Session/SessionController.cs | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Session/SessionController.cs b/src/Tgstation.Server.Host/Components/Session/SessionController.cs index 4672b63757..8e8a4715ec 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionController.cs @@ -167,6 +167,11 @@ namespace Tgstation.Server.Host.Components.Session /// readonly object synchronizationLock; + /// + /// If this session is meant to validate the presence of the DMAPI. + /// + readonly bool apiValidationSession; + /// /// The waits on when DreamDaemon currently has it's ports closed. /// @@ -244,7 +249,7 @@ namespace Tgstation.Server.Host.Components.Session /// The returning a to be run after the ends. /// The optional time to wait before failing the . /// If this is a reattached session. - /// If this is a DMAPI validation session. + /// The value of . public SessionController( ReattachInformation reattachInformation, Api.Models.Instance metadata, @@ -276,6 +281,8 @@ namespace Tgstation.Server.Host.Components.Session this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer)); + apiValidationSession = apiValidate; + portClosedForReboot = false; disposed = false; apiValidationStatus = ApiValidationStatus.NeverValidated; @@ -296,7 +303,7 @@ namespace Tgstation.Server.Host.Components.Session topicSendSemaphore = new FifoSemaphore(); synchronizationLock = new object(); - if (apiValidate || DMApiAvailable) + if (apiValidationSession || DMApiAvailable) { bridgeRegistration = bridgeRegistrar.RegisterHandler(this); this.chatTrackingContext.SetChannelSink(this); @@ -775,9 +782,16 @@ namespace Tgstation.Server.Host.Components.Session break; case BridgeCommandType.Startup: - var proceedTcs = new TaskCompletionSource(); - var firstValidationRequest = Interlocked.CompareExchange(ref postValidationShutdownTask, PostValidationShutdown(proceedTcs.Task), null) == null; - proceedTcs.SetResult(firstValidationRequest); + bool firstValidationRequest; + if (apiValidationSession) + { + var proceedTcs = new TaskCompletionSource(); + firstValidationRequest = Interlocked.CompareExchange(ref postValidationShutdownTask, PostValidationShutdown(proceedTcs.Task), null) == null; + proceedTcs.SetResult(firstValidationRequest); + } + else + firstValidationRequest = Interlocked.CompareExchange(ref postValidationShutdownTask, Task.CompletedTask, null) == null; + apiValidationStatus = ApiValidationStatus.BadValidationRequest; if (!firstValidationRequest) From 812d6a619cf7b75ed3dea2079883265efb949a42 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 22 Oct 2023 10:01:03 -0400 Subject: [PATCH 29/40] Remove the interactive test It's annoying to monitor --- tests/Tgstation.Server.Tests/Live/TestLiveServer.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index 21ab8be844..7ae5e35c6d 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -955,9 +955,6 @@ namespace Tgstation.Server.Tests.Live new LiveTestingServer(null, false).Dispose(); } - [TestMethod] - public async Task TestTgstationInteractive() => await TestTgstation(true); - [TestMethod] public async Task TestTgstationHeadless() => await TestTgstation(false); From c820296b4ca0e994edd7266118a35497d90cb9c5 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 22 Oct 2023 10:19:52 -0400 Subject: [PATCH 30/40] Fix initial bridge requests not expecting reboots --- .../Components/Session/SessionController.cs | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Session/SessionController.cs b/src/Tgstation.Server.Host/Components/Session/SessionController.cs index 8e8a4715ec..42054f24fb 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionController.cs @@ -782,20 +782,16 @@ namespace Tgstation.Server.Host.Components.Session break; case BridgeCommandType.Startup: - bool firstValidationRequest; + apiValidationStatus = ApiValidationStatus.BadValidationRequest; if (apiValidationSession) { var proceedTcs = new TaskCompletionSource(); - firstValidationRequest = Interlocked.CompareExchange(ref postValidationShutdownTask, PostValidationShutdown(proceedTcs.Task), null) == null; + var firstValidationRequest = Interlocked.CompareExchange(ref postValidationShutdownTask, PostValidationShutdown(proceedTcs.Task), null) == null; proceedTcs.SetResult(firstValidationRequest); + + if (!firstValidationRequest) + return BridgeError("Startup bridge request was repeated!"); } - else - firstValidationRequest = Interlocked.CompareExchange(ref postValidationShutdownTask, Task.CompletedTask, null) == null; - - apiValidationStatus = ApiValidationStatus.BadValidationRequest; - - if (!firstValidationRequest) - return BridgeError("Startup bridge request was repeated!"); if (parameters.Version == null) return BridgeError("Missing dmApiVersion field!"); From 4fac505a93a4d98892a6fbbc251a36ce5758b794 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 22 Oct 2023 10:37:00 -0400 Subject: [PATCH 31/40] Version bump to 5.16.4 --- build/Version.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/Version.props b/build/Version.props index c9c9e39ca0..3f21f8ad4b 100644 --- a/build/Version.props +++ b/build/Version.props @@ -3,7 +3,7 @@ - 5.16.3 + 5.16.4 4.7.1 9.12.0 6.0.1 From f0d5286a206f4ab1d9f289dea892d3bee00d04a7 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 22 Oct 2023 11:13:16 -0400 Subject: [PATCH 32/40] Disable this validation until BYOND bug is fixed See https://www.byond.com/forum/post/2894866 --- .../Components/Session/SessionController.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Tgstation.Server.Host/Components/Session/SessionController.cs b/src/Tgstation.Server.Host/Components/Session/SessionController.cs index 42054f24fb..b33d3628a1 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionController.cs @@ -783,6 +783,9 @@ namespace Tgstation.Server.Host.Components.Session break; case BridgeCommandType.Startup: apiValidationStatus = ApiValidationStatus.BadValidationRequest; + + // This business is is cancelled until this BYOND bug is resolved: https://www.byond.com/forum/post/2894866 +#if FALSE if (apiValidationSession) { var proceedTcs = new TaskCompletionSource(); @@ -792,6 +795,7 @@ namespace Tgstation.Server.Host.Components.Session if (!firstValidationRequest) return BridgeError("Startup bridge request was repeated!"); } +#endif if (parameters.Version == null) return BridgeError("Missing dmApiVersion field!"); From 720c42c522cb646fed39cff6d4f61a220bad8ca6 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 22 Oct 2023 11:44:07 -0400 Subject: [PATCH 33/40] Test that GameStaticFiles work in live tests --- tests/DMAPI/LongRunning/Test.dm | 17 ++++++ .../LongRunning/long_running_test_rooted.dme | 2 + .../Live/Instance/ConfigurationTest.cs | 38 +++++++++--- .../Live/Instance/WatchdogTest.cs | 61 ++++++++++++++++++- 4 files changed, 108 insertions(+), 10 deletions(-) diff --git a/tests/DMAPI/LongRunning/Test.dm b/tests/DMAPI/LongRunning/Test.dm index 657fadbfd0..f605c5ad59 100644 --- a/tests/DMAPI/LongRunning/Test.dm +++ b/tests/DMAPI/LongRunning/Test.dm @@ -31,6 +31,23 @@ if(!fexists("[DME_NAME].rsc")) FailTest("Failed to create .rsc!") +#ifdef RUN_STATIC_FILE_TESTS + if(params["expect_static_files"]) + if(!fexists("test2.txt")) + FailTest("Missing test2.txt") + + var/f2content = file2text("test2.txt") + if(f2content != "bbb") + FailTest("Unexpected test2.txt content: [f2content]") + + if(!fexists("data/test.txt")) + FailTest("Missing data/test.txt") + + var/f1content = file2text("data/test.txt") + if(f1content != "aaa") + FailTest("Unexpected data/test.txt content: [f1content]") +#endif + StartAsync() /proc/dab() diff --git a/tests/DMAPI/LongRunning/long_running_test_rooted.dme b/tests/DMAPI/LongRunning/long_running_test_rooted.dme index d8e9d61c96..a2ff783588 100644 --- a/tests/DMAPI/LongRunning/long_running_test_rooted.dme +++ b/tests/DMAPI/LongRunning/long_running_test_rooted.dme @@ -11,6 +11,8 @@ // END_PREFERENCES // BEGIN_INCLUDE +#define RUN_STATIC_FILE_TESTS +#define DME_NAME "long_running_test_rooted" #include "tests/DMAPI/LongRunning/Config.dm" #include "tests/DMAPI/test_prelude.dm" #include "tests/DMAPI/LongRunning/Test.dm" diff --git a/tests/Tgstation.Server.Tests/Live/Instance/ConfigurationTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/ConfigurationTest.cs index 2dea3e5b12..44c2f7246b 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/ConfigurationTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/ConfigurationTest.cs @@ -83,19 +83,40 @@ namespace Tgstation.Server.Tests.Live.Instance var path = Path.Combine(instance.Path, "Configuration", tmp); Assert.IsFalse(Directory.Exists(path)); - // leave a directory there to test the deployment process - var staticDir = new ConfigurationFileRequest - { - Path = "/GameStaticFiles/data" - }; - - await configurationClient.CreateDirectory(staticDir, cancellationToken); } public Task SetupDMApiTests(bool includingRoot, CancellationToken cancellationToken) { // just use an I/O manager here var ioManager = new DefaultIOManager(); + + async Task TestStaticFileAndDir() + { + // leave a file there to test the deployment process + var staticDir = new ConfigurationFileRequest + { + Path = "/GameStaticFiles/data" + }; + + await configurationClient.CreateDirectory(staticDir, cancellationToken); + + var staticFile = new ConfigurationFileRequest + { + Path = "/GameStaticFiles/data/test.txt" + }; + + await using var memoryStream = new MemoryStream(Encoding.UTF8.GetBytes("aaa")); + await configurationClient.Write(staticFile, memoryStream, cancellationToken); + + var staticFile2 = new ConfigurationFileRequest + { + Path = "/GameStaticFiles/test2.txt" + }; + + await using var memoryStream2 = new MemoryStream(Encoding.UTF8.GetBytes("bbb")); + await configurationClient.Write(staticFile2, memoryStream2, cancellationToken); + } + return Task.WhenAll( ioManager.CopyDirectory( Enumerable.Empty(), @@ -110,6 +131,9 @@ namespace Tgstation.Server.Tests.Live.Instance ioManager.ConcatPath(instance.Path, "Repository", "long_running_test_rooted.dme"), cancellationToken) : Task.CompletedTask, + includingRoot + ? TestStaticFileAndDir() + : Task.CompletedTask, ioManager.CopyDirectory( Enumerable.Empty(), null, diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index 784fce5f67..35bd51fd11 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -114,7 +114,7 @@ namespace Tgstation.Server.Tests.Live.Instance Port = ddPort, MapThreads = 2, LogOutput = false, - AdditionalParameters = "expect_chat_channels=1" + AdditionalParameters = "expect_chat_channels=1&expect_static_files=1" }, cancellationToken), CheckByondVersions(), ApiAssert.ThrowsException(() => instanceClient.DreamDaemon.Update(new DreamDaemonRequest @@ -157,8 +157,64 @@ namespace Tgstation.Server.Tests.Live.Instance System.Console.WriteLine($"TEST: END WATCHDOG TESTS {instanceClient.Metadata.Name}"); } + async ValueTask RegressionTest1686(CancellationToken cancellationToken) + { + async ValueTask RunTest(bool useTrusted) + { + System.Console.WriteLine($"TEST: RegressionTest1686 {useTrusted}..."); + var ddUpdateTask = instanceClient.DreamDaemon.Update(new DreamDaemonRequest + { + SecurityLevel = useTrusted ? DreamDaemonSecurity.Trusted : DreamDaemonSecurity.Safe, + AdditionalParameters = "expect_chat_channels=1&expect_static_files=1", + }, cancellationToken); + var currentStatus = await DeployTestDme("long_running_test_rooted", DreamDaemonSecurity.Trusted, true, cancellationToken); + await ddUpdateTask; + + Assert.AreEqual(WatchdogStatus.Offline, currentStatus.Status); + + var startJob = await StartDD(cancellationToken); + + await WaitForJob(startJob, 40, false, null, cancellationToken); + + currentStatus = await instanceClient.DreamDaemon.Update(new DreamDaemonRequest + { + SoftShutdown = true, + }, cancellationToken); + + Assert.AreEqual(WatchdogStatus.Online, currentStatus.Status); + + // reimplement TellWorldToReboot because it expects a new deployment and we don't care + System.Console.WriteLine("TEST: Hack world reboot topic..."); + var result = await topicClient.SendTopic(IPAddress.Loopback, "tgs_integration_test_special_tactics=1", ddPort, cancellationToken); + Assert.AreEqual("ack", result.StringData); + + using var tempCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + var tempToken = tempCts.Token; + using (tempToken.Register(() => System.Console.WriteLine("TEST ERROR: Timeout in RegressionTest1686!"))) + { + tempCts.CancelAfter(TimeSpan.FromMinutes(2)); + + do + { + await Task.Delay(TimeSpan.FromSeconds(1), tempToken); + currentStatus = await instanceClient.DreamDaemon.Read(tempToken); + } + while (currentStatus.Status != WatchdogStatus.Offline); + } + + await CheckDMApiFail(currentStatus.ActiveCompileJob, cancellationToken); + } + + await RunTest(true); + + if (new PlatformIdentifier().IsWindows || !usingBasicWatchdog) + await RunTest(false); + } + async Task InteropTestsForLongRunningDme(CancellationToken cancellationToken) { + await RegressionTest1686(cancellationToken); + await StartAndLeaveRunning(cancellationToken); await RegressionTest1550(cancellationToken); @@ -191,8 +247,7 @@ namespace Tgstation.Server.Tests.Live.Instance async ValueTask RegressionTest1550(CancellationToken cancellationToken) { // we need to cycle deployments twice because TGS holds the initial deployment - await DeployTestDme("LongRunning/long_running_test", DreamDaemonSecurity.Trusted, true, cancellationToken); - var currentStatus = await instanceClient.DreamDaemon.Read(cancellationToken); + var currentStatus = await DeployTestDme("LongRunning/long_running_test", DreamDaemonSecurity.Trusted, true, cancellationToken); Assert.AreEqual(WatchdogStatus.Online, currentStatus.Status); Assert.IsNotNull(currentStatus.StagedCompileJob); From 5f663b419eb4e671fff52909a53683ce826fdcef Mon Sep 17 00:00:00 2001 From: Hawk Date: Sun, 22 Oct 2023 19:02:37 +0100 Subject: [PATCH 34/40] Update Caddy Instructions Provide Caddy 2 Caddyfile. Update Caddy Documenation Link. Clarify PublicPath settings. --- README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index ab7a2556b7..dcceecf82b 100644 --- a/README.md +++ b/README.md @@ -401,14 +401,15 @@ Once complete, test that your configuration worked by visiting your proxy site f #### Caddy (Reccommended for Linux, or those unfamilar with configuring NGINX or Apache) 1. Setup a basic website configuration. Instructions on how to do so are out of scope. -2. In your Caddyfile, under a server entry, add the following (replace 8080 with the port TGS is hosted on): +2. In your Caddyfile, under a server entry, add the following (replace 5000 with the port TGS is hosted on): ``` -proxy /tgs localhost:8080 { - transparent +https://your.site.here { + reverse_proxy localhost:5000 } ``` +3. For this setup, your PublicPath needs to be blank. If you have a path in PublicPath, it needs to be in "reverse_proxy PublicPathHere localhost:5000". -See https://caddyserver.com/docs/proxy +See https://caddyserver.com/docs/caddyfile/directives/reverse_proxy #### NGINX (Reccommended for Linux) From 9d674618a90e7ca5deda34039fd48a13db7538e7 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 22 Oct 2023 14:09:42 -0400 Subject: [PATCH 35/40] Fix `HardLinkDmbProvider` not mirroring `GameStaticFiles` and other symlinks Also update readme about conditional requirements for GameStaticFiles to function Fixes #1686 --- README.md | 22 ++++++- .../Deployment/HardLinkDmbProvider.cs | 65 ++++++++++++++++--- .../Components/Watchdog/PosixWatchdog.cs | 1 + 3 files changed, 78 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index ab7a2556b7..b7636f5c85 100644 --- a/README.md +++ b/README.md @@ -570,7 +570,26 @@ This folder can contain anything. But, when certain events occur in the instance #### GameStaticFiles -Any files and folders contained in this root level of this folder will be symbolically linked to all deployments at the time they are created. This allows persistent game data (BYOND `.sav`s or code configuration files for example) to persist across all deployments. This folder contains a .tgsignore file which can be used to prevent symlinks from being generated by entering the names of files and folders (1 per line) +Any files and folders contained in this root level of this folder will be symbolically linked to all deployments at the time they are created. This allows persistent game data (BYOND `.sav`s or code configuration files for example) to persist across all deployments. This folder contains a .tgsignore file which can be used to prevent symlinks from being generated by entering the names of files and folders (1 per line). + +This functionality has the following prerequisites: + +- You are using Windows. + +**OR** + +- Your world uses the TGS DreamMaker API. +- Your world runs with the `Trusted` security level. + +**OR** + +- You are NOT using the basic watchdog. +- The contents of the `GameStaticFiles` directory are on the same filesystem as the instance's `Game` directory. + +**OR** + +- You are using the basic watchdog. +- Your world runs with the `Trusted` security level. ### Clients @@ -614,4 +633,3 @@ Feel free to ask for help [on the discussions page](https://github.com/tgstation * The remainder of the project is licensed under [GNU AGPL v3](http://www.gnu.org/licenses/agpl-3.0.html) See the files in the `/src/DMAPI` tree for the MIT license - diff --git a/src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs b/src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs index cd9a3f1185..263333f6bb 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs @@ -3,11 +3,13 @@ using System.Collections.Generic; using System.Diagnostics; using System.Globalization; using System.IO; +using System.Runtime.Versioning; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using Tgstation.Server.Api.Models; using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.Extensions; using Tgstation.Server.Host.IO; @@ -18,6 +20,7 @@ namespace Tgstation.Server.Host.Components.Deployment /// /// A that uses hard links. /// + [UnsupportedOSPlatform("windows")] sealed class HardLinkDmbProvider : SwappableDmbProvider { /// @@ -96,6 +99,7 @@ namespace Tgstation.Server.Host.Components.Deployment /// protected override async Task DoSwap(CancellationToken cancellationToken) { + logger.LogTrace("Begin DoSwap, mirroring task complete: {complete}...", mirroringTask.IsCompleted); var mirroredDir = await mirroringTask.WaitAsync(cancellationToken); var goAheadTcs = new TaskCompletionSource(); @@ -103,12 +107,15 @@ namespace Tgstation.Server.Host.Components.Deployment async void DisposeOfOldDirectory() { var directoryMoved = false; - var disposePath = Guid.NewGuid().ToString(); + var disposeGuid = Guid.NewGuid(); + var disposePath = disposeGuid.ToString(); + logger.LogTrace("Moving Live directory to {path} for deletion...", disposeGuid); try { await IOManager.MoveDirectory(LiveGameDirectory, disposePath, cancellationToken); directoryMoved = true; goAheadTcs.SetResult(); + logger.LogTrace("Deleting old Live directory {path}...", disposePath); await IOManager.DeleteDirectory(disposePath, CancellationToken.None); // DCT: We're detached at this point } catch (DirectoryNotFoundException ex) @@ -127,7 +134,9 @@ namespace Tgstation.Server.Host.Components.Deployment DisposeOfOldDirectory(); await goAheadTcs.Task; + logger.LogTrace("Moving mirror directory {path} to Live...", mirroredDir); await IOManager.MoveDirectory(mirroredDir, LiveGameDirectory, cancellationToken); + logger.LogTrace("Swap complete!"); } /// @@ -173,17 +182,40 @@ namespace Tgstation.Server.Host.Components.Deployment { var dir = new DirectoryInfo(src); Task subdirCreationTask = null; + var dreamDaemonWillAcceptOutOfDirectorySymlinks = CompileJob.MinimumSecurityLevel == DreamDaemonSecurity.Trusted; foreach (var subDirectory in dir.EnumerateDirectories()) { + var mirroredName = Path.Combine(dest, subDirectory.Name); + // check if we are a symbolic link - if (!subDirectory.Attributes.HasFlag(FileAttributes.Directory) || subDirectory.Attributes.HasFlag(FileAttributes.ReparsePoint)) - { - logger.LogTrace("Skipping symlink to {subdir}", subDirectory.Name); - continue; - } + if (subDirectory.Attributes.HasFlag(FileAttributes.ReparsePoint)) + if (dreamDaemonWillAcceptOutOfDirectorySymlinks) + { + var target = subDirectory.ResolveLinkTarget(false); + logger.LogDebug("Recreating directory {name} as symlink to {target}", subDirectory.Name, target); + if (subdirCreationTask == null) + { + subdirCreationTask = IOManager.CreateDirectory(dest, cancellationToken); + yield return subdirCreationTask; + } + + async Task CopyLink() + { + await subdirCreationTask.WaitAsync(cancellationToken); + using var lockContext = semaphore != null + ? await SemaphoreSlimContext.Lock(semaphore, cancellationToken) + : null; + await LinkFactory.CreateSymbolicLink(target.FullName, mirroredName, cancellationToken); + } + + yield return CopyLink(); + continue; + } + else + logger.LogDebug("Recreating symlinked directory {name} as hard links...", subDirectory.Name); var checkingSubdirCreationTask = true; - foreach (var copyTask in MirrorDirectoryImpl(subDirectory.FullName, Path.Combine(dest, subDirectory.Name), semaphore, cancellationToken)) + foreach (var copyTask in MirrorDirectoryImpl(subDirectory.FullName, mirroredName, semaphore, cancellationToken)) { if (subdirCreationTask == null) { @@ -214,7 +246,24 @@ namespace Tgstation.Server.Host.Components.Deployment using var lockContext = semaphore != null ? await SemaphoreSlimContext.Lock(semaphore, cancellationToken) : null; - await LinkFactory.CreateHardLink(sourceFile, destFile, cancellationToken); + + if (fileInfo.Attributes.HasFlag(FileAttributes.ReparsePoint)) + { + // AHHHHHHHHHHHHH + var target = fileInfo.ResolveLinkTarget(!dreamDaemonWillAcceptOutOfDirectorySymlinks); + if (dreamDaemonWillAcceptOutOfDirectorySymlinks) + { + logger.LogDebug("Recreating symlinked file {name} as symlink to {target}", fileInfo.Name, target.FullName); + await LinkFactory.CreateSymbolicLink(target.FullName, destFile, cancellationToken); + } + else + { + logger.LogDebug("Recreating symlinked file {name} as hard link to {target}", fileInfo.Name, target.FullName); + await LinkFactory.CreateHardLink(target.FullName, destFile, cancellationToken); + } + } + else + await LinkFactory.CreateHardLink(sourceFile, destFile, cancellationToken); } yield return LinkThisFile(); diff --git a/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs index eb563bd1fa..2fb5ad1d54 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs @@ -1,4 +1,5 @@ using System; +using System.Runtime.Versioning; using System.Threading; using System.Threading.Tasks; From 50072ae2227b48b7f0f638378e4087df765e01df Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 22 Oct 2023 14:20:24 -0400 Subject: [PATCH 36/40] Run milestone check on labeling --- .github/workflows/check-pr-has-milestone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-pr-has-milestone.yml b/.github/workflows/check-pr-has-milestone.yml index 954d539927..7f6ed97d2d 100644 --- a/.github/workflows/check-pr-has-milestone.yml +++ b/.github/workflows/check-pr-has-milestone.yml @@ -2,7 +2,7 @@ name: "Check PR Has Milestone" on: pull_request: - types: [ opened, edited, synchronize, reopened ] + types: [ opened, edited, synchronize, reopened, labeled ] branches: - dev - master From cc0b46205f0bf0fb663eabaf8fe74e61883bce60 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 22 Oct 2023 14:28:04 -0400 Subject: [PATCH 37/40] Fix warnings with OS restricted classes --- .../Components/Deployment/SymlinkDmbProvider.cs | 4 +++- .../Components/Watchdog/PosixWatchdog.cs | 1 + .../Components/Watchdog/PosixWatchdogFactory.cs | 2 ++ .../Components/Watchdog/WindowsWatchdog.cs | 4 +++- .../Components/Watchdog/WindowsWatchdogFactory.cs | 2 ++ 5 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Deployment/SymlinkDmbProvider.cs b/src/Tgstation.Server.Host/Components/Deployment/SymlinkDmbProvider.cs index 2edc58d37e..0054e7fa4b 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/SymlinkDmbProvider.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/SymlinkDmbProvider.cs @@ -1,4 +1,5 @@ -using System.Threading; +using System.Runtime.Versioning; +using System.Threading; using System.Threading.Tasks; using Tgstation.Server.Host.IO; @@ -8,6 +9,7 @@ namespace Tgstation.Server.Host.Components.Deployment /// /// A that uses symlinks. /// + [SupportedOSPlatform("windows")] sealed class SymlinkDmbProvider : SwappableDmbProvider { /// diff --git a/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs index 2fb5ad1d54..818dea9d7c 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs @@ -22,6 +22,7 @@ namespace Tgstation.Server.Host.Components.Watchdog /// /// A variant of the that works on POSIX systems. /// + [UnsupportedOSPlatform("windows")] sealed class PosixWatchdog : AdvancedWatchdog { /// diff --git a/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdogFactory.cs b/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdogFactory.cs index 64c8672262..1d0d140bc0 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdogFactory.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdogFactory.cs @@ -1,4 +1,5 @@ using System; +using System.Runtime.Versioning; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -20,6 +21,7 @@ namespace Tgstation.Server.Host.Components.Watchdog /// /// for creating s. /// + [UnsupportedOSPlatform("windows")] sealed class PosixWatchdogFactory : WindowsWatchdogFactory { /// diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs index 05c1f10088..8ff6bf3362 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs @@ -1,4 +1,5 @@ -using System.Threading; +using System.Runtime.Versioning; +using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -19,6 +20,7 @@ namespace Tgstation.Server.Host.Components.Watchdog /// /// A variant of the that works on Windows systems. /// + [SupportedOSPlatform("windows")] sealed class WindowsWatchdog : AdvancedWatchdog { /// diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs index 89ed6b145d..0888294aa4 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs @@ -1,4 +1,5 @@ using System; +using System.Runtime.Versioning; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -20,6 +21,7 @@ namespace Tgstation.Server.Host.Components.Watchdog /// /// for creating s. /// + [SupportedOSPlatform("windows")] class WindowsWatchdogFactory : WatchdogFactory { /// From ad2a5d2b9b131575a67ec2b709ec98fcd44f89c6 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 22 Oct 2023 14:30:13 -0400 Subject: [PATCH 38/40] Fix release build warnings --- .../Components/Session/SessionController.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Tgstation.Server.Host/Components/Session/SessionController.cs b/src/Tgstation.Server.Host/Components/Session/SessionController.cs index b33d3628a1..1ee53dfb57 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionController.cs @@ -795,6 +795,8 @@ namespace Tgstation.Server.Host.Components.Session if (!firstValidationRequest) return BridgeError("Startup bridge request was repeated!"); } +#else + postValidationShutdownTask = Task.CompletedTask; #endif if (parameters.Version == null) From a97001231ff6b1606b54adc16f4ee19a01e04069 Mon Sep 17 00:00:00 2001 From: Hawk-v3 Date: Sun, 22 Oct 2023 19:32:55 +0100 Subject: [PATCH 39/40] Update README.md Accept correction for definition of PublicPath Co-authored-by: Jordan Dominion --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index dcceecf82b..64221f5028 100644 --- a/README.md +++ b/README.md @@ -407,7 +407,7 @@ https://your.site.here { reverse_proxy localhost:5000 } ``` -3. For this setup, your PublicPath needs to be blank. If you have a path in PublicPath, it needs to be in "reverse_proxy PublicPathHere localhost:5000". +3. For this setup, your configuration's `ControlPanel:PublicPath` needs to be blank. If you have a path in `PublicPath`, it needs to be in "reverse_proxy PublicPathHere localhost:5000". See https://caddyserver.com/docs/caddyfile/directives/reverse_proxy From 5afc74c4872ca4c49fe7990c316fecd4b0f46603 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 22 Oct 2023 16:01:52 -0400 Subject: [PATCH 40/40] Fix more Release build warnings Not being able to build locally in release mode because of the .NET 8 RC SDK is painful --- .../Components/Deployment/SymlinkDmbProvider.cs | 4 +--- .../Components/Watchdog/WindowsWatchdog.cs | 4 +--- .../Components/Watchdog/WindowsWatchdogFactory.cs | 2 -- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Deployment/SymlinkDmbProvider.cs b/src/Tgstation.Server.Host/Components/Deployment/SymlinkDmbProvider.cs index 0054e7fa4b..2edc58d37e 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/SymlinkDmbProvider.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/SymlinkDmbProvider.cs @@ -1,5 +1,4 @@ -using System.Runtime.Versioning; -using System.Threading; +using System.Threading; using System.Threading.Tasks; using Tgstation.Server.Host.IO; @@ -9,7 +8,6 @@ namespace Tgstation.Server.Host.Components.Deployment /// /// A that uses symlinks. /// - [SupportedOSPlatform("windows")] sealed class SymlinkDmbProvider : SwappableDmbProvider { /// diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs index 8ff6bf3362..05c1f10088 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs @@ -1,5 +1,4 @@ -using System.Runtime.Versioning; -using System.Threading; +using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -20,7 +19,6 @@ namespace Tgstation.Server.Host.Components.Watchdog /// /// A variant of the that works on Windows systems. /// - [SupportedOSPlatform("windows")] sealed class WindowsWatchdog : AdvancedWatchdog { /// diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs index 0888294aa4..89ed6b145d 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs @@ -1,5 +1,4 @@ using System; -using System.Runtime.Versioning; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -21,7 +20,6 @@ namespace Tgstation.Server.Host.Components.Watchdog /// /// for creating s. /// - [SupportedOSPlatform("windows")] class WindowsWatchdogFactory : WatchdogFactory { ///