From 6d15f0ff02edbf79a71a8990e810a8045762d245 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 26 Apr 2025 16:20:59 -0400 Subject: [PATCH 01/52] Test token signing key is shared between swarm servers --- tests/Tgstation.Server.Tests/Live/TestLiveServer.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index ceb1f4bc6d..8df9a3d612 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -922,6 +922,12 @@ namespace Tgstation.Server.Tests.Live await using var node1Client = await CreateAdminClient(node1.ApiUrl, cancellationToken); await using var node2Client = await CreateAdminClient(node2.ApiUrl, cancellationToken); + // test a token signed from any one node will work on another + var token = node2Client.RestClient.Token; + var testNode1Client = restClientFactory.CreateFromToken(node1.ApiUrl, token); + + await testNode1Client.ServerInformation(cancellationToken); + var controllerInfo = await controllerClient.RestClient.ServerInformation(cancellationToken); async Task WaitForSwarmServerUpdate(IRestServerClient client, int currentServerCount) From c865caf9586686fdb10df3aa4bf28c97e114bd56 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 26 Apr 2025 22:12:29 -0400 Subject: [PATCH 02/52] Fix God's oldest typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8d0ac955b6..9b3eb0ad9b 100644 --- a/README.md +++ b/README.md @@ -639,7 +639,7 @@ Bots have a set of built-in commands that can be triggered via `!tgs`, mentionin All files in game code deployments are considered transient by default, meaning when new code is deployed, changes will be lost. Static files allow you to specify which files and folders stick around throughout all deployments. -The `StaticFiles` folder contains 3 root folders which cannot be deleted and operate under special rules +The `Configuration` folder contains 3 root folders which cannot be deleted and operate under special rules - `CodeModifications` - `EventScripts` - `GameStaticFiles` From 48d495e397940f4f5922f840f9e4dcee45ec09de Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Apr 2025 09:40:27 +0000 Subject: [PATCH 03/52] Bump Serilog.Sinks.File from 6.0.0 to 7.0.0 Bumps [Serilog.Sinks.File](https://github.com/serilog/serilog-sinks-file) from 6.0.0 to 7.0.0. - [Release notes](https://github.com/serilog/serilog-sinks-file/releases) - [Changelog](https://github.com/serilog/serilog-sinks-file/blob/dev/CHANGES.md) - [Commits](https://github.com/serilog/serilog-sinks-file/compare/v6.0.0...v7.0.0) --- updated-dependencies: - dependency-name: Serilog.Sinks.File dependency-version: 7.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- src/Tgstation.Server.Host/Tgstation.Server.Host.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index 9e4a3056c4..19695527da 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -154,7 +154,7 @@ - + From 9ba8b94e2ccbf477de3b5e7f4ab853e1a1a781a9 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Tue, 6 May 2025 18:52:09 -0400 Subject: [PATCH 04/52] Add API for queuing text deployment messages --- .../Components/Chat/ChatManager.cs | 54 +++++++++++-------- .../Components/Chat/IChatManager.cs | 6 +++ 2 files changed, 39 insertions(+), 21 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs b/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs index 5faa30dbd2..b6685b7460 100644 --- a/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs +++ b/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs @@ -351,28 +351,11 @@ namespace Tgstation.Server.Host.Components.Chat /// public void QueueWatchdogMessage(string message) - { - ArgumentNullException.ThrowIfNull(message); + => QueueMessageGeneric(mapping => mapping.IsWatchdogChannel, message); - message = String.Format(CultureInfo.InvariantCulture, "WD: {0}", message); - - if (!initialProviderConnectionsTask!.IsCompleted) - logger.LogTrace("Waiting for initial provider connections before sending watchdog message..."); - - // Reimplementing QueueMessage - QueueMessageInternal( - new MessageContent - { - Text = message, - }, - () => - { - // so it doesn't change while we're using it - lock (mappedChannels) - return mappedChannels.Where(x => x.Value.IsWatchdogChannel).Select(x => x.Key).ToList(); - }, - true); - } + /// + public void QueueDeploymentMessage(string message) + => QueueMessageGeneric(mapping => mapping.IsUpdatesChannel, message); /// public Func> QueueDeploymentMessage( @@ -1111,5 +1094,34 @@ namespace Tgstation.Server.Host.Components.Chat AddMessageTask(SendMessageTask()); } + + /// + /// Queues a message to a selected set of s. + /// + /// A for selecting the s to send to. + /// The message to send. + void QueueMessageGeneric(Predicate channelSelector, string message) + { + ArgumentNullException.ThrowIfNull(message); + + message = String.Format(CultureInfo.InvariantCulture, "WD: {0}", message); + + if (!initialProviderConnectionsTask!.IsCompleted) + logger.LogTrace("Waiting for initial provider connections before sending watchdog message..."); + + // Reimplementing QueueMessage + QueueMessageInternal( + new MessageContent + { + Text = message, + }, + () => + { + // so it doesn't change while we're using it + lock (mappedChannels) + return mappedChannels.Where(x => channelSelector(x.Value)).Select(x => x.Key).ToList(); + }, + true); + } } } diff --git a/src/Tgstation.Server.Host/Components/Chat/IChatManager.cs b/src/Tgstation.Server.Host/Components/Chat/IChatManager.cs index 669f21b305..2b2faf9496 100644 --- a/src/Tgstation.Server.Host/Components/Chat/IChatManager.cs +++ b/src/Tgstation.Server.Host/Components/Chat/IChatManager.cs @@ -57,6 +57,12 @@ namespace Tgstation.Server.Host.Components.Chat /// The message being sent. void QueueWatchdogMessage(string message); + /// + /// Queue a chat to configured deployment channels. + /// + /// The message being sent. + void QueueDeploymentMessage(string message); + /// /// Send the message for a deployment to configured deployment channels. /// From d5c3ddbd4ef216086ae72b0f3234d3a0f358d6ac Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Tue, 6 May 2025 18:53:18 -0400 Subject: [PATCH 05/52] Add a chat message when an automatic update fails due to a conflict --- src/Tgstation.Server.Host/Components/Instance.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Tgstation.Server.Host/Components/Instance.cs b/src/Tgstation.Server.Host/Components/Instance.cs index f0fd27f661..8957678a3e 100644 --- a/src/Tgstation.Server.Host/Components/Instance.cs +++ b/src/Tgstation.Server.Host/Components/Instance.cs @@ -531,7 +531,10 @@ namespace Tgstation.Server.Host.Components } } else if (preserveTestMerges) + { + Chat.QueueDeploymentMessage("Automatic update has failed due to a conflicting testmerge!"); throw new JobException(Api.Models.ErrorCode.InstanceUpdateTestMergeConflict); + } if (!preserveTestMerges) { From ac56e2c3fa8a96deb155d3ff6be4fd0a65390bd3 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Tue, 6 May 2025 18:53:42 -0400 Subject: [PATCH 06/52] Version bump to 6.18.0 --- build/Version.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/Version.props b/build/Version.props index 394ff58026..7312268ed1 100644 --- a/build/Version.props +++ b/build/Version.props @@ -3,7 +3,7 @@ - 6.17.0 + 6.18.0 5.7.0 10.13.0 0.6.0 From 6384254cdc1a4531e32ee703dfc94af32855efd4 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Tue, 6 May 2025 18:59:13 -0400 Subject: [PATCH 07/52] Fix bad prefix for auto-update fail message --- .../Components/Chat/ChatManager.cs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs b/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs index b6685b7460..4b42fb0a56 100644 --- a/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs +++ b/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs @@ -351,11 +351,11 @@ namespace Tgstation.Server.Host.Components.Chat /// public void QueueWatchdogMessage(string message) - => QueueMessageGeneric(mapping => mapping.IsWatchdogChannel, message); + => QueueMessageGeneric(mapping => mapping.IsWatchdogChannel, message, "WD"); /// public void QueueDeploymentMessage(string message) - => QueueMessageGeneric(mapping => mapping.IsUpdatesChannel, message); + => QueueMessageGeneric(mapping => mapping.IsUpdatesChannel, message, null); /// public Func> QueueDeploymentMessage( @@ -1100,14 +1100,18 @@ namespace Tgstation.Server.Host.Components.Chat /// /// A for selecting the s to send to. /// The message to send. - void QueueMessageGeneric(Predicate channelSelector, string message) + /// The optional prefix to the message to be sent. + void QueueMessageGeneric(Predicate channelSelector, string message, string prefix) { ArgumentNullException.ThrowIfNull(message); - message = String.Format(CultureInfo.InvariantCulture, "WD: {0}", message); + if (prefix != null) + { + message = $"{prefix}: {message}"; + } if (!initialProviderConnectionsTask!.IsCompleted) - logger.LogTrace("Waiting for initial provider connections before sending watchdog message..."); + logger.LogTrace("Waiting for initial provider connections before sending chat message..."); // Reimplementing QueueMessage QueueMessageInternal( From 1f4ac78d42a7231a444f79772932b12041be27bc Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Wed, 7 May 2025 17:56:33 -0400 Subject: [PATCH 08/52] Fix nullable error --- src/Tgstation.Server.Host/Components/Chat/ChatManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs b/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs index 4b42fb0a56..abd5ef90c3 100644 --- a/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs +++ b/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs @@ -1101,7 +1101,7 @@ namespace Tgstation.Server.Host.Components.Chat /// A for selecting the s to send to. /// The message to send. /// The optional prefix to the message to be sent. - void QueueMessageGeneric(Predicate channelSelector, string message, string prefix) + void QueueMessageGeneric(Predicate channelSelector, string message, string? prefix) { ArgumentNullException.ThrowIfNull(message); From cbbef1f8a0560280808fafc0ccba2792c2a9bbc1 Mon Sep 17 00:00:00 2001 From: Drulikar Date: Fri, 9 May 2025 11:03:39 -0500 Subject: [PATCH 09/52] Don't require https --- src/Tgstation.Server.Host/Core/Application.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index c4958767ee..a947eeb720 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -940,10 +940,7 @@ namespace Tgstation.Server.Host.Core options.Scope.Add(OpenIdConnectScope.OpenId); options.Scope.Add(OpenIdConnectScope.OfflineAccess); -#if DEBUG options.RequireHttpsMetadata = false; -#endif - options.SaveTokens = true; options.ResponseType = OpenIdConnectResponseType.Code; options.MapInboundClaims = false; From 721beaff68673573a387d66b4a0e4573e29fc352 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 9 May 2025 21:32:26 -0400 Subject: [PATCH 10/52] Fix conflicting name error --- src/Tgstation.Server.Host/Components/Chat/ChatManager.cs | 2 +- src/Tgstation.Server.Host/Components/Chat/IChatManager.cs | 2 +- src/Tgstation.Server.Host/Components/Instance.cs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs b/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs index abd5ef90c3..2aeca4f8f3 100644 --- a/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs +++ b/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs @@ -354,7 +354,7 @@ namespace Tgstation.Server.Host.Components.Chat => QueueMessageGeneric(mapping => mapping.IsWatchdogChannel, message, "WD"); /// - public void QueueDeploymentMessage(string message) + public void QueueRawDeploymentMessage(string message) => QueueMessageGeneric(mapping => mapping.IsUpdatesChannel, message, null); /// diff --git a/src/Tgstation.Server.Host/Components/Chat/IChatManager.cs b/src/Tgstation.Server.Host/Components/Chat/IChatManager.cs index 2b2faf9496..4f112afb69 100644 --- a/src/Tgstation.Server.Host/Components/Chat/IChatManager.cs +++ b/src/Tgstation.Server.Host/Components/Chat/IChatManager.cs @@ -61,7 +61,7 @@ namespace Tgstation.Server.Host.Components.Chat /// Queue a chat to configured deployment channels. /// /// The message being sent. - void QueueDeploymentMessage(string message); + void QueueRawDeploymentMessage(string message); /// /// Send the message for a deployment to configured deployment channels. diff --git a/src/Tgstation.Server.Host/Components/Instance.cs b/src/Tgstation.Server.Host/Components/Instance.cs index 8957678a3e..37454b7dbc 100644 --- a/src/Tgstation.Server.Host/Components/Instance.cs +++ b/src/Tgstation.Server.Host/Components/Instance.cs @@ -532,7 +532,7 @@ namespace Tgstation.Server.Host.Components } else if (preserveTestMerges) { - Chat.QueueDeploymentMessage("Automatic update has failed due to a conflicting testmerge!"); + Chat.QueueRawDeploymentMessage("Automatic update has failed due to a conflicting testmerge!"); throw new JobException(Api.Models.ErrorCode.InstanceUpdateTestMergeConflict); } From fb2b15134fd2fd66fa251b78015f2ae1a538629a Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 23 Jun 2025 21:35:28 -0400 Subject: [PATCH 11/52] Allow BYOND downloads to be configurable --- .../Components/Engine/ByondInstallerBase.cs | 81 +++++++++++++++++-- .../Components/Engine/PosixByondInstaller.cs | 8 +- .../Engine/WindowsByondInstaller.cs | 18 ++--- .../Configuration/GeneralConfiguration.cs | 11 +++ .../Engine/TestByondInstallerBase.cs | 23 ++++++ .../Engine/TestPosixByondInstaller.cs | 23 ++++-- 6 files changed, 137 insertions(+), 27 deletions(-) create mode 100644 tests/Tgstation.Server.Host.Tests/Components/Engine/TestByondInstallerBase.cs diff --git a/src/Tgstation.Server.Host/Components/Engine/ByondInstallerBase.cs b/src/Tgstation.Server.Host/Components/Engine/ByondInstallerBase.cs index e02c109e6b..527fcae985 100644 --- a/src/Tgstation.Server.Host/Components/Engine/ByondInstallerBase.cs +++ b/src/Tgstation.Server.Host/Components/Engine/ByondInstallerBase.cs @@ -6,8 +6,10 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Tgstation.Server.Api.Models; +using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.IO; using Tgstation.Server.Host.Jobs; @@ -41,6 +43,11 @@ namespace Tgstation.Server.Host.Components.Engine /// protected override EngineType TargetEngineType => EngineType.Byond; + /// + /// The for the . + /// + protected IOptionsMonitor GeneralConfigurationOptions { get; } + /// /// Path to the system user's local BYOND folder. /// @@ -52,25 +59,85 @@ namespace Tgstation.Server.Host.Components.Engine protected abstract string DreamMakerName { get; } /// - /// Gets the URL formatter string for downloading a byond version of {0:Major} {1:Minor}. + /// Template to do ${Marker:xxx} replacements in . /// - protected abstract string ByondRevisionsUrlTemplate { get; } + protected abstract string OSMarkerTemplate { get; } /// /// The for the . /// readonly IFileDownloader fileDownloader; + /// + /// Format a given . + /// + /// The BYOND version to download. + /// The template. + /// The . + /// The formatted byond download . + /// Exposed only for testability. + internal static Uri GetDownloadZipUrl(Version semver, string byondZipDownloadTemplate, string osMarkerTemplate) + { + // god forbid + var guardGuid = Guid.NewGuid(); + + var url = byondZipDownloadTemplate + .Replace("$$", guardGuid.ToString(), StringComparison.Ordinal) + .Replace("${Major}", semver.Major.ToString(CultureInfo.InvariantCulture), StringComparison.Ordinal) + .Replace("${Minor}", semver.Minor.ToString(CultureInfo.InvariantCulture), StringComparison.Ordinal); + + var osMarkerPrefix = $"${{{osMarkerTemplate}:"; + var osMarkerIndex = url.IndexOf(osMarkerPrefix); + while (osMarkerIndex != -1) + { + var start = osMarkerIndex + osMarkerPrefix.Length; + var end = url.IndexOf('}', start); + if (end == -1) + break; + + var substitution = url.Substring(start, end - start); + url = url.Replace($"{osMarkerPrefix}{substitution}}}", substitution, StringComparison.Ordinal); + + osMarkerIndex = url.IndexOf(osMarkerPrefix); + } + + // at this point, any other substitution attempts should be removed + var otherMarkerPrefix = "${"; + var otherMarkerIndex = url.IndexOf(otherMarkerPrefix); + while (otherMarkerIndex != -1) + { + var start = otherMarkerIndex + otherMarkerPrefix.Length; + var end = url.IndexOf('}', start); + if (end == -1) + break; + + var substitution = url.Substring(start, end - start); + url = url.Replace($"{otherMarkerPrefix}{substitution}}}", String.Empty, StringComparison.Ordinal); + + otherMarkerIndex = url.IndexOf(otherMarkerPrefix); + } + + url = url.Replace(guardGuid.ToString(), "$", StringComparison.Ordinal); + + return new Uri(url); + } + /// /// Initializes a new instance of the class. /// /// The for the . /// The for the . /// The value of . - protected ByondInstallerBase(IIOManager ioManager, ILogger logger, IFileDownloader fileDownloader) + /// The value of . + protected ByondInstallerBase( + IIOManager ioManager, + ILogger logger, + IFileDownloader fileDownloader, + IOptionsMonitor generalConfigurationOptions) : base(ioManager, logger) { this.fileDownloader = fileDownloader ?? throw new ArgumentNullException(nameof(fileDownloader)); + GeneralConfigurationOptions = generalConfigurationOptions ?? throw new ArgumentNullException(nameof(generalConfigurationOptions)); } /// @@ -195,8 +262,12 @@ namespace Tgstation.Server.Host.Components.Engine Uri GetDownloadZipUrl(EngineVersion version) { CheckVersionValidity(version); - var url = String.Format(CultureInfo.InvariantCulture, ByondRevisionsUrlTemplate, version.Version!.Major, version.Version.Minor); - return new Uri(url); + + var guardGuid = Guid.NewGuid(); + + var semver = version.Version!; + var template = GeneralConfigurationOptions.CurrentValue.ByondZipDownloadTemplate; + return GetDownloadZipUrl(semver, template, OSMarkerTemplate); } } } diff --git a/src/Tgstation.Server.Host/Components/Engine/PosixByondInstaller.cs b/src/Tgstation.Server.Host/Components/Engine/PosixByondInstaller.cs index fff62b3b3f..7e21f6a378 100644 --- a/src/Tgstation.Server.Host/Components/Engine/PosixByondInstaller.cs +++ b/src/Tgstation.Server.Host/Components/Engine/PosixByondInstaller.cs @@ -5,9 +5,11 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Tgstation.Server.Api.Models; using Tgstation.Server.Common.Extensions; +using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.IO; namespace Tgstation.Server.Host.Components.Engine @@ -39,7 +41,7 @@ namespace Tgstation.Server.Host.Components.Engine protected override string DreamMakerName => DreamMakerExecutableName + ShellScriptExtension; /// - protected override string ByondRevisionsUrlTemplate => "https://www.byond.com/download/build/{0}/{0}.{1}_byond_linux.zip"; + protected override string OSMarkerTemplate => "Linux"; /// /// The for the . @@ -52,13 +54,15 @@ namespace Tgstation.Server.Host.Components.Engine /// The value of . /// The for the . /// The for the . + /// The . /// The for the . public PosixByondInstaller( IPostWriteHandler postWriteHandler, IIOManager ioManager, IFileDownloader fileDownloader, + IOptionsMonitor generalConfigurationOptions, ILogger logger) - : base(ioManager, logger, fileDownloader) + : base(ioManager, logger, fileDownloader, generalConfigurationOptions) { this.postWriteHandler = postWriteHandler ?? throw new ArgumentNullException(nameof(postWriteHandler)); diff --git a/src/Tgstation.Server.Host/Components/Engine/WindowsByondInstaller.cs b/src/Tgstation.Server.Host/Components/Engine/WindowsByondInstaller.cs index ce2c7704e2..786d8df147 100644 --- a/src/Tgstation.Server.Host/Components/Engine/WindowsByondInstaller.cs +++ b/src/Tgstation.Server.Host/Components/Engine/WindowsByondInstaller.cs @@ -69,18 +69,13 @@ namespace Tgstation.Server.Host.Components.Engine protected override string PathToUserFolder { get; } /// - protected override string ByondRevisionsUrlTemplate => "https://www.byond.com/download/build/{0}/{0}.{1}_byond.zip"; + protected override string OSMarkerTemplate => "Windows"; /// /// The for the . /// readonly IProcessExecutor processExecutor; - /// - /// The for the . - /// - readonly GeneralConfiguration generalConfiguration; - /// /// The for the . /// @@ -100,7 +95,7 @@ namespace Tgstation.Server.Host.Components.Engine /// Initializes a new instance of the class. /// /// The value of . - /// The containing the value of . + /// The containing the . /// The containing the value of . /// The for the . /// The for the . @@ -109,13 +104,12 @@ namespace Tgstation.Server.Host.Components.Engine IProcessExecutor processExecutor, IIOManager ioManager, IFileDownloader fileDownloader, - IOptions generalConfigurationOptions, + IOptionsMonitor generalConfigurationOptions, IOptions sessionConfigurationOptions, ILogger logger) - : base(ioManager, logger, fileDownloader) + : base(ioManager, logger, fileDownloader, generalConfigurationOptions) { this.processExecutor = processExecutor ?? throw new ArgumentNullException(nameof(processExecutor)); - generalConfiguration = generalConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(generalConfigurationOptions)); sessionConfiguration = sessionConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(sessionConfigurationOptions)); var useServiceSpecialTactics = Environment.Is64BitProcess && Environment.UserName == $"{Environment.MachineName}$"; @@ -150,7 +144,7 @@ namespace Tgstation.Server.Host.Components.Engine installDirectXTask, }; - if (!generalConfiguration.SkipAddingByondFirewallException) + if (!GeneralConfigurationOptions.CurrentValue.SkipAddingByondFirewallException) { var firewallTask = AddDreamDaemonToFirewall(version, path, deploymentPipelineProcesses, cancellationToken); tasks.Add(firewallTask); @@ -165,7 +159,7 @@ namespace Tgstation.Server.Host.Components.Engine CheckVersionValidity(version); ArgumentNullException.ThrowIfNull(path); - if (generalConfiguration.SkipAddingByondFirewallException) + if (GeneralConfigurationOptions.CurrentValue.SkipAddingByondFirewallException) return; if (version.Version < DDExeVersion) diff --git a/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs b/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs index 3ba8b3e878..c47282ea80 100644 --- a/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs +++ b/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs @@ -148,6 +148,17 @@ namespace Tgstation.Server.Host.Configuration [YamlMember(SerializeAs = typeof(string))] public Uri OpenDreamGitUrl { get; set; } = new Uri(DefaultOpenDreamGitUrl); + /// + /// The formatter used to download official byond zip files for a given version + /// - ${Major} is substituted with the major version number + /// - ${Minor} is substituted with the minor version number + /// - ${Linux:xxx}, where xxx is any string, will be substituted with xxx if running under Linux. + /// - ${Windows:xxx}, where xxx is any string, will be substituted with xxx if running under Windows. + /// - $$ will evaluate to a literal $ and not be used for substitutions. + /// - Any inapplicable ${xxx} string will be removed. + /// + public string ByondZipDownloadTemplate { get; set; } = "https://www.byond.com/download/build/${Major}/${Major}.{Minor}_byond${Linux:_linux}.zip"; + /// /// The prefix to the OpenDream semver as tags appear in the git repository. /// diff --git a/tests/Tgstation.Server.Host.Tests/Components/Engine/TestByondInstallerBase.cs b/tests/Tgstation.Server.Host.Tests/Components/Engine/TestByondInstallerBase.cs new file mode 100644 index 0000000000..a3a4605877 --- /dev/null +++ b/tests/Tgstation.Server.Host.Tests/Components/Engine/TestByondInstallerBase.cs @@ -0,0 +1,23 @@ +using System; + +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace Tgstation.Server.Host.Components.Engine.Tests +{ + [TestClass] + public sealed class TestByondInstallerBase + { + [TestMethod] + public void TestUrlTemplateFormatting() + { + const string OSMarker = "TempleOS"; + + Assert.AreEqual( + new Uri("https://example.com/$515.1111_Hello Worl$d.zip"), + ByondInstallerBase.GetDownloadZipUrl( + new Version(515, 1111), + "https://example.com/$$${Major}.${Minor}_${TempleOS:Hello Worl$$d}.zip${Linux:Not this}${Or This}", + OSMarker)); + } + } +} diff --git a/tests/Tgstation.Server.Host.Tests/Components/Engine/TestPosixByondInstaller.cs b/tests/Tgstation.Server.Host.Tests/Components/Engine/TestPosixByondInstaller.cs index 60e86ad63e..c6288b6a7c 100644 --- a/tests/Tgstation.Server.Host.Tests/Components/Engine/TestPosixByondInstaller.cs +++ b/tests/Tgstation.Server.Host.Tests/Components/Engine/TestPosixByondInstaller.cs @@ -1,4 +1,5 @@ using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; using System; @@ -9,6 +10,7 @@ using System.Threading.Tasks; using Tgstation.Server.Api.Models; using Tgstation.Server.Api.Models.Internal; +using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.IO; namespace Tgstation.Server.Host.Components.Engine.Tests @@ -19,16 +21,18 @@ namespace Tgstation.Server.Host.Components.Engine.Tests [TestMethod] public void TestConstruction() { - Assert.ThrowsException(() => new PosixByondInstaller(null, null, null, null)); + Assert.ThrowsException(() => new PosixByondInstaller(null, null, null, null, null)); var mockPostWriteHandler = new Mock(); - Assert.ThrowsException(() => new PosixByondInstaller(mockPostWriteHandler.Object, null, null, null)); + Assert.ThrowsException(() => new PosixByondInstaller(mockPostWriteHandler.Object, null, null, null, null)); var mockIOManager = new Mock(); - Assert.ThrowsException(() => new PosixByondInstaller(mockPostWriteHandler.Object, mockIOManager.Object, null, null)); + Assert.ThrowsException(() => new PosixByondInstaller(mockPostWriteHandler.Object, mockIOManager.Object, null, null, null)); var mockFileDownloader = Mock.Of(); - Assert.ThrowsException(() => new PosixByondInstaller(mockPostWriteHandler.Object, mockIOManager.Object, mockFileDownloader, null)); + Assert.ThrowsException(() => new PosixByondInstaller(mockPostWriteHandler.Object, mockIOManager.Object, mockFileDownloader, null, null)); + var mockOptions = Mock.Of>(); + Assert.ThrowsException(() => new PosixByondInstaller(mockPostWriteHandler.Object, mockIOManager.Object, mockFileDownloader, mockOptions, null)); var mockLogger = new Mock>(); - _ = new PosixByondInstaller(mockPostWriteHandler.Object, mockIOManager.Object, mockFileDownloader, mockLogger.Object); + _ = new PosixByondInstaller(mockPostWriteHandler.Object, mockIOManager.Object, mockFileDownloader, mockOptions, mockLogger.Object); } [TestMethod] @@ -38,7 +42,8 @@ namespace Tgstation.Server.Host.Components.Engine.Tests var mockIOManager = new Mock(); var mockLogger = new Mock>(); var mockFileDownloader = Mock.Of(); - var installer = new PosixByondInstaller(mockPostWriteHandler.Object, mockIOManager.Object, mockFileDownloader, mockLogger.Object); + var mockOptions = Mock.Of>(); + var installer = new PosixByondInstaller(mockPostWriteHandler.Object, mockIOManager.Object, mockFileDownloader, mockOptions, mockLogger.Object); await installer.CleanCache(default); } @@ -50,7 +55,8 @@ namespace Tgstation.Server.Host.Components.Engine.Tests var mockPostWriteHandler = new Mock(); var mockLogger = new Mock>(); var mockFileDownloader = new Mock(); - var installer = new PosixByondInstaller(mockPostWriteHandler.Object, mockIOManager.Object, mockFileDownloader.Object, mockLogger.Object); + var mockOptions = Mock.Of>(); + var installer = new PosixByondInstaller(mockPostWriteHandler.Object, mockIOManager.Object, mockFileDownloader.Object, mockOptions, mockLogger.Object); await Assert.ThrowsExceptionAsync(() => installer.DownloadVersion(null, null, default).AsTask()); @@ -87,7 +93,8 @@ namespace Tgstation.Server.Host.Components.Engine.Tests var mockPostWriteHandler = new Mock(); var mockLogger = new Mock>(); var mockFileDownloader = Mock.Of(); - var installer = new PosixByondInstaller(mockPostWriteHandler.Object, mockIOManager.Object, mockFileDownloader, mockLogger.Object); + var mockOptions = Mock.Of>(); + var installer = new PosixByondInstaller(mockPostWriteHandler.Object, mockIOManager.Object, mockFileDownloader, mockOptions, mockLogger.Object); const string FakePath = "fake"; await Assert.ThrowsExceptionAsync(() => installer.Install(null, null, false, default).AsTask()); From 51d7aaec138cb4cc97061f796c7df53222a9996f Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 23 Jun 2025 21:36:08 -0400 Subject: [PATCH 12/52] Set default BYOND download to SS13 org mirror --- src/Tgstation.Server.Host/appsettings.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Tgstation.Server.Host/appsettings.yml b/src/Tgstation.Server.Host/appsettings.yml index 7d952c73d1..c06b79c667 100644 --- a/src/Tgstation.Server.Host/appsettings.yml +++ b/src/Tgstation.Server.Host/appsettings.yml @@ -17,6 +17,7 @@ General: HostApiDocumentation: false # Make HTTP API documentation available at /api/doc/tgs_api.json and /api/graphql SkipAddingByondFirewallException: false # Windows Only: Prevent running netsh.exe to add a firewall exception for installed engine binaries DeploymentDirectoryCopyTasksPerCore: 100 # Maximum number of concurrent file copy operations PER available CPU core + ByondZipDownloadTemplate: https://spacestation13.github.io/byond-builds/${Major}/${Major}.${Minor}_byond${Linux:_linux}.zip # URL of a plaintext file used to identify the curr OpenDreamGitUrl: https://github.com/OpenDreamProject/OpenDream # The repository to retrieve OpenDream from OpenDreamGitTagPrefix: v # The prefix to the OpenDream semver as tags appear in the git repository OpenDreamSuppressInstallOutput: false # Suppress the dotnet output of creating an OpenDream installation. Known to cause hangs in CI. From 6b696a4c1a6cf4bd6ef5b0d41127702c40881e70 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 23 Jun 2025 21:41:03 -0400 Subject: [PATCH 13/52] Give up on broken BYOND caching, download from SS13 mirror --- .github/workflows/ci-pipeline.yml | 39 +------------------ .../CachingFileDownloader.cs | 2 +- 2 files changed, 3 insertions(+), 38 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index fe0b463dce..7ad807b51f 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -166,19 +166,12 @@ jobs: sudo apt-get update sudo apt-get install -y -o APT::Immediate-Configure=0 libc6-i386 libstdc++6:i386 libgcc-s1:i386 - - name: Cache BYOND .zips - uses: actions/cache@v4 - id: cache-byond - with: - path: ~/byond-zips-cache - key: byond-zips - - name: Setup BYOND Cache if Necessary and Install run: | echo "Setting up BYOND." FULL_VERSION=${{ matrix.byond }} if [[ "$FULL_VERSION" = "EDGE" ]] ; then - VERSIONS=$(curl https://www.byond.com/download/version.txt) + VERSIONS=$(curl https://spacestation13.github.io/byond-builds/version.txt) FULL_VERSION=$(echo "$VERSIONS" | tail -n1) echo "EDGE version evaluated to $FULL_VERSION" @@ -193,7 +186,7 @@ jobs: if [[ ! -f $HOME/byond-zips-cache/linux/$FULL_VERSION.zip ]] ; then BYOND_MAJOR=${FULL_VERSION%.*} mkdir -p $HOME/byond-zips-cache/linux - curl "https://www.byond.com/download/build/$BYOND_MAJOR/${FULL_VERSION}_byond_linux.zip" -o $HOME/byond-zips-cache/linux/$FULL_VERSION.zip + curl "https://spacestation13.github.io/byond-builds/${FULL_VERSION}_byond_linux.zip" -o $HOME/byond-zips-cache/linux/$FULL_VERSION.zip fi mkdir -p "$HOME/BYOND" cd "$HOME/BYOND" @@ -509,13 +502,6 @@ jobs: if: always() run: rm -f ${{ env.TGS_TELEMETRY_KEY_FILE }} - - name: Cache BYOND .zips - uses: actions/cache@v4 - id: cache-byond - with: - path: ~/byond-zips-cache - key: byond-zips - - name: Run Unit Tests run: sudo dotnet test --no-build --logger "GitHubActions;summary.includePassedTests=true;summary.includeSkippedTests=true" --filter TestCategory!=RequiresDatabase -c ${{ matrix.configuration }}NoWindows --collect:"XPlat Code Coverage" --settings build/ci.runsettings --results-directory ./TestResults tgstation-server.sln env: @@ -578,13 +564,6 @@ jobs: if: always() run: rm -f ${{ env.TGS_TELEMETRY_KEY_FILE }} - - name: Cache BYOND .zips - uses: actions/cache@v4 - id: cache-byond - with: - path: ~/byond-zips-cache - key: byond-zips - - name: Run Unit Tests run: dotnet test --no-build --logger "GitHubActions;summary.includePassedTests=true;summary.includeSkippedTests=true" --filter TestCategory!=RequiresDatabase -c ${{ matrix.configuration }}NoWix --collect:"XPlat Code Coverage" --settings build/ci.runsettings --results-directory ./TestResults tgstation-server.sln env: @@ -717,13 +696,6 @@ jobs: if: always() run: rm -f ${{ env.TGS_TELEMETRY_KEY_FILE }} - - name: Cache BYOND .zips - uses: actions/cache@v4 - id: cache-byond - with: - path: ~/byond-zips-cache - key: byond-zips - - name: Run Live Tests # Logging here is weird because printing massive amounts of text on Windows runners is SLOW AS SHIT!!! id: live-tests shell: bash @@ -945,13 +917,6 @@ jobs: if: always() run: rm -f ${{ env.TGS_TELEMETRY_KEY_FILE }} - - name: Cache BYOND .zips - uses: actions/cache@v4 - id: cache-byond - with: - path: ~/byond-zips-cache - key: byond-zips - - name: Run Live Tests run: | cd tests/Tgstation.Server.Tests diff --git a/tests/Tgstation.Server.Tests/CachingFileDownloader.cs b/tests/Tgstation.Server.Tests/CachingFileDownloader.cs index de540b606d..8836f71232 100644 --- a/tests/Tgstation.Server.Tests/CachingFileDownloader.cs +++ b/tests/Tgstation.Server.Tests/CachingFileDownloader.cs @@ -87,7 +87,7 @@ namespace Tgstation.Server.Tests }; var url = new Uri( - $"https://www.byond.com/download/build/{version.Version.Major}/{version.Version.Major}.{version.Version.Minor}_byond{(!windows ? "_linux" : string.Empty)}.zip"); + $"https://spacestation13.github.io/byond-builds/{version.Version.Major}/{version.Version.Major}.{version.Version.Minor}_byond{(!windows ? "_linux" : string.Empty)}.zip"); string path = null; if (TestingUtils.RunningInGitHubActions) { From 7d90e0d2d153c2d5bc4ec4463dd15a2b1af725cb Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 23 Jun 2025 21:45:08 -0400 Subject: [PATCH 14/52] Bump main and config versions --- build/Version.props | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/Version.props b/build/Version.props index 394ff58026..80026e5567 100644 --- a/build/Version.props +++ b/build/Version.props @@ -3,8 +3,8 @@ - 6.17.0 - 5.7.0 + 6.18.0 + 5.8.0 10.13.0 0.6.0 7.0.0 From d514474fcecde79c9091a124f9901648025513dc Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 23 Jun 2025 21:45:51 -0400 Subject: [PATCH 15/52] Fix comment --- src/Tgstation.Server.Host/appsettings.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/appsettings.yml b/src/Tgstation.Server.Host/appsettings.yml index c06b79c667..58758dbc25 100644 --- a/src/Tgstation.Server.Host/appsettings.yml +++ b/src/Tgstation.Server.Host/appsettings.yml @@ -17,7 +17,7 @@ General: HostApiDocumentation: false # Make HTTP API documentation available at /api/doc/tgs_api.json and /api/graphql SkipAddingByondFirewallException: false # Windows Only: Prevent running netsh.exe to add a firewall exception for installed engine binaries DeploymentDirectoryCopyTasksPerCore: 100 # Maximum number of concurrent file copy operations PER available CPU core - ByondZipDownloadTemplate: https://spacestation13.github.io/byond-builds/${Major}/${Major}.${Minor}_byond${Linux:_linux}.zip # URL of a plaintext file used to identify the curr + ByondZipDownloadTemplate: https://spacestation13.github.io/byond-builds/${Major}/${Major}.${Minor}_byond${Linux:_linux}.zip # Template for downloading official byond versions OpenDreamGitUrl: https://github.com/OpenDreamProject/OpenDream # The repository to retrieve OpenDream from OpenDreamGitTagPrefix: v # The prefix to the OpenDream semver as tags appear in the git repository OpenDreamSuppressInstallOutput: false # Suppress the dotnet output of creating an OpenDream installation. Known to cause hangs in CI. From 13d68b7c5e66b72510f49a8f6ed004e486cdda9a Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 23 Jun 2025 22:24:39 -0400 Subject: [PATCH 16/52] Fix CI caching URL --- .github/workflows/ci-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 7ad807b51f..f8c957a876 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -186,7 +186,7 @@ jobs: if [[ ! -f $HOME/byond-zips-cache/linux/$FULL_VERSION.zip ]] ; then BYOND_MAJOR=${FULL_VERSION%.*} mkdir -p $HOME/byond-zips-cache/linux - curl "https://spacestation13.github.io/byond-builds/${FULL_VERSION}_byond_linux.zip" -o $HOME/byond-zips-cache/linux/$FULL_VERSION.zip + curl "https://spacestation13.github.io/byond-builds/$BYOND_MAJOR/${FULL_VERSION}_byond_linux.zip" -o $HOME/byond-zips-cache/linux/$FULL_VERSION.zip fi mkdir -p "$HOME/BYOND" cd "$HOME/BYOND" From da2732b2ac01d33c66e1127aff43dd3e83b54c08 Mon Sep 17 00:00:00 2001 From: Kashargul <144968721+Kashargul@users.noreply.github.com> Date: Thu, 26 Jun 2025 18:27:23 +0200 Subject: [PATCH 17/52] libcurl4 needed for 1664 --- build/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build/Dockerfile b/build/Dockerfile index e7cb447c2b..16bedcc9ab 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -59,11 +59,13 @@ RUN export TGS_TELEMETRY_KEY_FILE="../../${TGS_TELEMETRY_KEY_FILE}" \ FROM mcr.microsoft.com/dotnet/aspnet:8.0-bookworm-slim #needed for byond, curl for healthchecks -RUN apt-get update \ +RUN pkg --add-architecture i386 \ + && apt-get update \ && apt-get install -y \ gcc-multilib \ gdb \ curl \ + libcurl4:i386 \ && rm -rf /var/lib/apt/lists/* EXPOSE 5000 From 288a68d5fa7c519110c2de4fa9c6a65307034551 Mon Sep 17 00:00:00 2001 From: Kashargul <144968721+Kashargul@users.noreply.github.com> Date: Thu, 26 Jun 2025 18:50:19 +0200 Subject: [PATCH 18/52] build deps --- build/package/deb/debian/control | 1 + 1 file changed, 1 insertion(+) diff --git a/build/package/deb/debian/control b/build/package/deb/debian/control index 67356359b6..7ae12387a4 100644 --- a/build/package/deb/debian/control +++ b/build/package/deb/debian/control @@ -21,6 +21,7 @@ Depends: libstdc++6:i386 [amd64], libstdc++6 [i386], gcc-multilib [amd64], + libcurl4 [i386], Recommends: libsystemd0, gdb, From ce8248bb7606a617236aa2798c8daeac1d4a017b Mon Sep 17 00:00:00 2001 From: Kashargul <144968721+Kashargul@users.noreply.github.com> Date: Thu, 26 Jun 2025 23:55:42 +0200 Subject: [PATCH 19/52] Update build/Dockerfile Co-authored-by: Jordan Dominion --- build/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/Dockerfile b/build/Dockerfile index 16bedcc9ab..8a3a1d24f7 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -59,7 +59,7 @@ RUN export TGS_TELEMETRY_KEY_FILE="../../${TGS_TELEMETRY_KEY_FILE}" \ FROM mcr.microsoft.com/dotnet/aspnet:8.0-bookworm-slim #needed for byond, curl for healthchecks -RUN pkg --add-architecture i386 \ +RUN dpkg --add-architecture i386 \ && apt-get update \ && apt-get install -y \ gcc-multilib \ From a1a7ecbefd8dcb5154ddc9ab1b176acf4fc70699 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Jul 2025 11:12:17 +0000 Subject: [PATCH 20/52] Bump StrawberryShake.Server from 15.1.3 to 15.1.7 --- updated-dependencies: - dependency-name: StrawberryShake.Server dependency-version: 15.1.7 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: hotchocolate ... Signed-off-by: dependabot[bot] --- .../Tgstation.Server.Client.GraphQL.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Client.GraphQL/Tgstation.Server.Client.GraphQL.csproj b/src/Tgstation.Server.Client.GraphQL/Tgstation.Server.Client.GraphQL.csproj index d5b8544c62..9a575c625f 100644 --- a/src/Tgstation.Server.Client.GraphQL/Tgstation.Server.Client.GraphQL.csproj +++ b/src/Tgstation.Server.Client.GraphQL/Tgstation.Server.Client.GraphQL.csproj @@ -10,7 +10,7 @@ - + From 12632a8bb4f83f05121495735a5f67d11d9707f3 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 20 Jul 2025 20:07:42 -0400 Subject: [PATCH 21/52] AAAAA --- .github/CONTRIBUTING.md | 1 + .github/workflows/ci-pipeline.yml | 39 +++++++-- .../Configuration/GeneralConfiguration.cs | 7 +- src/Tgstation.Server.Host/appsettings.yml | 2 +- .../Engine/TestPosixByondInstaller.cs | 12 ++- .../CachingFileDownloader.cs | 25 ++++-- .../Live/Instance/EngineTest.cs | 39 +++------ .../Live/Instance/InstanceTest.cs | 15 ++-- .../Live/LiveTestingServer.cs | 3 +- .../Live/TestLiveServer.cs | 6 +- tests/Tgstation.Server.Tests/TestVersions.cs | 15 +++- tests/Tgstation.Server.Tests/TestingUtils.cs | 84 +++++++++++++++++++ 12 files changed, 191 insertions(+), 57 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index f116b2259d..c2912e0af5 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -50,6 +50,7 @@ You must also have the following environment variables set. To run them more acc - `TGS_TEST_DATABASE_TYPE`: `MySql`, `MariaDB`, `PostgresSql`, or `SqlServer`. - `TGS_TEST_CONNECTION_STRING`: To a valid database connection string. You can use the setup wizard to create one. - (Optional) `TGS_TEST_GITHUB_TOKEN`: A GitHub personal access token with no scopes used to bypass rate limits. +- (Optional) `TGS_TEST_BYOND_ZIP_DOWNLOAD_TEMPLATE`: Template URL for downloading BYOND zip files from a non-official mirror. - (Optional) The following variables are all interdependent, so if one is set they all must be. - `TGS_TEST_DISCORD_TOKEN`: To a valid discord bot token. - `TGS_TEST_DISCORD_CHANNEL`: To a valid discord channel ID that the above bot can access. diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index f8c957a876..9f9c9a5010 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -166,12 +166,12 @@ jobs: sudo apt-get update sudo apt-get install -y -o APT::Immediate-Configure=0 libc6-i386 libstdc++6:i386 libgcc-s1:i386 - - name: Setup BYOND Cache if Necessary and Install + - name: Evaluate EDGE BYOND version + id: edge_version_evaluation run: | - echo "Setting up BYOND." FULL_VERSION=${{ matrix.byond }} if [[ "$FULL_VERSION" = "EDGE" ]] ; then - VERSIONS=$(curl https://spacestation13.github.io/byond-builds/version.txt) + VERSIONS=$(curl https://www.byond.com/download/version.txt) FULL_VERSION=$(echo "$VERSIONS" | tail -n1) echo "EDGE version evaluated to $FULL_VERSION" @@ -183,14 +183,27 @@ jobs: FULL_VERSION=${bad_linux_releases[$FULL_VERSION]} fi fi - if [[ ! -f $HOME/byond-zips-cache/linux/$FULL_VERSION.zip ]] ; then + run: echo "EVALUATED_EDGE_VERSION=$FULL_VERSION" >> $GITHUB_OUTPUT + + - name: Cache BYOND .zips + uses: actions/cache@v4 + id: cache-byond + with: + path: ~/byond-zips-cache/linux/${{ steps.edge_version_evaluation.outputs.EVALUATED_EDGE_VERSION }} + key: byond-zips-linux-${{ steps.edge_version_evaluation.outputs.EVALUATED_EDGE_VERSION }} + + - name: Setup BYOND Cache if Necessary and Install + run: | + echo "Setting up BYOND." + FULL_VERSION=${{ steps.edge_version_evaluation.outputs.EVALUATED_EDGE_VERSION }} + if [[ ! -f $HOME/byond-zips-cache/linux/$FULL_VERSION/$FULL_VERSION.zip ]] ; then BYOND_MAJOR=${FULL_VERSION%.*} mkdir -p $HOME/byond-zips-cache/linux - curl "https://spacestation13.github.io/byond-builds/$BYOND_MAJOR/${FULL_VERSION}_byond_linux.zip" -o $HOME/byond-zips-cache/linux/$FULL_VERSION.zip + curl "https://www.byond.com/download/$BYOND_MAJOR/${FULL_VERSION}_byond_linux.zip" -o $HOME/byond-zips-cache/linux/$FULL_VERSION/$FULL_VERSION.zip fi mkdir -p "$HOME/BYOND" cd "$HOME/BYOND" - cp $HOME/byond-zips-cache/linux/$FULL_VERSION.zip byond.zip + cp $HOME/byond-zips-cache/linux/$FULL_VERSION/$FULL_VERSION.zip byond.zip unzip byond.zip cd byond make here @@ -612,6 +625,13 @@ jobs: with: node-version: ${{ env.TGS_NODE_VERSION }} + - name: Cache BYOND .zips + uses: actions/cache@v4 + id: cache-byond + with: + path: ~/byond-zips-cache/live/windows + key: byond-zips-windows-live + - name: Set TGS_TEST_DUMP_API_SPEC if: ${{ matrix.configuration == 'Release' && matrix.watchdog-type == 'Advanced' && matrix.database-type == 'SqlServer' }} run: echo "TGS_TEST_DUMP_API_SPEC=yes" >> $Env:GITHUB_ENV @@ -863,6 +883,13 @@ jobs: with: node-version: ${{ env.TGS_NODE_VERSION }} + - name: Cache BYOND .zips + uses: actions/cache@v4 + id: cache-byond + with: + path: ~/byond-zips-cache/live/linux + key: byond-zips-linux-live + - name: Set Sqlite Connection Info if: ${{ matrix.database-type == 'Sqlite' }} run: | diff --git a/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs b/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs index c47282ea80..651bb0bf85 100644 --- a/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs +++ b/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs @@ -31,6 +31,11 @@ namespace Tgstation.Server.Host.Configuration /// public const ushort DefaultApiPort = 5000; + /// + /// Default vale for . + /// + public const string DefaultByondZipDownloadTemplate = "https://www.byond.com/download/build/${Major}/${Major}.${Minor}_byond${Linux:_linux}.zip"; + /// /// The default value for . /// @@ -157,7 +162,7 @@ namespace Tgstation.Server.Host.Configuration /// - $$ will evaluate to a literal $ and not be used for substitutions. /// - Any inapplicable ${xxx} string will be removed. /// - public string ByondZipDownloadTemplate { get; set; } = "https://www.byond.com/download/build/${Major}/${Major}.{Minor}_byond${Linux:_linux}.zip"; + public string ByondZipDownloadTemplate { get; set; } = DefaultByondZipDownloadTemplate; /// /// The prefix to the OpenDream semver as tags appear in the git repository. diff --git a/src/Tgstation.Server.Host/appsettings.yml b/src/Tgstation.Server.Host/appsettings.yml index 58758dbc25..ff244ea7e4 100644 --- a/src/Tgstation.Server.Host/appsettings.yml +++ b/src/Tgstation.Server.Host/appsettings.yml @@ -17,7 +17,7 @@ General: HostApiDocumentation: false # Make HTTP API documentation available at /api/doc/tgs_api.json and /api/graphql SkipAddingByondFirewallException: false # Windows Only: Prevent running netsh.exe to add a firewall exception for installed engine binaries DeploymentDirectoryCopyTasksPerCore: 100 # Maximum number of concurrent file copy operations PER available CPU core - ByondZipDownloadTemplate: https://spacestation13.github.io/byond-builds/${Major}/${Major}.${Minor}_byond${Linux:_linux}.zip # Template for downloading official byond versions + ByondZipDownloadTemplate: https://www.byond.com/download/build/${Major}/${Major}.${Minor}_byond${Linux:_linux}.zip # Template for downloading official byond versions OpenDreamGitUrl: https://github.com/OpenDreamProject/OpenDream # The repository to retrieve OpenDream from OpenDreamGitTagPrefix: v # The prefix to the OpenDream semver as tags appear in the git repository OpenDreamSuppressInstallOutput: false # Suppress the dotnet output of creating an OpenDream installation. Known to cause hangs in CI. diff --git a/tests/Tgstation.Server.Host.Tests/Components/Engine/TestPosixByondInstaller.cs b/tests/Tgstation.Server.Host.Tests/Components/Engine/TestPosixByondInstaller.cs index c6288b6a7c..6dbfb155b4 100644 --- a/tests/Tgstation.Server.Host.Tests/Components/Engine/TestPosixByondInstaller.cs +++ b/tests/Tgstation.Server.Host.Tests/Components/Engine/TestPosixByondInstaller.cs @@ -55,8 +55,14 @@ namespace Tgstation.Server.Host.Components.Engine.Tests var mockPostWriteHandler = new Mock(); var mockLogger = new Mock>(); var mockFileDownloader = new Mock(); - var mockOptions = Mock.Of>(); - var installer = new PosixByondInstaller(mockPostWriteHandler.Object, mockIOManager.Object, mockFileDownloader.Object, mockOptions, mockLogger.Object); + var mockOptions = new Mock>(); + const string TestUrl = "https://chumb.is"; + mockOptions.SetupGet(x => x.CurrentValue).Returns(new GeneralConfiguration + { + ByondZipDownloadTemplate = TestUrl, + }); + + var installer = new PosixByondInstaller(mockPostWriteHandler.Object, mockIOManager.Object, mockFileDownloader.Object, mockOptions.Object, mockLogger.Object); await Assert.ThrowsExceptionAsync(() => installer.DownloadVersion(null, null, default).AsTask()); @@ -64,7 +70,7 @@ namespace Tgstation.Server.Host.Components.Engine.Tests mockFileDownloader .Setup( x => x.DownloadFile( - It.Is(uri => uri == new Uri("https://www.byond.com/download/build/511/511.1385_byond_linux.zip")), + It.Is(uri => uri == new Uri(TestUrl)), null)) .Returns( new BufferedFileStreamProvider( diff --git a/tests/Tgstation.Server.Tests/CachingFileDownloader.cs b/tests/Tgstation.Server.Tests/CachingFileDownloader.cs index 8836f71232..8fccd1234c 100644 --- a/tests/Tgstation.Server.Tests/CachingFileDownloader.cs +++ b/tests/Tgstation.Server.Tests/CachingFileDownloader.cs @@ -12,6 +12,8 @@ using Moq; using Tgstation.Server.Api.Models; using Tgstation.Server.Api.Models.Internal; using Tgstation.Server.Common.Http; +using Tgstation.Server.Host.Components.Engine; +using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.Extensions; using Tgstation.Server.Host.IO; using Tgstation.Server.Host.System; @@ -44,9 +46,9 @@ namespace Tgstation.Server.Tests var logger = loggerFactory.CreateLogger("CachingFileDownloader"); var cfd = new CachingFileDownloader(loggerFactory.CreateLogger()); - var edgeVersion = await EngineTest.GetEdgeVersion(Api.Models.EngineType.Byond, cfd, cancellationToken); - await InitializeByondVersion(logger, edgeVersion.Version, new PlatformIdentifier().IsWindows, cancellationToken); + // this also will inject the edge version + var edgeVersion = await EngineTest.GetEdgeVersion(Api.Models.EngineType.Byond, logger, cfd, cancellationToken); // predownload the target github release update asset var gitHubToken = Environment.GetEnvironmentVariable("TGS_TEST_GITHUB_TOKEN"); @@ -78,7 +80,7 @@ namespace Tgstation.Server.Tests ServiceCollectionExtensions.UseFileDownloader(); } - public static async ValueTask InitializeByondVersion(ILogger logger, Version byondVersion, bool windows, CancellationToken cancellationToken) + public static async ValueTask InitializeByondVersion(ILogger logger, Version byondVersion, bool windows, CancellationToken cancellationToken, string urlCacheOverrideTemplate = null) { var version = new EngineVersion { @@ -86,8 +88,9 @@ namespace Tgstation.Server.Tests Version = byondVersion, }; - var url = new Uri( - $"https://spacestation13.github.io/byond-builds/{version.Version.Major}/{version.Version.Major}.{version.Version.Minor}_byond{(!windows ? "_linux" : string.Empty)}.zip"); + var urlTemplate = TestingUtils.ByondZipDownloadTemplate; + + var url = ByondInstallerBase.GetDownloadZipUrl(byondVersion, urlTemplate, new PlatformIdentifier().IsWindows ? "Windows" : "Linux"); string path = null; if (TestingUtils.RunningInGitHubActions) { @@ -98,13 +101,23 @@ namespace Tgstation.Server.Tests Environment.SpecialFolder.UserProfile, Environment.SpecialFolderOption.DoNotVerify), "byond-zips-cache", + "live", windows ? "windows" : "linux"); path = Path.Combine( dir, + $"{version.Version.Major}.{version.Version.Minor}", $"{version.Version.Major}.{version.Version.Minor}.zip"); } - await (await CacheFile(logger, url, null, path, cancellationToken)).DisposeAsync(); + await (await CacheFile( + logger, + urlCacheOverrideTemplate != null + ? ByondInstallerBase.GetDownloadZipUrl(byondVersion, urlCacheOverrideTemplate, new PlatformIdentifier().IsWindows ? "Windows" : "Linux") + : url, + null, + path, + cancellationToken)) + .DisposeAsync(); } public static void Cleanup() diff --git a/tests/Tgstation.Server.Tests/Live/Instance/EngineTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/EngineTest.cs index ffdaeb63e4..1497054e7d 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/EngineTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/EngineTest.cs @@ -41,13 +41,13 @@ namespace Tgstation.Server.Tests.Live.Instance EngineVersion testVersion; readonly EngineType testEngine = engineType; - public Task Run(CancellationToken cancellationToken, out Task firstInstall) + public Task Run(ILogger logger, CancellationToken cancellationToken, out Task firstInstall) { - firstInstall = RunPartOne(cancellationToken); + firstInstall = RunPartOne(logger, cancellationToken); return RunContinued(firstInstall, cancellationToken); } - public static async ValueTask GetEdgeVersion(EngineType engineType, IFileDownloader fileDownloader, CancellationToken cancellationToken) + public static async ValueTask GetEdgeVersion(EngineType engineType, ILogger logger, IFileDownloader fileDownloader, CancellationToken cancellationToken) { var edgeVersion = edgeVersions[engineType]; @@ -57,26 +57,7 @@ namespace Tgstation.Server.Tests.Live.Instance EngineVersion engineVersion; if (engineType == EngineType.Byond) { - await using var provider = fileDownloader.DownloadFile(new Uri("https://www.byond.com/download/version.txt"), null); - var stream = await provider.GetResult(cancellationToken); - using var reader = new StreamReader(stream, Encoding.UTF8, false, -1, true); - var text = await reader.ReadToEndAsync(cancellationToken); - var splits = text.Split('\n', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries); - - var targetVersion = splits.Last(); - - var badVersionMap = new PlatformIdentifier().IsWindows - ? [] - // linux map also needs updating in CI - : new Dictionary() - { - { "515.1612", "515.1611" } - }; - - badVersionMap.Add("515.1617", "515.1616"); - - if (badVersionMap.TryGetValue(targetVersion, out var remappedVersion)) - targetVersion = remappedVersion; + var targetVersion = await TestingUtils.GetByondEdgeVersion(logger, fileDownloader, cancellationToken); Assert.IsTrue(EngineVersion.TryParse(targetVersion, out engineVersion), $"Bad version: {targetVersion}"); } @@ -112,9 +93,9 @@ namespace Tgstation.Server.Tests.Live.Instance return edgeVersions[engineType] = engineVersion; } - async Task RunPartOne(CancellationToken cancellationToken) + async Task RunPartOne(ILogger logger, CancellationToken cancellationToken) { - testVersion = await GetEdgeVersion(testEngine, fileDownloader, cancellationToken); + testVersion = await GetEdgeVersion(testEngine, logger, fileDownloader, cancellationToken); await TestNoVersion(cancellationToken); await TestInstallNullVersion(cancellationToken); await TestInstallStable(cancellationToken); @@ -284,8 +265,11 @@ namespace Tgstation.Server.Tests.Live.Instance async Task TestCustomInstalls(CancellationToken cancellationToken) { - var generalConfigOptionsMock = new Mock>(); - generalConfigOptionsMock.SetupGet(x => x.Value).Returns(new GeneralConfiguration()); + var generalConfigOptionsMock = new Mock>(); + generalConfigOptionsMock.SetupGet(x => x.CurrentValue).Returns(new GeneralConfiguration + { + ByondZipDownloadTemplate = TestingUtils.ByondZipDownloadTemplate, + }); var sessionConfigOptionsMock = new Mock>(); sessionConfigOptionsMock.SetupGet(x => x.Value).Returns(new SessionConfiguration()); @@ -303,6 +287,7 @@ namespace Tgstation.Server.Tests.Live.Instance Mock.Of(), Mock.Of(), fileDownloader, + generalConfigOptionsMock.Object, Mock.Of>()); using var windowsByondInstaller = byondInstaller as WindowsByondInstaller; diff --git a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs index f9188313d0..087c83a45d 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs @@ -36,6 +36,7 @@ namespace Tgstation.Server.Tests.Live.Instance readonly ushort serverPort = serverPort; public async Task RunTests( + ILogger logger, IInstanceClient instanceClient, ushort dmPort, ushort ddPort, @@ -44,14 +45,14 @@ namespace Tgstation.Server.Tests.Live.Instance bool usingBasicWatchdog, CancellationToken cancellationToken) { - var testVersion = await EngineTest.GetEdgeVersion(EngineType.Byond, fileDownloader, cancellationToken); + var testVersion = await EngineTest.GetEdgeVersion(EngineType.Byond, logger, fileDownloader, cancellationToken); await using var engineTest = new EngineTest(instanceClient.Engine, instanceClient.Jobs, fileDownloader, instanceClient.Metadata, testVersion.Engine.Value); await using var chatTest = new ChatTest(instanceClient.ChatBots, instanceManagerClient, instanceClient.Jobs, instanceClient.Metadata); var configTest = new ConfigurationTest(instanceClient.Configuration, instanceClient.Metadata); await using var repoTest = new RepositoryTest(instanceClient, instanceClient.Repository, instanceClient.Jobs); await using var dmTest = new DeploymentTest(instanceClient, instanceClient.Jobs, dmPort, ddPort, lowPrioDeployment, testVersion); - var byondTask = engineTest.Run(cancellationToken, out var firstInstall); + var byondTask = engineTest.Run(logger, cancellationToken, out var firstInstall); var chatTask = chatTest.RunPreWatchdog(cancellationToken); var repoLongJob = await repoTest.RunLongClone(cancellationToken); @@ -98,12 +99,13 @@ namespace Tgstation.Server.Tests.Live.Instance "OpenDreamRepository"); var odRepoIoManager = new ResolvingIOManager(ioManager, odRepoDir); - var mockOptions = new Mock>(); + var mockOptionsMonitor = new Mock>(); var genConfig = new GeneralConfiguration { OpenDreamGitUrl = openDreamUrl, + ByondZipDownloadTemplate = TestingUtils.ByondZipDownloadTemplate, }; - mockOptions.SetupGet(x => x.Value).Returns(genConfig); + mockOptionsMonitor.SetupGet(x => x.CurrentValue).Returns(genConfig); IEngineInstaller byondInstaller = compatVersion.Engine == EngineType.OpenDream ? new OpenDreamInstaller( @@ -124,20 +126,21 @@ namespace Tgstation.Server.Tests.Live.Instance genConfig), Mock.Of(), Mock.Of(), - mockOptions.Object, + Options.Create(genConfig), Options.Create(new SessionConfiguration())) : new PlatformIdentifier().IsWindows ? new WindowsByondInstaller( Mock.Of(), Mock.Of(), fileDownloader, - Options.Create(genConfig), + mockOptionsMonitor.Object, Options.Create(new SessionConfiguration()), Mock.Of>()) : new PosixByondInstaller( Mock.Of(), Mock.Of(), fileDownloader, + mockOptionsMonitor.Object, Mock.Of>()); using var windowsByondInstaller = byondInstaller as WindowsByondInstaller; diff --git a/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs b/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs index ac4f519fa6..e1d74d64a5 100644 --- a/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs +++ b/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs @@ -156,7 +156,8 @@ namespace Tgstation.Server.Tests.Live $"Security:TokenExpiryMinutes=120", // timeouts are useless for us $"General:OpenDreamSuppressInstallOutput={TestingUtils.RunningInGitHubActions}", "Telemetry:DisableVersionReporting=true", - $"General:PrometheusPort={port}" + $"General:PrometheusPort={port}", + $"General:ByondZipDownloadTemplate={TestingUtils.ByondZipDownloadTemplate}" }; if (MultiServerClient.UseGraphQL) diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index 8df9a3d612..7eeec04af3 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -1421,6 +1421,7 @@ namespace Tgstation.Server.Tests.Live await Task.Yield(); InstanceManager GetInstanceManager() => ((Host.Server)server.RealServer).Host.Services.GetRequiredService(); + ILogger GetLogger() => ((Host.Server)server.RealServer).Host.Services.GetRequiredService>(); // main run var serverTask = server.Run(cancellationToken).AsTask(); @@ -1590,7 +1591,7 @@ namespace Tgstation.Server.Tests.Live var testSerialized = TestingUtils.RunningInGitHubActions; // they only have 2 cores, can't handle intense parallelization async Task ODCompatTests() { - var edgeODVersionTask = EngineTest.GetEdgeVersion(EngineType.OpenDream, fileDownloader, cancellationToken); + var edgeODVersionTask = EngineTest.GetEdgeVersion(EngineType.OpenDream, GetLogger(), fileDownloader, cancellationToken); var ex = await Assert.ThrowsExceptionAsync( () => InstanceTest.DownloadEngineVersion( @@ -1649,6 +1650,7 @@ namespace Tgstation.Server.Tests.Live await FailFast( instanceTest .RunTests( + GetLogger(), instanceClient, mainDMPort.Value, mainDDPort.Value, @@ -1881,7 +1883,7 @@ namespace Tgstation.Server.Tests.Live preStartupTime = DateTimeOffset.UtcNow; serverTask = server.Run(cancellationToken).AsTask(); long expectedCompileJobId, expectedStaged; - var edgeVersion = await EngineTest.GetEdgeVersion(EngineType.Byond, fileDownloader, cancellationToken); + var edgeVersion = await EngineTest.GetEdgeVersion(EngineType.Byond, GetLogger(), fileDownloader, cancellationToken); await using (var adminClient = await CreateAdminClient(server.ApiUrl, cancellationToken)) { var restAdminClient = adminClient.RestClient; diff --git a/tests/Tgstation.Server.Tests/TestVersions.cs b/tests/Tgstation.Server.Tests/TestVersions.cs index a0809b78f6..2d9acd7974 100644 --- a/tests/Tgstation.Server.Tests/TestVersions.cs +++ b/tests/Tgstation.Server.Tests/TestVersions.cs @@ -103,10 +103,14 @@ namespace Tgstation.Server.Tests } [TestMethod] + [TestCategory("RequiresDatabase")] public async Task TestDDExeByondVersion() { - var mockGeneralConfigurationOptions = new Mock>(); - mockGeneralConfigurationOptions.SetupGet(x => x.Value).Returns(new GeneralConfiguration()); + var mockGeneralConfigurationOptions = new Mock>(); + mockGeneralConfigurationOptions.SetupGet(x => x.CurrentValue).Returns(new GeneralConfiguration + { + ByondZipDownloadTemplate = TestingUtils.ByondZipDownloadTemplate, + }); var mockSessionConfigurationOptions = new Mock>(); mockSessionConfigurationOptions.SetupGet(x => x.Value).Returns(new SessionConfiguration()); @@ -165,12 +169,14 @@ namespace Tgstation.Server.Tests static Version MapThreadsVersion() => (Version)typeof(ByondInstallerBase).GetField("MapThreadsVersion", BindingFlags.Static | BindingFlags.NonPublic).GetValue(null) ?? throw new InvalidOperationException("Couldn't find MapThreadsVersion"); [TestMethod] + [TestCategory("RequiresDatabase")] public async Task TestMapThreadsByondVersion() { - var mockGeneralConfigurationOptions = new Mock>(); - mockGeneralConfigurationOptions.SetupGet(x => x.Value).Returns(new GeneralConfiguration + var mockGeneralConfigurationOptions = new Mock>(); + mockGeneralConfigurationOptions.SetupGet(x => x.CurrentValue).Returns(new GeneralConfiguration { SkipAddingByondFirewallException = true, + ByondZipDownloadTemplate = TestingUtils.ByondZipDownloadTemplate, }); var mockSessionConfigurationOptions = new Mock>(); mockSessionConfigurationOptions.SetupGet(x => x.Value).Returns(new SessionConfiguration()); @@ -209,6 +215,7 @@ namespace Tgstation.Server.Tests new PosixPostWriteHandler(loggerFactory.CreateLogger()), new DefaultIOManager(), fileDownloader, + mockGeneralConfigurationOptions.Object, loggerFactory.CreateLogger()); using var disposable = byondInstaller as IDisposable; diff --git a/tests/Tgstation.Server.Tests/TestingUtils.cs b/tests/Tgstation.Server.Tests/TestingUtils.cs index 3167d2c3d3..9cc11f7966 100644 --- a/tests/Tgstation.Server.Tests/TestingUtils.cs +++ b/tests/Tgstation.Server.Tests/TestingUtils.cs @@ -1,16 +1,23 @@ using System; +using System.Collections.Generic; using System.IO; using System.IO.Compression; +using System.Linq; +using System.Net.Http; using System.Reflection; +using System.Text; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; + using Moq; using Tgstation.Server.Host.Components.Engine; +using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.IO; +using Tgstation.Server.Host.System; namespace Tgstation.Server.Tests { @@ -70,5 +77,82 @@ namespace Tgstation.Server.Tests await new DefaultIOManager().DeleteDirectory(tempFolder, cancellationToken); } } + + static string byondZipDownloadTemplate; + + public static string ByondZipDownloadTemplate + { + get + { + if (byondZipDownloadTemplate == null) + { + var envvar = Environment.GetEnvironmentVariable("TGS_TEST_BYOND_ZIP_DOWNLOAD_TEMPLATE"); + if (envvar != null) + byondZipDownloadTemplate = envvar; + else + byondZipDownloadTemplate = GeneralConfiguration.DefaultByondZipDownloadTemplate; + } + + return byondZipDownloadTemplate; + } + } + + static string edgeVersion = null; + public static async ValueTask GetByondEdgeVersion(ILogger logger, IFileDownloader fileDownloader, CancellationToken cancellationToken) + { + if (edgeVersion != null) + return edgeVersion; + + async ValueTask GetVersionFromResponse(string versionTxt) + { + await using var provider = fileDownloader.DownloadFile(new Uri(versionTxt), null); + var stream = await provider.GetResult(cancellationToken); + using var reader = new StreamReader(stream, Encoding.UTF8, false, -1, true); + var text = await reader.ReadToEndAsync(cancellationToken); + var splits = text.Split('\n', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries); + + var targetVersion = splits.Last(); + + var badVersionMap = new PlatformIdentifier().IsWindows + ? [] + // linux map also needs updating in CI + : new Dictionary() + { + { "515.1612", "515.1611" } + }; + + badVersionMap.Add("515.1617", "515.1616"); + + if (badVersionMap.TryGetValue(targetVersion, out var remappedVersion)) + targetVersion = remappedVersion; + + return targetVersion; + } + + var mirroredVersionTxt = Environment.GetEnvironmentVariable("TGS_TEST_BYOND_MIRROR_VERSION_TXT"); + try + { + // always check byond.com first for latest up-to-date, mirror should ALWAYS have stable versions + const string DefaultMirror = "https://www.byond.com/download/version.txt"; + edgeVersion = await GetVersionFromResponse(DefaultMirror); + + logger.LogInformation("Downloading edge version from BYOND.com"); + + // if we got the result from byond.com, make sure the cache grabs the zip from there as well + await CachingFileDownloader.InitializeByondVersion(logger, Version.Parse(edgeVersion), new PlatformIdentifier().IsWindows, cancellationToken, GeneralConfiguration.DefaultByondZipDownloadTemplate); + } + catch (Exception ex) + { + logger.LogWarning(ex, "Cannot download zip from byond.com!"); + if (ByondZipDownloadTemplate == GeneralConfiguration.DefaultByondZipDownloadTemplate || mirroredVersionTxt == null) + throw; + + // fall back to the mirrored version.txt + await using var provider = fileDownloader.DownloadFile(new Uri(mirroredVersionTxt), null); + edgeVersion = await GetVersionFromResponse(mirroredVersionTxt); + } + + return edgeVersion; + } } } From b2c0d697d5e9b4bc1576d9abf5798b3990824c9c Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 20 Jul 2025 20:39:55 -0400 Subject: [PATCH 22/52] Cleanup usage of System.IO types to go through IIOManager --- .../Components/Deployment/DmbFactory.cs | 3 +- .../Components/Deployment/DmbProvider.cs | 17 ++++- .../Deployment/HardLinkDmbProvider.cs | 4 +- .../Engine/ZipStreamEngineInstallationData.cs | 2 +- .../Components/InstanceManager.cs | 2 +- .../Components/Repository/Repository.cs | 5 +- .../Components/StaticFiles/Configuration.cs | 4 +- .../Configuration/GeneralConfiguration.cs | 7 +- .../IO/DefaultIOManager.cs | 29 ++++++-- src/Tgstation.Server.Host/IO/IIOManager.cs | 28 +++++++- .../IO/ResolvingIOManager.cs | 3 +- .../Setup/SetupWizard.cs | 2 +- .../System/PosixProcessFeatures.cs | 66 +++++++++---------- .../System/WindowsProcessFeatures.cs | 1 + 14 files changed, 111 insertions(+), 62 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs b/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs index 8c523d97d5..a9cd217f18 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; -using System.IO; using System.Linq; using System.Runtime.CompilerServices; using System.Text; @@ -431,7 +430,7 @@ namespace Tgstation.Server.Host.Components.Deployment // Don't dispose it logger.LogDebug("Creating legacy two folder .dmb provider targeting {aDirName} directory...", LegacyADirectoryName); #pragma warning disable CA2000 // Dispose objects before losing scope (false positive) - newProvider = new DmbProvider(compileJob, engineVersion, ioManager, new DisposeInvoker(CleanupAction), Path.DirectorySeparatorChar + LegacyADirectoryName); + newProvider = new DmbProvider(compileJob, engineVersion, ioManager, new DisposeInvoker(CleanupAction), LegacyADirectoryName); #pragma warning restore CA2000 // Dispose objects before losing scope } diff --git a/src/Tgstation.Server.Host/Components/Deployment/DmbProvider.cs b/src/Tgstation.Server.Host/Components/Deployment/DmbProvider.cs index be47aeb6b3..4da03c795e 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/DmbProvider.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/DmbProvider.cs @@ -12,7 +12,18 @@ namespace Tgstation.Server.Host.Components.Deployment sealed class DmbProvider : DmbProviderBase, IDmbProvider { /// - public override string Directory => ioManager.ResolvePath(CompileJob.DirectoryName!.Value.ToString() + directoryAppend); + public override string Directory + { + get + { + var stringifiedCompileJobDirectory = CompileJob.DirectoryName!.Value.ToString(); + + if (directoryAppend != null) + stringifiedCompileJobDirectory = ioManager.ConcatPath(stringifiedCompileJobDirectory, directoryAppend); + + return ioManager.ResolvePath(stringifiedCompileJobDirectory); + } + } /// public override Models.CompileJob CompileJob { get; } @@ -28,7 +39,7 @@ namespace Tgstation.Server.Host.Components.Deployment /// /// Extra path to add to the end of . /// - readonly string directoryAppend; + readonly string? directoryAppend; /// /// The to run when is called. @@ -49,7 +60,7 @@ namespace Tgstation.Server.Host.Components.Deployment EngineVersion = engineVersion ?? throw new ArgumentNullException(nameof(engineVersion)); this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); this.onDispose = onDispose ?? throw new ArgumentNullException(nameof(onDispose)); - this.directoryAppend = directoryAppend ?? String.Empty; + this.directoryAppend = directoryAppend; } /// diff --git a/src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs b/src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs index ed6e72b4b6..477e5f06c4 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs @@ -239,12 +239,12 @@ namespace Tgstation.Server.Host.Components.Deployment /// I genuinely don't know how this will work with symlinked files. Waiting for the issue report I guess. IEnumerable MirrorDirectoryImpl(string src, string dest, SemaphoreSlim? semaphore, DreamDaemonSecurity securityLevel, CancellationToken cancellationToken) { - var dir = new DirectoryInfo(src); + var dir = IOManager.DirectoryInfo(src); Task? subdirCreationTask = null; var dreamDaemonWillAcceptOutOfDirectorySymlinks = securityLevel == DreamDaemonSecurity.Trusted; foreach (var subDirectory in dir.EnumerateDirectories()) { - var mirroredName = Path.Combine(dest, subDirectory.Name); + var mirroredName = IOManager.ConcatPath(dest, subDirectory.Name); // check if we are a symbolic link if (subDirectory.Attributes.HasFlag(FileAttributes.ReparsePoint)) diff --git a/src/Tgstation.Server.Host/Components/Engine/ZipStreamEngineInstallationData.cs b/src/Tgstation.Server.Host/Components/Engine/ZipStreamEngineInstallationData.cs index 292b725981..116689f8de 100644 --- a/src/Tgstation.Server.Host/Components/Engine/ZipStreamEngineInstallationData.cs +++ b/src/Tgstation.Server.Host/Components/Engine/ZipStreamEngineInstallationData.cs @@ -18,7 +18,7 @@ namespace Tgstation.Server.Host.Components.Engine readonly IIOManager ioManager; /// - /// The containing the zip data of the engine. + /// The containing the zip data of the engine. /// readonly Stream zipStream; diff --git a/src/Tgstation.Server.Host/Components/InstanceManager.cs b/src/Tgstation.Server.Host/Components/InstanceManager.cs index 1ded8b3dad..e1fa9b8edd 100644 --- a/src/Tgstation.Server.Host/Components/InstanceManager.cs +++ b/src/Tgstation.Server.Host/Components/InstanceManager.cs @@ -702,7 +702,7 @@ namespace Tgstation.Server.Host.Components { logger.LogDebug("Running as user: {username}", Environment.UserName); - generalConfiguration.CheckCompatibility(logger); + generalConfiguration.CheckCompatibility(logger, ioManager); using (var systemIdentity = systemIdentityFactory.GetCurrent()) { diff --git a/src/Tgstation.Server.Host/Components/Repository/Repository.cs b/src/Tgstation.Server.Host/Components/Repository/Repository.cs index b62cfbaa50..636321b5c7 100644 --- a/src/Tgstation.Server.Host/Components/Repository/Repository.cs +++ b/src/Tgstation.Server.Host/Components/Repository/Repository.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.Globalization; -using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -1099,8 +1098,8 @@ namespace Tgstation.Server.Host.Components.Repository => ioManager.GetDirectoryName(libGitRepo .Info .Path - .TrimEnd(Path.DirectorySeparatorChar) - .TrimEnd(Path.AltDirectorySeparatorChar)); + .TrimEnd(ioManager.DirectorySeparatorChar) + .TrimEnd(ioManager.AltDirectorySeparatorChar)); /// /// Recusively update all s in the . diff --git a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs index f5d320ec2d..675187f464 100644 --- a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs +++ b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs @@ -333,7 +333,7 @@ namespace Tgstation.Server.Host.Components.StaticFiles }, async cancellationToken => { - FileStream? result = null; + Stream? result = null; void GetFileStream() { result = ioManager.GetFileStream(path, false); @@ -771,7 +771,7 @@ namespace Tgstation.Server.Host.Components.StaticFiles var nullOrEmptyCheck = String.IsNullOrEmpty(configurationRelativePath); if (nullOrEmptyCheck) configurationRelativePath = DefaultIOManager.CurrentDirectory; - if (configurationRelativePath![0] == Path.DirectorySeparatorChar || configurationRelativePath[0] == Path.AltDirectorySeparatorChar) + if (configurationRelativePath![0] == ioManager.DirectorySeparatorChar || configurationRelativePath[0] == ioManager.AltDirectorySeparatorChar) configurationRelativePath = DefaultIOManager.CurrentDirectory + configurationRelativePath; var resolved = ioManager.ResolvePath(configurationRelativePath); var local = !nullOrEmptyCheck ? ioManager.ResolvePath() : null; diff --git a/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs b/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs index 3ba8b3e878..0bcff6ac96 100644 --- a/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs +++ b/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.IO; using System.Linq; using Microsoft.Extensions.Logging; @@ -9,6 +8,7 @@ using Newtonsoft.Json.Converters; using Tgstation.Server.Api.Models.Internal; using Tgstation.Server.Host.Extensions; +using Tgstation.Server.Host.IO; using Tgstation.Server.Host.Properties; using Tgstation.Server.Host.Setup; @@ -178,7 +178,8 @@ namespace Tgstation.Server.Host.Configuration /// Validates the current 's compatibility and provides migration instructions. /// /// The to use. - public void CheckCompatibility(ILogger logger) + /// The to use. + public void CheckCompatibility(ILogger logger, IIOManager ioManager) { ArgumentNullException.ThrowIfNull(logger); @@ -204,7 +205,7 @@ namespace Tgstation.Server.Host.Configuration if (ByondTopicTimeout <= 1000) logger.LogWarning("The timeout for sending BYOND topics is very low ({ms}ms). Topic calls may fail to complete at all!", ByondTopicTimeout); - if (AdditionalEventScriptsDirectories?.Any(path => !Path.IsPathRooted(path)) == true) + if (AdditionalEventScriptsDirectories?.Any(path => !ioManager.IsPathRooted(path)) == true) logger.LogWarning($"Config option \"{nameof(AdditionalEventScriptsDirectories)}\" contains non-rooted paths. These will be evaluated relative to each instances \"Configuration\" directory!"); } } diff --git a/src/Tgstation.Server.Host/IO/DefaultIOManager.cs b/src/Tgstation.Server.Host/IO/DefaultIOManager.cs index b392bb7654..2536b43ae6 100644 --- a/src/Tgstation.Server.Host/IO/DefaultIOManager.cs +++ b/src/Tgstation.Server.Host/IO/DefaultIOManager.cs @@ -30,6 +30,12 @@ namespace Tgstation.Server.Host.IO /// public const TaskCreationOptions BlockingTaskCreationOptions = TaskCreationOptions.None; + /// + public char DirectorySeparatorChar => Path.DirectorySeparatorChar; + + /// + public char AltDirectorySeparatorChar => Path.AltDirectorySeparatorChar; + /// /// Recursively empty a directory. /// @@ -350,13 +356,14 @@ namespace Tgstation.Server.Host.IO TaskScheduler.Current); /// - public FileStream GetFileStream(string path, bool shareWrite) => new( - ResolvePath(path), - FileMode.Open, - FileAccess.Read, - FileShare.Read | FileShare.Delete | (shareWrite ? FileShare.Write : FileShare.None), - DefaultBufferSize, - true); + public Stream GetFileStream(string path, bool shareWrite) + => new FileStream( + ResolvePath(path), + FileMode.Open, + FileAccess.Read, + FileShare.Read | FileShare.Delete | (shareWrite ? FileShare.Write : FileShare.None), + DefaultBufferSize, + true); /// public Task PathIsChildOf(string parentPath, string childPath, CancellationToken cancellationToken) => Task.Factory.StartNew( @@ -385,6 +392,14 @@ namespace Tgstation.Server.Host.IO BlockingTaskCreationOptions, TaskScheduler.Current); + /// + public DirectoryInfo DirectoryInfo(string path) + => new(ResolvePath(path)); // Consider async + + /// + public bool IsPathRooted(string path) + => Path.IsPathRooted(path); + /// /// Copies a directory from to . /// diff --git a/src/Tgstation.Server.Host/IO/IIOManager.cs b/src/Tgstation.Server.Host/IO/IIOManager.cs index 13a29310e1..7bba4f0705 100644 --- a/src/Tgstation.Server.Host/IO/IIOManager.cs +++ b/src/Tgstation.Server.Host/IO/IIOManager.cs @@ -11,6 +11,16 @@ namespace Tgstation.Server.Host.IO /// public interface IIOManager { + /// + /// Gets the primary directory separator character. + /// + char DirectorySeparatorChar { get; } + + /// + /// Gets the alternative directory separator character. + /// + char AltDirectorySeparatorChar { get; } + /// /// Retrieve the full path of the current working directory. /// @@ -234,8 +244,22 @@ namespace Tgstation.Server.Host.IO /// /// The path of the file. /// If should be used. - /// The of the file. + /// The of the file. /// This function is sychronous. - FileStream GetFileStream(string path, bool shareWrite); + Stream GetFileStream(string path, bool shareWrite); + + /// + /// Gets a for the given . + /// + /// The path to get for. + /// A new . + DirectoryInfo DirectoryInfo(string path); + + /// + /// Check if a given is at the root level of the filesystem. + /// + /// The path to check. + /// if the path is rooted, otherwise. + bool IsPathRooted(string path); } } diff --git a/src/Tgstation.Server.Host/IO/ResolvingIOManager.cs b/src/Tgstation.Server.Host/IO/ResolvingIOManager.cs index 3b86b92c36..f05ce162e5 100644 --- a/src/Tgstation.Server.Host/IO/ResolvingIOManager.cs +++ b/src/Tgstation.Server.Host/IO/ResolvingIOManager.cs @@ -1,5 +1,4 @@ using System; -using System.IO; namespace Tgstation.Server.Host.IO { @@ -29,7 +28,7 @@ namespace Tgstation.Server.Host.IO /// public override string ResolvePath(string path) { - if (!Path.IsPathRooted(path)) + if (!IsPathRooted(path)) return base.ResolvePath(ConcatPath(subdirectory, path)); return path; } diff --git a/src/Tgstation.Server.Host/Setup/SetupWizard.cs b/src/Tgstation.Server.Host/Setup/SetupWizard.cs index 32e82877e5..ea1ba73d85 100644 --- a/src/Tgstation.Server.Host/Setup/SetupWizard.cs +++ b/src/Tgstation.Server.Host/Setup/SetupWizard.cs @@ -304,7 +304,7 @@ namespace Tgstation.Server.Host.Setup /// A resulting in the SQLite database path to store in the configuration. async ValueTask ValidateNonExistantSqliteDBName(string databaseName, CancellationToken cancellationToken) { - var dbPathIsRooted = Path.IsPathRooted(databaseName); + var dbPathIsRooted = ioManager.IsPathRooted(databaseName); var resolvedPath = ioManager.ResolvePath( dbPathIsRooted ? databaseName diff --git a/src/Tgstation.Server.Host/System/PosixProcessFeatures.cs b/src/Tgstation.Server.Host/System/PosixProcessFeatures.cs index 12a0164a1f..6477dc05d5 100644 --- a/src/Tgstation.Server.Host/System/PosixProcessFeatures.cs +++ b/src/Tgstation.Server.Host/System/PosixProcessFeatures.cs @@ -64,39 +64,6 @@ namespace Tgstation.Server.Host.System this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); } - /// - /// Gets potential paths to the gcore executable. - /// - /// The potential paths to the gcore executable. - static IEnumerable GetPotentialGCorePaths() - { - var enviromentPath = Environment.GetEnvironmentVariable("PATH"); - IEnumerable enumerator; - if (enviromentPath == null) - enumerator = Enumerable.Empty(); - else - { - var paths = enviromentPath.Split(';'); - enumerator = paths - .Select(x => x.Split(':')) - .SelectMany(x => x); - } - - var exeName = "gcore"; - - enumerator = enumerator - .Concat(new List(2) - { - "/usr/bin", - "/usr/share/bin", - "/bin", - }); - - enumerator = enumerator.Select(x => Path.Combine(x, exeName)); - - return enumerator; - } - /// public void ResumeProcess(global::System.Diagnostics.Process process) { @@ -255,5 +222,38 @@ namespace Tgstation.Server.Host.System Encoding.UTF8.GetBytes(adjustedValue.ToString(CultureInfo.InvariantCulture)), cancellationToken); } + + /// + /// Gets potential paths to the gcore executable. + /// + /// The potential paths to the gcore executable. + IEnumerable GetPotentialGCorePaths() + { + var enviromentPath = Environment.GetEnvironmentVariable("PATH"); + IEnumerable enumerator; + if (enviromentPath == null) + enumerator = Enumerable.Empty(); + else + { + var paths = enviromentPath.Split(';'); + enumerator = paths + .Select(x => x.Split(':')) + .SelectMany(x => x); + } + + var exeName = "gcore"; + + enumerator = enumerator + .Concat(new List(2) + { + "/usr/bin", + "/usr/share/bin", + "/bin", + }); + + enumerator = enumerator.Select(x => ioManager.ConcatPath(x, exeName)); + + return enumerator; + } } } diff --git a/src/Tgstation.Server.Host/System/WindowsProcessFeatures.cs b/src/Tgstation.Server.Host/System/WindowsProcessFeatures.cs index 21855c0062..a048736a22 100644 --- a/src/Tgstation.Server.Host/System/WindowsProcessFeatures.cs +++ b/src/Tgstation.Server.Host/System/WindowsProcessFeatures.cs @@ -145,6 +145,7 @@ namespace Tgstation.Server.Host.System throw new JobException(ErrorCode.GameServerOffline, ex); } + // Windows API so has to be a real FS await using var fileStream = new FileStream(outputFile, FileMode.CreateNew); await Task.Factory.StartNew( From 3b1473c7125659cc8bf34357fded57e60d112ee7 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 20 Jul 2025 21:46:53 -0400 Subject: [PATCH 23/52] Improve how `ResolvingIOManager`s are constructed --- .../Components/Engine/ByondInstallerBase.cs | 2 +- .../Components/Engine/OpenDreamInstaller.cs | 2 +- .../Components/InstanceFactory.cs | 12 ++++++------ src/Tgstation.Server.Host/Core/Application.cs | 3 +-- src/Tgstation.Server.Host/IO/DefaultIOManager.cs | 11 +++++++++++ src/Tgstation.Server.Host/IO/IIOManager.cs | 7 +++++++ src/Tgstation.Server.Host/IO/ResolvingIOManager.cs | 8 ++------ .../Components/StaticFiles/TestConfiguration.cs | 2 +- .../Live/Instance/InstanceTest.cs | 2 +- tests/Tgstation.Server.Tests/TestRepository.cs | 3 +-- 10 files changed, 32 insertions(+), 20 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Engine/ByondInstallerBase.cs b/src/Tgstation.Server.Host/Components/Engine/ByondInstallerBase.cs index e02c109e6b..d0ddb3a506 100644 --- a/src/Tgstation.Server.Host/Components/Engine/ByondInstallerBase.cs +++ b/src/Tgstation.Server.Host/Components/Engine/ByondInstallerBase.cs @@ -78,7 +78,7 @@ namespace Tgstation.Server.Host.Components.Engine { CheckVersionValidity(version); - var installationIOManager = new ResolvingIOManager(IOManager, path); + var installationIOManager = IOManager.CreateResolverForSubdirectory(path); var supportsMapThreads = version.Version >= MapThreadsVersion; return ValueTask.FromResult( diff --git a/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs b/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs index ad2fb20e95..87ad2cd721 100644 --- a/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs +++ b/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs @@ -126,7 +126,7 @@ namespace Tgstation.Server.Host.Components.Engine var dotnetPath = (await DotnetHelper.GetDotnetPath(platformIdentifier, IOManager, cancellationToken)) ?? throw new JobException("Failed to find dotnet path!"); return new OpenDreamInstallation( - new ResolvingIOManager(IOManager, path), + IOManager.CreateResolverForSubdirectory(path), asyncDelayer, httpClientFactory, dotnetPath, diff --git a/src/Tgstation.Server.Host/Components/InstanceFactory.cs b/src/Tgstation.Server.Host/Components/InstanceFactory.cs index c7d08ad4b1..04994acec5 100644 --- a/src/Tgstation.Server.Host/Components/InstanceFactory.cs +++ b/src/Tgstation.Server.Host/Components/InstanceFactory.cs @@ -164,7 +164,7 @@ namespace Tgstation.Server.Host.Components /// /// The instance's . /// The for the instance's "Game" directory. - static ResolvingIOManager CreateGameIOManager(IIOManager instanceIOManager) => new(instanceIOManager, "Game"); + static IIOManager CreateGameIOManager(IIOManager instanceIOManager) => instanceIOManager.CreateResolverForSubdirectory("Game"); #pragma warning disable CA1502 // TODO: Decomplexify /// @@ -270,11 +270,11 @@ namespace Tgstation.Server.Host.Components var instanceIoManager = CreateInstanceIOManager(metadata); // various other ioManagers - var repoIoManager = new ResolvingIOManager(instanceIoManager, "Repository"); - var byondIOManager = new ResolvingIOManager(instanceIoManager, "Byond"); + var repoIoManager = instanceIoManager.CreateResolverForSubdirectory("Repository"); + var byondIOManager = instanceIoManager.CreateResolverForSubdirectory("Byond"); var gameIoManager = CreateGameIOManager(instanceIoManager); - var diagnosticsIOManager = new ResolvingIOManager(instanceIoManager, "Diagnostics"); - var configurationIoManager = new ResolvingIOManager(instanceIoManager, "Configuration"); + var diagnosticsIOManager = instanceIoManager.CreateResolverForSubdirectory("Diagnostics"); + var configurationIoManager = instanceIoManager.CreateResolverForSubdirectory("Configuration"); var metricFactory = this.metricFactory.WithLabels( new Dictionary @@ -441,6 +441,6 @@ namespace Tgstation.Server.Host.Components /// /// The . /// The for the . - ResolvingIOManager CreateInstanceIOManager(Models.Instance metadata) => new(ioManager, metadata.Path!); + IIOManager CreateInstanceIOManager(Models.Instance metadata) => ioManager.CreateResolverForSubdirectory(metadata.Path!); } } diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index a947eeb720..e2d1d75dbf 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -485,8 +485,7 @@ namespace Tgstation.Server.Host.Core services => services .GetRequiredService() .CreateRepositoryManager( - new ResolvingIOManager( - services.GetRequiredService(), + services.GetRequiredService().CreateResolverForSubdirectory( openDreamRepositoryDirectory), new NoopEventConsumer())); diff --git a/src/Tgstation.Server.Host/IO/DefaultIOManager.cs b/src/Tgstation.Server.Host/IO/DefaultIOManager.cs index 2536b43ae6..38d63468c7 100644 --- a/src/Tgstation.Server.Host/IO/DefaultIOManager.cs +++ b/src/Tgstation.Server.Host/IO/DefaultIOManager.cs @@ -400,6 +400,17 @@ namespace Tgstation.Server.Host.IO public bool IsPathRooted(string path) => Path.IsPathRooted(path); + /// + public IIOManager CreateResolverForSubdirectory(string subdirectoryPath) + { + ArgumentNullException.ThrowIfNull(subdirectoryPath); + + return new ResolvingIOManager( + ConcatPath( + ResolvePath(), + subdirectoryPath)); + } + /// /// Copies a directory from to . /// diff --git a/src/Tgstation.Server.Host/IO/IIOManager.cs b/src/Tgstation.Server.Host/IO/IIOManager.cs index 7bba4f0705..a9f38c2a31 100644 --- a/src/Tgstation.Server.Host/IO/IIOManager.cs +++ b/src/Tgstation.Server.Host/IO/IIOManager.cs @@ -21,6 +21,13 @@ namespace Tgstation.Server.Host.IO /// char AltDirectorySeparatorChar { get; } + /// + /// Create a new that resolves paths to the specified . + /// + /// A relative or absolute path that the new will resolve as its current directory. + /// A new . + IIOManager CreateResolverForSubdirectory(string subdirectoryPath); + /// /// Retrieve the full path of the current working directory. /// diff --git a/src/Tgstation.Server.Host/IO/ResolvingIOManager.cs b/src/Tgstation.Server.Host/IO/ResolvingIOManager.cs index f05ce162e5..69e7feefba 100644 --- a/src/Tgstation.Server.Host/IO/ResolvingIOManager.cs +++ b/src/Tgstation.Server.Host/IO/ResolvingIOManager.cs @@ -15,14 +15,10 @@ namespace Tgstation.Server.Host.IO /// /// Initializes a new instance of the class. /// - /// The that resolves to the directory to work out of. /// The value of . - public ResolvingIOManager(IIOManager parent, string subdirectory) + public ResolvingIOManager(string subdirectory) { - ArgumentNullException.ThrowIfNull(parent); - ArgumentNullException.ThrowIfNull(subdirectory); - - this.subdirectory = ConcatPath(parent.ResolvePath(), subdirectory); + this.subdirectory = subdirectory ?? throw new ArgumentNullException(nameof(subdirectory)); } /// diff --git a/tests/Tgstation.Server.Host.Tests/Components/StaticFiles/TestConfiguration.cs b/tests/Tgstation.Server.Host.Tests/Components/StaticFiles/TestConfiguration.cs index f5e6b9bf5f..3dd2d6d13f 100644 --- a/tests/Tgstation.Server.Host.Tests/Components/StaticFiles/TestConfiguration.cs +++ b/tests/Tgstation.Server.Host.Tests/Components/StaticFiles/TestConfiguration.cs @@ -33,7 +33,7 @@ namespace Tgstation.Server.Host.Components.StaticFiles.Tests var tempPath = Path.GetTempFileName(); File.Delete(tempPath); - var ioManager = new ResolvingIOManager(new DefaultIOManager(), tempPath); + var ioManager = new DefaultIOManager().CreateResolverForSubdirectory(tempPath); await ioManager.CreateDirectory(".", CancellationToken.None); try { diff --git a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs index f9188313d0..8d667ebc41 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs @@ -96,7 +96,7 @@ namespace Tgstation.Server.Tests.Live.Instance Environment.SpecialFolderOption.DoNotVerify), new AssemblyInformationProvider().VersionPrefix, "OpenDreamRepository"); - var odRepoIoManager = new ResolvingIOManager(ioManager, odRepoDir); + var odRepoIoManager = ioManager.CreateResolverForSubdirectory(odRepoDir); var mockOptions = new Mock>(); var genConfig = new GeneralConfiguration diff --git a/tests/Tgstation.Server.Tests/TestRepository.cs b/tests/Tgstation.Server.Tests/TestRepository.cs index 59b4783994..89cf34daac 100644 --- a/tests/Tgstation.Server.Tests/TestRepository.cs +++ b/tests/Tgstation.Server.Tests/TestRepository.cs @@ -75,8 +75,7 @@ namespace Tgstation.Server.Tests using var manager = new RepositoryManager( repoFac, commands, - new ResolvingIOManager( - new DefaultIOManager(), + new DefaultIOManager().CreateResolverForSubdirectory( tempPath), Mock.Of(), new WindowsPostWriteHandler(), From f0cac0e81001c10bd8d5fbed9a60f84ac2742931 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 20 Jul 2025 22:11:42 -0400 Subject: [PATCH 24/52] Switch to using `IFileSystem` on the backend for file operations --- .../Deployment/HardLinkDmbProvider.cs | 18 +-- .../Components/StaticFiles/Configuration.cs | 2 +- src/Tgstation.Server.Host/Core/Application.cs | 12 +- .../Extensions/WebHostBuilderExtensions.cs | 10 +- .../IO/DefaultIOManager.cs | 122 ++++++++++-------- src/Tgstation.Server.Host/IO/IIOManager.cs | 29 ++--- .../IO/ISynchronousIOManager.cs | 7 + .../IO/PosixFilesystemLinkFactory.cs | 18 ++- .../IO/ResolvingIOManager.cs | 7 +- .../IO/SynchronousIOManager.cs | 52 +++++--- .../IO/WindowsFilesystemLinkFactory.cs | 18 ++- src/Tgstation.Server.Host/ServerFactory.cs | 19 ++- .../System/PosixProcessFeatures.cs | 4 +- .../Tgstation.Server.Host.csproj | 1 + .../Transfer/FileTransferService.cs | 2 +- .../StaticFiles/TestConfiguration.cs | 8 +- .../IO/TestFilesystemLinkFactory.cs | 6 +- .../IO/TestIOManager.cs | 100 ++++++++------ .../System/TestPosixSignalHandler.cs | 4 +- .../System/TestProcessFeatures.cs | 3 +- .../System/TestSymlinkFactory.cs | 7 +- .../TestProgram.cs | 12 +- .../TestServerFactory.cs | 9 +- .../Tgstation.Server.Host.Tests.csproj | 1 + .../CachingFileDownloader.cs | 3 +- .../Live/Instance/ConfigurationTest.cs | 6 +- .../Live/Instance/InstanceTest.cs | 5 +- .../Live/Instance/WatchdogTest.cs | 5 +- .../Live/LiveTestingServer.cs | 3 +- .../Live/TestLiveServer.cs | 5 +- .../Tgstation.Server.Tests/TestRepository.cs | 7 +- .../TestSystemInteraction.cs | 5 +- tests/Tgstation.Server.Tests/TestVersions.cs | 7 +- tests/Tgstation.Server.Tests/TestingUtils.cs | 3 +- 34 files changed, 326 insertions(+), 194 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs b/src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs index 477e5f06c4..df3c9cf2a1 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.Globalization; using System.IO; +using System.IO.Abstractions; using System.Runtime.Versioning; using System.Threading; using System.Threading.Tasks; @@ -187,8 +188,10 @@ namespace Tgstation.Server.Host.Components.Deployment dest = IOManager.ResolvePath(mirrorGuid.ToString()); using var semaphore = taskThrottle.HasValue ? new SemaphoreSlim(taskThrottle.Value) : null; + + var dir = await IOManager.DirectoryInfo(src, cancellationToken); await Task.WhenAll(MirrorDirectoryImpl( - src, + dir, dest, semaphore, securityLevel, @@ -230,19 +233,18 @@ namespace Tgstation.Server.Host.Components.Deployment /// /// Recursively create tasks to create a hard link directory mirror of to . /// - /// The source directory path. + /// The source . /// The destination directory path. /// Optional used to limit degree of parallelism. /// The launch level. /// The for the operation. - /// A of s representing the running operations. The first returned is always the necessary call to . + /// An of s representing the running operations. The first returned is always the necessary call to . /// I genuinely don't know how this will work with symlinked files. Waiting for the issue report I guess. - IEnumerable MirrorDirectoryImpl(string src, string dest, SemaphoreSlim? semaphore, DreamDaemonSecurity securityLevel, CancellationToken cancellationToken) + IEnumerable MirrorDirectoryImpl(IDirectoryInfo src, string dest, SemaphoreSlim? semaphore, DreamDaemonSecurity securityLevel, CancellationToken cancellationToken) { - var dir = IOManager.DirectoryInfo(src); Task? subdirCreationTask = null; var dreamDaemonWillAcceptOutOfDirectorySymlinks = securityLevel == DreamDaemonSecurity.Trusted; - foreach (var subDirectory in dir.EnumerateDirectories()) + foreach (var subDirectory in src.EnumerateDirectories()) { var mirroredName = IOManager.ConcatPath(dest, subDirectory.Name); @@ -275,7 +277,7 @@ namespace Tgstation.Server.Host.Components.Deployment logger.LogDebug("Recreating symlinked directory {name} as hard links...", subDirectory.Name); var checkingSubdirCreationTask = true; - foreach (var copyTask in MirrorDirectoryImpl(subDirectory.FullName, mirroredName, semaphore, securityLevel, cancellationToken)) + foreach (var copyTask in MirrorDirectoryImpl(subDirectory, mirroredName, semaphore, securityLevel, cancellationToken)) { if (subdirCreationTask == null) { @@ -289,7 +291,7 @@ namespace Tgstation.Server.Host.Components.Deployment } } - foreach (var fileInfo in dir.EnumerateFiles()) + foreach (var fileInfo in src.EnumerateFiles()) { if (subdirCreationTask == null) { diff --git a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs index 675187f464..6dbfc3cc9c 100644 --- a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs +++ b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs @@ -336,7 +336,7 @@ namespace Tgstation.Server.Host.Components.StaticFiles Stream? result = null; void GetFileStream() { - result = ioManager.GetFileStream(path, false); + result = synchronousIOManager.GetFileStream(path); } if (systemIdentity == null) diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index e2d1d75dbf..3246b8ba0f 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -2,6 +2,7 @@ using System.Collections.Frozen; using System.Collections.Generic; using System.Globalization; +using System.IO.Abstractions; using System.Threading.Tasks; using System.Web; @@ -105,10 +106,12 @@ namespace Tgstation.Server.Host.Core public static IServerFactory CreateDefaultServerFactory() { var assemblyInformationProvider = new AssemblyInformationProvider(); - var ioManager = new DefaultIOManager(); + var fileSystem = new FileSystem(); + var ioManager = new DefaultIOManager(fileSystem); return new ServerFactory( assemblyInformationProvider, - ioManager); + ioManager, + fileSystem); } /// @@ -154,11 +157,13 @@ namespace Tgstation.Server.Host.Core /// The needed for configuration. /// The needed for configuration. /// The needed for configuration. + /// The needed for configuration. public void ConfigureServices( IServiceCollection services, IAssemblyInformationProvider assemblyInformationProvider, IIOManager ioManager, - IPostSetupServices postSetupServices) + IPostSetupServices postSetupServices, + IFileSystem fileSystem) { ConfigureServices(services, assemblyInformationProvider, ioManager); @@ -541,6 +546,7 @@ namespace Tgstation.Server.Host.Core services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(fileSystem); services.AddHostedService(); services.AddHostedService(); diff --git a/src/Tgstation.Server.Host/Extensions/WebHostBuilderExtensions.cs b/src/Tgstation.Server.Host/Extensions/WebHostBuilderExtensions.cs index 3179c684f2..ba0316ad89 100644 --- a/src/Tgstation.Server.Host/Extensions/WebHostBuilderExtensions.cs +++ b/src/Tgstation.Server.Host/Extensions/WebHostBuilderExtensions.cs @@ -1,4 +1,5 @@ using System; +using System.IO.Abstractions; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; @@ -26,24 +27,27 @@ namespace Tgstation.Server.Host.Extensions /// The to configure. /// The to use. /// The to use. - /// The configured . /// The to use. + /// The to use. + /// The configured . public static IWebHostBuilder UseApplication( this IWebHostBuilder builder, IAssemblyInformationProvider assemblyInformationProvider, IIOManager ioManager, - IPostSetupServices postSetupServices) + IPostSetupServices postSetupServices, + IFileSystem fileSystem) { ArgumentNullException.ThrowIfNull(builder); ArgumentNullException.ThrowIfNull(assemblyInformationProvider); ArgumentNullException.ThrowIfNull(ioManager); ArgumentNullException.ThrowIfNull(postSetupServices); + ArgumentNullException.ThrowIfNull(fileSystem); return builder.ConfigureServices( (context, services) => { var application = new Application(context.Configuration, context.HostingEnvironment); - application.ConfigureServices(services, assemblyInformationProvider, ioManager, postSetupServices); + application.ConfigureServices(services, assemblyInformationProvider, ioManager, postSetupServices, fileSystem); services.AddSingleton(application); }) .Configure(ConfigureApplication); diff --git a/src/Tgstation.Server.Host/IO/DefaultIOManager.cs b/src/Tgstation.Server.Host/IO/DefaultIOManager.cs index 38d63468c7..470f437d7f 100644 --- a/src/Tgstation.Server.Host/IO/DefaultIOManager.cs +++ b/src/Tgstation.Server.Host/IO/DefaultIOManager.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.IO.Abstractions; using System.IO.Compression; using System.Linq; using System.Threading; @@ -31,17 +32,31 @@ namespace Tgstation.Server.Host.IO public const TaskCreationOptions BlockingTaskCreationOptions = TaskCreationOptions.None; /// - public char DirectorySeparatorChar => Path.DirectorySeparatorChar; + public char DirectorySeparatorChar => fileSystem.Path.DirectorySeparatorChar; /// - public char AltDirectorySeparatorChar => Path.AltDirectorySeparatorChar; + public char AltDirectorySeparatorChar => fileSystem.Path.AltDirectorySeparatorChar; + + /// + /// The backing . + /// + readonly IFileSystem fileSystem; + + /// + /// Initializes a new instance of the class. + /// + /// The value of . + public DefaultIOManager(IFileSystem fileSystem) + { + this.fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); + } /// /// Recursively empty a directory. /// /// of the directory to empty. /// The for the operation. - static void NormalizeAndDelete(DirectoryInfo dir, CancellationToken cancellationToken) + static void NormalizeAndDelete(IDirectoryInfo dir, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); @@ -117,7 +132,7 @@ namespace Tgstation.Server.Host.IO } /// - public string ConcatPath(params string[] paths) => Path.Combine(paths); + public string ConcatPath(params string[] paths) => fileSystem.Path.Combine(paths); /// public async ValueTask CopyFile(string src, string dest, CancellationToken cancellationToken) @@ -126,7 +141,7 @@ namespace Tgstation.Server.Host.IO ArgumentNullException.ThrowIfNull(dest); // tested to hell and back, these are the optimal buffer sizes - await using var srcStream = new FileStream( + await using var srcStream = fileSystem.FileStream.New( ResolvePath(src), FileMode.Open, FileAccess.Read, @@ -140,41 +155,39 @@ namespace Tgstation.Server.Host.IO } /// - public Task CreateDirectory(string path, CancellationToken cancellationToken) => Task.Factory.StartNew(() => Directory.CreateDirectory(ResolvePath(path)), cancellationToken, BlockingTaskCreationOptions, TaskScheduler.Current); + public Task CreateDirectory(string path, CancellationToken cancellationToken) => Task.Factory.StartNew(() => fileSystem.Directory.CreateDirectory(ResolvePath(path)), cancellationToken, BlockingTaskCreationOptions, TaskScheduler.Current); /// public Task DeleteDirectory(string path, CancellationToken cancellationToken) - { - path = ResolvePath(path); - var di = new DirectoryInfo(path); - if (!di.Exists) - return Task.CompletedTask; - - return Task.Factory.StartNew( - () => NormalizeAndDelete(di, cancellationToken), + => Task.Factory.StartNew( + () => + { + var di = fileSystem.DirectoryInfo.New(path); + if (di.Exists) + NormalizeAndDelete(di, cancellationToken); + }, cancellationToken, BlockingTaskCreationOptions, TaskScheduler.Current); - } /// - public Task DeleteFile(string path, CancellationToken cancellationToken) => Task.Factory.StartNew(() => File.Delete(ResolvePath(path)), cancellationToken, BlockingTaskCreationOptions, TaskScheduler.Current); + public Task DeleteFile(string path, CancellationToken cancellationToken) => Task.Factory.StartNew(() => fileSystem.File.Delete(ResolvePath(path)), cancellationToken, BlockingTaskCreationOptions, TaskScheduler.Current); /// - public Task FileExists(string path, CancellationToken cancellationToken) => Task.Factory.StartNew(() => File.Exists(ResolvePath(path)), cancellationToken, BlockingTaskCreationOptions, TaskScheduler.Current); + public Task FileExists(string path, CancellationToken cancellationToken) => Task.Factory.StartNew(() => fileSystem.File.Exists(ResolvePath(path)), cancellationToken, BlockingTaskCreationOptions, TaskScheduler.Current); /// - public Task DirectoryExists(string path, CancellationToken cancellationToken) => Task.Factory.StartNew(() => Directory.Exists(ResolvePath(path)), cancellationToken, BlockingTaskCreationOptions, TaskScheduler.Current); + public Task DirectoryExists(string path, CancellationToken cancellationToken) => Task.Factory.StartNew(() => fileSystem.Directory.Exists(ResolvePath(path)), cancellationToken, BlockingTaskCreationOptions, TaskScheduler.Current); /// - public string GetDirectoryName(string path) => Path.GetDirectoryName(path ?? throw new ArgumentNullException(nameof(path))) + public string GetDirectoryName(string path) => fileSystem.Path.GetDirectoryName(path ?? throw new ArgumentNullException(nameof(path))) ?? throw new InvalidOperationException($"Null was returned. Path ({path}) must be rooted. This is not supported!"); /// - public string GetFileName(string path) => Path.GetFileName(path ?? throw new ArgumentNullException(nameof(path))); + public string GetFileName(string path) => fileSystem.Path.GetFileName(path ?? throw new ArgumentNullException(nameof(path))); /// - public string GetFileNameWithoutExtension(string path) => Path.GetFileNameWithoutExtension(path ?? throw new ArgumentNullException(nameof(path))); + public string GetFileNameWithoutExtension(string path) => fileSystem.Path.GetFileNameWithoutExtension(path ?? throw new ArgumentNullException(nameof(path))); /// public Task> GetFilesWithExtension(string path, string extension, bool recursive, CancellationToken cancellationToken) => Task.Factory.StartNew( @@ -183,7 +196,7 @@ namespace Tgstation.Server.Host.IO path = ResolvePath(path); ArgumentNullException.ThrowIfNull(extension); var results = new List(); - foreach (var fileName in Directory.EnumerateFiles( + foreach (var fileName in fileSystem.Directory.EnumerateFiles( path, $"*.{extension}", recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly)) @@ -205,7 +218,7 @@ namespace Tgstation.Server.Host.IO ArgumentNullException.ThrowIfNull(destination); source = ResolvePath(source ?? throw new ArgumentNullException(nameof(source))); destination = ResolvePath(destination); - File.Move(source, destination); + fileSystem.File.Move(source, destination); }, cancellationToken, BlockingTaskCreationOptions, @@ -218,7 +231,7 @@ namespace Tgstation.Server.Host.IO ArgumentNullException.ThrowIfNull(destination); source = ResolvePath(source ?? throw new ArgumentNullException(nameof(source))); destination = ResolvePath(destination); - Directory.Move(source, destination); + fileSystem.Directory.Move(source, destination); }, cancellationToken, BlockingTaskCreationOptions, @@ -227,7 +240,7 @@ namespace Tgstation.Server.Host.IO /// public async ValueTask ReadAllBytes(string path, CancellationToken cancellationToken) { - await using var file = CreateAsyncSequentialReadStream(path); + await using var file = CreateAsyncReadStream(path, true, true); byte[] buf; buf = new byte[file.Length]; await file.ReadAsync(buf, cancellationToken); @@ -235,10 +248,12 @@ namespace Tgstation.Server.Host.IO } /// - public string ResolvePath() => ResolvePath(CurrentDirectory); + public string ResolvePath() + => ResolvePath(CurrentDirectory); /// - public virtual string ResolvePath(string path) => Path.GetFullPath(path ?? throw new ArgumentNullException(nameof(path))); + public virtual string ResolvePath(string path) + => fileSystem.Path.GetFullPath(path ?? throw new ArgumentNullException(nameof(path))); /// public async ValueTask WriteAllBytes(string path, byte[] contents, CancellationToken cancellationToken) @@ -248,10 +263,10 @@ namespace Tgstation.Server.Host.IO } /// - public FileStream CreateAsyncSequentialWriteStream(string path) + public Stream CreateAsyncSequentialWriteStream(string path) { path = ResolvePath(path); - return new FileStream( + return fileSystem.FileStream.New( path, FileMode.Create, FileAccess.Write, @@ -261,16 +276,18 @@ namespace Tgstation.Server.Host.IO } /// - public FileStream CreateAsyncSequentialReadStream(string path) + public Stream CreateAsyncReadStream(string path, bool sequental, bool shareWrite) { path = ResolvePath(path); - return new FileStream( + return fileSystem.FileStream.New( path, FileMode.Open, FileAccess.Read, - FileShare.ReadWrite | FileShare.Delete, + FileShare.ReadWrite | FileShare.Delete | (shareWrite ? FileShare.Write : FileShare.None), DefaultBufferSize, - FileOptions.Asynchronous | FileOptions.SequentialScan); + sequental + ? FileOptions.Asynchronous | FileOptions.SequentialScan + : FileOptions.Asynchronous); } /// @@ -280,7 +297,7 @@ namespace Tgstation.Server.Host.IO path = ResolvePath(path); var results = new List(); cancellationToken.ThrowIfCancellationRequested(); - foreach (var directoryName in Directory.EnumerateDirectories(path)) + foreach (var directoryName in fileSystem.Directory.EnumerateDirectories(path)) { results.Add(directoryName); cancellationToken.ThrowIfCancellationRequested(); @@ -299,7 +316,7 @@ namespace Tgstation.Server.Host.IO path = ResolvePath(path); var results = new List(); cancellationToken.ThrowIfCancellationRequested(); - foreach (var fileName in Directory.EnumerateFiles(path)) + foreach (var fileName in fileSystem.Directory.EnumerateFiles(path)) { results.Add(fileName); cancellationToken.ThrowIfCancellationRequested(); @@ -337,8 +354,8 @@ namespace Tgstation.Server.Host.IO public bool PathContainsParentAccess(string path) => path ?.Split( [ - Path.DirectorySeparatorChar, - Path.AltDirectorySeparatorChar, + fileSystem.Path.DirectorySeparatorChar, + fileSystem.Path.AltDirectorySeparatorChar, ]) .Any(x => x == "..") ?? throw new ArgumentNullException(nameof(path)); @@ -348,23 +365,13 @@ namespace Tgstation.Server.Host.IO () => { path = ResolvePath(path ?? throw new ArgumentNullException(nameof(path))); - var fileInfo = new FileInfo(path); + var fileInfo = fileSystem.FileInfo.New(path); return new DateTimeOffset(fileInfo.LastWriteTimeUtc); }, cancellationToken, BlockingTaskCreationOptions, TaskScheduler.Current); - /// - public Stream GetFileStream(string path, bool shareWrite) - => new FileStream( - ResolvePath(path), - FileMode.Open, - FileAccess.Read, - FileShare.Read | FileShare.Delete | (shareWrite ? FileShare.Write : FileShare.None), - DefaultBufferSize, - true); - /// public Task PathIsChildOf(string parentPath, string childPath, CancellationToken cancellationToken) => Task.Factory.StartNew( () => @@ -376,8 +383,8 @@ namespace Tgstation.Server.Host.IO return true; // https://stackoverflow.com/questions/5617320/given-full-path-check-if-path-is-subdirectory-of-some-other-path-or-otherwise?lq=1 - var di1 = new DirectoryInfo(parentPath); - var di2 = new DirectoryInfo(childPath); + var di1 = fileSystem.DirectoryInfo.New(parentPath); + var di2 = fileSystem.DirectoryInfo.New(childPath); while (di2.Parent != null) { if (di2.Parent.FullName == di1.FullName) @@ -393,12 +400,16 @@ namespace Tgstation.Server.Host.IO TaskScheduler.Current); /// - public DirectoryInfo DirectoryInfo(string path) - => new(ResolvePath(path)); // Consider async + public Task DirectoryInfo(string path, CancellationToken cancellationToken) + => Task.Factory.StartNew( + () => fileSystem.DirectoryInfo.New(ResolvePath(path)), + cancellationToken, + BlockingTaskCreationOptions, + TaskScheduler.Current); /// public bool IsPathRooted(string path) - => Path.IsPathRooted(path); + => fileSystem.Path.IsPathRooted(path); /// public IIOManager CreateResolverForSubdirectory(string subdirectoryPath) @@ -406,6 +417,7 @@ namespace Tgstation.Server.Host.IO ArgumentNullException.ThrowIfNull(subdirectoryPath); return new ResolvingIOManager( + fileSystem, ConcatPath( ResolvePath(), subdirectoryPath)); @@ -429,7 +441,7 @@ namespace Tgstation.Server.Host.IO SemaphoreSlim? semaphore, CancellationToken cancellationToken) { - var dir = new DirectoryInfo(src); + var dir = fileSystem.DirectoryInfo.New(src); Task? subdirCreationTask = null; foreach (var subDirectory in dir.EnumerateDirectories()) { @@ -437,7 +449,7 @@ namespace Tgstation.Server.Host.IO continue; var checkingSubdirCreationTask = true; - foreach (var copyTask in CopyDirectoryImpl(subDirectory.FullName, Path.Combine(dest, subDirectory.Name), null, postCopyCallback, semaphore, cancellationToken)) + foreach (var copyTask in CopyDirectoryImpl(subDirectory.FullName, fileSystem.Path.Combine(dest, subDirectory.Name), null, postCopyCallback, semaphore, cancellationToken)) { if (subdirCreationTask == null) { diff --git a/src/Tgstation.Server.Host/IO/IIOManager.cs b/src/Tgstation.Server.Host/IO/IIOManager.cs index a9f38c2a31..3b81869fb8 100644 --- a/src/Tgstation.Server.Host/IO/IIOManager.cs +++ b/src/Tgstation.Server.Host/IO/IIOManager.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.IO.Abstractions; using System.Threading; using System.Threading.Tasks; @@ -134,15 +135,17 @@ namespace Tgstation.Server.Host.IO /// Creates an asynchronous for sequential writing. /// /// The path of the file to write, will be truncated. - /// The open . - FileStream CreateAsyncSequentialWriteStream(string path); + /// The open . + Stream CreateAsyncSequentialWriteStream(string path); /// /// Creates an asynchronous for sequential reading. /// /// The path of the file to write, will be truncated. - /// The open . - FileStream CreateAsyncSequentialReadStream(string path); + /// If the sequential read flag should be added. + /// If should be used. + /// The open . + Stream CreateAsyncReadStream(string path, bool sequential, bool shareWrite); /// /// Writes some to a file at overwriting previous content. @@ -247,20 +250,12 @@ namespace Tgstation.Server.Host.IO Task GetLastModified(string path, CancellationToken cancellationToken); /// - /// Gets the for a given file . + /// Gets a for the given . /// - /// The path of the file. - /// If should be used. - /// The of the file. - /// This function is sychronous. - Stream GetFileStream(string path, bool shareWrite); - - /// - /// Gets a for the given . - /// - /// The path to get for. - /// A new . - DirectoryInfo DirectoryInfo(string path); + /// The path to get for. + /// The for the operation. + /// A resulting in the of the . + Task DirectoryInfo(string path, CancellationToken cancellationToken); /// /// Check if a given is at the root level of the filesystem. diff --git a/src/Tgstation.Server.Host/IO/ISynchronousIOManager.cs b/src/Tgstation.Server.Host/IO/ISynchronousIOManager.cs index 16e0ea370e..850f6941f2 100644 --- a/src/Tgstation.Server.Host/IO/ISynchronousIOManager.cs +++ b/src/Tgstation.Server.Host/IO/ISynchronousIOManager.cs @@ -63,5 +63,12 @@ namespace Tgstation.Server.Host.IO /// The path to check. /// if is a directory, otherwise. bool IsDirectory(string path); + + /// + /// Gets the for a given file without write share. + /// + /// The path of the file. + /// The of the file. + Stream GetFileStream(string path); } } diff --git a/src/Tgstation.Server.Host/IO/PosixFilesystemLinkFactory.cs b/src/Tgstation.Server.Host/IO/PosixFilesystemLinkFactory.cs index b80af9e122..c0fa195906 100644 --- a/src/Tgstation.Server.Host/IO/PosixFilesystemLinkFactory.cs +++ b/src/Tgstation.Server.Host/IO/PosixFilesystemLinkFactory.cs @@ -1,5 +1,5 @@ using System; -using System.IO; +using System.IO.Abstractions; using System.Threading; using System.Threading.Tasks; @@ -15,6 +15,20 @@ namespace Tgstation.Server.Host.IO /// public bool SymlinkedDirectoriesAreDeletedAsFiles => true; + /// + /// The to use. + /// + readonly IFileSystem fileSystem; + + /// + /// Initializes a new instance of the class. + /// + /// The value of . + public PosixFilesystemLinkFactory(IFileSystem fileSystem) + { + this.fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); + } + /// public Task CreateHardLink(string targetPath, string linkPath, CancellationToken cancellationToken) => Task.Factory.StartNew( () => @@ -39,7 +53,7 @@ namespace Tgstation.Server.Host.IO ArgumentNullException.ThrowIfNull(linkPath); UnixFileSystemInfo fsInfo; - var isFile = File.Exists(targetPath); + var isFile = fileSystem.File.Exists(targetPath); cancellationToken.ThrowIfCancellationRequested(); if (isFile) fsInfo = new UnixFileInfo(targetPath); diff --git a/src/Tgstation.Server.Host/IO/ResolvingIOManager.cs b/src/Tgstation.Server.Host/IO/ResolvingIOManager.cs index 69e7feefba..625d7da86e 100644 --- a/src/Tgstation.Server.Host/IO/ResolvingIOManager.cs +++ b/src/Tgstation.Server.Host/IO/ResolvingIOManager.cs @@ -1,4 +1,5 @@ using System; +using System.IO.Abstractions; namespace Tgstation.Server.Host.IO { @@ -15,8 +16,12 @@ namespace Tgstation.Server.Host.IO /// /// Initializes a new instance of the class. /// + /// The for the . /// The value of . - public ResolvingIOManager(string subdirectory) + public ResolvingIOManager( + IFileSystem fileSystem, + string subdirectory) + : base(fileSystem) { this.subdirectory = subdirectory ?? throw new ArgumentNullException(nameof(subdirectory)); } diff --git a/src/Tgstation.Server.Host/IO/SynchronousIOManager.cs b/src/Tgstation.Server.Host/IO/SynchronousIOManager.cs index b0388b51af..892b3edaec 100644 --- a/src/Tgstation.Server.Host/IO/SynchronousIOManager.cs +++ b/src/Tgstation.Server.Host/IO/SynchronousIOManager.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Globalization; using System.IO; +using System.IO.Abstractions; using System.Linq; using System.Security.Cryptography; using System.Threading; @@ -13,6 +14,11 @@ namespace Tgstation.Server.Host.IO /// sealed class SynchronousIOManager : ISynchronousIOManager { + /// + /// The to use. + /// + readonly IFileSystem fileSystem; + /// /// The for the . /// @@ -21,9 +27,11 @@ namespace Tgstation.Server.Host.IO /// /// Initializes a new instance of the class. /// + /// The value of . /// The value of . - public SynchronousIOManager(ILogger logger) + public SynchronousIOManager(IFileSystem fileSystem, ILogger logger) { + this.fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); } @@ -33,32 +41,32 @@ namespace Tgstation.Server.Host.IO if (IsDirectory(path)) return true; cancellationToken.ThrowIfCancellationRequested(); - Directory.CreateDirectory(path); + fileSystem.Directory.CreateDirectory(path); return false; } /// public bool DeleteDirectory(string path) { - if (File.Exists(path)) + if (fileSystem.File.Exists(path)) return false; - if (!Directory.Exists(path)) + if (!fileSystem.Directory.Exists(path)) return true; - if (Directory.EnumerateFileSystemEntries(path).Any()) + if (fileSystem.Directory.EnumerateFileSystemEntries(path).Any()) return false; - Directory.Delete(path); + fileSystem.Directory.Delete(path); return true; } /// public IEnumerable GetDirectories(string path, CancellationToken cancellationToken) { - foreach (var directoryName in Directory.EnumerateDirectories(path)) + foreach (var directoryName in fileSystem.Directory.EnumerateDirectories(path)) { - yield return Path.GetFileName(directoryName); + yield return fileSystem.Path.GetFileName(directoryName); cancellationToken.ThrowIfCancellationRequested(); } } @@ -66,9 +74,9 @@ namespace Tgstation.Server.Host.IO /// public IEnumerable GetFiles(string path, CancellationToken cancellationToken) { - foreach (var fileName in Directory.EnumerateFiles(path)) + foreach (var fileName in fileSystem.Directory.EnumerateFiles(path)) { - yield return Path.GetFileName(fileName); + yield return fileSystem.Path.GetFileName(fileName); cancellationToken.ThrowIfCancellationRequested(); } } @@ -77,14 +85,14 @@ namespace Tgstation.Server.Host.IO public bool IsDirectory(string path) { ArgumentNullException.ThrowIfNull(path); - return Directory.Exists(path); + return fileSystem.Directory.Exists(path); } /// public byte[] ReadFile(string path) { ArgumentNullException.ThrowIfNull(path); - return File.ReadAllBytes(path); + return fileSystem.File.ReadAllBytes(path); } /// @@ -94,16 +102,16 @@ namespace Tgstation.Server.Host.IO ArgumentNullException.ThrowIfNull(data); cancellationToken.ThrowIfCancellationRequested(); - var directory = Path.GetDirectoryName(path) ?? throw new ArgumentException("path cannot be rooted!", nameof(path)); - Directory.CreateDirectory(directory); + var directory = fileSystem.Path.GetDirectoryName(path) ?? throw new ArgumentException("path cannot be rooted!", nameof(path)); + fileSystem.Directory.CreateDirectory(directory); - var newFile = !File.Exists(path); + var newFile = !fileSystem.File.Exists(path); cancellationToken.ThrowIfCancellationRequested(); logger.LogTrace("Starting checked write to {path} ({fileType} file)", path, newFile ? "New" : "Pre-existing"); - using (var file = File.Open(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None)) + using (var file = fileSystem.File.Open(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None)) { cancellationToken.ThrowIfCancellationRequested(); @@ -160,10 +168,20 @@ namespace Tgstation.Server.Host.IO if (data.Length == 0) { logger.LogDebug("Stream is empty, deleting file"); - File.Delete(path); + fileSystem.File.Delete(path); } return true; } + + /// + public Stream GetFileStream(string path) + => fileSystem.FileStream.New( + path, + FileMode.Open, + FileAccess.Read, + FileShare.Read | FileShare.Delete, + DefaultIOManager.DefaultBufferSize, + true); } } diff --git a/src/Tgstation.Server.Host/IO/WindowsFilesystemLinkFactory.cs b/src/Tgstation.Server.Host/IO/WindowsFilesystemLinkFactory.cs index 54ff36b320..01cd4934d0 100644 --- a/src/Tgstation.Server.Host/IO/WindowsFilesystemLinkFactory.cs +++ b/src/Tgstation.Server.Host/IO/WindowsFilesystemLinkFactory.cs @@ -1,6 +1,6 @@ using System; using System.ComponentModel; -using System.IO; +using System.IO.Abstractions; using System.Threading; using System.Threading.Tasks; @@ -16,6 +16,20 @@ namespace Tgstation.Server.Host.IO /// public bool SymlinkedDirectoriesAreDeletedAsFiles => false; + /// + /// The to use. + /// + readonly IFileSystem fileSystem; + + /// + /// Initializes a new instance of the class. + /// + /// The value of . + public WindowsFilesystemLinkFactory(IFileSystem fileSystem) + { + this.fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); + } + /// public Task CreateHardLink(string targetPath, string linkPath, CancellationToken cancellationToken) => throw new NotSupportedException(); @@ -28,7 +42,7 @@ namespace Tgstation.Server.Host.IO ArgumentNullException.ThrowIfNull(linkPath); // check if its not a file - var flags = File.Exists(targetPath) ? NativeMethods.CreateSymbolicLinkFlags.None : NativeMethods.CreateSymbolicLinkFlags.Directory; + var flags = fileSystem.File.Exists(targetPath) ? NativeMethods.CreateSymbolicLinkFlags.None : NativeMethods.CreateSymbolicLinkFlags.Directory; /* * no don't fucking use this diff --git a/src/Tgstation.Server.Host/ServerFactory.cs b/src/Tgstation.Server.Host/ServerFactory.cs index 5306221a60..5f3230ec0a 100644 --- a/src/Tgstation.Server.Host/ServerFactory.cs +++ b/src/Tgstation.Server.Host/ServerFactory.cs @@ -1,4 +1,5 @@ using System; +using System.IO.Abstractions; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -29,23 +30,33 @@ namespace Tgstation.Server.Host /// public const string AppSettings = "appsettings"; + /// + public IIOManager IOManager { get; } + /// /// The for the . /// readonly IAssemblyInformationProvider assemblyInformationProvider; - /// - public IIOManager IOManager { get; } + /// + /// The for the . + /// + readonly IFileSystem fileSystem; /// /// Initializes a new instance of the class. /// /// The value of . /// The value of . - internal ServerFactory(IAssemblyInformationProvider assemblyInformationProvider, IIOManager ioManager) + /// The value of . + internal ServerFactory( + IAssemblyInformationProvider assemblyInformationProvider, + IIOManager ioManager, + IFileSystem fileSystem) { this.assemblyInformationProvider = assemblyInformationProvider ?? throw new ArgumentNullException(nameof(assemblyInformationProvider)); IOManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); + this.fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); } /// @@ -160,7 +171,7 @@ namespace Tgstation.Server.Host }) .UseIIS() .UseIISIntegration() - .UseApplication(assemblyInformationProvider, IOManager, postSetupServices) + .UseApplication(assemblyInformationProvider, IOManager, postSetupServices, fileSystem) .SuppressStatusMessages(true) .UseShutdownTimeout( TimeSpan.FromMinutes( diff --git a/src/Tgstation.Server.Host/System/PosixProcessFeatures.cs b/src/Tgstation.Server.Host/System/PosixProcessFeatures.cs index 6477dc05d5..ee6abe8a62 100644 --- a/src/Tgstation.Server.Host/System/PosixProcessFeatures.cs +++ b/src/Tgstation.Server.Host/System/PosixProcessFeatures.cs @@ -168,8 +168,8 @@ namespace Tgstation.Server.Host.System { // can't use ReadAllBytes here, /proc files have 0 length so the buffer is initialized to empty // https://stackoverflow.com/questions/12237712/how-can-i-show-the-size-of-files-in-proc-it-should-not-be-size-zero - await using var fileStream = ioManager.CreateAsyncSequentialReadStream( - "/proc/self/oom_score_adj"); + await using var fileStream = ioManager.CreateAsyncReadStream( + "/proc/self/oom_score_adj", true, true); using var reader = new StreamReader(fileStream, Encoding.UTF8, leaveOpen: true); originalString = await reader.ReadToEndAsync(cancellationToken); } diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index 19695527da..32fd89607a 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -161,6 +161,7 @@ + diff --git a/src/Tgstation.Server.Host/Transfer/FileTransferService.cs b/src/Tgstation.Server.Host/Transfer/FileTransferService.cs index bb1cafa171..ac8ce12a1b 100644 --- a/src/Tgstation.Server.Host/Transfer/FileTransferService.cs +++ b/src/Tgstation.Server.Host/Transfer/FileTransferService.cs @@ -212,7 +212,7 @@ namespace Tgstation.Server.Host.Transfer if (downloadProvider.StreamProvider != null) stream = await downloadProvider.StreamProvider(cancellationToken); else - stream = ioManager.GetFileStream(downloadProvider.FilePath, downloadProvider.ShareWrite); + stream = ioManager.CreateAsyncReadStream(downloadProvider.FilePath, false, downloadProvider.ShareWrite); } catch (IOException ex) { diff --git a/tests/Tgstation.Server.Host.Tests/Components/StaticFiles/TestConfiguration.cs b/tests/Tgstation.Server.Host.Tests/Components/StaticFiles/TestConfiguration.cs index 3dd2d6d13f..7314b0783a 100644 --- a/tests/Tgstation.Server.Host.Tests/Components/StaticFiles/TestConfiguration.cs +++ b/tests/Tgstation.Server.Host.Tests/Components/StaticFiles/TestConfiguration.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using System.IO.Abstractions.TestingHelpers; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -31,9 +32,10 @@ namespace Tgstation.Server.Host.Components.StaticFiles.Tests builder.SetMinimumLevel(LogLevel.Trace); }); + var mockFs = new MockFileSystem(); var tempPath = Path.GetTempFileName(); File.Delete(tempPath); - var ioManager = new DefaultIOManager().CreateResolverForSubdirectory(tempPath); + var ioManager = new DefaultIOManager(mockFs).CreateResolverForSubdirectory(tempPath); await ioManager.CreateDirectory(".", CancellationToken.None); try { @@ -46,7 +48,9 @@ namespace Tgstation.Server.Host.Components.StaticFiles.Tests var configuration = new Configuration( ioManager, - new SynchronousIOManager(loggerFactory.CreateLogger()), + new SynchronousIOManager( + mockFs, + loggerFactory.CreateLogger()), Mock.Of(), Mock.Of(), Mock.Of(), diff --git a/tests/Tgstation.Server.Host.Tests/IO/TestFilesystemLinkFactory.cs b/tests/Tgstation.Server.Host.Tests/IO/TestFilesystemLinkFactory.cs index dbf2aaf2a8..16ab28a73c 100644 --- a/tests/Tgstation.Server.Host.Tests/IO/TestFilesystemLinkFactory.cs +++ b/tests/Tgstation.Server.Host.Tests/IO/TestFilesystemLinkFactory.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using System.IO.Abstractions; using System.Runtime.InteropServices; using System.Security.Principal; using System.Threading; @@ -17,10 +18,11 @@ namespace Tgstation.Server.Host.IO.Tests [ClassInitialize] public static void SelectFactory(TestContext _) { + var fileSystem = new FileSystem(); if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - linkFactory = new WindowsFilesystemLinkFactory(); + linkFactory = new WindowsFilesystemLinkFactory(fileSystem); else - linkFactory = new PosixFilesystemLinkFactory(); + linkFactory = new PosixFilesystemLinkFactory(fileSystem); } public static bool HasPermissionToMakeSymlinks() diff --git a/tests/Tgstation.Server.Host.Tests/IO/TestIOManager.cs b/tests/Tgstation.Server.Host.Tests/IO/TestIOManager.cs index 8cd5424a4f..60fa057449 100644 --- a/tests/Tgstation.Server.Host.Tests/IO/TestIOManager.cs +++ b/tests/Tgstation.Server.Host.Tests/IO/TestIOManager.cs @@ -1,5 +1,7 @@ using System; using System.IO; +using System.IO.Abstractions; +using System.IO.Abstractions.TestingHelpers; using System.Linq; using System.Text; using System.Threading; @@ -14,27 +16,34 @@ namespace Tgstation.Server.Host.IO.Tests [TestClass] public sealed class TestIOManager { - readonly IIOManager ioManager = new DefaultIOManager(); + readonly IFileSystem fileSystem; + readonly IIOManager ioManager; + + public TestIOManager() + { + fileSystem = new MockFileSystem(); + ioManager = new DefaultIOManager(fileSystem); + } [TestMethod] public async Task TestDeleteDirectory() { - var tempPath = Path.GetTempFileName(); - File.Delete(tempPath); - Directory.CreateDirectory(tempPath); + var tempPath = fileSystem.Path.Combine(fileSystem.Path.GetTempPath(), fileSystem.Path.GetRandomFileName()); + fileSystem.File.Delete(tempPath); + fileSystem.Directory.CreateDirectory(tempPath); try { - await File.WriteAllTextAsync(Path.Combine(tempPath, "file.txt"), "asdf"); - var subDir = Path.Combine(tempPath, "subdir"); - Directory.CreateDirectory(subDir); - await File.WriteAllTextAsync(Path.Combine(subDir, "file2.txt"), "fdsa"); + await fileSystem.File.WriteAllTextAsync(Path.Combine(tempPath, "file.txt"), "asdf"); + var subDir = fileSystem.Path.Combine(tempPath, "subdir"); + fileSystem.Directory.CreateDirectory(subDir); + await fileSystem.File.WriteAllTextAsync(Path.Combine(subDir, "file2.txt"), "fdsa"); await ioManager.DeleteDirectory(tempPath, default); - Assert.IsFalse(Directory.Exists(tempPath)); + Assert.IsFalse(fileSystem.Directory.Exists(tempPath)); } catch { - Directory.Delete(tempPath, true); + fileSystem.Directory.Delete(tempPath, true); throw; } } @@ -42,13 +51,17 @@ namespace Tgstation.Server.Host.IO.Tests [TestMethod] public async Task TestDeleteDirectoryWithSymlinkInsideDoesntRecurse() { - var linkFactory = (IFilesystemLinkFactory)(new PlatformIdentifier().IsWindows - ? new WindowsFilesystemLinkFactory() - : new PosixFilesystemLinkFactory()); + // need a real FS here + var fileSystem = new FileSystem(); + var ioManager = new DefaultIOManager(fileSystem); - var tempPath = Path.GetTempFileName(); - File.Delete(tempPath); - Directory.CreateDirectory(tempPath); + var linkFactory = (IFilesystemLinkFactory)(new PlatformIdentifier().IsWindows + ? new WindowsFilesystemLinkFactory(fileSystem) + : new PosixFilesystemLinkFactory(fileSystem)); + + var tempPath = fileSystem.Path.Combine(fileSystem.Path.GetTempPath(), fileSystem.Path.GetRandomFileName()); + fileSystem.File.Delete(tempPath); + fileSystem.Directory.CreateDirectory(tempPath); try { var targetDir = ioManager.ConcatPath(tempPath, "targetdir"); @@ -81,7 +94,7 @@ namespace Tgstation.Server.Host.IO.Tests } catch { - Directory.Delete(tempPath, true); + fileSystem.Directory.Delete(tempPath, true); throw; } } @@ -89,14 +102,15 @@ namespace Tgstation.Server.Host.IO.Tests [TestMethod] public async Task TestFileExists() { - var tempPath = Path.GetTempFileName(); + var tempPath = fileSystem.Path.Combine(fileSystem.Path.GetTempPath(), fileSystem.Path.GetRandomFileName()); + await fileSystem.File.WriteAllBytesAsync(tempPath, Array.Empty()); try { Assert.IsTrue(await ioManager.FileExists(tempPath, default)); } finally { - File.Delete(tempPath); + fileSystem.File.Delete(tempPath); } Assert.IsFalse(await ioManager.FileExists(tempPath, default)); @@ -105,12 +119,12 @@ namespace Tgstation.Server.Host.IO.Tests [TestMethod] public async Task TestDirectoryExists() { - var tempPath = Path.GetTempFileName(); - File.Delete(tempPath); + var tempPath = fileSystem.Path.Combine(fileSystem.Path.GetTempPath(), fileSystem.Path.GetRandomFileName()); + fileSystem.File.Delete(tempPath); Assert.IsFalse(await ioManager.DirectoryExists(tempPath, default)); - Directory.CreateDirectory(tempPath); + fileSystem.Directory.CreateDirectory(tempPath); try { @@ -118,7 +132,7 @@ namespace Tgstation.Server.Host.IO.Tests } catch { - Directory.Delete(tempPath); + fileSystem.Directory.Delete(tempPath); throw; } } @@ -180,18 +194,18 @@ namespace Tgstation.Server.Host.IO.Tests async Task TestCopyDirectory(int? throttle) { - var tempPath = Path.GetTempFileName(); - File.Delete(tempPath); - Directory.CreateDirectory(tempPath); + var tempPath = fileSystem.Path.Combine(fileSystem.Path.GetTempPath(), fileSystem.Path.GetRandomFileName()); + fileSystem.File.Delete(tempPath); + fileSystem.Directory.CreateDirectory(tempPath); try { - var tempPath2 = Path.GetTempFileName(); - File.Delete(tempPath2); + var tempPath2 = fileSystem.Path.Combine(fileSystem.Path.GetTempPath(), fileSystem.Path.GetRandomFileName()); + fileSystem.File.Delete(tempPath2); - await File.WriteAllTextAsync(Path.Combine(tempPath, "file.txt"), "asdf"); - var subDir = Path.Combine(tempPath, "subdir"); - Directory.CreateDirectory(subDir); - await File.WriteAllTextAsync(Path.Combine(subDir, "file2.txt"), "fdsa"); + await fileSystem.File.WriteAllTextAsync(fileSystem.Path.Combine(tempPath, "file.txt"), "asdf"); + var subDir = fileSystem.Path.Combine(tempPath, "subdir"); + fileSystem.Directory.CreateDirectory(subDir); + await fileSystem.File.WriteAllTextAsync(fileSystem.Path.Combine(subDir, "file2.txt"), "fdsa"); try { @@ -203,26 +217,26 @@ namespace Tgstation.Server.Host.IO.Tests throttle, default); - Assert.IsTrue(Directory.Exists(tempPath2)); - var newFilePath = Path.Combine(tempPath2, "file.txt"); - Assert.IsTrue(File.Exists(newFilePath)); - var newFileText = await File.ReadAllTextAsync(newFilePath); + Assert.IsTrue(fileSystem.Directory.Exists(tempPath2)); + var newFilePath = fileSystem.Path.Combine(tempPath2, "file.txt"); + Assert.IsTrue(fileSystem.File.Exists(newFilePath)); + var newFileText = await fileSystem.File.ReadAllTextAsync(newFilePath); Assert.AreEqual("asdf", newFileText); - var newDirPath = Path.Combine(tempPath2, "subdir"); - Assert.IsTrue(Directory.Exists(newDirPath)); - var newFile2Path = Path.Combine(newDirPath, "file2.txt"); - Assert.IsTrue(File.Exists(newFile2Path)); - var newFile2Text = await File.ReadAllTextAsync(newFile2Path); + var newDirPath = fileSystem.Path.Combine(tempPath2, "subdir"); + Assert.IsTrue(fileSystem.Directory.Exists(newDirPath)); + var newFile2Path = fileSystem.Path.Combine(newDirPath, "file2.txt"); + Assert.IsTrue(fileSystem.File.Exists(newFile2Path)); + var newFile2Text = await fileSystem.File.ReadAllTextAsync(newFile2Path); Assert.AreEqual("fdsa", newFile2Text); } finally { - Directory.Delete(tempPath2, true); + fileSystem.Directory.Delete(tempPath2, true); } } finally { - Directory.Delete(tempPath, true); + fileSystem.Directory.Delete(tempPath, true); } } } diff --git a/tests/Tgstation.Server.Host.Tests/System/TestPosixSignalHandler.cs b/tests/Tgstation.Server.Host.Tests/System/TestPosixSignalHandler.cs index dd0f5fd5fb..dd4f2debf3 100644 --- a/tests/Tgstation.Server.Host.Tests/System/TestPosixSignalHandler.cs +++ b/tests/Tgstation.Server.Host.Tests/System/TestPosixSignalHandler.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using System.IO.Abstractions; using System.Reflection; using System.Threading; using System.Threading.Tasks; @@ -58,7 +59,8 @@ namespace Tgstation.Server.Host.System.Tests processExecutor = new ProcessExecutor( new PosixProcessFeatures( new Lazy(() => processExecutor), - new DefaultIOManager(), + new DefaultIOManager( + new FileSystem()), loggerFactory.CreateLogger()), Mock.Of(), loggerFactory.CreateLogger(), diff --git a/tests/Tgstation.Server.Host.Tests/System/TestProcessFeatures.cs b/tests/Tgstation.Server.Host.Tests/System/TestProcessFeatures.cs index a32b9e389c..34d902e8e8 100644 --- a/tests/Tgstation.Server.Host.Tests/System/TestProcessFeatures.cs +++ b/tests/Tgstation.Server.Host.Tests/System/TestProcessFeatures.cs @@ -2,6 +2,7 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; using System; +using System.IO.Abstractions.TestingHelpers; using System.Runtime.InteropServices; using Tgstation.Server.Host.IO; @@ -21,7 +22,7 @@ namespace Tgstation.Server.Host.System.Tests { features = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? (IProcessFeatures)new WindowsProcessFeatures(Mock.Of>()) - : new PosixProcessFeatures(new Lazy(() => null), new DefaultIOManager(), Mock.Of>()); + : new PosixProcessFeatures(new Lazy(() => null), new DefaultIOManager(new MockFileSystem()), Mock.Of>()); } [TestMethod] diff --git a/tests/Tgstation.Server.Host.Tests/System/TestSymlinkFactory.cs b/tests/Tgstation.Server.Host.Tests/System/TestSymlinkFactory.cs index 4d3c5be749..222e870cd0 100644 --- a/tests/Tgstation.Server.Host.Tests/System/TestSymlinkFactory.cs +++ b/tests/Tgstation.Server.Host.Tests/System/TestSymlinkFactory.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using System.IO.Abstractions; using System.Threading; using System.Threading.Tasks; @@ -13,8 +14,10 @@ namespace Tgstation.Server.Host.System.Tests public sealed class TestSymlinkFactory { readonly IFilesystemLinkFactory factory = new PlatformIdentifier().IsWindows - ? new WindowsFilesystemLinkFactory() - : new PosixFilesystemLinkFactory(); + ? new WindowsFilesystemLinkFactory( + new FileSystem()) + : new PosixFilesystemLinkFactory( + new FileSystem()); [TestMethod] public async Task TestSymlinks() diff --git a/tests/Tgstation.Server.Host.Tests/TestProgram.cs b/tests/Tgstation.Server.Host.Tests/TestProgram.cs index f04531154f..a4f9318e13 100644 --- a/tests/Tgstation.Server.Host.Tests/TestProgram.cs +++ b/tests/Tgstation.Server.Host.Tests/TestProgram.cs @@ -2,6 +2,7 @@ using Moq; using System; using System.IO; +using System.IO.Abstractions.TestingHelpers; using System.Threading; using System.Threading.Tasks; @@ -80,28 +81,29 @@ namespace Tgstation.Server.Host.Tests public async Task TestStandardRunWithExceptionAndWatchdog() { var mockServer = new Mock(); + var mockFs = new MockFileSystem(); var exception = new DivideByZeroException(); mockServer.Setup(x => x.Run(It.IsAny())).Throws(exception); mockServer.SetupGet(x => x.RestartRequested).Returns(true); var mockServerFactory = new Mock(); - mockServerFactory.SetupGet(x => x.IOManager).Returns(new DefaultIOManager()); + mockServerFactory.SetupGet(x => x.IOManager).Returns(new DefaultIOManager(mockFs)); mockServerFactory.Setup(x => x.CreateServer(It.IsNotNull(), It.IsAny(), It.IsAny())).ReturnsAsync(mockServer.Object); var program = new Program { ServerFactory = mockServerFactory.Object }; - var tempFileName = Path.GetTempFileName(); - File.Delete(tempFileName); + var tempFileName = mockFs.Path.Combine(mockFs.Path.GetTempPath(), mockFs.Path.GetRandomFileName()); + mockFs.File.Delete(tempFileName); try { var result = await program.Main(Array.Empty(), tempFileName); Assert.AreEqual(HostExitCode.Error, result); - Assert.AreEqual(exception.ToString(), File.ReadAllText(tempFileName)); + Assert.AreEqual(exception.ToString(), mockFs.File.ReadAllText(tempFileName)); } finally { - File.Delete(tempFileName); + mockFs.File.Delete(tempFileName); } } } diff --git a/tests/Tgstation.Server.Host.Tests/TestServerFactory.cs b/tests/Tgstation.Server.Host.Tests/TestServerFactory.cs index 9a96a8332f..a520286782 100644 --- a/tests/Tgstation.Server.Host.Tests/TestServerFactory.cs +++ b/tests/Tgstation.Server.Host.Tests/TestServerFactory.cs @@ -1,6 +1,7 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; using System; +using System.IO.Abstractions.TestingHelpers; using System.Threading.Tasks; using Tgstation.Server.Host.Core; @@ -20,11 +21,13 @@ namespace Tgstation.Server.Host.Tests [TestMethod] public void TestConstructor() { - Assert.ThrowsException(() => new ServerFactory(null, null)); + Assert.ThrowsException(() => new ServerFactory(null, null, null)); IAssemblyInformationProvider assemblyInformationProvider = Mock.Of(); - Assert.ThrowsException(() => new ServerFactory(assemblyInformationProvider, null)); + Assert.ThrowsException(() => new ServerFactory(assemblyInformationProvider, null, null)); IIOManager ioManager = Mock.Of(); - _ = new ServerFactory(assemblyInformationProvider, ioManager); + Assert.ThrowsException(() => new ServerFactory(assemblyInformationProvider, ioManager, null)); + var mockFileSystem = new MockFileSystem(); + _ = new ServerFactory(assemblyInformationProvider, ioManager, mockFileSystem); } [TestMethod] diff --git a/tests/Tgstation.Server.Host.Tests/Tgstation.Server.Host.Tests.csproj b/tests/Tgstation.Server.Host.Tests/Tgstation.Server.Host.Tests.csproj index b6cb2167ad..f696ba76b7 100644 --- a/tests/Tgstation.Server.Host.Tests/Tgstation.Server.Host.Tests.csproj +++ b/tests/Tgstation.Server.Host.Tests/Tgstation.Server.Host.Tests.csproj @@ -8,6 +8,7 @@ + diff --git a/tests/Tgstation.Server.Tests/CachingFileDownloader.cs b/tests/Tgstation.Server.Tests/CachingFileDownloader.cs index de540b606d..76bdba28ee 100644 --- a/tests/Tgstation.Server.Tests/CachingFileDownloader.cs +++ b/tests/Tgstation.Server.Tests/CachingFileDownloader.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.IO.Abstractions; using System.Threading; using System.Threading.Tasks; @@ -195,7 +196,7 @@ namespace Tgstation.Server.Tests try { Directory.CreateDirectory(Path.GetDirectoryName(path)); - await using var fs = new DefaultIOManager().CreateAsyncSequentialWriteStream(path); + await using var fs = new DefaultIOManager(new FileSystem()).CreateAsyncSequentialWriteStream(path); await ms.CopyToAsync(fs, cancellationToken); cachedPaths.Add(url.ToString(), Tuple.Create(path, temporal)); diff --git a/tests/Tgstation.Server.Tests/Live/Instance/ConfigurationTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/ConfigurationTest.cs index eb39799414..52217388c1 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/ConfigurationTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/ConfigurationTest.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using System.IO.Abstractions; using System.Linq; using System.Text; using System.Threading; @@ -90,7 +91,8 @@ namespace Tgstation.Server.Tests.Live.Instance public ValueTask SetupDMApiTests(bool includingRoot, CancellationToken cancellationToken) { // just use an I/O manager here - var ioManager = new DefaultIOManager(); + var ioManager = new DefaultIOManager( + new FileSystem()); async ValueTask TestStaticFileAndDir() { @@ -127,7 +129,7 @@ namespace Tgstation.Server.Tests.Live.Instance Path = $"/EventScripts/{scriptName}" }; - await using var readStream = ioManager.GetFileStream($"../../../../DMAPI/{(basic ? "BasicOperation" : "LongRunning")}/{scriptName}", false); + await using var readStream = ioManager.CreateAsyncReadStream($"../../../../DMAPI/{(basic ? "BasicOperation" : "LongRunning")}/{scriptName}", true, false); await configurationClient.Write( resourcingScript, readStream, diff --git a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs index 8d667ebc41..1c77c35c1a 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.IO.Abstractions; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -89,7 +90,7 @@ namespace Tgstation.Server.Tests.Live.Instance Uri openDreamUrl, CancellationToken cancellationToken) { - var ioManager = new DefaultIOManager(); + var ioManager = new DefaultIOManager(new FileSystem()); var odRepoDir = ioManager.ConcatPath( Environment.GetFolderPath( Environment.SpecialFolder.LocalApplicationData, @@ -107,7 +108,7 @@ namespace Tgstation.Server.Tests.Live.Instance IEngineInstaller byondInstaller = compatVersion.Engine == EngineType.OpenDream ? new OpenDreamInstaller( - new DefaultIOManager(), + ioManager, Mock.Of>(), new PlatformIdentifier(), Mock.Of(), diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index df2476444b..dab176cdc5 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -14,6 +14,7 @@ using System; using System.Collections.Generic; using System.Globalization; using System.IO; +using System.IO.Abstractions; using System.Linq; using System.Net; using System.Net.Sockets; @@ -809,7 +810,7 @@ namespace Tgstation.Server.Tests.Live.Instance var features = new PosixProcessFeatures( new Lazy(Mock.Of()), - new DefaultIOManager(), + new DefaultIOManager(new FileSystem()), Mock.Of>()); features.SuspendProcess(proc); @@ -876,7 +877,7 @@ namespace Tgstation.Server.Tests.Live.Instance executor = new ProcessExecutor( RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? new WindowsProcessFeatures(Mock.Of>()) - : new PosixProcessFeatures(new Lazy(() => executor), new DefaultIOManager(), Mock.Of>()), + : new PosixProcessFeatures(new Lazy(() => executor), new DefaultIOManager(new FileSystem()), Mock.Of>()), Mock.Of(), Mock.Of>(), LoggerFactory.Create(x => { })); diff --git a/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs b/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs index ac4f519fa6..09bf058c1d 100644 --- a/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs +++ b/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs @@ -6,6 +6,7 @@ using System; using System.Collections.Generic; using System.Globalization; using System.IO; +using System.IO.Abstractions; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -43,7 +44,7 @@ namespace Tgstation.Server.Tests.Live for (int i = 0; i < 5; ++i) try { - new DefaultIOManager().DeleteDirectory(directory, default).GetAwaiter().GetResult(); + new DefaultIOManager(new FileSystem()).DeleteDirectory(directory, default).GetAwaiter().GetResult(); } catch { diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index 8df9a3d612..c872d9ab29 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; +using System.IO.Abstractions; using System.Linq; using System.Management; using System.Net; @@ -415,7 +416,7 @@ namespace Tgstation.Server.Tests.Live var gitHubToken = Environment.GetEnvironmentVariable("TGS_TEST_GITHUB_TOKEN"); if (String.IsNullOrWhiteSpace(gitHubToken)) gitHubToken = null; - await new Host.IO.DefaultIOManager().DeleteDirectory(server.UpdatePath, cancellationToken); + await new Host.IO.DefaultIOManager(new FileSystem()).DeleteDirectory(server.UpdatePath, cancellationToken); serverTask = server.Run(cancellationToken).AsTask(); await using (var adminClient = await CreateAdminClient(server.ApiUrl, cancellationToken)) @@ -1135,7 +1136,7 @@ namespace Tgstation.Server.Tests.Live ApiValidationSecurityLevel = DreamDaemonSecurity.Trusted, }, cancellationToken); - var ioManager = new Host.IO.DefaultIOManager(); + var ioManager = new Host.IO.DefaultIOManager(new FileSystem()); var repoPath = ioManager.ConcatPath(instance.Path, "Repository"); await using var jobsTest = new JobsRequiredTest(instanceClient.Jobs); var postWriteHandler = (Host.IO.IPostWriteHandler)(new PlatformIdentifier().IsWindows diff --git a/tests/Tgstation.Server.Tests/TestRepository.cs b/tests/Tgstation.Server.Tests/TestRepository.cs index 89cf34daac..96a2349bea 100644 --- a/tests/Tgstation.Server.Tests/TestRepository.cs +++ b/tests/Tgstation.Server.Tests/TestRepository.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using System.IO.Abstractions; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -58,7 +59,7 @@ namespace Tgstation.Server.Tests } finally { - await new DefaultIOManager().DeleteDirectory( + await new DefaultIOManager(new FileSystem()).DeleteDirectory( Path.GetDirectoryName(tempPath), CancellationToken.None); } @@ -75,7 +76,7 @@ namespace Tgstation.Server.Tests using var manager = new RepositoryManager( repoFac, commands, - new DefaultIOManager().CreateResolverForSubdirectory( + new DefaultIOManager(new FileSystem()).CreateResolverForSubdirectory( tempPath), Mock.Of(), new WindowsPostWriteHandler(), @@ -130,7 +131,7 @@ namespace Tgstation.Server.Tests } finally { - await new DefaultIOManager().DeleteDirectory( + await new DefaultIOManager(new FileSystem()).DeleteDirectory( Path.GetDirectoryName(tempPath), CancellationToken.None); } diff --git a/tests/Tgstation.Server.Tests/TestSystemInteraction.cs b/tests/Tgstation.Server.Tests/TestSystemInteraction.cs index e7c8e28962..f2601acdee 100644 --- a/tests/Tgstation.Server.Tests/TestSystemInteraction.cs +++ b/tests/Tgstation.Server.Tests/TestSystemInteraction.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using System.IO.Abstractions; using System.Threading; using System.Threading.Tasks; @@ -24,7 +25,7 @@ namespace Tgstation.Server.Tests var platformIdentifier = new PlatformIdentifier(); var processExecutor = new ProcessExecutor( Mock.Of(), - new DefaultIOManager(), + new DefaultIOManager(new FileSystem()), Mock.Of>(), loggerFactory); @@ -52,7 +53,7 @@ namespace Tgstation.Server.Tests var platformIdentifier = new PlatformIdentifier(); var processExecutor = new ProcessExecutor( Mock.Of(), - new DefaultIOManager(), + new DefaultIOManager(new FileSystem()), loggerFactory.CreateLogger(), loggerFactory); diff --git a/tests/Tgstation.Server.Tests/TestVersions.cs b/tests/Tgstation.Server.Tests/TestVersions.cs index a0809b78f6..047de71987 100644 --- a/tests/Tgstation.Server.Tests/TestVersions.cs +++ b/tests/Tgstation.Server.Tests/TestVersions.cs @@ -33,6 +33,7 @@ using Tgstation.Server.Host.System; using Tgstation.Server.Api.Models; using Tgstation.Server.Tests.Live; using Tgstation.Server.Host.Properties; +using System.IO.Abstractions; namespace Tgstation.Server.Tests { @@ -207,7 +208,7 @@ namespace Tgstation.Server.Tests loggerFactory.CreateLogger()) : new PosixByondInstaller( new PosixPostWriteHandler(loggerFactory.CreateLogger()), - new DefaultIOManager(), + new DefaultIOManager(new FileSystem()), fileDownloader, loggerFactory.CreateLogger()); using var disposable = byondInstaller as IDisposable; @@ -217,13 +218,13 @@ namespace Tgstation.Server.Tests ? new WindowsProcessFeatures(Mock.Of>()) : new PosixProcessFeatures( new Lazy(() => null), - new DefaultIOManager(), + new DefaultIOManager(new FileSystem()), loggerFactory.CreateLogger()), Mock.Of(), loggerFactory.CreateLogger(), loggerFactory); - var ioManager = new DefaultIOManager(); + var ioManager = new DefaultIOManager(new FileSystem()); var tempPath = ioManager.ConcatPath(LiveTestingServer.BaseDirectory, "mapthreads"); await ioManager.CreateDirectory(tempPath, default); try diff --git a/tests/Tgstation.Server.Tests/TestingUtils.cs b/tests/Tgstation.Server.Tests/TestingUtils.cs index 3167d2c3d3..3e1a2a3267 100644 --- a/tests/Tgstation.Server.Tests/TestingUtils.cs +++ b/tests/Tgstation.Server.Tests/TestingUtils.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using System.IO.Abstractions; using System.IO.Compression; using System.Reflection; using System.Threading; @@ -67,7 +68,7 @@ namespace Tgstation.Server.Tests } finally { - await new DefaultIOManager().DeleteDirectory(tempFolder, cancellationToken); + await new DefaultIOManager(new FileSystem()).DeleteDirectory(tempFolder, cancellationToken); } } } From 933b9d9fe84a7af89b4a2ccc20bc840f908298bf Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 21 Jul 2025 18:03:39 -0400 Subject: [PATCH 25/52] Finish properly setting up mirror and caching courtesy of @AffectedArc07 --- .github/CONTRIBUTING.md | 2 ++ .github/workflows/ci-pipeline.yml | 2 ++ .../CachingFileDownloader.cs | 30 ++++++++++++++----- .../Live/TestLiveServer.cs | 14 +++++++-- tests/Tgstation.Server.Tests/TestingUtils.cs | 2 +- 5 files changed, 39 insertions(+), 11 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index c2912e0af5..54178ff953 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -51,6 +51,8 @@ You must also have the following environment variables set. To run them more acc - `TGS_TEST_CONNECTION_STRING`: To a valid database connection string. You can use the setup wizard to create one. - (Optional) `TGS_TEST_GITHUB_TOKEN`: A GitHub personal access token with no scopes used to bypass rate limits. - (Optional) `TGS_TEST_BYOND_ZIP_DOWNLOAD_TEMPLATE`: Template URL for downloading BYOND zip files from a non-official mirror. +- (Optional) `TGS_TEST_BYOND_MIRROR_VERSION_TXT`: version.txt for a BYOND zip mirror. Requires `TGS_TEST_BYOND_ZIP_DOWNLOAD_TEMPLATE`. +- (Optional) `TGS_TEST_BYOND_ZIPS_BASE_PATH`: Directory on disk to cache BYOND zip files. - (Optional) The following variables are all interdependent, so if one is set they all must be. - `TGS_TEST_DISCORD_TOKEN`: To a valid discord bot token. - `TGS_TEST_DISCORD_CHANNEL`: To a valid discord channel ID that the above bot can access. diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 9f9c9a5010..0e098dc503 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -42,6 +42,8 @@ env: TGS_NODE_VERSION: 20.x TGS_TEST_GITHUB_TOKEN: ${{ secrets.LIVE_TESTS_TOKEN }} PACKAGING_PRIVATE_KEY_PASSPHRASE: ${{ secrets.PACKAGING_PRIVATE_KEY_PASSPHRASE }} + TGS_TEST_BYOND_ZIP_DOWNLOAD_TEMPLATE: https://mocha.affectedarc07.co.uk/tgs_byond_mirrors/${Major}.${Minor}_byond${Linux:_linux}.zip + TGS_TEST_BYOND_MIRROR_VERSION_TXT: https://mocha.affectedarc07.co.uk/tgs_byond_mirrors/version.txt concurrency: group: "ci-${{ (github.event_name != 'push' && github.event_name != 'schedule' && github.event.inputs.pull_request_number) || github.run_id }}-${{ github.event_name }}" diff --git a/tests/Tgstation.Server.Tests/CachingFileDownloader.cs b/tests/Tgstation.Server.Tests/CachingFileDownloader.cs index 8fccd1234c..f2897cf808 100644 --- a/tests/Tgstation.Server.Tests/CachingFileDownloader.cs +++ b/tests/Tgstation.Server.Tests/CachingFileDownloader.cs @@ -92,15 +92,21 @@ namespace Tgstation.Server.Tests var url = ByondInstallerBase.GetDownloadZipUrl(byondVersion, urlTemplate, new PlatformIdentifier().IsWindows ? "Windows" : "Linux"); string path = null; - if (TestingUtils.RunningInGitHubActions) + string basePath = Environment.GetEnvironmentVariable("TGS_TEST_BYOND_ZIPS_BASE_PATH"); + if (basePath == null && TestingUtils.RunningInGitHubActions) { - // actions is supposed to cache BYOND for us - - var dir = Path.Combine( + // actions is supposed to cache BYOND for us here + basePath = Path.Combine( Environment.GetFolderPath( Environment.SpecialFolder.UserProfile, Environment.SpecialFolderOption.DoNotVerify), - "byond-zips-cache", + "byond-zips-cache"); + } + + if (basePath != null) + { + var dir = Path.Combine( + basePath, "live", windows ? "windows" : "linux"); path = Path.Combine( @@ -109,15 +115,23 @@ namespace Tgstation.Server.Tests $"{version.Version.Major}.{version.Version.Minor}.zip"); } + Uri overrideUrl = null; + if (urlCacheOverrideTemplate != null) + { + overrideUrl = url; + url = ByondInstallerBase.GetDownloadZipUrl(byondVersion, urlCacheOverrideTemplate, new PlatformIdentifier().IsWindows ? "Windows" : "Linux"); + } + await (await CacheFile( logger, - urlCacheOverrideTemplate != null - ? ByondInstallerBase.GetDownloadZipUrl(byondVersion, urlCacheOverrideTemplate, new PlatformIdentifier().IsWindows ? "Windows" : "Linux") - : url, + url, null, path, cancellationToken)) .DisposeAsync(); + + if (overrideUrl != null) + cachedPaths[overrideUrl.ToString()] = cachedPaths[url.ToString()]; } public static void Cleanup() diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index 7eeec04af3..61013ea17e 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -1626,6 +1626,16 @@ namespace Tgstation.Server.Tests.Live if (openDreamOnly) return; + var windowsMinCompat = new Version(510, 1346); + var linuxMinCompat = new Version(512, 1451); // http://www.byond.com/forum/?forum=5&command=search&scope=local&text=resolved%3a512.1451 + await CachingFileDownloader.InitializeByondVersion( + GetLogger(), + new PlatformIdentifier().IsWindows + ? windowsMinCompat + : linuxMinCompat, + new PlatformIdentifier().IsWindows, + cancellationToken); + var compatTests = FailFast( instanceTest .RunCompatTests( @@ -1633,8 +1643,8 @@ namespace Tgstation.Server.Tests.Live { Engine = EngineType.Byond, Version = new PlatformIdentifier().IsWindows - ? new Version(510, 1346) - : new Version(512, 1451) // http://www.byond.com/forum/?forum=5&command=search&scope=local&text=resolved%3a512.1451 + ? windowsMinCompat + : linuxMinCompat, }, server.OpenDreamUrl, firstAdminRestClient.Instances.CreateClient(compatInstance), diff --git a/tests/Tgstation.Server.Tests/TestingUtils.cs b/tests/Tgstation.Server.Tests/TestingUtils.cs index 9cc11f7966..40925352dd 100644 --- a/tests/Tgstation.Server.Tests/TestingUtils.cs +++ b/tests/Tgstation.Server.Tests/TestingUtils.cs @@ -136,7 +136,7 @@ namespace Tgstation.Server.Tests const string DefaultMirror = "https://www.byond.com/download/version.txt"; edgeVersion = await GetVersionFromResponse(DefaultMirror); - logger.LogInformation("Downloading edge version from BYOND.com"); + logger.LogInformation("Downloading edge version from BYOND.com {edge}", edgeVersion); // if we got the result from byond.com, make sure the cache grabs the zip from there as well await CachingFileDownloader.InitializeByondVersion(logger, Version.Parse(edgeVersion), new PlatformIdentifier().IsWindows, cancellationToken, GeneralConfiguration.DefaultByondZipDownloadTemplate); From cf82c4be7497645887a0428065135a8aaad1055d Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 21 Jul 2025 18:09:01 -0400 Subject: [PATCH 26/52] Fix EDGE version evaluation in scripts --- .github/workflows/ci-pipeline.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 0e098dc503..8a52bba815 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -169,6 +169,7 @@ jobs: sudo apt-get install -y -o APT::Immediate-Configure=0 libc6-i386 libstdc++6:i386 libgcc-s1:i386 - name: Evaluate EDGE BYOND version + if: ${{ matrix.byond == 'EDGE' }} id: edge_version_evaluation run: | FULL_VERSION=${{ matrix.byond }} @@ -185,7 +186,7 @@ jobs: FULL_VERSION=${bad_linux_releases[$FULL_VERSION]} fi fi - run: echo "EVALUATED_EDGE_VERSION=$FULL_VERSION" >> $GITHUB_OUTPUT + echo "EVALUATED_EDGE_VERSION=$FULL_VERSION" >> $GITHUB_OUTPUT - name: Cache BYOND .zips uses: actions/cache@v4 @@ -197,7 +198,10 @@ jobs: - name: Setup BYOND Cache if Necessary and Install run: | echo "Setting up BYOND." - FULL_VERSION=${{ steps.edge_version_evaluation.outputs.EVALUATED_EDGE_VERSION }} + FULL_VERSION=${{ matrix.byond }} + if [[ "$FULL_VERSION" = "EDGE" ]] ; then + FULL_VERSION=${{ steps.edge_version_evaluation.outputs.EVALUATED_EDGE_VERSION }} + fi if [[ ! -f $HOME/byond-zips-cache/linux/$FULL_VERSION/$FULL_VERSION.zip ]] ; then BYOND_MAJOR=${FULL_VERSION%.*} mkdir -p $HOME/byond-zips-cache/linux From 0e86acbaa3f3372da378314777969351c9ec431f Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 21 Jul 2025 18:11:30 -0400 Subject: [PATCH 27/52] Fix version evaluation --- .github/workflows/ci-pipeline.yml | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 8a52bba815..68bec6e0f2 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -168,9 +168,8 @@ jobs: sudo apt-get update sudo apt-get install -y -o APT::Immediate-Configure=0 libc6-i386 libstdc++6:i386 libgcc-s1:i386 - - name: Evaluate EDGE BYOND version - if: ${{ matrix.byond == 'EDGE' }} - id: edge_version_evaluation + - name: Evaluate BYOND version + id: version_evaluation run: | FULL_VERSION=${{ matrix.byond }} if [[ "$FULL_VERSION" = "EDGE" ]] ; then @@ -186,22 +185,19 @@ jobs: FULL_VERSION=${bad_linux_releases[$FULL_VERSION]} fi fi - echo "EVALUATED_EDGE_VERSION=$FULL_VERSION" >> $GITHUB_OUTPUT + echo "EVALUATED_VERSION=$FULL_VERSION" >> $GITHUB_OUTPUT - name: Cache BYOND .zips uses: actions/cache@v4 id: cache-byond with: - path: ~/byond-zips-cache/linux/${{ steps.edge_version_evaluation.outputs.EVALUATED_EDGE_VERSION }} - key: byond-zips-linux-${{ steps.edge_version_evaluation.outputs.EVALUATED_EDGE_VERSION }} + path: ~/byond-zips-cache/linux/${{ steps.version_evaluation.outputs.EVALUATED_VERSION }} + key: byond-zips-linux-${{ steps.version_evaluation.outputs.EVALUATED_VERSION }} - name: Setup BYOND Cache if Necessary and Install run: | echo "Setting up BYOND." - FULL_VERSION=${{ matrix.byond }} - if [[ "$FULL_VERSION" = "EDGE" ]] ; then - FULL_VERSION=${{ steps.edge_version_evaluation.outputs.EVALUATED_EDGE_VERSION }} - fi + FULL_VERSION=${{ steps.version_evaluation.outputs.EVALUATED_VERSION }} if [[ ! -f $HOME/byond-zips-cache/linux/$FULL_VERSION/$FULL_VERSION.zip ]] ; then BYOND_MAJOR=${FULL_VERSION%.*} mkdir -p $HOME/byond-zips-cache/linux From 8ef960a5bff5ec3aa6c9e0671a4f32aa09f2b4ff Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 21 Jul 2025 18:21:31 -0400 Subject: [PATCH 28/52] Mirror everything except EDGE --- .github/workflows/ci-pipeline.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 68bec6e0f2..dc5060ad24 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -44,6 +44,7 @@ env: PACKAGING_PRIVATE_KEY_PASSPHRASE: ${{ secrets.PACKAGING_PRIVATE_KEY_PASSPHRASE }} TGS_TEST_BYOND_ZIP_DOWNLOAD_TEMPLATE: https://mocha.affectedarc07.co.uk/tgs_byond_mirrors/${Major}.${Minor}_byond${Linux:_linux}.zip TGS_TEST_BYOND_MIRROR_VERSION_TXT: https://mocha.affectedarc07.co.uk/tgs_byond_mirrors/version.txt + DMAPI_BYOND_BUILD_MIRROR_DOWNLOAD_TEMPLATE: https://mocha.affectedarc07.co.uk/tgs_byond_mirrors/${FULL_VERSION}_byond_linux.zip # Available vars are $BYOND_MAJOR (i.e. 516) and $FULL_VERSION (i.e. 516.1666) concurrency: group: "ci-${{ (github.event_name != 'push' && github.event_name != 'schedule' && github.event.inputs.pull_request_number) || github.run_id }}-${{ github.event_name }}" @@ -200,8 +201,12 @@ jobs: FULL_VERSION=${{ steps.version_evaluation.outputs.EVALUATED_VERSION }} if [[ ! -f $HOME/byond-zips-cache/linux/$FULL_VERSION/$FULL_VERSION.zip ]] ; then BYOND_MAJOR=${FULL_VERSION%.*} - mkdir -p $HOME/byond-zips-cache/linux - curl "https://www.byond.com/download/$BYOND_MAJOR/${FULL_VERSION}_byond_linux.zip" -o $HOME/byond-zips-cache/linux/$FULL_VERSION/$FULL_VERSION.zip + mkdir -p $HOME/byond-zips-cache/linux/$FULL_VERSION + DOWNLOAD_URL="${{ env.DMAPI_BYOND_BUILD_MIRROR_DOWNLOAD_TEMPLATE }}" + if [[ "$FULL_VERSION" = "EDGE" ]] ; then + DOWNLOAD_URL="https://www.byond.com/download/$BYOND_MAJOR/${FULL_VERSION}_byond_linux.zip" + fi + curl "$DOWNLOAD_URL" -o $HOME/byond-zips-cache/linux/$FULL_VERSION/$FULL_VERSION.zip fi mkdir -p "$HOME/BYOND" cd "$HOME/BYOND" From 0904014028ba200ef42e05367cd9069bef3e0960 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 22 Jul 2025 05:26:00 +0000 Subject: [PATCH 29/52] Bump form-data Bumps the npm_and_yarn group with 1 update in the /src/Tgstation.Server.Host.Utils.GitLab.GraphQL directory: [form-data](https://github.com/form-data/form-data). Updates `form-data` from 4.0.2 to 4.0.4 - [Release notes](https://github.com/form-data/form-data/releases) - [Changelog](https://github.com/form-data/form-data/blob/master/CHANGELOG.md) - [Commits](https://github.com/form-data/form-data/compare/v4.0.2...v4.0.4) --- updated-dependencies: - dependency-name: form-data dependency-version: 4.0.4 dependency-type: indirect dependency-group: npm_and_yarn ... Signed-off-by: dependabot[bot] --- src/Tgstation.Server.Host.Utils.GitLab.GraphQL/yarn.lock | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Tgstation.Server.Host.Utils.GitLab.GraphQL/yarn.lock b/src/Tgstation.Server.Host.Utils.GitLab.GraphQL/yarn.lock index d1d93cd55d..5e4dca122d 100644 --- a/src/Tgstation.Server.Host.Utils.GitLab.GraphQL/yarn.lock +++ b/src/Tgstation.Server.Host.Utils.GitLab.GraphQL/yarn.lock @@ -1855,14 +1855,15 @@ __metadata: linkType: hard "form-data@npm:^4.0.0": - version: 4.0.2 - resolution: "form-data@npm:4.0.2" + version: 4.0.4 + resolution: "form-data@npm:4.0.4" dependencies: asynckit: "npm:^0.4.0" combined-stream: "npm:^1.0.8" es-set-tostringtag: "npm:^2.1.0" + hasown: "npm:^2.0.2" mime-types: "npm:^2.1.12" - checksum: 10/82c65b426af4a40090e517a1bc9057f76970b4c6043e37aa49859c447d88553e77d4cc5626395079a53d2b0889ba5f2a49f3900db3ad3f3f1bf76613532572fb + checksum: 10/a4b62e21932f48702bc468cc26fb276d186e6b07b557e3dd7cc455872bdbb82db7db066844a64ad3cf40eaf3a753c830538183570462d3649fdfd705601cbcfb languageName: node linkType: hard From 1d0e30bc7fef7102f917c14704f449a169e30e1c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 25 Jul 2025 09:49:31 +0000 Subject: [PATCH 30/52] Bump the mstest group with 2 updates Bumps MSTest.TestAdapter from 3.8.3 to 3.9.3 Bumps MSTest.TestFramework from 3.8.3 to 3.9.3 --- updated-dependencies: - dependency-name: MSTest.TestAdapter dependency-version: 3.9.3 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: mstest - dependency-name: MSTest.TestFramework dependency-version: 3.9.3 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: mstest ... Signed-off-by: dependabot[bot] --- build/TestCommon.props | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/TestCommon.props b/build/TestCommon.props index 52ba145ada..44c2a0c2e4 100644 --- a/build/TestCommon.props +++ b/build/TestCommon.props @@ -18,9 +18,9 @@ - + - + From 23f02e95976858b2a4d4f74e6ab9aa56157d9a35 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 26 Jul 2025 10:23:02 -0400 Subject: [PATCH 31/52] Hopefully use Spacestation13 BYOND builds mirror --- .github/workflows/ci-pipeline.yml | 53 +++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index dc5060ad24..9e1c37201e 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -174,7 +174,7 @@ jobs: run: | FULL_VERSION=${{ matrix.byond }} if [[ "$FULL_VERSION" = "EDGE" ]] ; then - VERSIONS=$(curl https://www.byond.com/download/version.txt) + VERSIONS=$(curl https://spacestation13.github.io/byond-builds/version.txt) FULL_VERSION=$(echo "$VERSIONS" | tail -n1) echo "EDGE version evaluated to $FULL_VERSION" @@ -203,8 +203,8 @@ jobs: BYOND_MAJOR=${FULL_VERSION%.*} mkdir -p $HOME/byond-zips-cache/linux/$FULL_VERSION DOWNLOAD_URL="${{ env.DMAPI_BYOND_BUILD_MIRROR_DOWNLOAD_TEMPLATE }}" - if [[ "$FULL_VERSION" = "EDGE" ]] ; then - DOWNLOAD_URL="https://www.byond.com/download/$BYOND_MAJOR/${FULL_VERSION}_byond_linux.zip" + if [[ "${{ matrix.byond }}" = "EDGE" ]] ; then + DOWNLOAD_URL="https://spacestation13.github.io/byond-builds/$BYOND_MAJOR/${FULL_VERSION}_byond_linux.zip" fi curl "$DOWNLOAD_URL" -o $HOME/byond-zips-cache/linux/$FULL_VERSION/$FULL_VERSION.zip fi @@ -597,9 +597,50 @@ jobs: name: windows-unit-test-coverage-${{ matrix.configuration }} path: ./TestResults/ + prep-edge-versions: + name: Prepare Live Tests Cache of EDGE Versions + needs: start-gate + runs-on: ubuntu-latest + steps: + - name: Cache BYOND .zips (Linux) + uses: actions/cache@v4 + with: + path: ~/byond-zips-cache/live/linux + key: byond-zips-linux-live + + - name: Cache BYOND .zips (Windows) + uses: actions/cache@v4 + with: + path: ~/byond-zips-cache/live/windows + key: byond-zips-windows-live + + - name: Evaluate BYOND version + id: version_evaluation + run: | + VERSIONS=$(curl https://spacestation13.github.io/byond-builds/version.txt) + FULL_VERSION=$(echo "$VERSIONS" | tail -n1) + echo "EDGE version evaluated to $FULL_VERSION" + echo "EVALUATED_VERSION=$FULL_VERSION" >> $GITHUB_OUTPUT + + - name: Setup BYOND Cache if Necessary and Install + run: | + echo "Downloading BYOND." + FULL_VERSION=${{ steps.version_evaluation.outputs.EVALUATED_VERSION }} + BYOND_MAJOR=${FULL_VERSION%.*} + if [[ ! -f $HOME/byond-zips-cache/live/linux/$FULL_VERSION/$FULL_VERSION.zip ]] ; then + mkdir -p $HOME/byond-zips-cache/live/linux/$FULL_VERSION + DOWNLOAD_URL="https://spacestation13.github.io/byond-builds/$BYOND_MAJOR/${FULL_VERSION}_byond_linux.zip" + curl "$DOWNLOAD_URL" -o $HOME/byond-zips-cache/live/linux/$FULL_VERSION/$FULL_VERSION.zip + fi + if [[ ! -f $HOME/byond-zips-cache/live/windows/$FULL_VERSION/$FULL_VERSION.zip ]] ; then + mkdir -p $HOME/byond-zips-cache/live/windows/$FULL_VERSION + DOWNLOAD_URL="https://spacestation13.github.io/byond-builds/$BYOND_MAJOR/${FULL_VERSION}_byond.zip" + curl "$DOWNLOAD_URL" -o $HOME/byond-zips-cache/live/windows/$FULL_VERSION/$FULL_VERSION.zip + fi + windows-integration-tests: name: Windows Live Tests - needs: [dmapi-build, opendream-build] + needs: [dmapi-build, opendream-build, prep-edge-versions] strategy: fail-fast: false matrix: @@ -634,7 +675,6 @@ jobs: - name: Cache BYOND .zips uses: actions/cache@v4 - id: cache-byond with: path: ~/byond-zips-cache/live/windows key: byond-zips-windows-live @@ -815,7 +855,7 @@ jobs: linux-integration-tests: name: Linux Live Tests - needs: [dmapi-build, opendream-build] + needs: [dmapi-build, opendream-build, prep-edge-versions] services: # We start all dbs here so we can just code the stuff once mssql: image: ${{ (matrix.database-type == 'SqlServer') && 'mcr.microsoft.com/mssql/server:2019-latest' || '' }} @@ -892,7 +932,6 @@ jobs: - name: Cache BYOND .zips uses: actions/cache@v4 - id: cache-byond with: path: ~/byond-zips-cache/live/linux key: byond-zips-linux-live From ce35c4e64e500ce3b9700361879d3f3fb76204ac Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 26 Jul 2025 10:27:34 -0400 Subject: [PATCH 32/52] Add libcurl4 to CI --- .github/workflows/ci-pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 9e1c37201e..19794e284a 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -167,7 +167,7 @@ jobs: run: | sudo dpkg --add-architecture i386 sudo apt-get update - sudo apt-get install -y -o APT::Immediate-Configure=0 libc6-i386 libstdc++6:i386 libgcc-s1:i386 + sudo apt-get install -y -o APT::Immediate-Configure=0 libc6-i386 libstdc++6:i386 libgcc-s1:i386 libcurl4:i386 - name: Evaluate BYOND version id: version_evaluation @@ -923,7 +923,7 @@ jobs: run: | sudo dpkg --add-architecture i386 sudo apt-get update - sudo apt-get install -y -o APT::Immediate-Configure=0 libc6-i386 libstdc++6:i386 gdb libgcc-s1:i386 libgdiplus + sudo apt-get install -y -o APT::Immediate-Configure=0 libc6-i386 libstdc++6:i386 gdb libgcc-s1:i386 libgdiplus libcurl4:i386 - name: Setup Node.JS uses: actions/setup-node@v4 From 5026cd31619c009dd39ee1cec78709cd71a2eb84 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 26 Jul 2025 10:47:20 -0400 Subject: [PATCH 33/52] Update nix setup, add flake validation job --- .github/workflows/ci-pipeline.yml | 26 ++++++++++++++++++++++++++ .github/workflows/nix-deployment.yml | 2 +- build/package/nix/flake.nix | 1 + 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 19794e284a..cd6ce47071 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -93,6 +93,31 @@ jobs: name: release_notes_bins path: ./release_notes_bins/ + validate-nix-flake: + name: Validate Nix Flake + needs: start-gate + runs-on: ubuntu-latest + steps: + - name: Setup Nix + uses: cachix/install-nix-action@v31 + with: + nix_path: nixpkgs=channel:nixos-unstable + + - name: Checkout (Branch) + uses: actions/checkout@v4 + if: github.event_name == 'push' || github.event_name == 'schedule' + + - name: Checkout (PR Merge) + uses: actions/checkout@v4 + if: github.event_name != 'push' && github.event_name != 'schedule' + with: + ref: "refs/pull/${{ inputs.pull_request_number }}/merge" + + - name: Check Flake + run: | + cd build/package/nix + nix flake check + code-scanning: name: Run CodeQL needs: start-gate @@ -1695,6 +1720,7 @@ jobs: check-winget-pr-template, efcore-version-match, code-scanning, + validate-nix-flake, ] runs-on: ubuntu-latest steps: diff --git a/.github/workflows/nix-deployment.yml b/.github/workflows/nix-deployment.yml index 16c62fe453..470913346e 100644 --- a/.github/workflows/nix-deployment.yml +++ b/.github/workflows/nix-deployment.yml @@ -15,7 +15,7 @@ jobs: sudo apt-get install -y xmlstarlet - name: Setup Nix - uses: cachix/install-nix-action@v30 + uses: cachix/install-nix-action@v31 with: nix_path: nixpkgs=channel:nixos-unstable diff --git a/build/package/nix/flake.nix b/build/package/nix/flake.nix index f2bf8c14a9..16c7c70a89 100644 --- a/build/package/nix/flake.nix +++ b/build/package/nix/flake.nix @@ -9,5 +9,6 @@ imports = [ ./tgstation-server.nix ]; }; }; + checks.x86_64-linux.flake-build = self.packages.x86_64-linux.default; }; } From e0a96a630cbf3c07c09eb82a092a06036fe05a8a Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 26 Jul 2025 10:50:02 -0400 Subject: [PATCH 34/52] Breaking this temporarily --- .github/workflows/ci-security.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-security.yml b/.github/workflows/ci-security.yml index eb7806f7d1..28efbaf9d5 100644 --- a/.github/workflows/ci-security.yml +++ b/.github/workflows/ci-security.yml @@ -1,4 +1,5 @@ -name: CI Security +na*(@$&UQ!*@)(*$!U)U$! + ASLDKme: CI Security on: pull_request: @@ -21,7 +22,7 @@ concurrency: group: "ci-security-${{ github.head_ref || github.run_id }}-${{ github.event_name }}" cancel-in-progress: true -jobs: +job1231s: security-checkpoint: name: Check CI Clearance if: github.event_name == 'pull_request_target' && (github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id || github.event.pull_request.user.id == 49699333) && github.event.pull_request.state == 'open' From 2c9591a7f4ef378eecd2ee723ebe13f32e062c48 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 26 Jul 2025 11:02:26 -0400 Subject: [PATCH 35/52] Try this --- build/package/nix/flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/package/nix/flake.nix b/build/package/nix/flake.nix index 16c7c70a89..a67c9fcaec 100644 --- a/build/package/nix/flake.nix +++ b/build/package/nix/flake.nix @@ -9,6 +9,6 @@ imports = [ ./tgstation-server.nix ]; }; }; - checks.x86_64-linux.flake-build = self.packages.x86_64-linux.default; + checks.x86_64-linux.package-build = pkgs.callPackage ./package.nix { } }; } From ab8a3f1f5224ea313e279dccc4b7b5bee6b50c6e Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 26 Jul 2025 11:02:43 -0400 Subject: [PATCH 36/52] Add missing semicolon --- build/package/nix/flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/package/nix/flake.nix b/build/package/nix/flake.nix index a67c9fcaec..11bb337751 100644 --- a/build/package/nix/flake.nix +++ b/build/package/nix/flake.nix @@ -9,6 +9,6 @@ imports = [ ./tgstation-server.nix ]; }; }; - checks.x86_64-linux.package-build = pkgs.callPackage ./package.nix { } + checks.x86_64-linux.package-build = pkgs.callPackage ./package.nix { }; }; } From 93c570bd2e3e2e6e200528cae6642ef4dba36229 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 26 Jul 2025 11:03:59 -0400 Subject: [PATCH 37/52] Try with nixpkgs --- build/package/nix/flake.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/package/nix/flake.nix b/build/package/nix/flake.nix index 11bb337751..52ef59cd67 100644 --- a/build/package/nix/flake.nix +++ b/build/package/nix/flake.nix @@ -3,12 +3,12 @@ inputs = {}; - outputs = { ... }: { + outputs = { nixpkgs, ... }: { nixosModules = { default = { ... }: { imports = [ ./tgstation-server.nix ]; }; }; - checks.x86_64-linux.package-build = pkgs.callPackage ./package.nix { }; + checks.x86_64-linux.package-build = nixpkgs.legacyPackages.x86_64-linux.callPackage ./package.nix { }; }; } From cd6619b7d894bde2d7cf5ec595293e29daa60834 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 26 Jul 2025 11:15:06 -0400 Subject: [PATCH 38/52] Should get things where we need them --- .github/workflows/ci-pipeline.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index cd6ce47071..7f342a125b 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -97,6 +97,8 @@ jobs: name: Validate Nix Flake needs: start-gate runs-on: ubuntu-latest + env: + TEST_TGS_VERSION: "6.17.0" # Version we use here doesn't matter as it won't be executed. Just used to download a zip and calc hash steps: - name: Setup Nix uses: cachix/install-nix-action@v31 @@ -113,6 +115,24 @@ jobs: with: ref: "refs/pull/${{ inputs.pull_request_number }}/merge" + - name: Replace current TGS version with test version + run: + CURRENT_TGS_VERSION="$(xmlstarlet sel -N X="http://schemas.microsoft.com/developer/msbuild/2003" --template --value-of /X:Project/X:PropertyGroup/X:TgsCoreVersion build/Version.props)" + sed -i -e "s/${CURRENT_TGS_VERSION}/${{ env.TEST_TGS_VERSION }}" build/Version.props + + - name: Retrieve ServerConsole.zip Artifact + run: | + mkdir release + curl -L https://github.com/tgstation/tgstation-server/releases/download/tgstation-server-v${{ env.TEST_TGS_VERSION }}/ServerConsole.zip -f -o release/ServerConsole.zip + + - name: Regenerate Nix Hash + run: | + nix hash path ./release > build/package/nix/ServerConsole.sha256 + cat build/package/nix/ServerConsole.sha256 + + - name: Cleanup Download + run: rm -rf ./release + - name: Check Flake run: | cd build/package/nix From 9921e26c7c206b416dc085a8b74019bb6ce01a60 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 26 Jul 2025 11:15:14 -0400 Subject: [PATCH 39/52] Revert "Breaking this temporarily" This reverts commit e0a96a630cbf3c07c09eb82a092a06036fe05a8a. --- .github/workflows/ci-security.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-security.yml b/.github/workflows/ci-security.yml index 28efbaf9d5..eb7806f7d1 100644 --- a/.github/workflows/ci-security.yml +++ b/.github/workflows/ci-security.yml @@ -1,5 +1,4 @@ -na*(@$&UQ!*@)(*$!U)U$! - ASLDKme: CI Security +name: CI Security on: pull_request: @@ -22,7 +21,7 @@ concurrency: group: "ci-security-${{ github.head_ref || github.run_id }}-${{ github.event_name }}" cancel-in-progress: true -job1231s: +jobs: security-checkpoint: name: Check CI Clearance if: github.event_name == 'pull_request_target' && (github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id || github.event.pull_request.user.id == 49699333) && github.event.pull_request.state == 'open' From 5ed5d2aafc70533d0273401ec70e9caac2f4b557 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 26 Jul 2025 11:16:52 -0400 Subject: [PATCH 40/52] Add missing `xmlstarlet` dep --- .github/workflows/ci-pipeline.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 7f342a125b..16e237c9d5 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -100,6 +100,11 @@ jobs: env: TEST_TGS_VERSION: "6.17.0" # Version we use here doesn't matter as it won't be executed. Just used to download a zip and calc hash steps: + - name: Install Native Packages # Name checked in rerunFlakyTests.js + run: | + sudo apt-get update + sudo apt-get install -y xmlstarlet + - name: Setup Nix uses: cachix/install-nix-action@v31 with: From bed489af78f9326e0f34cbc81820ef3fee6cc867 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 26 Jul 2025 11:19:17 -0400 Subject: [PATCH 41/52] Add libcurl as a nix package dependency --- build/package/nix/package.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/build/package/nix/package.nix b/build/package/nix/package.nix index 848ebe91f8..23e25072e3 100644 --- a/build/package/nix/package.nix +++ b/build/package/nix/package.nix @@ -86,6 +86,7 @@ stdenv.mkDerivation { gcc_multi glibc bash + libcurl4 ]; nativeBuildInputs = with pkgs; [ makeWrapper From 731b65d9dfcbda0b33ccf1a4fb70efed1865c063 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 26 Jul 2025 11:19:50 -0400 Subject: [PATCH 42/52] Fix sed command --- .github/workflows/ci-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 16e237c9d5..ffc2dc8ff4 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -123,7 +123,7 @@ jobs: - name: Replace current TGS version with test version run: CURRENT_TGS_VERSION="$(xmlstarlet sel -N X="http://schemas.microsoft.com/developer/msbuild/2003" --template --value-of /X:Project/X:PropertyGroup/X:TgsCoreVersion build/Version.props)" - sed -i -e "s/${CURRENT_TGS_VERSION}/${{ env.TEST_TGS_VERSION }}" build/Version.props + sed -i -e "s/${CURRENT_TGS_VERSION}/${{ env.TEST_TGS_VERSION }}/g" build/Version.props - name: Retrieve ServerConsole.zip Artifact run: | From cac2a9d90fd5fea99d9fab15e9c8aa3928ea0aa3 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 26 Jul 2025 11:22:16 -0400 Subject: [PATCH 43/52] Okay, fuck it, just get full curl --- build/package/nix/package.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/package/nix/package.nix b/build/package/nix/package.nix index 23e25072e3..9deeba9fb8 100644 --- a/build/package/nix/package.nix +++ b/build/package/nix/package.nix @@ -86,7 +86,7 @@ stdenv.mkDerivation { gcc_multi glibc bash - libcurl4 + curl ]; nativeBuildInputs = with pkgs; [ makeWrapper From 29b58366a23f462939c3b90ac9490510471227a3 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 26 Jul 2025 12:31:31 -0400 Subject: [PATCH 44/52] Uhh what the hell? --- .github/workflows/ci-pipeline.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index ffc2dc8ff4..b79e92a130 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -124,6 +124,7 @@ jobs: run: CURRENT_TGS_VERSION="$(xmlstarlet sel -N X="http://schemas.microsoft.com/developer/msbuild/2003" --template --value-of /X:Project/X:PropertyGroup/X:TgsCoreVersion build/Version.props)" sed -i -e "s/${CURRENT_TGS_VERSION}/${{ env.TEST_TGS_VERSION }}/g" build/Version.props + cat build/Version.props - name: Retrieve ServerConsole.zip Artifact run: | From dc2656190da74ab3dea9097b536fd8eaaefdee54 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 26 Jul 2025 12:43:36 -0400 Subject: [PATCH 45/52] Excuse me wtf? --- .github/workflows/ci-pipeline.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index b79e92a130..12ea98db29 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -122,6 +122,8 @@ jobs: - name: Replace current TGS version with test version run: + ls -al + ls -al build CURRENT_TGS_VERSION="$(xmlstarlet sel -N X="http://schemas.microsoft.com/developer/msbuild/2003" --template --value-of /X:Project/X:PropertyGroup/X:TgsCoreVersion build/Version.props)" sed -i -e "s/${CURRENT_TGS_VERSION}/${{ env.TEST_TGS_VERSION }}/g" build/Version.props cat build/Version.props From ee7d44a23e2bbd8a87111107283e3ffa7e59eb85 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 26 Jul 2025 14:04:31 -0400 Subject: [PATCH 46/52] Ah --- .github/workflows/ci-pipeline.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 12ea98db29..4651095d0a 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -121,12 +121,9 @@ jobs: ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - name: Replace current TGS version with test version - run: - ls -al - ls -al build + run: | CURRENT_TGS_VERSION="$(xmlstarlet sel -N X="http://schemas.microsoft.com/developer/msbuild/2003" --template --value-of /X:Project/X:PropertyGroup/X:TgsCoreVersion build/Version.props)" sed -i -e "s/${CURRENT_TGS_VERSION}/${{ env.TEST_TGS_VERSION }}/g" build/Version.props - cat build/Version.props - name: Retrieve ServerConsole.zip Artifact run: | From dfb9c199e0a6558d0da4877019aa25594c01aa6f Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 26 Jul 2025 14:12:13 -0400 Subject: [PATCH 47/52] Microsoft has stopped supporting their package repo https://learn.microsoft.com/en-ca/dotnet/core/install/linux-ubuntu-decision#register-the-microsoft-package-repository --- build/package/deb/build_package.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/build/package/deb/build_package.sh b/build/package/deb/build_package.sh index 29d828f1bb..58ee50cdfe 100755 --- a/build/package/deb/build_package.sh +++ b/build/package/deb/build_package.sh @@ -22,11 +22,6 @@ apt-get install -y \ xmlstarlet \ libgdiplus -declare repo_version=$(if command -v lsb_release &> /dev/null; then lsb_release -r -s; else grep -oP '(?<=^VERSION_ID=).+' /etc/os-release | tr -d '"'; fi) -curl -L https://packages.microsoft.com/config/ubuntu/$repo_version/packages-microsoft-prod.deb -o packages-microsoft-prod.deb -dpkg -i ./packages-microsoft-prod.deb -rm packages-microsoft-prod.deb - # https://github.com/nodesource/distributions mkdir -p /etc/apt/keyrings curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg From 1b8994cf2b81b93e5f2780e381c89b6ffe72e02e Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 26 Jul 2025 14:19:58 -0400 Subject: [PATCH 48/52] I'm tired of fighting you BYOND! --- tests/Tgstation.Server.Tests/TestingUtils.cs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/Tgstation.Server.Tests/TestingUtils.cs b/tests/Tgstation.Server.Tests/TestingUtils.cs index 40925352dd..f710141b36 100644 --- a/tests/Tgstation.Server.Tests/TestingUtils.cs +++ b/tests/Tgstation.Server.Tests/TestingUtils.cs @@ -133,13 +133,19 @@ namespace Tgstation.Server.Tests try { // always check byond.com first for latest up-to-date, mirror should ALWAYS have stable versions - const string DefaultMirror = "https://www.byond.com/download/version.txt"; + // except byond hates all CI runners now + const string DefaultMirror = "https://spacestation13.github.io/byond-builds/version.txt"; edgeVersion = await GetVersionFromResponse(DefaultMirror); - logger.LogInformation("Downloading edge version from BYOND.com {edge}", edgeVersion); + logger.LogInformation("Downloading edge version from SS13 mirror {edge}", edgeVersion); // if we got the result from byond.com, make sure the cache grabs the zip from there as well - await CachingFileDownloader.InitializeByondVersion(logger, Version.Parse(edgeVersion), new PlatformIdentifier().IsWindows, cancellationToken, GeneralConfiguration.DefaultByondZipDownloadTemplate); + await CachingFileDownloader.InitializeByondVersion( + logger, + Version.Parse(edgeVersion), + new PlatformIdentifier().IsWindows, + cancellationToken, + "https://spacestation13.github.io/byond-builds/${Major}/${Major}.${Minor}_byond${Linux:_linux}.zip"); } catch (Exception ex) { From 1e0c888da21a8c380a3f3b60286955bed5d728ce Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 26 Jul 2025 19:15:15 -0400 Subject: [PATCH 49/52] Bring all nuget packages up to date and trim the fat --- build/TestCommon.props | 2 +- .../Tgstation.Server.Api.csproj | 2 +- .../Tgstation.Server.Client.GraphQL.csproj | 3 +- .../Tgstation.Server.Client.csproj | 4 +- .../Tgstation.Server.Host.Console.csproj | 4 +- .../Tgstation.Server.Host.Service.csproj | 10 ++--- ...on.Server.Host.Utils.GitLab.GraphQL.csproj | 3 +- .../Tgstation.Server.Host.Watchdog.csproj | 2 +- .../.config/dotnet-tools.json | 2 +- .../Tgstation.Server.Host.csproj | 43 ++++++++++--------- .../Tgstation.Server.Shared.csproj | 2 +- .../Tgstation.Server.Client.Tests.csproj | 2 +- .../Tgstation.Server.Host.Tests.csproj | 3 +- .../Tgstation.Server.Tests.csproj | 4 -- 14 files changed, 42 insertions(+), 44 deletions(-) diff --git a/build/TestCommon.props b/build/TestCommon.props index 44c2a0c2e4..8a6c5d675f 100644 --- a/build/TestCommon.props +++ b/build/TestCommon.props @@ -13,7 +13,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/src/Tgstation.Server.Api/Tgstation.Server.Api.csproj b/src/Tgstation.Server.Api/Tgstation.Server.Api.csproj index 0e83f44424..6f32743090 100644 --- a/src/Tgstation.Server.Api/Tgstation.Server.Api.csproj +++ b/src/Tgstation.Server.Api/Tgstation.Server.Api.csproj @@ -28,7 +28,7 @@ - + diff --git a/src/Tgstation.Server.Client.GraphQL/Tgstation.Server.Client.GraphQL.csproj b/src/Tgstation.Server.Client.GraphQL/Tgstation.Server.Client.GraphQL.csproj index 9a575c625f..67e9d74754 100644 --- a/src/Tgstation.Server.Client.GraphQL/Tgstation.Server.Client.GraphQL.csproj +++ b/src/Tgstation.Server.Client.GraphQL/Tgstation.Server.Client.GraphQL.csproj @@ -9,8 +9,7 @@ - - + diff --git a/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj b/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj index 83d41dfff1..f681ecff8f 100644 --- a/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj +++ b/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj @@ -11,9 +11,9 @@ - + - + diff --git a/src/Tgstation.Server.Host.Console/Tgstation.Server.Host.Console.csproj b/src/Tgstation.Server.Host.Console/Tgstation.Server.Host.Console.csproj index 4942cfd2ff..75f657b562 100644 --- a/src/Tgstation.Server.Host.Console/Tgstation.Server.Host.Console.csproj +++ b/src/Tgstation.Server.Host.Console/Tgstation.Server.Host.Console.csproj @@ -13,9 +13,9 @@ - + - + diff --git a/src/Tgstation.Server.Host.Service/Tgstation.Server.Host.Service.csproj b/src/Tgstation.Server.Host.Service/Tgstation.Server.Host.Service.csproj index 6a31f318db..59ab045d5a 100644 --- a/src/Tgstation.Server.Host.Service/Tgstation.Server.Host.Service.csproj +++ b/src/Tgstation.Server.Host.Service/Tgstation.Server.Host.Service.csproj @@ -21,21 +21,21 @@ - + - + - + - + - + diff --git a/src/Tgstation.Server.Host.Utils.GitLab.GraphQL/Tgstation.Server.Host.Utils.GitLab.GraphQL.csproj b/src/Tgstation.Server.Host.Utils.GitLab.GraphQL/Tgstation.Server.Host.Utils.GitLab.GraphQL.csproj index 41df3014e0..ab2f3f9dbc 100644 --- a/src/Tgstation.Server.Host.Utils.GitLab.GraphQL/Tgstation.Server.Host.Utils.GitLab.GraphQL.csproj +++ b/src/Tgstation.Server.Host.Utils.GitLab.GraphQL/Tgstation.Server.Host.Utils.GitLab.GraphQL.csproj @@ -38,7 +38,8 @@ - + + diff --git a/src/Tgstation.Server.Host.Watchdog/Tgstation.Server.Host.Watchdog.csproj b/src/Tgstation.Server.Host.Watchdog/Tgstation.Server.Host.Watchdog.csproj index af47f499f3..5f31ab855f 100644 --- a/src/Tgstation.Server.Host.Watchdog/Tgstation.Server.Host.Watchdog.csproj +++ b/src/Tgstation.Server.Host.Watchdog/Tgstation.Server.Host.Watchdog.csproj @@ -10,7 +10,7 @@ - + diff --git a/src/Tgstation.Server.Host/.config/dotnet-tools.json b/src/Tgstation.Server.Host/.config/dotnet-tools.json index 76e6bce7b3..7cf60e92f7 100644 --- a/src/Tgstation.Server.Host/.config/dotnet-tools.json +++ b/src/Tgstation.Server.Host/.config/dotnet-tools.json @@ -3,7 +3,7 @@ "isRoot": true, "tools": { "dotnet-ef": { - "version": "9.0.4", + "version": "9.0.7", "commands": [ "dotnet-ef" ] diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index 32fd89607a..91dcae4c65 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -96,41 +96,41 @@ - + - + - + - + - + - + - + - + - + - + - + runtime; build; native; contentfiles; analyzers; buildtransitive - + - + - + - + @@ -144,9 +144,9 @@ - + - + @@ -156,14 +156,15 @@ - + - + - + + - + diff --git a/src/Tgstation.Server.Shared/Tgstation.Server.Shared.csproj b/src/Tgstation.Server.Shared/Tgstation.Server.Shared.csproj index 861618a8a9..e636c80a73 100644 --- a/src/Tgstation.Server.Shared/Tgstation.Server.Shared.csproj +++ b/src/Tgstation.Server.Shared/Tgstation.Server.Shared.csproj @@ -10,7 +10,7 @@ - + diff --git a/tests/Tgstation.Server.Client.Tests/Tgstation.Server.Client.Tests.csproj b/tests/Tgstation.Server.Client.Tests/Tgstation.Server.Client.Tests.csproj index 725b7c8bcd..66888f4430 100644 --- a/tests/Tgstation.Server.Client.Tests/Tgstation.Server.Client.Tests.csproj +++ b/tests/Tgstation.Server.Client.Tests/Tgstation.Server.Client.Tests.csproj @@ -6,7 +6,7 @@ - + diff --git a/tests/Tgstation.Server.Host.Tests/Tgstation.Server.Host.Tests.csproj b/tests/Tgstation.Server.Host.Tests/Tgstation.Server.Host.Tests.csproj index f696ba76b7..2b96d6fd6a 100644 --- a/tests/Tgstation.Server.Host.Tests/Tgstation.Server.Host.Tests.csproj +++ b/tests/Tgstation.Server.Host.Tests/Tgstation.Server.Host.Tests.csproj @@ -7,7 +7,8 @@ - + + diff --git a/tests/Tgstation.Server.Tests/Tgstation.Server.Tests.csproj b/tests/Tgstation.Server.Tests/Tgstation.Server.Tests.csproj index 8b87acdc61..b2e176f1e3 100644 --- a/tests/Tgstation.Server.Tests/Tgstation.Server.Tests.csproj +++ b/tests/Tgstation.Server.Tests/Tgstation.Server.Tests.csproj @@ -5,10 +5,6 @@ $(TgsFrameworkVersion) - - - - From fd050f068d391cf66716fb6d512c94a80f664362 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 26 Jul 2025 19:19:11 -0400 Subject: [PATCH 50/52] Update wix version --- build/package/winget/.config/dotnet-tools.json | 2 +- .../Tgstation.Server.Host.Service.Wix.Bundle.wixproj | 6 +++--- .../Tgstation.Server.Host.Service.Wix.Extensions.csproj | 2 +- .../Tgstation.Server.Host.Service.Wix.wixproj | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/build/package/winget/.config/dotnet-tools.json b/build/package/winget/.config/dotnet-tools.json index 588fcd1095..998101e103 100644 --- a/build/package/winget/.config/dotnet-tools.json +++ b/build/package/winget/.config/dotnet-tools.json @@ -3,7 +3,7 @@ "isRoot": true, "tools": { "wix": { - "version": "5.0.2", + "version": "6.0.1", "commands": [ "wix" ] diff --git a/build/package/winget/Tgstation.Server.Host.Service.Wix.Bundle/Tgstation.Server.Host.Service.Wix.Bundle.wixproj b/build/package/winget/Tgstation.Server.Host.Service.Wix.Bundle/Tgstation.Server.Host.Service.Wix.Bundle.wixproj index df2363b5ac..42304195a2 100644 --- a/build/package/winget/Tgstation.Server.Host.Service.Wix.Bundle/Tgstation.Server.Host.Service.Wix.Bundle.wixproj +++ b/build/package/winget/Tgstation.Server.Host.Service.Wix.Bundle/Tgstation.Server.Host.Service.Wix.Bundle.wixproj @@ -1,4 +1,4 @@ - + ProductVersion=$(TgsCoreVersion);NetMajorVersion=$(TgsNetMajorVersion);DotnetRedistUrl=$(TgsDotnetRedistUrl);MariaDBRedistUrl=https://github.com/tgstation/tgstation-server/releases/download/tgstation-server-v$(TgsCoreVersion)/mariadb-$(TgsMariaDBRedistVersion)-winx64.msi @@ -24,8 +24,8 @@ - - + + diff --git a/build/package/winget/Tgstation.Server.Host.Service.Wix.Extensions/Tgstation.Server.Host.Service.Wix.Extensions.csproj b/build/package/winget/Tgstation.Server.Host.Service.Wix.Extensions/Tgstation.Server.Host.Service.Wix.Extensions.csproj index fcdbd512a6..1e53056e15 100644 --- a/build/package/winget/Tgstation.Server.Host.Service.Wix.Extensions/Tgstation.Server.Host.Service.Wix.Extensions.csproj +++ b/build/package/winget/Tgstation.Server.Host.Service.Wix.Extensions/Tgstation.Server.Host.Service.Wix.Extensions.csproj @@ -7,7 +7,7 @@ - + diff --git a/build/package/winget/Tgstation.Server.Host.Service.Wix/Tgstation.Server.Host.Service.Wix.wixproj b/build/package/winget/Tgstation.Server.Host.Service.Wix/Tgstation.Server.Host.Service.Wix.wixproj index 272694a58e..8e32edfa4b 100644 --- a/build/package/winget/Tgstation.Server.Host.Service.Wix/Tgstation.Server.Host.Service.Wix.wixproj +++ b/build/package/winget/Tgstation.Server.Host.Service.Wix/Tgstation.Server.Host.Service.Wix.wixproj @@ -1,4 +1,4 @@ - + ProductVersion=$(TgsCoreVersion) @@ -20,8 +20,8 @@ - - + + From 5a37c36c6588bb3a0533108acf89bd200017930e Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 26 Jul 2025 19:27:39 -0400 Subject: [PATCH 51/52] Revert "Update wix version" This reverts commit fd050f068d391cf66716fb6d512c94a80f664362. --- build/package/winget/.config/dotnet-tools.json | 2 +- .../Tgstation.Server.Host.Service.Wix.Bundle.wixproj | 6 +++--- .../Tgstation.Server.Host.Service.Wix.Extensions.csproj | 2 +- .../Tgstation.Server.Host.Service.Wix.wixproj | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/build/package/winget/.config/dotnet-tools.json b/build/package/winget/.config/dotnet-tools.json index 998101e103..588fcd1095 100644 --- a/build/package/winget/.config/dotnet-tools.json +++ b/build/package/winget/.config/dotnet-tools.json @@ -3,7 +3,7 @@ "isRoot": true, "tools": { "wix": { - "version": "6.0.1", + "version": "5.0.2", "commands": [ "wix" ] diff --git a/build/package/winget/Tgstation.Server.Host.Service.Wix.Bundle/Tgstation.Server.Host.Service.Wix.Bundle.wixproj b/build/package/winget/Tgstation.Server.Host.Service.Wix.Bundle/Tgstation.Server.Host.Service.Wix.Bundle.wixproj index 42304195a2..df2363b5ac 100644 --- a/build/package/winget/Tgstation.Server.Host.Service.Wix.Bundle/Tgstation.Server.Host.Service.Wix.Bundle.wixproj +++ b/build/package/winget/Tgstation.Server.Host.Service.Wix.Bundle/Tgstation.Server.Host.Service.Wix.Bundle.wixproj @@ -1,4 +1,4 @@ - + ProductVersion=$(TgsCoreVersion);NetMajorVersion=$(TgsNetMajorVersion);DotnetRedistUrl=$(TgsDotnetRedistUrl);MariaDBRedistUrl=https://github.com/tgstation/tgstation-server/releases/download/tgstation-server-v$(TgsCoreVersion)/mariadb-$(TgsMariaDBRedistVersion)-winx64.msi @@ -24,8 +24,8 @@ - - + + diff --git a/build/package/winget/Tgstation.Server.Host.Service.Wix.Extensions/Tgstation.Server.Host.Service.Wix.Extensions.csproj b/build/package/winget/Tgstation.Server.Host.Service.Wix.Extensions/Tgstation.Server.Host.Service.Wix.Extensions.csproj index 1e53056e15..fcdbd512a6 100644 --- a/build/package/winget/Tgstation.Server.Host.Service.Wix.Extensions/Tgstation.Server.Host.Service.Wix.Extensions.csproj +++ b/build/package/winget/Tgstation.Server.Host.Service.Wix.Extensions/Tgstation.Server.Host.Service.Wix.Extensions.csproj @@ -7,7 +7,7 @@ - + diff --git a/build/package/winget/Tgstation.Server.Host.Service.Wix/Tgstation.Server.Host.Service.Wix.wixproj b/build/package/winget/Tgstation.Server.Host.Service.Wix/Tgstation.Server.Host.Service.Wix.wixproj index 8e32edfa4b..272694a58e 100644 --- a/build/package/winget/Tgstation.Server.Host.Service.Wix/Tgstation.Server.Host.Service.Wix.wixproj +++ b/build/package/winget/Tgstation.Server.Host.Service.Wix/Tgstation.Server.Host.Service.Wix.wixproj @@ -1,4 +1,4 @@ - + ProductVersion=$(TgsCoreVersion) @@ -20,8 +20,8 @@ - - + + From 0d4d68d7362d81a8d065045e048346dd1c207c22 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 26 Jul 2025 20:08:31 -0400 Subject: [PATCH 52/52] Fix `DefaultIOManager` bugs - Fix `CreateResolverForSubdirectory` not considering if the input path was rooted. - Fix `DeleteDirectory` not resolving the input path. --- src/Tgstation.Server.Host/IO/DefaultIOManager.cs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/Tgstation.Server.Host/IO/DefaultIOManager.cs b/src/Tgstation.Server.Host/IO/DefaultIOManager.cs index 470f437d7f..eaf2b27736 100644 --- a/src/Tgstation.Server.Host/IO/DefaultIOManager.cs +++ b/src/Tgstation.Server.Host/IO/DefaultIOManager.cs @@ -162,7 +162,8 @@ namespace Tgstation.Server.Host.IO => Task.Factory.StartNew( () => { - var di = fileSystem.DirectoryInfo.New(path); + var di = fileSystem.DirectoryInfo.New( + ResolvePath(path)); if (di.Exists) NormalizeAndDelete(di, cancellationToken); }, @@ -416,11 +417,14 @@ namespace Tgstation.Server.Host.IO { ArgumentNullException.ThrowIfNull(subdirectoryPath); + if (!Path.IsPathRooted(subdirectoryPath)) + subdirectoryPath = ConcatPath( + ResolvePath(), + subdirectoryPath); + return new ResolvingIOManager( fileSystem, - ConcatPath( - ResolvePath(), - subdirectoryPath)); + subdirectoryPath); } ///