From 5b08cacbb7b3a407ec84da1c9856a7b1cd95e521 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 25 Apr 2025 10:02:00 +0000 Subject: [PATCH 01/52] Bump Microsoft.IdentityModel.JsonWebTokens and Microsoft.Bcl.AsyncInterfaces Bumps [Microsoft.IdentityModel.JsonWebTokens](https://github.com/AzureAD/azure-activedirectory-identitymodel-extensions-for-dotnet) and [Microsoft.Bcl.AsyncInterfaces](https://github.com/dotnet/runtime). These dependencies needed to be updated together. Updates `Microsoft.IdentityModel.JsonWebTokens` from 8.8.0 to 8.9.0 - [Release notes](https://github.com/AzureAD/azure-activedirectory-identitymodel-extensions-for-dotnet/releases) - [Changelog](https://github.com/AzureAD/azure-activedirectory-identitymodel-extensions-for-dotnet/blob/dev/CHANGELOG.md) - [Commits](https://github.com/AzureAD/azure-activedirectory-identitymodel-extensions-for-dotnet/compare/8.8.0...8.9.0) Updates `Microsoft.Bcl.AsyncInterfaces` from 9.0.4 to 8.0.0 - [Release notes](https://github.com/dotnet/runtime/releases) - [Commits](https://github.com/dotnet/runtime/compare/v9.0.4...v8.0.0) --- updated-dependencies: - dependency-name: Microsoft.IdentityModel.JsonWebTokens dependency-version: 8.9.0 dependency-type: direct:production update-type: version-update:semver-minor - dependency-name: Microsoft.Bcl.AsyncInterfaces dependency-version: 8.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- src/Tgstation.Server.Api/Tgstation.Server.Api.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Api/Tgstation.Server.Api.csproj b/src/Tgstation.Server.Api/Tgstation.Server.Api.csproj index d5a111921d..0e83f44424 100644 --- a/src/Tgstation.Server.Api/Tgstation.Server.Api.csproj +++ b/src/Tgstation.Server.Api/Tgstation.Server.Api.csproj @@ -28,7 +28,7 @@ - + From 2d62fb2e1d8b608b5a6cc35b73612cfc7b6a95de Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 25 Apr 2025 10:05:21 +0000 Subject: [PATCH 02/52] Bump Microsoft.IdentityModel.Protocols.OpenIdConnect from 8.8.0 to 8.9.0 Bumps [Microsoft.IdentityModel.Protocols.OpenIdConnect](https://github.com/AzureAD/azure-activedirectory-identitymodel-extensions-for-dotnet) from 8.8.0 to 8.9.0. - [Release notes](https://github.com/AzureAD/azure-activedirectory-identitymodel-extensions-for-dotnet/releases) - [Changelog](https://github.com/AzureAD/azure-activedirectory-identitymodel-extensions-for-dotnet/blob/dev/CHANGELOG.md) - [Commits](https://github.com/AzureAD/azure-activedirectory-identitymodel-extensions-for-dotnet/compare/8.8.0...8.9.0) --- updated-dependencies: - dependency-name: Microsoft.IdentityModel.Protocols.OpenIdConnect dependency-version: 8.9.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- src/Tgstation.Server.Host/Tgstation.Server.Host.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index 4422e33221..9e4a3056c4 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -130,7 +130,7 @@ - + From 48d495e397940f4f5922f840f9e4dcee45ec09de Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Apr 2025 09:40:27 +0000 Subject: [PATCH 03/52] Bump Serilog.Sinks.File from 6.0.0 to 7.0.0 Bumps [Serilog.Sinks.File](https://github.com/serilog/serilog-sinks-file) from 6.0.0 to 7.0.0. - [Release notes](https://github.com/serilog/serilog-sinks-file/releases) - [Changelog](https://github.com/serilog/serilog-sinks-file/blob/dev/CHANGES.md) - [Commits](https://github.com/serilog/serilog-sinks-file/compare/v6.0.0...v7.0.0) --- updated-dependencies: - dependency-name: Serilog.Sinks.File dependency-version: 7.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- src/Tgstation.Server.Host/Tgstation.Server.Host.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index 9e4a3056c4..19695527da 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -154,7 +154,7 @@ - + From 9ba8b94e2ccbf477de3b5e7f4ab853e1a1a781a9 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Tue, 6 May 2025 18:52:09 -0400 Subject: [PATCH 04/52] Add API for queuing text deployment messages --- .../Components/Chat/ChatManager.cs | 54 +++++++++++-------- .../Components/Chat/IChatManager.cs | 6 +++ 2 files changed, 39 insertions(+), 21 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs b/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs index 5faa30dbd2..b6685b7460 100644 --- a/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs +++ b/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs @@ -351,28 +351,11 @@ namespace Tgstation.Server.Host.Components.Chat /// public void QueueWatchdogMessage(string message) - { - ArgumentNullException.ThrowIfNull(message); + => QueueMessageGeneric(mapping => mapping.IsWatchdogChannel, message); - message = String.Format(CultureInfo.InvariantCulture, "WD: {0}", message); - - if (!initialProviderConnectionsTask!.IsCompleted) - logger.LogTrace("Waiting for initial provider connections before sending watchdog message..."); - - // Reimplementing QueueMessage - QueueMessageInternal( - new MessageContent - { - Text = message, - }, - () => - { - // so it doesn't change while we're using it - lock (mappedChannels) - return mappedChannels.Where(x => x.Value.IsWatchdogChannel).Select(x => x.Key).ToList(); - }, - true); - } + /// + public void QueueDeploymentMessage(string message) + => QueueMessageGeneric(mapping => mapping.IsUpdatesChannel, message); /// public Func> QueueDeploymentMessage( @@ -1111,5 +1094,34 @@ namespace Tgstation.Server.Host.Components.Chat AddMessageTask(SendMessageTask()); } + + /// + /// Queues a message to a selected set of s. + /// + /// A for selecting the s to send to. + /// The message to send. + void QueueMessageGeneric(Predicate channelSelector, string message) + { + ArgumentNullException.ThrowIfNull(message); + + message = String.Format(CultureInfo.InvariantCulture, "WD: {0}", message); + + if (!initialProviderConnectionsTask!.IsCompleted) + logger.LogTrace("Waiting for initial provider connections before sending watchdog message..."); + + // Reimplementing QueueMessage + QueueMessageInternal( + new MessageContent + { + Text = message, + }, + () => + { + // so it doesn't change while we're using it + lock (mappedChannels) + return mappedChannels.Where(x => channelSelector(x.Value)).Select(x => x.Key).ToList(); + }, + true); + } } } diff --git a/src/Tgstation.Server.Host/Components/Chat/IChatManager.cs b/src/Tgstation.Server.Host/Components/Chat/IChatManager.cs index 669f21b305..2b2faf9496 100644 --- a/src/Tgstation.Server.Host/Components/Chat/IChatManager.cs +++ b/src/Tgstation.Server.Host/Components/Chat/IChatManager.cs @@ -57,6 +57,12 @@ namespace Tgstation.Server.Host.Components.Chat /// The message being sent. void QueueWatchdogMessage(string message); + /// + /// Queue a chat to configured deployment channels. + /// + /// The message being sent. + void QueueDeploymentMessage(string message); + /// /// Send the message for a deployment to configured deployment channels. /// From d5c3ddbd4ef216086ae72b0f3234d3a0f358d6ac Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Tue, 6 May 2025 18:53:18 -0400 Subject: [PATCH 05/52] Add a chat message when an automatic update fails due to a conflict --- src/Tgstation.Server.Host/Components/Instance.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Tgstation.Server.Host/Components/Instance.cs b/src/Tgstation.Server.Host/Components/Instance.cs index f0fd27f661..8957678a3e 100644 --- a/src/Tgstation.Server.Host/Components/Instance.cs +++ b/src/Tgstation.Server.Host/Components/Instance.cs @@ -531,7 +531,10 @@ namespace Tgstation.Server.Host.Components } } else if (preserveTestMerges) + { + Chat.QueueDeploymentMessage("Automatic update has failed due to a conflicting testmerge!"); throw new JobException(Api.Models.ErrorCode.InstanceUpdateTestMergeConflict); + } if (!preserveTestMerges) { From ac56e2c3fa8a96deb155d3ff6be4fd0a65390bd3 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Tue, 6 May 2025 18:53:42 -0400 Subject: [PATCH 06/52] Version bump to 6.18.0 --- build/Version.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/Version.props b/build/Version.props index 394ff58026..7312268ed1 100644 --- a/build/Version.props +++ b/build/Version.props @@ -3,7 +3,7 @@ - 6.17.0 + 6.18.0 5.7.0 10.13.0 0.6.0 From 6384254cdc1a4531e32ee703dfc94af32855efd4 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Tue, 6 May 2025 18:59:13 -0400 Subject: [PATCH 07/52] Fix bad prefix for auto-update fail message --- .../Components/Chat/ChatManager.cs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs b/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs index b6685b7460..4b42fb0a56 100644 --- a/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs +++ b/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs @@ -351,11 +351,11 @@ namespace Tgstation.Server.Host.Components.Chat /// public void QueueWatchdogMessage(string message) - => QueueMessageGeneric(mapping => mapping.IsWatchdogChannel, message); + => QueueMessageGeneric(mapping => mapping.IsWatchdogChannel, message, "WD"); /// public void QueueDeploymentMessage(string message) - => QueueMessageGeneric(mapping => mapping.IsUpdatesChannel, message); + => QueueMessageGeneric(mapping => mapping.IsUpdatesChannel, message, null); /// public Func> QueueDeploymentMessage( @@ -1100,14 +1100,18 @@ namespace Tgstation.Server.Host.Components.Chat /// /// A for selecting the s to send to. /// The message to send. - void QueueMessageGeneric(Predicate channelSelector, string message) + /// The optional prefix to the message to be sent. + void QueueMessageGeneric(Predicate channelSelector, string message, string prefix) { ArgumentNullException.ThrowIfNull(message); - message = String.Format(CultureInfo.InvariantCulture, "WD: {0}", message); + if (prefix != null) + { + message = $"{prefix}: {message}"; + } if (!initialProviderConnectionsTask!.IsCompleted) - logger.LogTrace("Waiting for initial provider connections before sending watchdog message..."); + logger.LogTrace("Waiting for initial provider connections before sending chat message..."); // Reimplementing QueueMessage QueueMessageInternal( From 1f4ac78d42a7231a444f79772932b12041be27bc Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Wed, 7 May 2025 17:56:33 -0400 Subject: [PATCH 08/52] Fix nullable error --- src/Tgstation.Server.Host/Components/Chat/ChatManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs b/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs index 4b42fb0a56..abd5ef90c3 100644 --- a/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs +++ b/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs @@ -1101,7 +1101,7 @@ namespace Tgstation.Server.Host.Components.Chat /// A for selecting the s to send to. /// The message to send. /// The optional prefix to the message to be sent. - void QueueMessageGeneric(Predicate channelSelector, string message, string prefix) + void QueueMessageGeneric(Predicate channelSelector, string message, string? prefix) { ArgumentNullException.ThrowIfNull(message); From cbbef1f8a0560280808fafc0ccba2792c2a9bbc1 Mon Sep 17 00:00:00 2001 From: Drulikar Date: Fri, 9 May 2025 11:03:39 -0500 Subject: [PATCH 09/52] Don't require https --- src/Tgstation.Server.Host/Core/Application.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index c4958767ee..a947eeb720 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -940,10 +940,7 @@ namespace Tgstation.Server.Host.Core options.Scope.Add(OpenIdConnectScope.OpenId); options.Scope.Add(OpenIdConnectScope.OfflineAccess); -#if DEBUG options.RequireHttpsMetadata = false; -#endif - options.SaveTokens = true; options.ResponseType = OpenIdConnectResponseType.Code; options.MapInboundClaims = false; From 721beaff68673573a387d66b4a0e4573e29fc352 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 9 May 2025 21:32:26 -0400 Subject: [PATCH 10/52] Fix conflicting name error --- src/Tgstation.Server.Host/Components/Chat/ChatManager.cs | 2 +- src/Tgstation.Server.Host/Components/Chat/IChatManager.cs | 2 +- src/Tgstation.Server.Host/Components/Instance.cs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs b/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs index abd5ef90c3..2aeca4f8f3 100644 --- a/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs +++ b/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs @@ -354,7 +354,7 @@ namespace Tgstation.Server.Host.Components.Chat => QueueMessageGeneric(mapping => mapping.IsWatchdogChannel, message, "WD"); /// - public void QueueDeploymentMessage(string message) + public void QueueRawDeploymentMessage(string message) => QueueMessageGeneric(mapping => mapping.IsUpdatesChannel, message, null); /// diff --git a/src/Tgstation.Server.Host/Components/Chat/IChatManager.cs b/src/Tgstation.Server.Host/Components/Chat/IChatManager.cs index 2b2faf9496..4f112afb69 100644 --- a/src/Tgstation.Server.Host/Components/Chat/IChatManager.cs +++ b/src/Tgstation.Server.Host/Components/Chat/IChatManager.cs @@ -61,7 +61,7 @@ namespace Tgstation.Server.Host.Components.Chat /// Queue a chat to configured deployment channels. /// /// The message being sent. - void QueueDeploymentMessage(string message); + void QueueRawDeploymentMessage(string message); /// /// Send the message for a deployment to configured deployment channels. diff --git a/src/Tgstation.Server.Host/Components/Instance.cs b/src/Tgstation.Server.Host/Components/Instance.cs index 8957678a3e..37454b7dbc 100644 --- a/src/Tgstation.Server.Host/Components/Instance.cs +++ b/src/Tgstation.Server.Host/Components/Instance.cs @@ -532,7 +532,7 @@ namespace Tgstation.Server.Host.Components } else if (preserveTestMerges) { - Chat.QueueDeploymentMessage("Automatic update has failed due to a conflicting testmerge!"); + Chat.QueueRawDeploymentMessage("Automatic update has failed due to a conflicting testmerge!"); throw new JobException(Api.Models.ErrorCode.InstanceUpdateTestMergeConflict); } From fb2b15134fd2fd66fa251b78015f2ae1a538629a Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 23 Jun 2025 21:35:28 -0400 Subject: [PATCH 11/52] Allow BYOND downloads to be configurable --- .../Components/Engine/ByondInstallerBase.cs | 81 +++++++++++++++++-- .../Components/Engine/PosixByondInstaller.cs | 8 +- .../Engine/WindowsByondInstaller.cs | 18 ++--- .../Configuration/GeneralConfiguration.cs | 11 +++ .../Engine/TestByondInstallerBase.cs | 23 ++++++ .../Engine/TestPosixByondInstaller.cs | 23 ++++-- 6 files changed, 137 insertions(+), 27 deletions(-) create mode 100644 tests/Tgstation.Server.Host.Tests/Components/Engine/TestByondInstallerBase.cs diff --git a/src/Tgstation.Server.Host/Components/Engine/ByondInstallerBase.cs b/src/Tgstation.Server.Host/Components/Engine/ByondInstallerBase.cs index e02c109e6b..527fcae985 100644 --- a/src/Tgstation.Server.Host/Components/Engine/ByondInstallerBase.cs +++ b/src/Tgstation.Server.Host/Components/Engine/ByondInstallerBase.cs @@ -6,8 +6,10 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Tgstation.Server.Api.Models; +using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.IO; using Tgstation.Server.Host.Jobs; @@ -41,6 +43,11 @@ namespace Tgstation.Server.Host.Components.Engine /// protected override EngineType TargetEngineType => EngineType.Byond; + /// + /// The for the . + /// + protected IOptionsMonitor GeneralConfigurationOptions { get; } + /// /// Path to the system user's local BYOND folder. /// @@ -52,25 +59,85 @@ namespace Tgstation.Server.Host.Components.Engine protected abstract string DreamMakerName { get; } /// - /// Gets the URL formatter string for downloading a byond version of {0:Major} {1:Minor}. + /// Template to do ${Marker:xxx} replacements in . /// - protected abstract string ByondRevisionsUrlTemplate { get; } + protected abstract string OSMarkerTemplate { get; } /// /// The for the . /// readonly IFileDownloader fileDownloader; + /// + /// Format a given . + /// + /// The BYOND version to download. + /// The template. + /// The . + /// The formatted byond download . + /// Exposed only for testability. + internal static Uri GetDownloadZipUrl(Version semver, string byondZipDownloadTemplate, string osMarkerTemplate) + { + // god forbid + var guardGuid = Guid.NewGuid(); + + var url = byondZipDownloadTemplate + .Replace("$$", guardGuid.ToString(), StringComparison.Ordinal) + .Replace("${Major}", semver.Major.ToString(CultureInfo.InvariantCulture), StringComparison.Ordinal) + .Replace("${Minor}", semver.Minor.ToString(CultureInfo.InvariantCulture), StringComparison.Ordinal); + + var osMarkerPrefix = $"${{{osMarkerTemplate}:"; + var osMarkerIndex = url.IndexOf(osMarkerPrefix); + while (osMarkerIndex != -1) + { + var start = osMarkerIndex + osMarkerPrefix.Length; + var end = url.IndexOf('}', start); + if (end == -1) + break; + + var substitution = url.Substring(start, end - start); + url = url.Replace($"{osMarkerPrefix}{substitution}}}", substitution, StringComparison.Ordinal); + + osMarkerIndex = url.IndexOf(osMarkerPrefix); + } + + // at this point, any other substitution attempts should be removed + var otherMarkerPrefix = "${"; + var otherMarkerIndex = url.IndexOf(otherMarkerPrefix); + while (otherMarkerIndex != -1) + { + var start = otherMarkerIndex + otherMarkerPrefix.Length; + var end = url.IndexOf('}', start); + if (end == -1) + break; + + var substitution = url.Substring(start, end - start); + url = url.Replace($"{otherMarkerPrefix}{substitution}}}", String.Empty, StringComparison.Ordinal); + + otherMarkerIndex = url.IndexOf(otherMarkerPrefix); + } + + url = url.Replace(guardGuid.ToString(), "$", StringComparison.Ordinal); + + return new Uri(url); + } + /// /// Initializes a new instance of the class. /// /// The for the . /// The for the . /// The value of . - protected ByondInstallerBase(IIOManager ioManager, ILogger logger, IFileDownloader fileDownloader) + /// The value of . + protected ByondInstallerBase( + IIOManager ioManager, + ILogger logger, + IFileDownloader fileDownloader, + IOptionsMonitor generalConfigurationOptions) : base(ioManager, logger) { this.fileDownloader = fileDownloader ?? throw new ArgumentNullException(nameof(fileDownloader)); + GeneralConfigurationOptions = generalConfigurationOptions ?? throw new ArgumentNullException(nameof(generalConfigurationOptions)); } /// @@ -195,8 +262,12 @@ namespace Tgstation.Server.Host.Components.Engine Uri GetDownloadZipUrl(EngineVersion version) { CheckVersionValidity(version); - var url = String.Format(CultureInfo.InvariantCulture, ByondRevisionsUrlTemplate, version.Version!.Major, version.Version.Minor); - return new Uri(url); + + var guardGuid = Guid.NewGuid(); + + var semver = version.Version!; + var template = GeneralConfigurationOptions.CurrentValue.ByondZipDownloadTemplate; + return GetDownloadZipUrl(semver, template, OSMarkerTemplate); } } } diff --git a/src/Tgstation.Server.Host/Components/Engine/PosixByondInstaller.cs b/src/Tgstation.Server.Host/Components/Engine/PosixByondInstaller.cs index fff62b3b3f..7e21f6a378 100644 --- a/src/Tgstation.Server.Host/Components/Engine/PosixByondInstaller.cs +++ b/src/Tgstation.Server.Host/Components/Engine/PosixByondInstaller.cs @@ -5,9 +5,11 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Tgstation.Server.Api.Models; using Tgstation.Server.Common.Extensions; +using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.IO; namespace Tgstation.Server.Host.Components.Engine @@ -39,7 +41,7 @@ namespace Tgstation.Server.Host.Components.Engine protected override string DreamMakerName => DreamMakerExecutableName + ShellScriptExtension; /// - protected override string ByondRevisionsUrlTemplate => "https://www.byond.com/download/build/{0}/{0}.{1}_byond_linux.zip"; + protected override string OSMarkerTemplate => "Linux"; /// /// The for the . @@ -52,13 +54,15 @@ namespace Tgstation.Server.Host.Components.Engine /// The value of . /// The for the . /// The for the . + /// The . /// The for the . public PosixByondInstaller( IPostWriteHandler postWriteHandler, IIOManager ioManager, IFileDownloader fileDownloader, + IOptionsMonitor generalConfigurationOptions, ILogger logger) - : base(ioManager, logger, fileDownloader) + : base(ioManager, logger, fileDownloader, generalConfigurationOptions) { this.postWriteHandler = postWriteHandler ?? throw new ArgumentNullException(nameof(postWriteHandler)); diff --git a/src/Tgstation.Server.Host/Components/Engine/WindowsByondInstaller.cs b/src/Tgstation.Server.Host/Components/Engine/WindowsByondInstaller.cs index ce2c7704e2..786d8df147 100644 --- a/src/Tgstation.Server.Host/Components/Engine/WindowsByondInstaller.cs +++ b/src/Tgstation.Server.Host/Components/Engine/WindowsByondInstaller.cs @@ -69,18 +69,13 @@ namespace Tgstation.Server.Host.Components.Engine protected override string PathToUserFolder { get; } /// - protected override string ByondRevisionsUrlTemplate => "https://www.byond.com/download/build/{0}/{0}.{1}_byond.zip"; + protected override string OSMarkerTemplate => "Windows"; /// /// The for the . /// readonly IProcessExecutor processExecutor; - /// - /// The for the . - /// - readonly GeneralConfiguration generalConfiguration; - /// /// The for the . /// @@ -100,7 +95,7 @@ namespace Tgstation.Server.Host.Components.Engine /// Initializes a new instance of the class. /// /// The value of . - /// The containing the value of . + /// The containing the . /// The containing the value of . /// The for the . /// The for the . @@ -109,13 +104,12 @@ namespace Tgstation.Server.Host.Components.Engine IProcessExecutor processExecutor, IIOManager ioManager, IFileDownloader fileDownloader, - IOptions generalConfigurationOptions, + IOptionsMonitor generalConfigurationOptions, IOptions sessionConfigurationOptions, ILogger logger) - : base(ioManager, logger, fileDownloader) + : base(ioManager, logger, fileDownloader, generalConfigurationOptions) { this.processExecutor = processExecutor ?? throw new ArgumentNullException(nameof(processExecutor)); - generalConfiguration = generalConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(generalConfigurationOptions)); sessionConfiguration = sessionConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(sessionConfigurationOptions)); var useServiceSpecialTactics = Environment.Is64BitProcess && Environment.UserName == $"{Environment.MachineName}$"; @@ -150,7 +144,7 @@ namespace Tgstation.Server.Host.Components.Engine installDirectXTask, }; - if (!generalConfiguration.SkipAddingByondFirewallException) + if (!GeneralConfigurationOptions.CurrentValue.SkipAddingByondFirewallException) { var firewallTask = AddDreamDaemonToFirewall(version, path, deploymentPipelineProcesses, cancellationToken); tasks.Add(firewallTask); @@ -165,7 +159,7 @@ namespace Tgstation.Server.Host.Components.Engine CheckVersionValidity(version); ArgumentNullException.ThrowIfNull(path); - if (generalConfiguration.SkipAddingByondFirewallException) + if (GeneralConfigurationOptions.CurrentValue.SkipAddingByondFirewallException) return; if (version.Version < DDExeVersion) diff --git a/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs b/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs index 3ba8b3e878..c47282ea80 100644 --- a/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs +++ b/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs @@ -148,6 +148,17 @@ namespace Tgstation.Server.Host.Configuration [YamlMember(SerializeAs = typeof(string))] public Uri OpenDreamGitUrl { get; set; } = new Uri(DefaultOpenDreamGitUrl); + /// + /// The formatter used to download official byond zip files for a given version + /// - ${Major} is substituted with the major version number + /// - ${Minor} is substituted with the minor version number + /// - ${Linux:xxx}, where xxx is any string, will be substituted with xxx if running under Linux. + /// - ${Windows:xxx}, where xxx is any string, will be substituted with xxx if running under Windows. + /// - $$ will evaluate to a literal $ and not be used for substitutions. + /// - Any inapplicable ${xxx} string will be removed. + /// + public string ByondZipDownloadTemplate { get; set; } = "https://www.byond.com/download/build/${Major}/${Major}.{Minor}_byond${Linux:_linux}.zip"; + /// /// The prefix to the OpenDream semver as tags appear in the git repository. /// diff --git a/tests/Tgstation.Server.Host.Tests/Components/Engine/TestByondInstallerBase.cs b/tests/Tgstation.Server.Host.Tests/Components/Engine/TestByondInstallerBase.cs new file mode 100644 index 0000000000..a3a4605877 --- /dev/null +++ b/tests/Tgstation.Server.Host.Tests/Components/Engine/TestByondInstallerBase.cs @@ -0,0 +1,23 @@ +using System; + +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace Tgstation.Server.Host.Components.Engine.Tests +{ + [TestClass] + public sealed class TestByondInstallerBase + { + [TestMethod] + public void TestUrlTemplateFormatting() + { + const string OSMarker = "TempleOS"; + + Assert.AreEqual( + new Uri("https://example.com/$515.1111_Hello Worl$d.zip"), + ByondInstallerBase.GetDownloadZipUrl( + new Version(515, 1111), + "https://example.com/$$${Major}.${Minor}_${TempleOS:Hello Worl$$d}.zip${Linux:Not this}${Or This}", + OSMarker)); + } + } +} diff --git a/tests/Tgstation.Server.Host.Tests/Components/Engine/TestPosixByondInstaller.cs b/tests/Tgstation.Server.Host.Tests/Components/Engine/TestPosixByondInstaller.cs index 60e86ad63e..c6288b6a7c 100644 --- a/tests/Tgstation.Server.Host.Tests/Components/Engine/TestPosixByondInstaller.cs +++ b/tests/Tgstation.Server.Host.Tests/Components/Engine/TestPosixByondInstaller.cs @@ -1,4 +1,5 @@ using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; using System; @@ -9,6 +10,7 @@ using System.Threading.Tasks; using Tgstation.Server.Api.Models; using Tgstation.Server.Api.Models.Internal; +using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.IO; namespace Tgstation.Server.Host.Components.Engine.Tests @@ -19,16 +21,18 @@ namespace Tgstation.Server.Host.Components.Engine.Tests [TestMethod] public void TestConstruction() { - Assert.ThrowsException(() => new PosixByondInstaller(null, null, null, null)); + Assert.ThrowsException(() => new PosixByondInstaller(null, null, null, null, null)); var mockPostWriteHandler = new Mock(); - Assert.ThrowsException(() => new PosixByondInstaller(mockPostWriteHandler.Object, null, null, null)); + Assert.ThrowsException(() => new PosixByondInstaller(mockPostWriteHandler.Object, null, null, null, null)); var mockIOManager = new Mock(); - Assert.ThrowsException(() => new PosixByondInstaller(mockPostWriteHandler.Object, mockIOManager.Object, null, null)); + Assert.ThrowsException(() => new PosixByondInstaller(mockPostWriteHandler.Object, mockIOManager.Object, null, null, null)); var mockFileDownloader = Mock.Of(); - Assert.ThrowsException(() => new PosixByondInstaller(mockPostWriteHandler.Object, mockIOManager.Object, mockFileDownloader, null)); + Assert.ThrowsException(() => new PosixByondInstaller(mockPostWriteHandler.Object, mockIOManager.Object, mockFileDownloader, null, null)); + var mockOptions = Mock.Of>(); + Assert.ThrowsException(() => new PosixByondInstaller(mockPostWriteHandler.Object, mockIOManager.Object, mockFileDownloader, mockOptions, null)); var mockLogger = new Mock>(); - _ = new PosixByondInstaller(mockPostWriteHandler.Object, mockIOManager.Object, mockFileDownloader, mockLogger.Object); + _ = new PosixByondInstaller(mockPostWriteHandler.Object, mockIOManager.Object, mockFileDownloader, mockOptions, mockLogger.Object); } [TestMethod] @@ -38,7 +42,8 @@ namespace Tgstation.Server.Host.Components.Engine.Tests var mockIOManager = new Mock(); var mockLogger = new Mock>(); var mockFileDownloader = Mock.Of(); - var installer = new PosixByondInstaller(mockPostWriteHandler.Object, mockIOManager.Object, mockFileDownloader, mockLogger.Object); + var mockOptions = Mock.Of>(); + var installer = new PosixByondInstaller(mockPostWriteHandler.Object, mockIOManager.Object, mockFileDownloader, mockOptions, mockLogger.Object); await installer.CleanCache(default); } @@ -50,7 +55,8 @@ namespace Tgstation.Server.Host.Components.Engine.Tests var mockPostWriteHandler = new Mock(); var mockLogger = new Mock>(); var mockFileDownloader = new Mock(); - var installer = new PosixByondInstaller(mockPostWriteHandler.Object, mockIOManager.Object, mockFileDownloader.Object, mockLogger.Object); + var mockOptions = Mock.Of>(); + var installer = new PosixByondInstaller(mockPostWriteHandler.Object, mockIOManager.Object, mockFileDownloader.Object, mockOptions, mockLogger.Object); await Assert.ThrowsExceptionAsync(() => installer.DownloadVersion(null, null, default).AsTask()); @@ -87,7 +93,8 @@ namespace Tgstation.Server.Host.Components.Engine.Tests var mockPostWriteHandler = new Mock(); var mockLogger = new Mock>(); var mockFileDownloader = Mock.Of(); - var installer = new PosixByondInstaller(mockPostWriteHandler.Object, mockIOManager.Object, mockFileDownloader, mockLogger.Object); + var mockOptions = Mock.Of>(); + var installer = new PosixByondInstaller(mockPostWriteHandler.Object, mockIOManager.Object, mockFileDownloader, mockOptions, mockLogger.Object); const string FakePath = "fake"; await Assert.ThrowsExceptionAsync(() => installer.Install(null, null, false, default).AsTask()); From 51d7aaec138cb4cc97061f796c7df53222a9996f Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 23 Jun 2025 21:36:08 -0400 Subject: [PATCH 12/52] Set default BYOND download to SS13 org mirror --- src/Tgstation.Server.Host/appsettings.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Tgstation.Server.Host/appsettings.yml b/src/Tgstation.Server.Host/appsettings.yml index 7d952c73d1..c06b79c667 100644 --- a/src/Tgstation.Server.Host/appsettings.yml +++ b/src/Tgstation.Server.Host/appsettings.yml @@ -17,6 +17,7 @@ General: HostApiDocumentation: false # Make HTTP API documentation available at /api/doc/tgs_api.json and /api/graphql SkipAddingByondFirewallException: false # Windows Only: Prevent running netsh.exe to add a firewall exception for installed engine binaries DeploymentDirectoryCopyTasksPerCore: 100 # Maximum number of concurrent file copy operations PER available CPU core + ByondZipDownloadTemplate: https://spacestation13.github.io/byond-builds/${Major}/${Major}.${Minor}_byond${Linux:_linux}.zip # URL of a plaintext file used to identify the curr OpenDreamGitUrl: https://github.com/OpenDreamProject/OpenDream # The repository to retrieve OpenDream from OpenDreamGitTagPrefix: v # The prefix to the OpenDream semver as tags appear in the git repository OpenDreamSuppressInstallOutput: false # Suppress the dotnet output of creating an OpenDream installation. Known to cause hangs in CI. From 6b696a4c1a6cf4bd6ef5b0d41127702c40881e70 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 23 Jun 2025 21:41:03 -0400 Subject: [PATCH 13/52] Give up on broken BYOND caching, download from SS13 mirror --- .github/workflows/ci-pipeline.yml | 39 +------------------ .../CachingFileDownloader.cs | 2 +- 2 files changed, 3 insertions(+), 38 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index fe0b463dce..7ad807b51f 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -166,19 +166,12 @@ jobs: sudo apt-get update sudo apt-get install -y -o APT::Immediate-Configure=0 libc6-i386 libstdc++6:i386 libgcc-s1:i386 - - name: Cache BYOND .zips - uses: actions/cache@v4 - id: cache-byond - with: - path: ~/byond-zips-cache - key: byond-zips - - name: Setup BYOND Cache if Necessary and Install run: | echo "Setting up BYOND." FULL_VERSION=${{ matrix.byond }} if [[ "$FULL_VERSION" = "EDGE" ]] ; then - VERSIONS=$(curl https://www.byond.com/download/version.txt) + VERSIONS=$(curl https://spacestation13.github.io/byond-builds/version.txt) FULL_VERSION=$(echo "$VERSIONS" | tail -n1) echo "EDGE version evaluated to $FULL_VERSION" @@ -193,7 +186,7 @@ jobs: if [[ ! -f $HOME/byond-zips-cache/linux/$FULL_VERSION.zip ]] ; then BYOND_MAJOR=${FULL_VERSION%.*} mkdir -p $HOME/byond-zips-cache/linux - curl "https://www.byond.com/download/build/$BYOND_MAJOR/${FULL_VERSION}_byond_linux.zip" -o $HOME/byond-zips-cache/linux/$FULL_VERSION.zip + curl "https://spacestation13.github.io/byond-builds/${FULL_VERSION}_byond_linux.zip" -o $HOME/byond-zips-cache/linux/$FULL_VERSION.zip fi mkdir -p "$HOME/BYOND" cd "$HOME/BYOND" @@ -509,13 +502,6 @@ jobs: if: always() run: rm -f ${{ env.TGS_TELEMETRY_KEY_FILE }} - - name: Cache BYOND .zips - uses: actions/cache@v4 - id: cache-byond - with: - path: ~/byond-zips-cache - key: byond-zips - - name: Run Unit Tests run: sudo dotnet test --no-build --logger "GitHubActions;summary.includePassedTests=true;summary.includeSkippedTests=true" --filter TestCategory!=RequiresDatabase -c ${{ matrix.configuration }}NoWindows --collect:"XPlat Code Coverage" --settings build/ci.runsettings --results-directory ./TestResults tgstation-server.sln env: @@ -578,13 +564,6 @@ jobs: if: always() run: rm -f ${{ env.TGS_TELEMETRY_KEY_FILE }} - - name: Cache BYOND .zips - uses: actions/cache@v4 - id: cache-byond - with: - path: ~/byond-zips-cache - key: byond-zips - - name: Run Unit Tests run: dotnet test --no-build --logger "GitHubActions;summary.includePassedTests=true;summary.includeSkippedTests=true" --filter TestCategory!=RequiresDatabase -c ${{ matrix.configuration }}NoWix --collect:"XPlat Code Coverage" --settings build/ci.runsettings --results-directory ./TestResults tgstation-server.sln env: @@ -717,13 +696,6 @@ jobs: if: always() run: rm -f ${{ env.TGS_TELEMETRY_KEY_FILE }} - - name: Cache BYOND .zips - uses: actions/cache@v4 - id: cache-byond - with: - path: ~/byond-zips-cache - key: byond-zips - - name: Run Live Tests # Logging here is weird because printing massive amounts of text on Windows runners is SLOW AS SHIT!!! id: live-tests shell: bash @@ -945,13 +917,6 @@ jobs: if: always() run: rm -f ${{ env.TGS_TELEMETRY_KEY_FILE }} - - name: Cache BYOND .zips - uses: actions/cache@v4 - id: cache-byond - with: - path: ~/byond-zips-cache - key: byond-zips - - name: Run Live Tests run: | cd tests/Tgstation.Server.Tests diff --git a/tests/Tgstation.Server.Tests/CachingFileDownloader.cs b/tests/Tgstation.Server.Tests/CachingFileDownloader.cs index de540b606d..8836f71232 100644 --- a/tests/Tgstation.Server.Tests/CachingFileDownloader.cs +++ b/tests/Tgstation.Server.Tests/CachingFileDownloader.cs @@ -87,7 +87,7 @@ namespace Tgstation.Server.Tests }; var url = new Uri( - $"https://www.byond.com/download/build/{version.Version.Major}/{version.Version.Major}.{version.Version.Minor}_byond{(!windows ? "_linux" : string.Empty)}.zip"); + $"https://spacestation13.github.io/byond-builds/{version.Version.Major}/{version.Version.Major}.{version.Version.Minor}_byond{(!windows ? "_linux" : string.Empty)}.zip"); string path = null; if (TestingUtils.RunningInGitHubActions) { From 7d90e0d2d153c2d5bc4ec4463dd15a2b1af725cb Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 23 Jun 2025 21:45:08 -0400 Subject: [PATCH 14/52] Bump main and config versions --- build/Version.props | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/Version.props b/build/Version.props index 394ff58026..80026e5567 100644 --- a/build/Version.props +++ b/build/Version.props @@ -3,8 +3,8 @@ - 6.17.0 - 5.7.0 + 6.18.0 + 5.8.0 10.13.0 0.6.0 7.0.0 From d514474fcecde79c9091a124f9901648025513dc Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 23 Jun 2025 21:45:51 -0400 Subject: [PATCH 15/52] Fix comment --- src/Tgstation.Server.Host/appsettings.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/appsettings.yml b/src/Tgstation.Server.Host/appsettings.yml index c06b79c667..58758dbc25 100644 --- a/src/Tgstation.Server.Host/appsettings.yml +++ b/src/Tgstation.Server.Host/appsettings.yml @@ -17,7 +17,7 @@ General: HostApiDocumentation: false # Make HTTP API documentation available at /api/doc/tgs_api.json and /api/graphql SkipAddingByondFirewallException: false # Windows Only: Prevent running netsh.exe to add a firewall exception for installed engine binaries DeploymentDirectoryCopyTasksPerCore: 100 # Maximum number of concurrent file copy operations PER available CPU core - ByondZipDownloadTemplate: https://spacestation13.github.io/byond-builds/${Major}/${Major}.${Minor}_byond${Linux:_linux}.zip # URL of a plaintext file used to identify the curr + ByondZipDownloadTemplate: https://spacestation13.github.io/byond-builds/${Major}/${Major}.${Minor}_byond${Linux:_linux}.zip # Template for downloading official byond versions OpenDreamGitUrl: https://github.com/OpenDreamProject/OpenDream # The repository to retrieve OpenDream from OpenDreamGitTagPrefix: v # The prefix to the OpenDream semver as tags appear in the git repository OpenDreamSuppressInstallOutput: false # Suppress the dotnet output of creating an OpenDream installation. Known to cause hangs in CI. From 13d68b7c5e66b72510f49a8f6ed004e486cdda9a Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 23 Jun 2025 22:24:39 -0400 Subject: [PATCH 16/52] Fix CI caching URL --- .github/workflows/ci-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 7ad807b51f..f8c957a876 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -186,7 +186,7 @@ jobs: if [[ ! -f $HOME/byond-zips-cache/linux/$FULL_VERSION.zip ]] ; then BYOND_MAJOR=${FULL_VERSION%.*} mkdir -p $HOME/byond-zips-cache/linux - curl "https://spacestation13.github.io/byond-builds/${FULL_VERSION}_byond_linux.zip" -o $HOME/byond-zips-cache/linux/$FULL_VERSION.zip + curl "https://spacestation13.github.io/byond-builds/$BYOND_MAJOR/${FULL_VERSION}_byond_linux.zip" -o $HOME/byond-zips-cache/linux/$FULL_VERSION.zip fi mkdir -p "$HOME/BYOND" cd "$HOME/BYOND" From da2732b2ac01d33c66e1127aff43dd3e83b54c08 Mon Sep 17 00:00:00 2001 From: Kashargul <144968721+Kashargul@users.noreply.github.com> Date: Thu, 26 Jun 2025 18:27:23 +0200 Subject: [PATCH 17/52] libcurl4 needed for 1664 --- build/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build/Dockerfile b/build/Dockerfile index e7cb447c2b..16bedcc9ab 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -59,11 +59,13 @@ RUN export TGS_TELEMETRY_KEY_FILE="../../${TGS_TELEMETRY_KEY_FILE}" \ FROM mcr.microsoft.com/dotnet/aspnet:8.0-bookworm-slim #needed for byond, curl for healthchecks -RUN apt-get update \ +RUN pkg --add-architecture i386 \ + && apt-get update \ && apt-get install -y \ gcc-multilib \ gdb \ curl \ + libcurl4:i386 \ && rm -rf /var/lib/apt/lists/* EXPOSE 5000 From 288a68d5fa7c519110c2de4fa9c6a65307034551 Mon Sep 17 00:00:00 2001 From: Kashargul <144968721+Kashargul@users.noreply.github.com> Date: Thu, 26 Jun 2025 18:50:19 +0200 Subject: [PATCH 18/52] build deps --- build/package/deb/debian/control | 1 + 1 file changed, 1 insertion(+) diff --git a/build/package/deb/debian/control b/build/package/deb/debian/control index 67356359b6..7ae12387a4 100644 --- a/build/package/deb/debian/control +++ b/build/package/deb/debian/control @@ -21,6 +21,7 @@ Depends: libstdc++6:i386 [amd64], libstdc++6 [i386], gcc-multilib [amd64], + libcurl4 [i386], Recommends: libsystemd0, gdb, From ce8248bb7606a617236aa2798c8daeac1d4a017b Mon Sep 17 00:00:00 2001 From: Kashargul <144968721+Kashargul@users.noreply.github.com> Date: Thu, 26 Jun 2025 23:55:42 +0200 Subject: [PATCH 19/52] Update build/Dockerfile Co-authored-by: Jordan Dominion --- build/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/Dockerfile b/build/Dockerfile index 16bedcc9ab..8a3a1d24f7 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -59,7 +59,7 @@ RUN export TGS_TELEMETRY_KEY_FILE="../../${TGS_TELEMETRY_KEY_FILE}" \ FROM mcr.microsoft.com/dotnet/aspnet:8.0-bookworm-slim #needed for byond, curl for healthchecks -RUN pkg --add-architecture i386 \ +RUN dpkg --add-architecture i386 \ && apt-get update \ && apt-get install -y \ gcc-multilib \ From a1a7ecbefd8dcb5154ddc9ab1b176acf4fc70699 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Jul 2025 11:12:17 +0000 Subject: [PATCH 20/52] Bump StrawberryShake.Server from 15.1.3 to 15.1.7 --- updated-dependencies: - dependency-name: StrawberryShake.Server dependency-version: 15.1.7 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: hotchocolate ... Signed-off-by: dependabot[bot] --- .../Tgstation.Server.Client.GraphQL.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Client.GraphQL/Tgstation.Server.Client.GraphQL.csproj b/src/Tgstation.Server.Client.GraphQL/Tgstation.Server.Client.GraphQL.csproj index d5b8544c62..9a575c625f 100644 --- a/src/Tgstation.Server.Client.GraphQL/Tgstation.Server.Client.GraphQL.csproj +++ b/src/Tgstation.Server.Client.GraphQL/Tgstation.Server.Client.GraphQL.csproj @@ -10,7 +10,7 @@ - + From 12632a8bb4f83f05121495735a5f67d11d9707f3 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 20 Jul 2025 20:07:42 -0400 Subject: [PATCH 21/52] AAAAA --- .github/CONTRIBUTING.md | 1 + .github/workflows/ci-pipeline.yml | 39 +++++++-- .../Configuration/GeneralConfiguration.cs | 7 +- src/Tgstation.Server.Host/appsettings.yml | 2 +- .../Engine/TestPosixByondInstaller.cs | 12 ++- .../CachingFileDownloader.cs | 25 ++++-- .../Live/Instance/EngineTest.cs | 39 +++------ .../Live/Instance/InstanceTest.cs | 15 ++-- .../Live/LiveTestingServer.cs | 3 +- .../Live/TestLiveServer.cs | 6 +- tests/Tgstation.Server.Tests/TestVersions.cs | 15 +++- tests/Tgstation.Server.Tests/TestingUtils.cs | 84 +++++++++++++++++++ 12 files changed, 191 insertions(+), 57 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index f116b2259d..c2912e0af5 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -50,6 +50,7 @@ You must also have the following environment variables set. To run them more acc - `TGS_TEST_DATABASE_TYPE`: `MySql`, `MariaDB`, `PostgresSql`, or `SqlServer`. - `TGS_TEST_CONNECTION_STRING`: To a valid database connection string. You can use the setup wizard to create one. - (Optional) `TGS_TEST_GITHUB_TOKEN`: A GitHub personal access token with no scopes used to bypass rate limits. +- (Optional) `TGS_TEST_BYOND_ZIP_DOWNLOAD_TEMPLATE`: Template URL for downloading BYOND zip files from a non-official mirror. - (Optional) The following variables are all interdependent, so if one is set they all must be. - `TGS_TEST_DISCORD_TOKEN`: To a valid discord bot token. - `TGS_TEST_DISCORD_CHANNEL`: To a valid discord channel ID that the above bot can access. diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index f8c957a876..9f9c9a5010 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -166,12 +166,12 @@ jobs: sudo apt-get update sudo apt-get install -y -o APT::Immediate-Configure=0 libc6-i386 libstdc++6:i386 libgcc-s1:i386 - - name: Setup BYOND Cache if Necessary and Install + - name: Evaluate EDGE BYOND version + id: edge_version_evaluation run: | - echo "Setting up BYOND." FULL_VERSION=${{ matrix.byond }} if [[ "$FULL_VERSION" = "EDGE" ]] ; then - VERSIONS=$(curl https://spacestation13.github.io/byond-builds/version.txt) + VERSIONS=$(curl https://www.byond.com/download/version.txt) FULL_VERSION=$(echo "$VERSIONS" | tail -n1) echo "EDGE version evaluated to $FULL_VERSION" @@ -183,14 +183,27 @@ jobs: FULL_VERSION=${bad_linux_releases[$FULL_VERSION]} fi fi - if [[ ! -f $HOME/byond-zips-cache/linux/$FULL_VERSION.zip ]] ; then + run: echo "EVALUATED_EDGE_VERSION=$FULL_VERSION" >> $GITHUB_OUTPUT + + - name: Cache BYOND .zips + uses: actions/cache@v4 + id: cache-byond + with: + path: ~/byond-zips-cache/linux/${{ steps.edge_version_evaluation.outputs.EVALUATED_EDGE_VERSION }} + key: byond-zips-linux-${{ steps.edge_version_evaluation.outputs.EVALUATED_EDGE_VERSION }} + + - name: Setup BYOND Cache if Necessary and Install + run: | + echo "Setting up BYOND." + FULL_VERSION=${{ steps.edge_version_evaluation.outputs.EVALUATED_EDGE_VERSION }} + if [[ ! -f $HOME/byond-zips-cache/linux/$FULL_VERSION/$FULL_VERSION.zip ]] ; then BYOND_MAJOR=${FULL_VERSION%.*} mkdir -p $HOME/byond-zips-cache/linux - curl "https://spacestation13.github.io/byond-builds/$BYOND_MAJOR/${FULL_VERSION}_byond_linux.zip" -o $HOME/byond-zips-cache/linux/$FULL_VERSION.zip + curl "https://www.byond.com/download/$BYOND_MAJOR/${FULL_VERSION}_byond_linux.zip" -o $HOME/byond-zips-cache/linux/$FULL_VERSION/$FULL_VERSION.zip fi mkdir -p "$HOME/BYOND" cd "$HOME/BYOND" - cp $HOME/byond-zips-cache/linux/$FULL_VERSION.zip byond.zip + cp $HOME/byond-zips-cache/linux/$FULL_VERSION/$FULL_VERSION.zip byond.zip unzip byond.zip cd byond make here @@ -612,6 +625,13 @@ jobs: with: node-version: ${{ env.TGS_NODE_VERSION }} + - name: Cache BYOND .zips + uses: actions/cache@v4 + id: cache-byond + with: + path: ~/byond-zips-cache/live/windows + key: byond-zips-windows-live + - name: Set TGS_TEST_DUMP_API_SPEC if: ${{ matrix.configuration == 'Release' && matrix.watchdog-type == 'Advanced' && matrix.database-type == 'SqlServer' }} run: echo "TGS_TEST_DUMP_API_SPEC=yes" >> $Env:GITHUB_ENV @@ -863,6 +883,13 @@ jobs: with: node-version: ${{ env.TGS_NODE_VERSION }} + - name: Cache BYOND .zips + uses: actions/cache@v4 + id: cache-byond + with: + path: ~/byond-zips-cache/live/linux + key: byond-zips-linux-live + - name: Set Sqlite Connection Info if: ${{ matrix.database-type == 'Sqlite' }} run: | diff --git a/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs b/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs index c47282ea80..651bb0bf85 100644 --- a/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs +++ b/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs @@ -31,6 +31,11 @@ namespace Tgstation.Server.Host.Configuration /// public const ushort DefaultApiPort = 5000; + /// + /// Default vale for . + /// + public const string DefaultByondZipDownloadTemplate = "https://www.byond.com/download/build/${Major}/${Major}.${Minor}_byond${Linux:_linux}.zip"; + /// /// The default value for . /// @@ -157,7 +162,7 @@ namespace Tgstation.Server.Host.Configuration /// - $$ will evaluate to a literal $ and not be used for substitutions. /// - Any inapplicable ${xxx} string will be removed. /// - public string ByondZipDownloadTemplate { get; set; } = "https://www.byond.com/download/build/${Major}/${Major}.{Minor}_byond${Linux:_linux}.zip"; + public string ByondZipDownloadTemplate { get; set; } = DefaultByondZipDownloadTemplate; /// /// The prefix to the OpenDream semver as tags appear in the git repository. diff --git a/src/Tgstation.Server.Host/appsettings.yml b/src/Tgstation.Server.Host/appsettings.yml index 58758dbc25..ff244ea7e4 100644 --- a/src/Tgstation.Server.Host/appsettings.yml +++ b/src/Tgstation.Server.Host/appsettings.yml @@ -17,7 +17,7 @@ General: HostApiDocumentation: false # Make HTTP API documentation available at /api/doc/tgs_api.json and /api/graphql SkipAddingByondFirewallException: false # Windows Only: Prevent running netsh.exe to add a firewall exception for installed engine binaries DeploymentDirectoryCopyTasksPerCore: 100 # Maximum number of concurrent file copy operations PER available CPU core - ByondZipDownloadTemplate: https://spacestation13.github.io/byond-builds/${Major}/${Major}.${Minor}_byond${Linux:_linux}.zip # Template for downloading official byond versions + ByondZipDownloadTemplate: https://www.byond.com/download/build/${Major}/${Major}.${Minor}_byond${Linux:_linux}.zip # Template for downloading official byond versions OpenDreamGitUrl: https://github.com/OpenDreamProject/OpenDream # The repository to retrieve OpenDream from OpenDreamGitTagPrefix: v # The prefix to the OpenDream semver as tags appear in the git repository OpenDreamSuppressInstallOutput: false # Suppress the dotnet output of creating an OpenDream installation. Known to cause hangs in CI. diff --git a/tests/Tgstation.Server.Host.Tests/Components/Engine/TestPosixByondInstaller.cs b/tests/Tgstation.Server.Host.Tests/Components/Engine/TestPosixByondInstaller.cs index c6288b6a7c..6dbfb155b4 100644 --- a/tests/Tgstation.Server.Host.Tests/Components/Engine/TestPosixByondInstaller.cs +++ b/tests/Tgstation.Server.Host.Tests/Components/Engine/TestPosixByondInstaller.cs @@ -55,8 +55,14 @@ namespace Tgstation.Server.Host.Components.Engine.Tests var mockPostWriteHandler = new Mock(); var mockLogger = new Mock>(); var mockFileDownloader = new Mock(); - var mockOptions = Mock.Of>(); - var installer = new PosixByondInstaller(mockPostWriteHandler.Object, mockIOManager.Object, mockFileDownloader.Object, mockOptions, mockLogger.Object); + var mockOptions = new Mock>(); + const string TestUrl = "https://chumb.is"; + mockOptions.SetupGet(x => x.CurrentValue).Returns(new GeneralConfiguration + { + ByondZipDownloadTemplate = TestUrl, + }); + + var installer = new PosixByondInstaller(mockPostWriteHandler.Object, mockIOManager.Object, mockFileDownloader.Object, mockOptions.Object, mockLogger.Object); await Assert.ThrowsExceptionAsync(() => installer.DownloadVersion(null, null, default).AsTask()); @@ -64,7 +70,7 @@ namespace Tgstation.Server.Host.Components.Engine.Tests mockFileDownloader .Setup( x => x.DownloadFile( - It.Is(uri => uri == new Uri("https://www.byond.com/download/build/511/511.1385_byond_linux.zip")), + It.Is(uri => uri == new Uri(TestUrl)), null)) .Returns( new BufferedFileStreamProvider( diff --git a/tests/Tgstation.Server.Tests/CachingFileDownloader.cs b/tests/Tgstation.Server.Tests/CachingFileDownloader.cs index 8836f71232..8fccd1234c 100644 --- a/tests/Tgstation.Server.Tests/CachingFileDownloader.cs +++ b/tests/Tgstation.Server.Tests/CachingFileDownloader.cs @@ -12,6 +12,8 @@ using Moq; using Tgstation.Server.Api.Models; using Tgstation.Server.Api.Models.Internal; using Tgstation.Server.Common.Http; +using Tgstation.Server.Host.Components.Engine; +using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.Extensions; using Tgstation.Server.Host.IO; using Tgstation.Server.Host.System; @@ -44,9 +46,9 @@ namespace Tgstation.Server.Tests var logger = loggerFactory.CreateLogger("CachingFileDownloader"); var cfd = new CachingFileDownloader(loggerFactory.CreateLogger()); - var edgeVersion = await EngineTest.GetEdgeVersion(Api.Models.EngineType.Byond, cfd, cancellationToken); - await InitializeByondVersion(logger, edgeVersion.Version, new PlatformIdentifier().IsWindows, cancellationToken); + // this also will inject the edge version + var edgeVersion = await EngineTest.GetEdgeVersion(Api.Models.EngineType.Byond, logger, cfd, cancellationToken); // predownload the target github release update asset var gitHubToken = Environment.GetEnvironmentVariable("TGS_TEST_GITHUB_TOKEN"); @@ -78,7 +80,7 @@ namespace Tgstation.Server.Tests ServiceCollectionExtensions.UseFileDownloader(); } - public static async ValueTask InitializeByondVersion(ILogger logger, Version byondVersion, bool windows, CancellationToken cancellationToken) + public static async ValueTask InitializeByondVersion(ILogger logger, Version byondVersion, bool windows, CancellationToken cancellationToken, string urlCacheOverrideTemplate = null) { var version = new EngineVersion { @@ -86,8 +88,9 @@ namespace Tgstation.Server.Tests Version = byondVersion, }; - var url = new Uri( - $"https://spacestation13.github.io/byond-builds/{version.Version.Major}/{version.Version.Major}.{version.Version.Minor}_byond{(!windows ? "_linux" : string.Empty)}.zip"); + var urlTemplate = TestingUtils.ByondZipDownloadTemplate; + + var url = ByondInstallerBase.GetDownloadZipUrl(byondVersion, urlTemplate, new PlatformIdentifier().IsWindows ? "Windows" : "Linux"); string path = null; if (TestingUtils.RunningInGitHubActions) { @@ -98,13 +101,23 @@ namespace Tgstation.Server.Tests Environment.SpecialFolder.UserProfile, Environment.SpecialFolderOption.DoNotVerify), "byond-zips-cache", + "live", windows ? "windows" : "linux"); path = Path.Combine( dir, + $"{version.Version.Major}.{version.Version.Minor}", $"{version.Version.Major}.{version.Version.Minor}.zip"); } - await (await CacheFile(logger, url, null, path, cancellationToken)).DisposeAsync(); + await (await CacheFile( + logger, + urlCacheOverrideTemplate != null + ? ByondInstallerBase.GetDownloadZipUrl(byondVersion, urlCacheOverrideTemplate, new PlatformIdentifier().IsWindows ? "Windows" : "Linux") + : url, + null, + path, + cancellationToken)) + .DisposeAsync(); } public static void Cleanup() diff --git a/tests/Tgstation.Server.Tests/Live/Instance/EngineTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/EngineTest.cs index ffdaeb63e4..1497054e7d 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/EngineTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/EngineTest.cs @@ -41,13 +41,13 @@ namespace Tgstation.Server.Tests.Live.Instance EngineVersion testVersion; readonly EngineType testEngine = engineType; - public Task Run(CancellationToken cancellationToken, out Task firstInstall) + public Task Run(ILogger logger, CancellationToken cancellationToken, out Task firstInstall) { - firstInstall = RunPartOne(cancellationToken); + firstInstall = RunPartOne(logger, cancellationToken); return RunContinued(firstInstall, cancellationToken); } - public static async ValueTask GetEdgeVersion(EngineType engineType, IFileDownloader fileDownloader, CancellationToken cancellationToken) + public static async ValueTask GetEdgeVersion(EngineType engineType, ILogger logger, IFileDownloader fileDownloader, CancellationToken cancellationToken) { var edgeVersion = edgeVersions[engineType]; @@ -57,26 +57,7 @@ namespace Tgstation.Server.Tests.Live.Instance EngineVersion engineVersion; if (engineType == EngineType.Byond) { - await using var provider = fileDownloader.DownloadFile(new Uri("https://www.byond.com/download/version.txt"), null); - var stream = await provider.GetResult(cancellationToken); - using var reader = new StreamReader(stream, Encoding.UTF8, false, -1, true); - var text = await reader.ReadToEndAsync(cancellationToken); - var splits = text.Split('\n', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries); - - var targetVersion = splits.Last(); - - var badVersionMap = new PlatformIdentifier().IsWindows - ? [] - // linux map also needs updating in CI - : new Dictionary() - { - { "515.1612", "515.1611" } - }; - - badVersionMap.Add("515.1617", "515.1616"); - - if (badVersionMap.TryGetValue(targetVersion, out var remappedVersion)) - targetVersion = remappedVersion; + var targetVersion = await TestingUtils.GetByondEdgeVersion(logger, fileDownloader, cancellationToken); Assert.IsTrue(EngineVersion.TryParse(targetVersion, out engineVersion), $"Bad version: {targetVersion}"); } @@ -112,9 +93,9 @@ namespace Tgstation.Server.Tests.Live.Instance return edgeVersions[engineType] = engineVersion; } - async Task RunPartOne(CancellationToken cancellationToken) + async Task RunPartOne(ILogger logger, CancellationToken cancellationToken) { - testVersion = await GetEdgeVersion(testEngine, fileDownloader, cancellationToken); + testVersion = await GetEdgeVersion(testEngine, logger, fileDownloader, cancellationToken); await TestNoVersion(cancellationToken); await TestInstallNullVersion(cancellationToken); await TestInstallStable(cancellationToken); @@ -284,8 +265,11 @@ namespace Tgstation.Server.Tests.Live.Instance async Task TestCustomInstalls(CancellationToken cancellationToken) { - var generalConfigOptionsMock = new Mock>(); - generalConfigOptionsMock.SetupGet(x => x.Value).Returns(new GeneralConfiguration()); + var generalConfigOptionsMock = new Mock>(); + generalConfigOptionsMock.SetupGet(x => x.CurrentValue).Returns(new GeneralConfiguration + { + ByondZipDownloadTemplate = TestingUtils.ByondZipDownloadTemplate, + }); var sessionConfigOptionsMock = new Mock>(); sessionConfigOptionsMock.SetupGet(x => x.Value).Returns(new SessionConfiguration()); @@ -303,6 +287,7 @@ namespace Tgstation.Server.Tests.Live.Instance Mock.Of(), Mock.Of(), fileDownloader, + generalConfigOptionsMock.Object, Mock.Of>()); using var windowsByondInstaller = byondInstaller as WindowsByondInstaller; diff --git a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs index f9188313d0..087c83a45d 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs @@ -36,6 +36,7 @@ namespace Tgstation.Server.Tests.Live.Instance readonly ushort serverPort = serverPort; public async Task RunTests( + ILogger logger, IInstanceClient instanceClient, ushort dmPort, ushort ddPort, @@ -44,14 +45,14 @@ namespace Tgstation.Server.Tests.Live.Instance bool usingBasicWatchdog, CancellationToken cancellationToken) { - var testVersion = await EngineTest.GetEdgeVersion(EngineType.Byond, fileDownloader, cancellationToken); + var testVersion = await EngineTest.GetEdgeVersion(EngineType.Byond, logger, fileDownloader, cancellationToken); await using var engineTest = new EngineTest(instanceClient.Engine, instanceClient.Jobs, fileDownloader, instanceClient.Metadata, testVersion.Engine.Value); await using var chatTest = new ChatTest(instanceClient.ChatBots, instanceManagerClient, instanceClient.Jobs, instanceClient.Metadata); var configTest = new ConfigurationTest(instanceClient.Configuration, instanceClient.Metadata); await using var repoTest = new RepositoryTest(instanceClient, instanceClient.Repository, instanceClient.Jobs); await using var dmTest = new DeploymentTest(instanceClient, instanceClient.Jobs, dmPort, ddPort, lowPrioDeployment, testVersion); - var byondTask = engineTest.Run(cancellationToken, out var firstInstall); + var byondTask = engineTest.Run(logger, cancellationToken, out var firstInstall); var chatTask = chatTest.RunPreWatchdog(cancellationToken); var repoLongJob = await repoTest.RunLongClone(cancellationToken); @@ -98,12 +99,13 @@ namespace Tgstation.Server.Tests.Live.Instance "OpenDreamRepository"); var odRepoIoManager = new ResolvingIOManager(ioManager, odRepoDir); - var mockOptions = new Mock>(); + var mockOptionsMonitor = new Mock>(); var genConfig = new GeneralConfiguration { OpenDreamGitUrl = openDreamUrl, + ByondZipDownloadTemplate = TestingUtils.ByondZipDownloadTemplate, }; - mockOptions.SetupGet(x => x.Value).Returns(genConfig); + mockOptionsMonitor.SetupGet(x => x.CurrentValue).Returns(genConfig); IEngineInstaller byondInstaller = compatVersion.Engine == EngineType.OpenDream ? new OpenDreamInstaller( @@ -124,20 +126,21 @@ namespace Tgstation.Server.Tests.Live.Instance genConfig), Mock.Of(), Mock.Of(), - mockOptions.Object, + Options.Create(genConfig), Options.Create(new SessionConfiguration())) : new PlatformIdentifier().IsWindows ? new WindowsByondInstaller( Mock.Of(), Mock.Of(), fileDownloader, - Options.Create(genConfig), + mockOptionsMonitor.Object, Options.Create(new SessionConfiguration()), Mock.Of>()) : new PosixByondInstaller( Mock.Of(), Mock.Of(), fileDownloader, + mockOptionsMonitor.Object, Mock.Of>()); using var windowsByondInstaller = byondInstaller as WindowsByondInstaller; diff --git a/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs b/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs index ac4f519fa6..e1d74d64a5 100644 --- a/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs +++ b/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs @@ -156,7 +156,8 @@ namespace Tgstation.Server.Tests.Live $"Security:TokenExpiryMinutes=120", // timeouts are useless for us $"General:OpenDreamSuppressInstallOutput={TestingUtils.RunningInGitHubActions}", "Telemetry:DisableVersionReporting=true", - $"General:PrometheusPort={port}" + $"General:PrometheusPort={port}", + $"General:ByondZipDownloadTemplate={TestingUtils.ByondZipDownloadTemplate}" }; if (MultiServerClient.UseGraphQL) diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index 8df9a3d612..7eeec04af3 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -1421,6 +1421,7 @@ namespace Tgstation.Server.Tests.Live await Task.Yield(); InstanceManager GetInstanceManager() => ((Host.Server)server.RealServer).Host.Services.GetRequiredService(); + ILogger GetLogger() => ((Host.Server)server.RealServer).Host.Services.GetRequiredService>(); // main run var serverTask = server.Run(cancellationToken).AsTask(); @@ -1590,7 +1591,7 @@ namespace Tgstation.Server.Tests.Live var testSerialized = TestingUtils.RunningInGitHubActions; // they only have 2 cores, can't handle intense parallelization async Task ODCompatTests() { - var edgeODVersionTask = EngineTest.GetEdgeVersion(EngineType.OpenDream, fileDownloader, cancellationToken); + var edgeODVersionTask = EngineTest.GetEdgeVersion(EngineType.OpenDream, GetLogger(), fileDownloader, cancellationToken); var ex = await Assert.ThrowsExceptionAsync( () => InstanceTest.DownloadEngineVersion( @@ -1649,6 +1650,7 @@ namespace Tgstation.Server.Tests.Live await FailFast( instanceTest .RunTests( + GetLogger(), instanceClient, mainDMPort.Value, mainDDPort.Value, @@ -1881,7 +1883,7 @@ namespace Tgstation.Server.Tests.Live preStartupTime = DateTimeOffset.UtcNow; serverTask = server.Run(cancellationToken).AsTask(); long expectedCompileJobId, expectedStaged; - var edgeVersion = await EngineTest.GetEdgeVersion(EngineType.Byond, fileDownloader, cancellationToken); + var edgeVersion = await EngineTest.GetEdgeVersion(EngineType.Byond, GetLogger(), fileDownloader, cancellationToken); await using (var adminClient = await CreateAdminClient(server.ApiUrl, cancellationToken)) { var restAdminClient = adminClient.RestClient; diff --git a/tests/Tgstation.Server.Tests/TestVersions.cs b/tests/Tgstation.Server.Tests/TestVersions.cs index a0809b78f6..2d9acd7974 100644 --- a/tests/Tgstation.Server.Tests/TestVersions.cs +++ b/tests/Tgstation.Server.Tests/TestVersions.cs @@ -103,10 +103,14 @@ namespace Tgstation.Server.Tests } [TestMethod] + [TestCategory("RequiresDatabase")] public async Task TestDDExeByondVersion() { - var mockGeneralConfigurationOptions = new Mock>(); - mockGeneralConfigurationOptions.SetupGet(x => x.Value).Returns(new GeneralConfiguration()); + var mockGeneralConfigurationOptions = new Mock>(); + mockGeneralConfigurationOptions.SetupGet(x => x.CurrentValue).Returns(new GeneralConfiguration + { + ByondZipDownloadTemplate = TestingUtils.ByondZipDownloadTemplate, + }); var mockSessionConfigurationOptions = new Mock>(); mockSessionConfigurationOptions.SetupGet(x => x.Value).Returns(new SessionConfiguration()); @@ -165,12 +169,14 @@ namespace Tgstation.Server.Tests static Version MapThreadsVersion() => (Version)typeof(ByondInstallerBase).GetField("MapThreadsVersion", BindingFlags.Static | BindingFlags.NonPublic).GetValue(null) ?? throw new InvalidOperationException("Couldn't find MapThreadsVersion"); [TestMethod] + [TestCategory("RequiresDatabase")] public async Task TestMapThreadsByondVersion() { - var mockGeneralConfigurationOptions = new Mock>(); - mockGeneralConfigurationOptions.SetupGet(x => x.Value).Returns(new GeneralConfiguration + var mockGeneralConfigurationOptions = new Mock>(); + mockGeneralConfigurationOptions.SetupGet(x => x.CurrentValue).Returns(new GeneralConfiguration { SkipAddingByondFirewallException = true, + ByondZipDownloadTemplate = TestingUtils.ByondZipDownloadTemplate, }); var mockSessionConfigurationOptions = new Mock>(); mockSessionConfigurationOptions.SetupGet(x => x.Value).Returns(new SessionConfiguration()); @@ -209,6 +215,7 @@ namespace Tgstation.Server.Tests new PosixPostWriteHandler(loggerFactory.CreateLogger()), new DefaultIOManager(), fileDownloader, + mockGeneralConfigurationOptions.Object, loggerFactory.CreateLogger()); using var disposable = byondInstaller as IDisposable; diff --git a/tests/Tgstation.Server.Tests/TestingUtils.cs b/tests/Tgstation.Server.Tests/TestingUtils.cs index 3167d2c3d3..9cc11f7966 100644 --- a/tests/Tgstation.Server.Tests/TestingUtils.cs +++ b/tests/Tgstation.Server.Tests/TestingUtils.cs @@ -1,16 +1,23 @@ using System; +using System.Collections.Generic; using System.IO; using System.IO.Compression; +using System.Linq; +using System.Net.Http; using System.Reflection; +using System.Text; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; + using Moq; using Tgstation.Server.Host.Components.Engine; +using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.IO; +using Tgstation.Server.Host.System; namespace Tgstation.Server.Tests { @@ -70,5 +77,82 @@ namespace Tgstation.Server.Tests await new DefaultIOManager().DeleteDirectory(tempFolder, cancellationToken); } } + + static string byondZipDownloadTemplate; + + public static string ByondZipDownloadTemplate + { + get + { + if (byondZipDownloadTemplate == null) + { + var envvar = Environment.GetEnvironmentVariable("TGS_TEST_BYOND_ZIP_DOWNLOAD_TEMPLATE"); + if (envvar != null) + byondZipDownloadTemplate = envvar; + else + byondZipDownloadTemplate = GeneralConfiguration.DefaultByondZipDownloadTemplate; + } + + return byondZipDownloadTemplate; + } + } + + static string edgeVersion = null; + public static async ValueTask GetByondEdgeVersion(ILogger logger, IFileDownloader fileDownloader, CancellationToken cancellationToken) + { + if (edgeVersion != null) + return edgeVersion; + + async ValueTask GetVersionFromResponse(string versionTxt) + { + await using var provider = fileDownloader.DownloadFile(new Uri(versionTxt), null); + var stream = await provider.GetResult(cancellationToken); + using var reader = new StreamReader(stream, Encoding.UTF8, false, -1, true); + var text = await reader.ReadToEndAsync(cancellationToken); + var splits = text.Split('\n', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries); + + var targetVersion = splits.Last(); + + var badVersionMap = new PlatformIdentifier().IsWindows + ? [] + // linux map also needs updating in CI + : new Dictionary() + { + { "515.1612", "515.1611" } + }; + + badVersionMap.Add("515.1617", "515.1616"); + + if (badVersionMap.TryGetValue(targetVersion, out var remappedVersion)) + targetVersion = remappedVersion; + + return targetVersion; + } + + var mirroredVersionTxt = Environment.GetEnvironmentVariable("TGS_TEST_BYOND_MIRROR_VERSION_TXT"); + try + { + // always check byond.com first for latest up-to-date, mirror should ALWAYS have stable versions + const string DefaultMirror = "https://www.byond.com/download/version.txt"; + edgeVersion = await GetVersionFromResponse(DefaultMirror); + + logger.LogInformation("Downloading edge version from BYOND.com"); + + // if we got the result from byond.com, make sure the cache grabs the zip from there as well + await CachingFileDownloader.InitializeByondVersion(logger, Version.Parse(edgeVersion), new PlatformIdentifier().IsWindows, cancellationToken, GeneralConfiguration.DefaultByondZipDownloadTemplate); + } + catch (Exception ex) + { + logger.LogWarning(ex, "Cannot download zip from byond.com!"); + if (ByondZipDownloadTemplate == GeneralConfiguration.DefaultByondZipDownloadTemplate || mirroredVersionTxt == null) + throw; + + // fall back to the mirrored version.txt + await using var provider = fileDownloader.DownloadFile(new Uri(mirroredVersionTxt), null); + edgeVersion = await GetVersionFromResponse(mirroredVersionTxt); + } + + return edgeVersion; + } } } From b2c0d697d5e9b4bc1576d9abf5798b3990824c9c Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 20 Jul 2025 20:39:55 -0400 Subject: [PATCH 22/52] Cleanup usage of System.IO types to go through IIOManager --- .../Components/Deployment/DmbFactory.cs | 3 +- .../Components/Deployment/DmbProvider.cs | 17 ++++- .../Deployment/HardLinkDmbProvider.cs | 4 +- .../Engine/ZipStreamEngineInstallationData.cs | 2 +- .../Components/InstanceManager.cs | 2 +- .../Components/Repository/Repository.cs | 5 +- .../Components/StaticFiles/Configuration.cs | 4 +- .../Configuration/GeneralConfiguration.cs | 7 +- .../IO/DefaultIOManager.cs | 29 ++++++-- src/Tgstation.Server.Host/IO/IIOManager.cs | 28 +++++++- .../IO/ResolvingIOManager.cs | 3 +- .../Setup/SetupWizard.cs | 2 +- .../System/PosixProcessFeatures.cs | 66 +++++++++---------- .../System/WindowsProcessFeatures.cs | 1 + 14 files changed, 111 insertions(+), 62 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs b/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs index 8c523d97d5..a9cd217f18 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; -using System.IO; using System.Linq; using System.Runtime.CompilerServices; using System.Text; @@ -431,7 +430,7 @@ namespace Tgstation.Server.Host.Components.Deployment // Don't dispose it logger.LogDebug("Creating legacy two folder .dmb provider targeting {aDirName} directory...", LegacyADirectoryName); #pragma warning disable CA2000 // Dispose objects before losing scope (false positive) - newProvider = new DmbProvider(compileJob, engineVersion, ioManager, new DisposeInvoker(CleanupAction), Path.DirectorySeparatorChar + LegacyADirectoryName); + newProvider = new DmbProvider(compileJob, engineVersion, ioManager, new DisposeInvoker(CleanupAction), LegacyADirectoryName); #pragma warning restore CA2000 // Dispose objects before losing scope } diff --git a/src/Tgstation.Server.Host/Components/Deployment/DmbProvider.cs b/src/Tgstation.Server.Host/Components/Deployment/DmbProvider.cs index be47aeb6b3..4da03c795e 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/DmbProvider.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/DmbProvider.cs @@ -12,7 +12,18 @@ namespace Tgstation.Server.Host.Components.Deployment sealed class DmbProvider : DmbProviderBase, IDmbProvider { /// - public override string Directory => ioManager.ResolvePath(CompileJob.DirectoryName!.Value.ToString() + directoryAppend); + public override string Directory + { + get + { + var stringifiedCompileJobDirectory = CompileJob.DirectoryName!.Value.ToString(); + + if (directoryAppend != null) + stringifiedCompileJobDirectory = ioManager.ConcatPath(stringifiedCompileJobDirectory, directoryAppend); + + return ioManager.ResolvePath(stringifiedCompileJobDirectory); + } + } /// public override Models.CompileJob CompileJob { get; } @@ -28,7 +39,7 @@ namespace Tgstation.Server.Host.Components.Deployment /// /// Extra path to add to the end of . /// - readonly string directoryAppend; + readonly string? directoryAppend; /// /// The to run when is called. @@ -49,7 +60,7 @@ namespace Tgstation.Server.Host.Components.Deployment EngineVersion = engineVersion ?? throw new ArgumentNullException(nameof(engineVersion)); this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); this.onDispose = onDispose ?? throw new ArgumentNullException(nameof(onDispose)); - this.directoryAppend = directoryAppend ?? String.Empty; + this.directoryAppend = directoryAppend; } /// diff --git a/src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs b/src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs index ed6e72b4b6..477e5f06c4 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs @@ -239,12 +239,12 @@ namespace Tgstation.Server.Host.Components.Deployment /// I genuinely don't know how this will work with symlinked files. Waiting for the issue report I guess. IEnumerable MirrorDirectoryImpl(string src, string dest, SemaphoreSlim? semaphore, DreamDaemonSecurity securityLevel, CancellationToken cancellationToken) { - var dir = new DirectoryInfo(src); + var dir = IOManager.DirectoryInfo(src); Task? subdirCreationTask = null; var dreamDaemonWillAcceptOutOfDirectorySymlinks = securityLevel == DreamDaemonSecurity.Trusted; foreach (var subDirectory in dir.EnumerateDirectories()) { - var mirroredName = Path.Combine(dest, subDirectory.Name); + var mirroredName = IOManager.ConcatPath(dest, subDirectory.Name); // check if we are a symbolic link if (subDirectory.Attributes.HasFlag(FileAttributes.ReparsePoint)) diff --git a/src/Tgstation.Server.Host/Components/Engine/ZipStreamEngineInstallationData.cs b/src/Tgstation.Server.Host/Components/Engine/ZipStreamEngineInstallationData.cs index 292b725981..116689f8de 100644 --- a/src/Tgstation.Server.Host/Components/Engine/ZipStreamEngineInstallationData.cs +++ b/src/Tgstation.Server.Host/Components/Engine/ZipStreamEngineInstallationData.cs @@ -18,7 +18,7 @@ namespace Tgstation.Server.Host.Components.Engine readonly IIOManager ioManager; /// - /// The containing the zip data of the engine. + /// The containing the zip data of the engine. /// readonly Stream zipStream; diff --git a/src/Tgstation.Server.Host/Components/InstanceManager.cs b/src/Tgstation.Server.Host/Components/InstanceManager.cs index 1ded8b3dad..e1fa9b8edd 100644 --- a/src/Tgstation.Server.Host/Components/InstanceManager.cs +++ b/src/Tgstation.Server.Host/Components/InstanceManager.cs @@ -702,7 +702,7 @@ namespace Tgstation.Server.Host.Components { logger.LogDebug("Running as user: {username}", Environment.UserName); - generalConfiguration.CheckCompatibility(logger); + generalConfiguration.CheckCompatibility(logger, ioManager); using (var systemIdentity = systemIdentityFactory.GetCurrent()) { diff --git a/src/Tgstation.Server.Host/Components/Repository/Repository.cs b/src/Tgstation.Server.Host/Components/Repository/Repository.cs index b62cfbaa50..636321b5c7 100644 --- a/src/Tgstation.Server.Host/Components/Repository/Repository.cs +++ b/src/Tgstation.Server.Host/Components/Repository/Repository.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.Globalization; -using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -1099,8 +1098,8 @@ namespace Tgstation.Server.Host.Components.Repository => ioManager.GetDirectoryName(libGitRepo .Info .Path - .TrimEnd(Path.DirectorySeparatorChar) - .TrimEnd(Path.AltDirectorySeparatorChar)); + .TrimEnd(ioManager.DirectorySeparatorChar) + .TrimEnd(ioManager.AltDirectorySeparatorChar)); /// /// Recusively update all s in the . diff --git a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs index f5d320ec2d..675187f464 100644 --- a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs +++ b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs @@ -333,7 +333,7 @@ namespace Tgstation.Server.Host.Components.StaticFiles }, async cancellationToken => { - FileStream? result = null; + Stream? result = null; void GetFileStream() { result = ioManager.GetFileStream(path, false); @@ -771,7 +771,7 @@ namespace Tgstation.Server.Host.Components.StaticFiles var nullOrEmptyCheck = String.IsNullOrEmpty(configurationRelativePath); if (nullOrEmptyCheck) configurationRelativePath = DefaultIOManager.CurrentDirectory; - if (configurationRelativePath![0] == Path.DirectorySeparatorChar || configurationRelativePath[0] == Path.AltDirectorySeparatorChar) + if (configurationRelativePath![0] == ioManager.DirectorySeparatorChar || configurationRelativePath[0] == ioManager.AltDirectorySeparatorChar) configurationRelativePath = DefaultIOManager.CurrentDirectory + configurationRelativePath; var resolved = ioManager.ResolvePath(configurationRelativePath); var local = !nullOrEmptyCheck ? ioManager.ResolvePath() : null; diff --git a/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs b/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs index 3ba8b3e878..0bcff6ac96 100644 --- a/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs +++ b/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.IO; using System.Linq; using Microsoft.Extensions.Logging; @@ -9,6 +8,7 @@ using Newtonsoft.Json.Converters; using Tgstation.Server.Api.Models.Internal; using Tgstation.Server.Host.Extensions; +using Tgstation.Server.Host.IO; using Tgstation.Server.Host.Properties; using Tgstation.Server.Host.Setup; @@ -178,7 +178,8 @@ namespace Tgstation.Server.Host.Configuration /// Validates the current 's compatibility and provides migration instructions. /// /// The to use. - public void CheckCompatibility(ILogger logger) + /// The to use. + public void CheckCompatibility(ILogger logger, IIOManager ioManager) { ArgumentNullException.ThrowIfNull(logger); @@ -204,7 +205,7 @@ namespace Tgstation.Server.Host.Configuration if (ByondTopicTimeout <= 1000) logger.LogWarning("The timeout for sending BYOND topics is very low ({ms}ms). Topic calls may fail to complete at all!", ByondTopicTimeout); - if (AdditionalEventScriptsDirectories?.Any(path => !Path.IsPathRooted(path)) == true) + if (AdditionalEventScriptsDirectories?.Any(path => !ioManager.IsPathRooted(path)) == true) logger.LogWarning($"Config option \"{nameof(AdditionalEventScriptsDirectories)}\" contains non-rooted paths. These will be evaluated relative to each instances \"Configuration\" directory!"); } } diff --git a/src/Tgstation.Server.Host/IO/DefaultIOManager.cs b/src/Tgstation.Server.Host/IO/DefaultIOManager.cs index b392bb7654..2536b43ae6 100644 --- a/src/Tgstation.Server.Host/IO/DefaultIOManager.cs +++ b/src/Tgstation.Server.Host/IO/DefaultIOManager.cs @@ -30,6 +30,12 @@ namespace Tgstation.Server.Host.IO /// public const TaskCreationOptions BlockingTaskCreationOptions = TaskCreationOptions.None; + /// + public char DirectorySeparatorChar => Path.DirectorySeparatorChar; + + /// + public char AltDirectorySeparatorChar => Path.AltDirectorySeparatorChar; + /// /// Recursively empty a directory. /// @@ -350,13 +356,14 @@ namespace Tgstation.Server.Host.IO TaskScheduler.Current); /// - public FileStream GetFileStream(string path, bool shareWrite) => new( - ResolvePath(path), - FileMode.Open, - FileAccess.Read, - FileShare.Read | FileShare.Delete | (shareWrite ? FileShare.Write : FileShare.None), - DefaultBufferSize, - true); + public Stream GetFileStream(string path, bool shareWrite) + => new FileStream( + ResolvePath(path), + FileMode.Open, + FileAccess.Read, + FileShare.Read | FileShare.Delete | (shareWrite ? FileShare.Write : FileShare.None), + DefaultBufferSize, + true); /// public Task PathIsChildOf(string parentPath, string childPath, CancellationToken cancellationToken) => Task.Factory.StartNew( @@ -385,6 +392,14 @@ namespace Tgstation.Server.Host.IO BlockingTaskCreationOptions, TaskScheduler.Current); + /// + public DirectoryInfo DirectoryInfo(string path) + => new(ResolvePath(path)); // Consider async + + /// + public bool IsPathRooted(string path) + => Path.IsPathRooted(path); + /// /// Copies a directory from to . /// diff --git a/src/Tgstation.Server.Host/IO/IIOManager.cs b/src/Tgstation.Server.Host/IO/IIOManager.cs index 13a29310e1..7bba4f0705 100644 --- a/src/Tgstation.Server.Host/IO/IIOManager.cs +++ b/src/Tgstation.Server.Host/IO/IIOManager.cs @@ -11,6 +11,16 @@ namespace Tgstation.Server.Host.IO /// public interface IIOManager { + /// + /// Gets the primary directory separator character. + /// + char DirectorySeparatorChar { get; } + + /// + /// Gets the alternative directory separator character. + /// + char AltDirectorySeparatorChar { get; } + /// /// Retrieve the full path of the current working directory. /// @@ -234,8 +244,22 @@ namespace Tgstation.Server.Host.IO /// /// The path of the file. /// If should be used. - /// The of the file. + /// The of the file. /// This function is sychronous. - FileStream GetFileStream(string path, bool shareWrite); + Stream GetFileStream(string path, bool shareWrite); + + /// + /// Gets a for the given . + /// + /// The path to get for. + /// A new . + DirectoryInfo DirectoryInfo(string path); + + /// + /// Check if a given is at the root level of the filesystem. + /// + /// The path to check. + /// if the path is rooted, otherwise. + bool IsPathRooted(string path); } } diff --git a/src/Tgstation.Server.Host/IO/ResolvingIOManager.cs b/src/Tgstation.Server.Host/IO/ResolvingIOManager.cs index 3b86b92c36..f05ce162e5 100644 --- a/src/Tgstation.Server.Host/IO/ResolvingIOManager.cs +++ b/src/Tgstation.Server.Host/IO/ResolvingIOManager.cs @@ -1,5 +1,4 @@ using System; -using System.IO; namespace Tgstation.Server.Host.IO { @@ -29,7 +28,7 @@ namespace Tgstation.Server.Host.IO /// public override string ResolvePath(string path) { - if (!Path.IsPathRooted(path)) + if (!IsPathRooted(path)) return base.ResolvePath(ConcatPath(subdirectory, path)); return path; } diff --git a/src/Tgstation.Server.Host/Setup/SetupWizard.cs b/src/Tgstation.Server.Host/Setup/SetupWizard.cs index 32e82877e5..ea1ba73d85 100644 --- a/src/Tgstation.Server.Host/Setup/SetupWizard.cs +++ b/src/Tgstation.Server.Host/Setup/SetupWizard.cs @@ -304,7 +304,7 @@ namespace Tgstation.Server.Host.Setup /// A resulting in the SQLite database path to store in the configuration. async ValueTask ValidateNonExistantSqliteDBName(string databaseName, CancellationToken cancellationToken) { - var dbPathIsRooted = Path.IsPathRooted(databaseName); + var dbPathIsRooted = ioManager.IsPathRooted(databaseName); var resolvedPath = ioManager.ResolvePath( dbPathIsRooted ? databaseName diff --git a/src/Tgstation.Server.Host/System/PosixProcessFeatures.cs b/src/Tgstation.Server.Host/System/PosixProcessFeatures.cs index 12a0164a1f..6477dc05d5 100644 --- a/src/Tgstation.Server.Host/System/PosixProcessFeatures.cs +++ b/src/Tgstation.Server.Host/System/PosixProcessFeatures.cs @@ -64,39 +64,6 @@ namespace Tgstation.Server.Host.System this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); } - /// - /// Gets potential paths to the gcore executable. - /// - /// The potential paths to the gcore executable. - static IEnumerable GetPotentialGCorePaths() - { - var enviromentPath = Environment.GetEnvironmentVariable("PATH"); - IEnumerable enumerator; - if (enviromentPath == null) - enumerator = Enumerable.Empty(); - else - { - var paths = enviromentPath.Split(';'); - enumerator = paths - .Select(x => x.Split(':')) - .SelectMany(x => x); - } - - var exeName = "gcore"; - - enumerator = enumerator - .Concat(new List(2) - { - "/usr/bin", - "/usr/share/bin", - "/bin", - }); - - enumerator = enumerator.Select(x => Path.Combine(x, exeName)); - - return enumerator; - } - /// public void ResumeProcess(global::System.Diagnostics.Process process) { @@ -255,5 +222,38 @@ namespace Tgstation.Server.Host.System Encoding.UTF8.GetBytes(adjustedValue.ToString(CultureInfo.InvariantCulture)), cancellationToken); } + + /// + /// Gets potential paths to the gcore executable. + /// + /// The potential paths to the gcore executable. + IEnumerable GetPotentialGCorePaths() + { + var enviromentPath = Environment.GetEnvironmentVariable("PATH"); + IEnumerable enumerator; + if (enviromentPath == null) + enumerator = Enumerable.Empty(); + else + { + var paths = enviromentPath.Split(';'); + enumerator = paths + .Select(x => x.Split(':')) + .SelectMany(x => x); + } + + var exeName = "gcore"; + + enumerator = enumerator + .Concat(new List(2) + { + "/usr/bin", + "/usr/share/bin", + "/bin", + }); + + enumerator = enumerator.Select(x => ioManager.ConcatPath(x, exeName)); + + return enumerator; + } } } diff --git a/src/Tgstation.Server.Host/System/WindowsProcessFeatures.cs b/src/Tgstation.Server.Host/System/WindowsProcessFeatures.cs index 21855c0062..a048736a22 100644 --- a/src/Tgstation.Server.Host/System/WindowsProcessFeatures.cs +++ b/src/Tgstation.Server.Host/System/WindowsProcessFeatures.cs @@ -145,6 +145,7 @@ namespace Tgstation.Server.Host.System throw new JobException(ErrorCode.GameServerOffline, ex); } + // Windows API so has to be a real FS await using var fileStream = new FileStream(outputFile, FileMode.CreateNew); await Task.Factory.StartNew( From 3b1473c7125659cc8bf34357fded57e60d112ee7 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 20 Jul 2025 21:46:53 -0400 Subject: [PATCH 23/52] Improve how `ResolvingIOManager`s are constructed --- .../Components/Engine/ByondInstallerBase.cs | 2 +- .../Components/Engine/OpenDreamInstaller.cs | 2 +- .../Components/InstanceFactory.cs | 12 ++++++------ src/Tgstation.Server.Host/Core/Application.cs | 3 +-- src/Tgstation.Server.Host/IO/DefaultIOManager.cs | 11 +++++++++++ src/Tgstation.Server.Host/IO/IIOManager.cs | 7 +++++++ src/Tgstation.Server.Host/IO/ResolvingIOManager.cs | 8 ++------ .../Components/StaticFiles/TestConfiguration.cs | 2 +- .../Live/Instance/InstanceTest.cs | 2 +- tests/Tgstation.Server.Tests/TestRepository.cs | 3 +-- 10 files changed, 32 insertions(+), 20 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Engine/ByondInstallerBase.cs b/src/Tgstation.Server.Host/Components/Engine/ByondInstallerBase.cs index e02c109e6b..d0ddb3a506 100644 --- a/src/Tgstation.Server.Host/Components/Engine/ByondInstallerBase.cs +++ b/src/Tgstation.Server.Host/Components/Engine/ByondInstallerBase.cs @@ -78,7 +78,7 @@ namespace Tgstation.Server.Host.Components.Engine { CheckVersionValidity(version); - var installationIOManager = new ResolvingIOManager(IOManager, path); + var installationIOManager = IOManager.CreateResolverForSubdirectory(path); var supportsMapThreads = version.Version >= MapThreadsVersion; return ValueTask.FromResult( diff --git a/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs b/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs index ad2fb20e95..87ad2cd721 100644 --- a/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs +++ b/src/Tgstation.Server.Host/Components/Engine/OpenDreamInstaller.cs @@ -126,7 +126,7 @@ namespace Tgstation.Server.Host.Components.Engine var dotnetPath = (await DotnetHelper.GetDotnetPath(platformIdentifier, IOManager, cancellationToken)) ?? throw new JobException("Failed to find dotnet path!"); return new OpenDreamInstallation( - new ResolvingIOManager(IOManager, path), + IOManager.CreateResolverForSubdirectory(path), asyncDelayer, httpClientFactory, dotnetPath, diff --git a/src/Tgstation.Server.Host/Components/InstanceFactory.cs b/src/Tgstation.Server.Host/Components/InstanceFactory.cs index c7d08ad4b1..04994acec5 100644 --- a/src/Tgstation.Server.Host/Components/InstanceFactory.cs +++ b/src/Tgstation.Server.Host/Components/InstanceFactory.cs @@ -164,7 +164,7 @@ namespace Tgstation.Server.Host.Components /// /// The instance's . /// The for the instance's "Game" directory. - static ResolvingIOManager CreateGameIOManager(IIOManager instanceIOManager) => new(instanceIOManager, "Game"); + static IIOManager CreateGameIOManager(IIOManager instanceIOManager) => instanceIOManager.CreateResolverForSubdirectory("Game"); #pragma warning disable CA1502 // TODO: Decomplexify /// @@ -270,11 +270,11 @@ namespace Tgstation.Server.Host.Components var instanceIoManager = CreateInstanceIOManager(metadata); // various other ioManagers - var repoIoManager = new ResolvingIOManager(instanceIoManager, "Repository"); - var byondIOManager = new ResolvingIOManager(instanceIoManager, "Byond"); + var repoIoManager = instanceIoManager.CreateResolverForSubdirectory("Repository"); + var byondIOManager = instanceIoManager.CreateResolverForSubdirectory("Byond"); var gameIoManager = CreateGameIOManager(instanceIoManager); - var diagnosticsIOManager = new ResolvingIOManager(instanceIoManager, "Diagnostics"); - var configurationIoManager = new ResolvingIOManager(instanceIoManager, "Configuration"); + var diagnosticsIOManager = instanceIoManager.CreateResolverForSubdirectory("Diagnostics"); + var configurationIoManager = instanceIoManager.CreateResolverForSubdirectory("Configuration"); var metricFactory = this.metricFactory.WithLabels( new Dictionary @@ -441,6 +441,6 @@ namespace Tgstation.Server.Host.Components /// /// The . /// The for the . - ResolvingIOManager CreateInstanceIOManager(Models.Instance metadata) => new(ioManager, metadata.Path!); + IIOManager CreateInstanceIOManager(Models.Instance metadata) => ioManager.CreateResolverForSubdirectory(metadata.Path!); } } diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index a947eeb720..e2d1d75dbf 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -485,8 +485,7 @@ namespace Tgstation.Server.Host.Core services => services .GetRequiredService() .CreateRepositoryManager( - new ResolvingIOManager( - services.GetRequiredService(), + services.GetRequiredService().CreateResolverForSubdirectory( openDreamRepositoryDirectory), new NoopEventConsumer())); diff --git a/src/Tgstation.Server.Host/IO/DefaultIOManager.cs b/src/Tgstation.Server.Host/IO/DefaultIOManager.cs index 2536b43ae6..38d63468c7 100644 --- a/src/Tgstation.Server.Host/IO/DefaultIOManager.cs +++ b/src/Tgstation.Server.Host/IO/DefaultIOManager.cs @@ -400,6 +400,17 @@ namespace Tgstation.Server.Host.IO public bool IsPathRooted(string path) => Path.IsPathRooted(path); + /// + public IIOManager CreateResolverForSubdirectory(string subdirectoryPath) + { + ArgumentNullException.ThrowIfNull(subdirectoryPath); + + return new ResolvingIOManager( + ConcatPath( + ResolvePath(), + subdirectoryPath)); + } + /// /// Copies a directory from to . /// diff --git a/src/Tgstation.Server.Host/IO/IIOManager.cs b/src/Tgstation.Server.Host/IO/IIOManager.cs index 7bba4f0705..a9f38c2a31 100644 --- a/src/Tgstation.Server.Host/IO/IIOManager.cs +++ b/src/Tgstation.Server.Host/IO/IIOManager.cs @@ -21,6 +21,13 @@ namespace Tgstation.Server.Host.IO /// char AltDirectorySeparatorChar { get; } + /// + /// Create a new that resolves paths to the specified . + /// + /// A relative or absolute path that the new will resolve as its current directory. + /// A new . + IIOManager CreateResolverForSubdirectory(string subdirectoryPath); + /// /// Retrieve the full path of the current working directory. /// diff --git a/src/Tgstation.Server.Host/IO/ResolvingIOManager.cs b/src/Tgstation.Server.Host/IO/ResolvingIOManager.cs index f05ce162e5..69e7feefba 100644 --- a/src/Tgstation.Server.Host/IO/ResolvingIOManager.cs +++ b/src/Tgstation.Server.Host/IO/ResolvingIOManager.cs @@ -15,14 +15,10 @@ namespace Tgstation.Server.Host.IO /// /// Initializes a new instance of the class. /// - /// The that resolves to the directory to work out of. /// The value of . - public ResolvingIOManager(IIOManager parent, string subdirectory) + public ResolvingIOManager(string subdirectory) { - ArgumentNullException.ThrowIfNull(parent); - ArgumentNullException.ThrowIfNull(subdirectory); - - this.subdirectory = ConcatPath(parent.ResolvePath(), subdirectory); + this.subdirectory = subdirectory ?? throw new ArgumentNullException(nameof(subdirectory)); } /// diff --git a/tests/Tgstation.Server.Host.Tests/Components/StaticFiles/TestConfiguration.cs b/tests/Tgstation.Server.Host.Tests/Components/StaticFiles/TestConfiguration.cs index f5e6b9bf5f..3dd2d6d13f 100644 --- a/tests/Tgstation.Server.Host.Tests/Components/StaticFiles/TestConfiguration.cs +++ b/tests/Tgstation.Server.Host.Tests/Components/StaticFiles/TestConfiguration.cs @@ -33,7 +33,7 @@ namespace Tgstation.Server.Host.Components.StaticFiles.Tests var tempPath = Path.GetTempFileName(); File.Delete(tempPath); - var ioManager = new ResolvingIOManager(new DefaultIOManager(), tempPath); + var ioManager = new DefaultIOManager().CreateResolverForSubdirectory(tempPath); await ioManager.CreateDirectory(".", CancellationToken.None); try { diff --git a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs index f9188313d0..8d667ebc41 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs @@ -96,7 +96,7 @@ namespace Tgstation.Server.Tests.Live.Instance Environment.SpecialFolderOption.DoNotVerify), new AssemblyInformationProvider().VersionPrefix, "OpenDreamRepository"); - var odRepoIoManager = new ResolvingIOManager(ioManager, odRepoDir); + var odRepoIoManager = ioManager.CreateResolverForSubdirectory(odRepoDir); var mockOptions = new Mock>(); var genConfig = new GeneralConfiguration diff --git a/tests/Tgstation.Server.Tests/TestRepository.cs b/tests/Tgstation.Server.Tests/TestRepository.cs index 59b4783994..89cf34daac 100644 --- a/tests/Tgstation.Server.Tests/TestRepository.cs +++ b/tests/Tgstation.Server.Tests/TestRepository.cs @@ -75,8 +75,7 @@ namespace Tgstation.Server.Tests using var manager = new RepositoryManager( repoFac, commands, - new ResolvingIOManager( - new DefaultIOManager(), + new DefaultIOManager().CreateResolverForSubdirectory( tempPath), Mock.Of(), new WindowsPostWriteHandler(), From f0cac0e81001c10bd8d5fbed9a60f84ac2742931 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 20 Jul 2025 22:11:42 -0400 Subject: [PATCH 24/52] Switch to using `IFileSystem` on the backend for file operations --- .../Deployment/HardLinkDmbProvider.cs | 18 +-- .../Components/StaticFiles/Configuration.cs | 2 +- src/Tgstation.Server.Host/Core/Application.cs | 12 +- .../Extensions/WebHostBuilderExtensions.cs | 10 +- .../IO/DefaultIOManager.cs | 122 ++++++++++-------- src/Tgstation.Server.Host/IO/IIOManager.cs | 29 ++--- .../IO/ISynchronousIOManager.cs | 7 + .../IO/PosixFilesystemLinkFactory.cs | 18 ++- .../IO/ResolvingIOManager.cs | 7 +- .../IO/SynchronousIOManager.cs | 52 +++++--- .../IO/WindowsFilesystemLinkFactory.cs | 18 ++- src/Tgstation.Server.Host/ServerFactory.cs | 19 ++- .../System/PosixProcessFeatures.cs | 4 +- .../Tgstation.Server.Host.csproj | 1 + .../Transfer/FileTransferService.cs | 2 +- .../StaticFiles/TestConfiguration.cs | 8 +- .../IO/TestFilesystemLinkFactory.cs | 6 +- .../IO/TestIOManager.cs | 100 ++++++++------ .../System/TestPosixSignalHandler.cs | 4 +- .../System/TestProcessFeatures.cs | 3 +- .../System/TestSymlinkFactory.cs | 7 +- .../TestProgram.cs | 12 +- .../TestServerFactory.cs | 9 +- .../Tgstation.Server.Host.Tests.csproj | 1 + .../CachingFileDownloader.cs | 3 +- .../Live/Instance/ConfigurationTest.cs | 6 +- .../Live/Instance/InstanceTest.cs | 5 +- .../Live/Instance/WatchdogTest.cs | 5 +- .../Live/LiveTestingServer.cs | 3 +- .../Live/TestLiveServer.cs | 5 +- .../Tgstation.Server.Tests/TestRepository.cs | 7 +- .../TestSystemInteraction.cs | 5 +- tests/Tgstation.Server.Tests/TestVersions.cs | 7 +- tests/Tgstation.Server.Tests/TestingUtils.cs | 3 +- 34 files changed, 326 insertions(+), 194 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs b/src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs index 477e5f06c4..df3c9cf2a1 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.Globalization; using System.IO; +using System.IO.Abstractions; using System.Runtime.Versioning; using System.Threading; using System.Threading.Tasks; @@ -187,8 +188,10 @@ namespace Tgstation.Server.Host.Components.Deployment dest = IOManager.ResolvePath(mirrorGuid.ToString()); using var semaphore = taskThrottle.HasValue ? new SemaphoreSlim(taskThrottle.Value) : null; + + var dir = await IOManager.DirectoryInfo(src, cancellationToken); await Task.WhenAll(MirrorDirectoryImpl( - src, + dir, dest, semaphore, securityLevel, @@ -230,19 +233,18 @@ namespace Tgstation.Server.Host.Components.Deployment /// /// Recursively create tasks to create a hard link directory mirror of to . /// - /// The source directory path. + /// The source . /// The destination directory path. /// Optional used to limit degree of parallelism. /// The launch level. /// The for the operation. - /// A of s representing the running operations. The first returned is always the necessary call to . + /// An of s representing the running operations. The first returned is always the necessary call to . /// I genuinely don't know how this will work with symlinked files. Waiting for the issue report I guess. - IEnumerable MirrorDirectoryImpl(string src, string dest, SemaphoreSlim? semaphore, DreamDaemonSecurity securityLevel, CancellationToken cancellationToken) + IEnumerable MirrorDirectoryImpl(IDirectoryInfo src, string dest, SemaphoreSlim? semaphore, DreamDaemonSecurity securityLevel, CancellationToken cancellationToken) { - var dir = IOManager.DirectoryInfo(src); Task? subdirCreationTask = null; var dreamDaemonWillAcceptOutOfDirectorySymlinks = securityLevel == DreamDaemonSecurity.Trusted; - foreach (var subDirectory in dir.EnumerateDirectories()) + foreach (var subDirectory in src.EnumerateDirectories()) { var mirroredName = IOManager.ConcatPath(dest, subDirectory.Name); @@ -275,7 +277,7 @@ namespace Tgstation.Server.Host.Components.Deployment logger.LogDebug("Recreating symlinked directory {name} as hard links...", subDirectory.Name); var checkingSubdirCreationTask = true; - foreach (var copyTask in MirrorDirectoryImpl(subDirectory.FullName, mirroredName, semaphore, securityLevel, cancellationToken)) + foreach (var copyTask in MirrorDirectoryImpl(subDirectory, mirroredName, semaphore, securityLevel, cancellationToken)) { if (subdirCreationTask == null) { @@ -289,7 +291,7 @@ namespace Tgstation.Server.Host.Components.Deployment } } - foreach (var fileInfo in dir.EnumerateFiles()) + foreach (var fileInfo in src.EnumerateFiles()) { if (subdirCreationTask == null) { diff --git a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs index 675187f464..6dbfc3cc9c 100644 --- a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs +++ b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs @@ -336,7 +336,7 @@ namespace Tgstation.Server.Host.Components.StaticFiles Stream? result = null; void GetFileStream() { - result = ioManager.GetFileStream(path, false); + result = synchronousIOManager.GetFileStream(path); } if (systemIdentity == null) diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index e2d1d75dbf..3246b8ba0f 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -2,6 +2,7 @@ using System.Collections.Frozen; using System.Collections.Generic; using System.Globalization; +using System.IO.Abstractions; using System.Threading.Tasks; using System.Web; @@ -105,10 +106,12 @@ namespace Tgstation.Server.Host.Core public static IServerFactory CreateDefaultServerFactory() { var assemblyInformationProvider = new AssemblyInformationProvider(); - var ioManager = new DefaultIOManager(); + var fileSystem = new FileSystem(); + var ioManager = new DefaultIOManager(fileSystem); return new ServerFactory( assemblyInformationProvider, - ioManager); + ioManager, + fileSystem); } /// @@ -154,11 +157,13 @@ namespace Tgstation.Server.Host.Core /// The needed for configuration. /// The needed for configuration. /// The needed for configuration. + /// The needed for configuration. public void ConfigureServices( IServiceCollection services, IAssemblyInformationProvider assemblyInformationProvider, IIOManager ioManager, - IPostSetupServices postSetupServices) + IPostSetupServices postSetupServices, + IFileSystem fileSystem) { ConfigureServices(services, assemblyInformationProvider, ioManager); @@ -541,6 +546,7 @@ namespace Tgstation.Server.Host.Core services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(fileSystem); services.AddHostedService(); services.AddHostedService(); diff --git a/src/Tgstation.Server.Host/Extensions/WebHostBuilderExtensions.cs b/src/Tgstation.Server.Host/Extensions/WebHostBuilderExtensions.cs index 3179c684f2..ba0316ad89 100644 --- a/src/Tgstation.Server.Host/Extensions/WebHostBuilderExtensions.cs +++ b/src/Tgstation.Server.Host/Extensions/WebHostBuilderExtensions.cs @@ -1,4 +1,5 @@ using System; +using System.IO.Abstractions; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; @@ -26,24 +27,27 @@ namespace Tgstation.Server.Host.Extensions /// The to configure. /// The to use. /// The to use. - /// The configured . /// The to use. + /// The to use. + /// The configured . public static IWebHostBuilder UseApplication( this IWebHostBuilder builder, IAssemblyInformationProvider assemblyInformationProvider, IIOManager ioManager, - IPostSetupServices postSetupServices) + IPostSetupServices postSetupServices, + IFileSystem fileSystem) { ArgumentNullException.ThrowIfNull(builder); ArgumentNullException.ThrowIfNull(assemblyInformationProvider); ArgumentNullException.ThrowIfNull(ioManager); ArgumentNullException.ThrowIfNull(postSetupServices); + ArgumentNullException.ThrowIfNull(fileSystem); return builder.ConfigureServices( (context, services) => { var application = new Application(context.Configuration, context.HostingEnvironment); - application.ConfigureServices(services, assemblyInformationProvider, ioManager, postSetupServices); + application.ConfigureServices(services, assemblyInformationProvider, ioManager, postSetupServices, fileSystem); services.AddSingleton(application); }) .Configure(ConfigureApplication); diff --git a/src/Tgstation.Server.Host/IO/DefaultIOManager.cs b/src/Tgstation.Server.Host/IO/DefaultIOManager.cs index 38d63468c7..470f437d7f 100644 --- a/src/Tgstation.Server.Host/IO/DefaultIOManager.cs +++ b/src/Tgstation.Server.Host/IO/DefaultIOManager.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.IO.Abstractions; using System.IO.Compression; using System.Linq; using System.Threading; @@ -31,17 +32,31 @@ namespace Tgstation.Server.Host.IO public const TaskCreationOptions BlockingTaskCreationOptions = TaskCreationOptions.None; /// - public char DirectorySeparatorChar => Path.DirectorySeparatorChar; + public char DirectorySeparatorChar => fileSystem.Path.DirectorySeparatorChar; /// - public char AltDirectorySeparatorChar => Path.AltDirectorySeparatorChar; + public char AltDirectorySeparatorChar => fileSystem.Path.AltDirectorySeparatorChar; + + /// + /// The backing . + /// + readonly IFileSystem fileSystem; + + /// + /// Initializes a new instance of the class. + /// + /// The value of . + public DefaultIOManager(IFileSystem fileSystem) + { + this.fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); + } /// /// Recursively empty a directory. /// /// of the directory to empty. /// The for the operation. - static void NormalizeAndDelete(DirectoryInfo dir, CancellationToken cancellationToken) + static void NormalizeAndDelete(IDirectoryInfo dir, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); @@ -117,7 +132,7 @@ namespace Tgstation.Server.Host.IO } /// - public string ConcatPath(params string[] paths) => Path.Combine(paths); + public string ConcatPath(params string[] paths) => fileSystem.Path.Combine(paths); /// public async ValueTask CopyFile(string src, string dest, CancellationToken cancellationToken) @@ -126,7 +141,7 @@ namespace Tgstation.Server.Host.IO ArgumentNullException.ThrowIfNull(dest); // tested to hell and back, these are the optimal buffer sizes - await using var srcStream = new FileStream( + await using var srcStream = fileSystem.FileStream.New( ResolvePath(src), FileMode.Open, FileAccess.Read, @@ -140,41 +155,39 @@ namespace Tgstation.Server.Host.IO } /// - public Task CreateDirectory(string path, CancellationToken cancellationToken) => Task.Factory.StartNew(() => Directory.CreateDirectory(ResolvePath(path)), cancellationToken, BlockingTaskCreationOptions, TaskScheduler.Current); + public Task CreateDirectory(string path, CancellationToken cancellationToken) => Task.Factory.StartNew(() => fileSystem.Directory.CreateDirectory(ResolvePath(path)), cancellationToken, BlockingTaskCreationOptions, TaskScheduler.Current); /// public Task DeleteDirectory(string path, CancellationToken cancellationToken) - { - path = ResolvePath(path); - var di = new DirectoryInfo(path); - if (!di.Exists) - return Task.CompletedTask; - - return Task.Factory.StartNew( - () => NormalizeAndDelete(di, cancellationToken), + => Task.Factory.StartNew( + () => + { + var di = fileSystem.DirectoryInfo.New(path); + if (di.Exists) + NormalizeAndDelete(di, cancellationToken); + }, cancellationToken, BlockingTaskCreationOptions, TaskScheduler.Current); - } /// - public Task DeleteFile(string path, CancellationToken cancellationToken) => Task.Factory.StartNew(() => File.Delete(ResolvePath(path)), cancellationToken, BlockingTaskCreationOptions, TaskScheduler.Current); + public Task DeleteFile(string path, CancellationToken cancellationToken) => Task.Factory.StartNew(() => fileSystem.File.Delete(ResolvePath(path)), cancellationToken, BlockingTaskCreationOptions, TaskScheduler.Current); /// - public Task FileExists(string path, CancellationToken cancellationToken) => Task.Factory.StartNew(() => File.Exists(ResolvePath(path)), cancellationToken, BlockingTaskCreationOptions, TaskScheduler.Current); + public Task FileExists(string path, CancellationToken cancellationToken) => Task.Factory.StartNew(() => fileSystem.File.Exists(ResolvePath(path)), cancellationToken, BlockingTaskCreationOptions, TaskScheduler.Current); /// - public Task DirectoryExists(string path, CancellationToken cancellationToken) => Task.Factory.StartNew(() => Directory.Exists(ResolvePath(path)), cancellationToken, BlockingTaskCreationOptions, TaskScheduler.Current); + public Task DirectoryExists(string path, CancellationToken cancellationToken) => Task.Factory.StartNew(() => fileSystem.Directory.Exists(ResolvePath(path)), cancellationToken, BlockingTaskCreationOptions, TaskScheduler.Current); /// - public string GetDirectoryName(string path) => Path.GetDirectoryName(path ?? throw new ArgumentNullException(nameof(path))) + public string GetDirectoryName(string path) => fileSystem.Path.GetDirectoryName(path ?? throw new ArgumentNullException(nameof(path))) ?? throw new InvalidOperationException($"Null was returned. Path ({path}) must be rooted. This is not supported!"); /// - public string GetFileName(string path) => Path.GetFileName(path ?? throw new ArgumentNullException(nameof(path))); + public string GetFileName(string path) => fileSystem.Path.GetFileName(path ?? throw new ArgumentNullException(nameof(path))); /// - public string GetFileNameWithoutExtension(string path) => Path.GetFileNameWithoutExtension(path ?? throw new ArgumentNullException(nameof(path))); + public string GetFileNameWithoutExtension(string path) => fileSystem.Path.GetFileNameWithoutExtension(path ?? throw new ArgumentNullException(nameof(path))); /// public Task> GetFilesWithExtension(string path, string extension, bool recursive, CancellationToken cancellationToken) => Task.Factory.StartNew( @@ -183,7 +196,7 @@ namespace Tgstation.Server.Host.IO path = ResolvePath(path); ArgumentNullException.ThrowIfNull(extension); var results = new List(); - foreach (var fileName in Directory.EnumerateFiles( + foreach (var fileName in fileSystem.Directory.EnumerateFiles( path, $"*.{extension}", recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly)) @@ -205,7 +218,7 @@ namespace Tgstation.Server.Host.IO ArgumentNullException.ThrowIfNull(destination); source = ResolvePath(source ?? throw new ArgumentNullException(nameof(source))); destination = ResolvePath(destination); - File.Move(source, destination); + fileSystem.File.Move(source, destination); }, cancellationToken, BlockingTaskCreationOptions, @@ -218,7 +231,7 @@ namespace Tgstation.Server.Host.IO ArgumentNullException.ThrowIfNull(destination); source = ResolvePath(source ?? throw new ArgumentNullException(nameof(source))); destination = ResolvePath(destination); - Directory.Move(source, destination); + fileSystem.Directory.Move(source, destination); }, cancellationToken, BlockingTaskCreationOptions, @@ -227,7 +240,7 @@ namespace Tgstation.Server.Host.IO /// public async ValueTask ReadAllBytes(string path, CancellationToken cancellationToken) { - await using var file = CreateAsyncSequentialReadStream(path); + await using var file = CreateAsyncReadStream(path, true, true); byte[] buf; buf = new byte[file.Length]; await file.ReadAsync(buf, cancellationToken); @@ -235,10 +248,12 @@ namespace Tgstation.Server.Host.IO } /// - public string ResolvePath() => ResolvePath(CurrentDirectory); + public string ResolvePath() + => ResolvePath(CurrentDirectory); /// - public virtual string ResolvePath(string path) => Path.GetFullPath(path ?? throw new ArgumentNullException(nameof(path))); + public virtual string ResolvePath(string path) + => fileSystem.Path.GetFullPath(path ?? throw new ArgumentNullException(nameof(path))); /// public async ValueTask WriteAllBytes(string path, byte[] contents, CancellationToken cancellationToken) @@ -248,10 +263,10 @@ namespace Tgstation.Server.Host.IO } /// - public FileStream CreateAsyncSequentialWriteStream(string path) + public Stream CreateAsyncSequentialWriteStream(string path) { path = ResolvePath(path); - return new FileStream( + return fileSystem.FileStream.New( path, FileMode.Create, FileAccess.Write, @@ -261,16 +276,18 @@ namespace Tgstation.Server.Host.IO } /// - public FileStream CreateAsyncSequentialReadStream(string path) + public Stream CreateAsyncReadStream(string path, bool sequental, bool shareWrite) { path = ResolvePath(path); - return new FileStream( + return fileSystem.FileStream.New( path, FileMode.Open, FileAccess.Read, - FileShare.ReadWrite | FileShare.Delete, + FileShare.ReadWrite | FileShare.Delete | (shareWrite ? FileShare.Write : FileShare.None), DefaultBufferSize, - FileOptions.Asynchronous | FileOptions.SequentialScan); + sequental + ? FileOptions.Asynchronous | FileOptions.SequentialScan + : FileOptions.Asynchronous); } /// @@ -280,7 +297,7 @@ namespace Tgstation.Server.Host.IO path = ResolvePath(path); var results = new List(); cancellationToken.ThrowIfCancellationRequested(); - foreach (var directoryName in Directory.EnumerateDirectories(path)) + foreach (var directoryName in fileSystem.Directory.EnumerateDirectories(path)) { results.Add(directoryName); cancellationToken.ThrowIfCancellationRequested(); @@ -299,7 +316,7 @@ namespace Tgstation.Server.Host.IO path = ResolvePath(path); var results = new List(); cancellationToken.ThrowIfCancellationRequested(); - foreach (var fileName in Directory.EnumerateFiles(path)) + foreach (var fileName in fileSystem.Directory.EnumerateFiles(path)) { results.Add(fileName); cancellationToken.ThrowIfCancellationRequested(); @@ -337,8 +354,8 @@ namespace Tgstation.Server.Host.IO public bool PathContainsParentAccess(string path) => path ?.Split( [ - Path.DirectorySeparatorChar, - Path.AltDirectorySeparatorChar, + fileSystem.Path.DirectorySeparatorChar, + fileSystem.Path.AltDirectorySeparatorChar, ]) .Any(x => x == "..") ?? throw new ArgumentNullException(nameof(path)); @@ -348,23 +365,13 @@ namespace Tgstation.Server.Host.IO () => { path = ResolvePath(path ?? throw new ArgumentNullException(nameof(path))); - var fileInfo = new FileInfo(path); + var fileInfo = fileSystem.FileInfo.New(path); return new DateTimeOffset(fileInfo.LastWriteTimeUtc); }, cancellationToken, BlockingTaskCreationOptions, TaskScheduler.Current); - /// - public Stream GetFileStream(string path, bool shareWrite) - => new FileStream( - ResolvePath(path), - FileMode.Open, - FileAccess.Read, - FileShare.Read | FileShare.Delete | (shareWrite ? FileShare.Write : FileShare.None), - DefaultBufferSize, - true); - /// public Task PathIsChildOf(string parentPath, string childPath, CancellationToken cancellationToken) => Task.Factory.StartNew( () => @@ -376,8 +383,8 @@ namespace Tgstation.Server.Host.IO return true; // https://stackoverflow.com/questions/5617320/given-full-path-check-if-path-is-subdirectory-of-some-other-path-or-otherwise?lq=1 - var di1 = new DirectoryInfo(parentPath); - var di2 = new DirectoryInfo(childPath); + var di1 = fileSystem.DirectoryInfo.New(parentPath); + var di2 = fileSystem.DirectoryInfo.New(childPath); while (di2.Parent != null) { if (di2.Parent.FullName == di1.FullName) @@ -393,12 +400,16 @@ namespace Tgstation.Server.Host.IO TaskScheduler.Current); /// - public DirectoryInfo DirectoryInfo(string path) - => new(ResolvePath(path)); // Consider async + public Task DirectoryInfo(string path, CancellationToken cancellationToken) + => Task.Factory.StartNew( + () => fileSystem.DirectoryInfo.New(ResolvePath(path)), + cancellationToken, + BlockingTaskCreationOptions, + TaskScheduler.Current); /// public bool IsPathRooted(string path) - => Path.IsPathRooted(path); + => fileSystem.Path.IsPathRooted(path); /// public IIOManager CreateResolverForSubdirectory(string subdirectoryPath) @@ -406,6 +417,7 @@ namespace Tgstation.Server.Host.IO ArgumentNullException.ThrowIfNull(subdirectoryPath); return new ResolvingIOManager( + fileSystem, ConcatPath( ResolvePath(), subdirectoryPath)); @@ -429,7 +441,7 @@ namespace Tgstation.Server.Host.IO SemaphoreSlim? semaphore, CancellationToken cancellationToken) { - var dir = new DirectoryInfo(src); + var dir = fileSystem.DirectoryInfo.New(src); Task? subdirCreationTask = null; foreach (var subDirectory in dir.EnumerateDirectories()) { @@ -437,7 +449,7 @@ namespace Tgstation.Server.Host.IO continue; var checkingSubdirCreationTask = true; - foreach (var copyTask in CopyDirectoryImpl(subDirectory.FullName, Path.Combine(dest, subDirectory.Name), null, postCopyCallback, semaphore, cancellationToken)) + foreach (var copyTask in CopyDirectoryImpl(subDirectory.FullName, fileSystem.Path.Combine(dest, subDirectory.Name), null, postCopyCallback, semaphore, cancellationToken)) { if (subdirCreationTask == null) { diff --git a/src/Tgstation.Server.Host/IO/IIOManager.cs b/src/Tgstation.Server.Host/IO/IIOManager.cs index a9f38c2a31..3b81869fb8 100644 --- a/src/Tgstation.Server.Host/IO/IIOManager.cs +++ b/src/Tgstation.Server.Host/IO/IIOManager.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.IO.Abstractions; using System.Threading; using System.Threading.Tasks; @@ -134,15 +135,17 @@ namespace Tgstation.Server.Host.IO /// Creates an asynchronous for sequential writing. /// /// The path of the file to write, will be truncated. - /// The open . - FileStream CreateAsyncSequentialWriteStream(string path); + /// The open . + Stream CreateAsyncSequentialWriteStream(string path); /// /// Creates an asynchronous for sequential reading. /// /// The path of the file to write, will be truncated. - /// The open . - FileStream CreateAsyncSequentialReadStream(string path); + /// If the sequential read flag should be added. + /// If should be used. + /// The open . + Stream CreateAsyncReadStream(string path, bool sequential, bool shareWrite); /// /// Writes some to a file at overwriting previous content. @@ -247,20 +250,12 @@ namespace Tgstation.Server.Host.IO Task GetLastModified(string path, CancellationToken cancellationToken); /// - /// Gets the for a given file . + /// Gets a for the given . /// - /// The path of the file. - /// If should be used. - /// The of the file. - /// This function is sychronous. - Stream GetFileStream(string path, bool shareWrite); - - /// - /// Gets a for the given . - /// - /// The path to get for. - /// A new . - DirectoryInfo DirectoryInfo(string path); + /// The path to get for. + /// The for the operation. + /// A resulting in the of the . + Task DirectoryInfo(string path, CancellationToken cancellationToken); /// /// Check if a given is at the root level of the filesystem. diff --git a/src/Tgstation.Server.Host/IO/ISynchronousIOManager.cs b/src/Tgstation.Server.Host/IO/ISynchronousIOManager.cs index 16e0ea370e..850f6941f2 100644 --- a/src/Tgstation.Server.Host/IO/ISynchronousIOManager.cs +++ b/src/Tgstation.Server.Host/IO/ISynchronousIOManager.cs @@ -63,5 +63,12 @@ namespace Tgstation.Server.Host.IO /// The path to check. /// if is a directory, otherwise. bool IsDirectory(string path); + + /// + /// Gets the for a given file without write share. + /// + /// The path of the file. + /// The of the file. + Stream GetFileStream(string path); } } diff --git a/src/Tgstation.Server.Host/IO/PosixFilesystemLinkFactory.cs b/src/Tgstation.Server.Host/IO/PosixFilesystemLinkFactory.cs index b80af9e122..c0fa195906 100644 --- a/src/Tgstation.Server.Host/IO/PosixFilesystemLinkFactory.cs +++ b/src/Tgstation.Server.Host/IO/PosixFilesystemLinkFactory.cs @@ -1,5 +1,5 @@ using System; -using System.IO; +using System.IO.Abstractions; using System.Threading; using System.Threading.Tasks; @@ -15,6 +15,20 @@ namespace Tgstation.Server.Host.IO /// public bool SymlinkedDirectoriesAreDeletedAsFiles => true; + /// + /// The to use. + /// + readonly IFileSystem fileSystem; + + /// + /// Initializes a new instance of the class. + /// + /// The value of . + public PosixFilesystemLinkFactory(IFileSystem fileSystem) + { + this.fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); + } + /// public Task CreateHardLink(string targetPath, string linkPath, CancellationToken cancellationToken) => Task.Factory.StartNew( () => @@ -39,7 +53,7 @@ namespace Tgstation.Server.Host.IO ArgumentNullException.ThrowIfNull(linkPath); UnixFileSystemInfo fsInfo; - var isFile = File.Exists(targetPath); + var isFile = fileSystem.File.Exists(targetPath); cancellationToken.ThrowIfCancellationRequested(); if (isFile) fsInfo = new UnixFileInfo(targetPath); diff --git a/src/Tgstation.Server.Host/IO/ResolvingIOManager.cs b/src/Tgstation.Server.Host/IO/ResolvingIOManager.cs index 69e7feefba..625d7da86e 100644 --- a/src/Tgstation.Server.Host/IO/ResolvingIOManager.cs +++ b/src/Tgstation.Server.Host/IO/ResolvingIOManager.cs @@ -1,4 +1,5 @@ using System; +using System.IO.Abstractions; namespace Tgstation.Server.Host.IO { @@ -15,8 +16,12 @@ namespace Tgstation.Server.Host.IO /// /// Initializes a new instance of the class. /// + /// The for the . /// The value of . - public ResolvingIOManager(string subdirectory) + public ResolvingIOManager( + IFileSystem fileSystem, + string subdirectory) + : base(fileSystem) { this.subdirectory = subdirectory ?? throw new ArgumentNullException(nameof(subdirectory)); } diff --git a/src/Tgstation.Server.Host/IO/SynchronousIOManager.cs b/src/Tgstation.Server.Host/IO/SynchronousIOManager.cs index b0388b51af..892b3edaec 100644 --- a/src/Tgstation.Server.Host/IO/SynchronousIOManager.cs +++ b/src/Tgstation.Server.Host/IO/SynchronousIOManager.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Globalization; using System.IO; +using System.IO.Abstractions; using System.Linq; using System.Security.Cryptography; using System.Threading; @@ -13,6 +14,11 @@ namespace Tgstation.Server.Host.IO /// sealed class SynchronousIOManager : ISynchronousIOManager { + /// + /// The to use. + /// + readonly IFileSystem fileSystem; + /// /// The for the . /// @@ -21,9 +27,11 @@ namespace Tgstation.Server.Host.IO /// /// Initializes a new instance of the class. /// + /// The value of . /// The value of . - public SynchronousIOManager(ILogger logger) + public SynchronousIOManager(IFileSystem fileSystem, ILogger logger) { + this.fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); } @@ -33,32 +41,32 @@ namespace Tgstation.Server.Host.IO if (IsDirectory(path)) return true; cancellationToken.ThrowIfCancellationRequested(); - Directory.CreateDirectory(path); + fileSystem.Directory.CreateDirectory(path); return false; } /// public bool DeleteDirectory(string path) { - if (File.Exists(path)) + if (fileSystem.File.Exists(path)) return false; - if (!Directory.Exists(path)) + if (!fileSystem.Directory.Exists(path)) return true; - if (Directory.EnumerateFileSystemEntries(path).Any()) + if (fileSystem.Directory.EnumerateFileSystemEntries(path).Any()) return false; - Directory.Delete(path); + fileSystem.Directory.Delete(path); return true; } /// public IEnumerable GetDirectories(string path, CancellationToken cancellationToken) { - foreach (var directoryName in Directory.EnumerateDirectories(path)) + foreach (var directoryName in fileSystem.Directory.EnumerateDirectories(path)) { - yield return Path.GetFileName(directoryName); + yield return fileSystem.Path.GetFileName(directoryName); cancellationToken.ThrowIfCancellationRequested(); } } @@ -66,9 +74,9 @@ namespace Tgstation.Server.Host.IO /// public IEnumerable GetFiles(string path, CancellationToken cancellationToken) { - foreach (var fileName in Directory.EnumerateFiles(path)) + foreach (var fileName in fileSystem.Directory.EnumerateFiles(path)) { - yield return Path.GetFileName(fileName); + yield return fileSystem.Path.GetFileName(fileName); cancellationToken.ThrowIfCancellationRequested(); } } @@ -77,14 +85,14 @@ namespace Tgstation.Server.Host.IO public bool IsDirectory(string path) { ArgumentNullException.ThrowIfNull(path); - return Directory.Exists(path); + return fileSystem.Directory.Exists(path); } /// public byte[] ReadFile(string path) { ArgumentNullException.ThrowIfNull(path); - return File.ReadAllBytes(path); + return fileSystem.File.ReadAllBytes(path); } /// @@ -94,16 +102,16 @@ namespace Tgstation.Server.Host.IO ArgumentNullException.ThrowIfNull(data); cancellationToken.ThrowIfCancellationRequested(); - var directory = Path.GetDirectoryName(path) ?? throw new ArgumentException("path cannot be rooted!", nameof(path)); - Directory.CreateDirectory(directory); + var directory = fileSystem.Path.GetDirectoryName(path) ?? throw new ArgumentException("path cannot be rooted!", nameof(path)); + fileSystem.Directory.CreateDirectory(directory); - var newFile = !File.Exists(path); + var newFile = !fileSystem.File.Exists(path); cancellationToken.ThrowIfCancellationRequested(); logger.LogTrace("Starting checked write to {path} ({fileType} file)", path, newFile ? "New" : "Pre-existing"); - using (var file = File.Open(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None)) + using (var file = fileSystem.File.Open(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None)) { cancellationToken.ThrowIfCancellationRequested(); @@ -160,10 +168,20 @@ namespace Tgstation.Server.Host.IO if (data.Length == 0) { logger.LogDebug("Stream is empty, deleting file"); - File.Delete(path); + fileSystem.File.Delete(path); } return true; } + + /// + public Stream GetFileStream(string path) + => fileSystem.FileStream.New( + path, + FileMode.Open, + FileAccess.Read, + FileShare.Read | FileShare.Delete, + DefaultIOManager.DefaultBufferSize, + true); } } diff --git a/src/Tgstation.Server.Host/IO/WindowsFilesystemLinkFactory.cs b/src/Tgstation.Server.Host/IO/WindowsFilesystemLinkFactory.cs index 54ff36b320..01cd4934d0 100644 --- a/src/Tgstation.Server.Host/IO/WindowsFilesystemLinkFactory.cs +++ b/src/Tgstation.Server.Host/IO/WindowsFilesystemLinkFactory.cs @@ -1,6 +1,6 @@ using System; using System.ComponentModel; -using System.IO; +using System.IO.Abstractions; using System.Threading; using System.Threading.Tasks; @@ -16,6 +16,20 @@ namespace Tgstation.Server.Host.IO /// public bool SymlinkedDirectoriesAreDeletedAsFiles => false; + /// + /// The to use. + /// + readonly IFileSystem fileSystem; + + /// + /// Initializes a new instance of the class. + /// + /// The value of . + public WindowsFilesystemLinkFactory(IFileSystem fileSystem) + { + this.fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); + } + /// public Task CreateHardLink(string targetPath, string linkPath, CancellationToken cancellationToken) => throw new NotSupportedException(); @@ -28,7 +42,7 @@ namespace Tgstation.Server.Host.IO ArgumentNullException.ThrowIfNull(linkPath); // check if its not a file - var flags = File.Exists(targetPath) ? NativeMethods.CreateSymbolicLinkFlags.None : NativeMethods.CreateSymbolicLinkFlags.Directory; + var flags = fileSystem.File.Exists(targetPath) ? NativeMethods.CreateSymbolicLinkFlags.None : NativeMethods.CreateSymbolicLinkFlags.Directory; /* * no don't fucking use this diff --git a/src/Tgstation.Server.Host/ServerFactory.cs b/src/Tgstation.Server.Host/ServerFactory.cs index 5306221a60..5f3230ec0a 100644 --- a/src/Tgstation.Server.Host/ServerFactory.cs +++ b/src/Tgstation.Server.Host/ServerFactory.cs @@ -1,4 +1,5 @@ using System; +using System.IO.Abstractions; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -29,23 +30,33 @@ namespace Tgstation.Server.Host /// public const string AppSettings = "appsettings"; + /// + public IIOManager IOManager { get; } + /// /// The for the . /// readonly IAssemblyInformationProvider assemblyInformationProvider; - /// - public IIOManager IOManager { get; } + /// + /// The for the . + /// + readonly IFileSystem fileSystem; /// /// Initializes a new instance of the class. /// /// The value of . /// The value of . - internal ServerFactory(IAssemblyInformationProvider assemblyInformationProvider, IIOManager ioManager) + /// The value of . + internal ServerFactory( + IAssemblyInformationProvider assemblyInformationProvider, + IIOManager ioManager, + IFileSystem fileSystem) { this.assemblyInformationProvider = assemblyInformationProvider ?? throw new ArgumentNullException(nameof(assemblyInformationProvider)); IOManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); + this.fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); } /// @@ -160,7 +171,7 @@ namespace Tgstation.Server.Host }) .UseIIS() .UseIISIntegration() - .UseApplication(assemblyInformationProvider, IOManager, postSetupServices) + .UseApplication(assemblyInformationProvider, IOManager, postSetupServices, fileSystem) .SuppressStatusMessages(true) .UseShutdownTimeout( TimeSpan.FromMinutes( diff --git a/src/Tgstation.Server.Host/System/PosixProcessFeatures.cs b/src/Tgstation.Server.Host/System/PosixProcessFeatures.cs index 6477dc05d5..ee6abe8a62 100644 --- a/src/Tgstation.Server.Host/System/PosixProcessFeatures.cs +++ b/src/Tgstation.Server.Host/System/PosixProcessFeatures.cs @@ -168,8 +168,8 @@ namespace Tgstation.Server.Host.System { // can't use ReadAllBytes here, /proc files have 0 length so the buffer is initialized to empty // https://stackoverflow.com/questions/12237712/how-can-i-show-the-size-of-files-in-proc-it-should-not-be-size-zero - await using var fileStream = ioManager.CreateAsyncSequentialReadStream( - "/proc/self/oom_score_adj"); + await using var fileStream = ioManager.CreateAsyncReadStream( + "/proc/self/oom_score_adj", true, true); using var reader = new StreamReader(fileStream, Encoding.UTF8, leaveOpen: true); originalString = await reader.ReadToEndAsync(cancellationToken); } diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index 19695527da..32fd89607a 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -161,6 +161,7 @@ + diff --git a/src/Tgstation.Server.Host/Transfer/FileTransferService.cs b/src/Tgstation.Server.Host/Transfer/FileTransferService.cs index bb1cafa171..ac8ce12a1b 100644 --- a/src/Tgstation.Server.Host/Transfer/FileTransferService.cs +++ b/src/Tgstation.Server.Host/Transfer/FileTransferService.cs @@ -212,7 +212,7 @@ namespace Tgstation.Server.Host.Transfer if (downloadProvider.StreamProvider != null) stream = await downloadProvider.StreamProvider(cancellationToken); else - stream = ioManager.GetFileStream(downloadProvider.FilePath, downloadProvider.ShareWrite); + stream = ioManager.CreateAsyncReadStream(downloadProvider.FilePath, false, downloadProvider.ShareWrite); } catch (IOException ex) { diff --git a/tests/Tgstation.Server.Host.Tests/Components/StaticFiles/TestConfiguration.cs b/tests/Tgstation.Server.Host.Tests/Components/StaticFiles/TestConfiguration.cs index 3dd2d6d13f..7314b0783a 100644 --- a/tests/Tgstation.Server.Host.Tests/Components/StaticFiles/TestConfiguration.cs +++ b/tests/Tgstation.Server.Host.Tests/Components/StaticFiles/TestConfiguration.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using System.IO.Abstractions.TestingHelpers; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -31,9 +32,10 @@ namespace Tgstation.Server.Host.Components.StaticFiles.Tests builder.SetMinimumLevel(LogLevel.Trace); }); + var mockFs = new MockFileSystem(); var tempPath = Path.GetTempFileName(); File.Delete(tempPath); - var ioManager = new DefaultIOManager().CreateResolverForSubdirectory(tempPath); + var ioManager = new DefaultIOManager(mockFs).CreateResolverForSubdirectory(tempPath); await ioManager.CreateDirectory(".", CancellationToken.None); try { @@ -46,7 +48,9 @@ namespace Tgstation.Server.Host.Components.StaticFiles.Tests var configuration = new Configuration( ioManager, - new SynchronousIOManager(loggerFactory.CreateLogger()), + new SynchronousIOManager( + mockFs, + loggerFactory.CreateLogger()), Mock.Of(), Mock.Of(), Mock.Of(), diff --git a/tests/Tgstation.Server.Host.Tests/IO/TestFilesystemLinkFactory.cs b/tests/Tgstation.Server.Host.Tests/IO/TestFilesystemLinkFactory.cs index dbf2aaf2a8..16ab28a73c 100644 --- a/tests/Tgstation.Server.Host.Tests/IO/TestFilesystemLinkFactory.cs +++ b/tests/Tgstation.Server.Host.Tests/IO/TestFilesystemLinkFactory.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using System.IO.Abstractions; using System.Runtime.InteropServices; using System.Security.Principal; using System.Threading; @@ -17,10 +18,11 @@ namespace Tgstation.Server.Host.IO.Tests [ClassInitialize] public static void SelectFactory(TestContext _) { + var fileSystem = new FileSystem(); if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - linkFactory = new WindowsFilesystemLinkFactory(); + linkFactory = new WindowsFilesystemLinkFactory(fileSystem); else - linkFactory = new PosixFilesystemLinkFactory(); + linkFactory = new PosixFilesystemLinkFactory(fileSystem); } public static bool HasPermissionToMakeSymlinks() diff --git a/tests/Tgstation.Server.Host.Tests/IO/TestIOManager.cs b/tests/Tgstation.Server.Host.Tests/IO/TestIOManager.cs index 8cd5424a4f..60fa057449 100644 --- a/tests/Tgstation.Server.Host.Tests/IO/TestIOManager.cs +++ b/tests/Tgstation.Server.Host.Tests/IO/TestIOManager.cs @@ -1,5 +1,7 @@ using System; using System.IO; +using System.IO.Abstractions; +using System.IO.Abstractions.TestingHelpers; using System.Linq; using System.Text; using System.Threading; @@ -14,27 +16,34 @@ namespace Tgstation.Server.Host.IO.Tests [TestClass] public sealed class TestIOManager { - readonly IIOManager ioManager = new DefaultIOManager(); + readonly IFileSystem fileSystem; + readonly IIOManager ioManager; + + public TestIOManager() + { + fileSystem = new MockFileSystem(); + ioManager = new DefaultIOManager(fileSystem); + } [TestMethod] public async Task TestDeleteDirectory() { - var tempPath = Path.GetTempFileName(); - File.Delete(tempPath); - Directory.CreateDirectory(tempPath); + var tempPath = fileSystem.Path.Combine(fileSystem.Path.GetTempPath(), fileSystem.Path.GetRandomFileName()); + fileSystem.File.Delete(tempPath); + fileSystem.Directory.CreateDirectory(tempPath); try { - await File.WriteAllTextAsync(Path.Combine(tempPath, "file.txt"), "asdf"); - var subDir = Path.Combine(tempPath, "subdir"); - Directory.CreateDirectory(subDir); - await File.WriteAllTextAsync(Path.Combine(subDir, "file2.txt"), "fdsa"); + await fileSystem.File.WriteAllTextAsync(Path.Combine(tempPath, "file.txt"), "asdf"); + var subDir = fileSystem.Path.Combine(tempPath, "subdir"); + fileSystem.Directory.CreateDirectory(subDir); + await fileSystem.File.WriteAllTextAsync(Path.Combine(subDir, "file2.txt"), "fdsa"); await ioManager.DeleteDirectory(tempPath, default); - Assert.IsFalse(Directory.Exists(tempPath)); + Assert.IsFalse(fileSystem.Directory.Exists(tempPath)); } catch { - Directory.Delete(tempPath, true); + fileSystem.Directory.Delete(tempPath, true); throw; } } @@ -42,13 +51,17 @@ namespace Tgstation.Server.Host.IO.Tests [TestMethod] public async Task TestDeleteDirectoryWithSymlinkInsideDoesntRecurse() { - var linkFactory = (IFilesystemLinkFactory)(new PlatformIdentifier().IsWindows - ? new WindowsFilesystemLinkFactory() - : new PosixFilesystemLinkFactory()); + // need a real FS here + var fileSystem = new FileSystem(); + var ioManager = new DefaultIOManager(fileSystem); - var tempPath = Path.GetTempFileName(); - File.Delete(tempPath); - Directory.CreateDirectory(tempPath); + var linkFactory = (IFilesystemLinkFactory)(new PlatformIdentifier().IsWindows + ? new WindowsFilesystemLinkFactory(fileSystem) + : new PosixFilesystemLinkFactory(fileSystem)); + + var tempPath = fileSystem.Path.Combine(fileSystem.Path.GetTempPath(), fileSystem.Path.GetRandomFileName()); + fileSystem.File.Delete(tempPath); + fileSystem.Directory.CreateDirectory(tempPath); try { var targetDir = ioManager.ConcatPath(tempPath, "targetdir"); @@ -81,7 +94,7 @@ namespace Tgstation.Server.Host.IO.Tests } catch { - Directory.Delete(tempPath, true); + fileSystem.Directory.Delete(tempPath, true); throw; } } @@ -89,14 +102,15 @@ namespace Tgstation.Server.Host.IO.Tests [TestMethod] public async Task TestFileExists() { - var tempPath = Path.GetTempFileName(); + var tempPath = fileSystem.Path.Combine(fileSystem.Path.GetTempPath(), fileSystem.Path.GetRandomFileName()); + await fileSystem.File.WriteAllBytesAsync(tempPath, Array.Empty()); try { Assert.IsTrue(await ioManager.FileExists(tempPath, default)); } finally { - File.Delete(tempPath); + fileSystem.File.Delete(tempPath); } Assert.IsFalse(await ioManager.FileExists(tempPath, default)); @@ -105,12 +119,12 @@ namespace Tgstation.Server.Host.IO.Tests [TestMethod] public async Task TestDirectoryExists() { - var tempPath = Path.GetTempFileName(); - File.Delete(tempPath); + var tempPath = fileSystem.Path.Combine(fileSystem.Path.GetTempPath(), fileSystem.Path.GetRandomFileName()); + fileSystem.File.Delete(tempPath); Assert.IsFalse(await ioManager.DirectoryExists(tempPath, default)); - Directory.CreateDirectory(tempPath); + fileSystem.Directory.CreateDirectory(tempPath); try { @@ -118,7 +132,7 @@ namespace Tgstation.Server.Host.IO.Tests } catch { - Directory.Delete(tempPath); + fileSystem.Directory.Delete(tempPath); throw; } } @@ -180,18 +194,18 @@ namespace Tgstation.Server.Host.IO.Tests async Task TestCopyDirectory(int? throttle) { - var tempPath = Path.GetTempFileName(); - File.Delete(tempPath); - Directory.CreateDirectory(tempPath); + var tempPath = fileSystem.Path.Combine(fileSystem.Path.GetTempPath(), fileSystem.Path.GetRandomFileName()); + fileSystem.File.Delete(tempPath); + fileSystem.Directory.CreateDirectory(tempPath); try { - var tempPath2 = Path.GetTempFileName(); - File.Delete(tempPath2); + var tempPath2 = fileSystem.Path.Combine(fileSystem.Path.GetTempPath(), fileSystem.Path.GetRandomFileName()); + fileSystem.File.Delete(tempPath2); - await File.WriteAllTextAsync(Path.Combine(tempPath, "file.txt"), "asdf"); - var subDir = Path.Combine(tempPath, "subdir"); - Directory.CreateDirectory(subDir); - await File.WriteAllTextAsync(Path.Combine(subDir, "file2.txt"), "fdsa"); + await fileSystem.File.WriteAllTextAsync(fileSystem.Path.Combine(tempPath, "file.txt"), "asdf"); + var subDir = fileSystem.Path.Combine(tempPath, "subdir"); + fileSystem.Directory.CreateDirectory(subDir); + await fileSystem.File.WriteAllTextAsync(fileSystem.Path.Combine(subDir, "file2.txt"), "fdsa"); try { @@ -203,26 +217,26 @@ namespace Tgstation.Server.Host.IO.Tests throttle, default); - Assert.IsTrue(Directory.Exists(tempPath2)); - var newFilePath = Path.Combine(tempPath2, "file.txt"); - Assert.IsTrue(File.Exists(newFilePath)); - var newFileText = await File.ReadAllTextAsync(newFilePath); + Assert.IsTrue(fileSystem.Directory.Exists(tempPath2)); + var newFilePath = fileSystem.Path.Combine(tempPath2, "file.txt"); + Assert.IsTrue(fileSystem.File.Exists(newFilePath)); + var newFileText = await fileSystem.File.ReadAllTextAsync(newFilePath); Assert.AreEqual("asdf", newFileText); - var newDirPath = Path.Combine(tempPath2, "subdir"); - Assert.IsTrue(Directory.Exists(newDirPath)); - var newFile2Path = Path.Combine(newDirPath, "file2.txt"); - Assert.IsTrue(File.Exists(newFile2Path)); - var newFile2Text = await File.ReadAllTextAsync(newFile2Path); + var newDirPath = fileSystem.Path.Combine(tempPath2, "subdir"); + Assert.IsTrue(fileSystem.Directory.Exists(newDirPath)); + var newFile2Path = fileSystem.Path.Combine(newDirPath, "file2.txt"); + Assert.IsTrue(fileSystem.File.Exists(newFile2Path)); + var newFile2Text = await fileSystem.File.ReadAllTextAsync(newFile2Path); Assert.AreEqual("fdsa", newFile2Text); } finally { - Directory.Delete(tempPath2, true); + fileSystem.Directory.Delete(tempPath2, true); } } finally { - Directory.Delete(tempPath, true); + fileSystem.Directory.Delete(tempPath, true); } } } diff --git a/tests/Tgstation.Server.Host.Tests/System/TestPosixSignalHandler.cs b/tests/Tgstation.Server.Host.Tests/System/TestPosixSignalHandler.cs index dd0f5fd5fb..dd4f2debf3 100644 --- a/tests/Tgstation.Server.Host.Tests/System/TestPosixSignalHandler.cs +++ b/tests/Tgstation.Server.Host.Tests/System/TestPosixSignalHandler.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using System.IO.Abstractions; using System.Reflection; using System.Threading; using System.Threading.Tasks; @@ -58,7 +59,8 @@ namespace Tgstation.Server.Host.System.Tests processExecutor = new ProcessExecutor( new PosixProcessFeatures( new Lazy(() => processExecutor), - new DefaultIOManager(), + new DefaultIOManager( + new FileSystem()), loggerFactory.CreateLogger()), Mock.Of(), loggerFactory.CreateLogger(), diff --git a/tests/Tgstation.Server.Host.Tests/System/TestProcessFeatures.cs b/tests/Tgstation.Server.Host.Tests/System/TestProcessFeatures.cs index a32b9e389c..34d902e8e8 100644 --- a/tests/Tgstation.Server.Host.Tests/System/TestProcessFeatures.cs +++ b/tests/Tgstation.Server.Host.Tests/System/TestProcessFeatures.cs @@ -2,6 +2,7 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; using System; +using System.IO.Abstractions.TestingHelpers; using System.Runtime.InteropServices; using Tgstation.Server.Host.IO; @@ -21,7 +22,7 @@ namespace Tgstation.Server.Host.System.Tests { features = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? (IProcessFeatures)new WindowsProcessFeatures(Mock.Of>()) - : new PosixProcessFeatures(new Lazy(() => null), new DefaultIOManager(), Mock.Of>()); + : new PosixProcessFeatures(new Lazy(() => null), new DefaultIOManager(new MockFileSystem()), Mock.Of>()); } [TestMethod] diff --git a/tests/Tgstation.Server.Host.Tests/System/TestSymlinkFactory.cs b/tests/Tgstation.Server.Host.Tests/System/TestSymlinkFactory.cs index 4d3c5be749..222e870cd0 100644 --- a/tests/Tgstation.Server.Host.Tests/System/TestSymlinkFactory.cs +++ b/tests/Tgstation.Server.Host.Tests/System/TestSymlinkFactory.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using System.IO.Abstractions; using System.Threading; using System.Threading.Tasks; @@ -13,8 +14,10 @@ namespace Tgstation.Server.Host.System.Tests public sealed class TestSymlinkFactory { readonly IFilesystemLinkFactory factory = new PlatformIdentifier().IsWindows - ? new WindowsFilesystemLinkFactory() - : new PosixFilesystemLinkFactory(); + ? new WindowsFilesystemLinkFactory( + new FileSystem()) + : new PosixFilesystemLinkFactory( + new FileSystem()); [TestMethod] public async Task TestSymlinks() diff --git a/tests/Tgstation.Server.Host.Tests/TestProgram.cs b/tests/Tgstation.Server.Host.Tests/TestProgram.cs index f04531154f..a4f9318e13 100644 --- a/tests/Tgstation.Server.Host.Tests/TestProgram.cs +++ b/tests/Tgstation.Server.Host.Tests/TestProgram.cs @@ -2,6 +2,7 @@ using Moq; using System; using System.IO; +using System.IO.Abstractions.TestingHelpers; using System.Threading; using System.Threading.Tasks; @@ -80,28 +81,29 @@ namespace Tgstation.Server.Host.Tests public async Task TestStandardRunWithExceptionAndWatchdog() { var mockServer = new Mock(); + var mockFs = new MockFileSystem(); var exception = new DivideByZeroException(); mockServer.Setup(x => x.Run(It.IsAny())).Throws(exception); mockServer.SetupGet(x => x.RestartRequested).Returns(true); var mockServerFactory = new Mock(); - mockServerFactory.SetupGet(x => x.IOManager).Returns(new DefaultIOManager()); + mockServerFactory.SetupGet(x => x.IOManager).Returns(new DefaultIOManager(mockFs)); mockServerFactory.Setup(x => x.CreateServer(It.IsNotNull(), It.IsAny(), It.IsAny())).ReturnsAsync(mockServer.Object); var program = new Program { ServerFactory = mockServerFactory.Object }; - var tempFileName = Path.GetTempFileName(); - File.Delete(tempFileName); + var tempFileName = mockFs.Path.Combine(mockFs.Path.GetTempPath(), mockFs.Path.GetRandomFileName()); + mockFs.File.Delete(tempFileName); try { var result = await program.Main(Array.Empty(), tempFileName); Assert.AreEqual(HostExitCode.Error, result); - Assert.AreEqual(exception.ToString(), File.ReadAllText(tempFileName)); + Assert.AreEqual(exception.ToString(), mockFs.File.ReadAllText(tempFileName)); } finally { - File.Delete(tempFileName); + mockFs.File.Delete(tempFileName); } } } diff --git a/tests/Tgstation.Server.Host.Tests/TestServerFactory.cs b/tests/Tgstation.Server.Host.Tests/TestServerFactory.cs index 9a96a8332f..a520286782 100644 --- a/tests/Tgstation.Server.Host.Tests/TestServerFactory.cs +++ b/tests/Tgstation.Server.Host.Tests/TestServerFactory.cs @@ -1,6 +1,7 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; using System; +using System.IO.Abstractions.TestingHelpers; using System.Threading.Tasks; using Tgstation.Server.Host.Core; @@ -20,11 +21,13 @@ namespace Tgstation.Server.Host.Tests [TestMethod] public void TestConstructor() { - Assert.ThrowsException(() => new ServerFactory(null, null)); + Assert.ThrowsException(() => new ServerFactory(null, null, null)); IAssemblyInformationProvider assemblyInformationProvider = Mock.Of(); - Assert.ThrowsException(() => new ServerFactory(assemblyInformationProvider, null)); + Assert.ThrowsException(() => new ServerFactory(assemblyInformationProvider, null, null)); IIOManager ioManager = Mock.Of(); - _ = new ServerFactory(assemblyInformationProvider, ioManager); + Assert.ThrowsException(() => new ServerFactory(assemblyInformationProvider, ioManager, null)); + var mockFileSystem = new MockFileSystem(); + _ = new ServerFactory(assemblyInformationProvider, ioManager, mockFileSystem); } [TestMethod] diff --git a/tests/Tgstation.Server.Host.Tests/Tgstation.Server.Host.Tests.csproj b/tests/Tgstation.Server.Host.Tests/Tgstation.Server.Host.Tests.csproj index b6cb2167ad..f696ba76b7 100644 --- a/tests/Tgstation.Server.Host.Tests/Tgstation.Server.Host.Tests.csproj +++ b/tests/Tgstation.Server.Host.Tests/Tgstation.Server.Host.Tests.csproj @@ -8,6 +8,7 @@ + diff --git a/tests/Tgstation.Server.Tests/CachingFileDownloader.cs b/tests/Tgstation.Server.Tests/CachingFileDownloader.cs index de540b606d..76bdba28ee 100644 --- a/tests/Tgstation.Server.Tests/CachingFileDownloader.cs +++ b/tests/Tgstation.Server.Tests/CachingFileDownloader.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.IO.Abstractions; using System.Threading; using System.Threading.Tasks; @@ -195,7 +196,7 @@ namespace Tgstation.Server.Tests try { Directory.CreateDirectory(Path.GetDirectoryName(path)); - await using var fs = new DefaultIOManager().CreateAsyncSequentialWriteStream(path); + await using var fs = new DefaultIOManager(new FileSystem()).CreateAsyncSequentialWriteStream(path); await ms.CopyToAsync(fs, cancellationToken); cachedPaths.Add(url.ToString(), Tuple.Create(path, temporal)); diff --git a/tests/Tgstation.Server.Tests/Live/Instance/ConfigurationTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/ConfigurationTest.cs index eb39799414..52217388c1 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/ConfigurationTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/ConfigurationTest.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using System.IO.Abstractions; using System.Linq; using System.Text; using System.Threading; @@ -90,7 +91,8 @@ namespace Tgstation.Server.Tests.Live.Instance public ValueTask SetupDMApiTests(bool includingRoot, CancellationToken cancellationToken) { // just use an I/O manager here - var ioManager = new DefaultIOManager(); + var ioManager = new DefaultIOManager( + new FileSystem()); async ValueTask TestStaticFileAndDir() { @@ -127,7 +129,7 @@ namespace Tgstation.Server.Tests.Live.Instance Path = $"/EventScripts/{scriptName}" }; - await using var readStream = ioManager.GetFileStream($"../../../../DMAPI/{(basic ? "BasicOperation" : "LongRunning")}/{scriptName}", false); + await using var readStream = ioManager.CreateAsyncReadStream($"../../../../DMAPI/{(basic ? "BasicOperation" : "LongRunning")}/{scriptName}", true, false); await configurationClient.Write( resourcingScript, readStream, diff --git a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs index 8d667ebc41..1c77c35c1a 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.IO.Abstractions; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -89,7 +90,7 @@ namespace Tgstation.Server.Tests.Live.Instance Uri openDreamUrl, CancellationToken cancellationToken) { - var ioManager = new DefaultIOManager(); + var ioManager = new DefaultIOManager(new FileSystem()); var odRepoDir = ioManager.ConcatPath( Environment.GetFolderPath( Environment.SpecialFolder.LocalApplicationData, @@ -107,7 +108,7 @@ namespace Tgstation.Server.Tests.Live.Instance IEngineInstaller byondInstaller = compatVersion.Engine == EngineType.OpenDream ? new OpenDreamInstaller( - new DefaultIOManager(), + ioManager, Mock.Of>(), new PlatformIdentifier(), Mock.Of(), diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index df2476444b..dab176cdc5 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -14,6 +14,7 @@ using System; using System.Collections.Generic; using System.Globalization; using System.IO; +using System.IO.Abstractions; using System.Linq; using System.Net; using System.Net.Sockets; @@ -809,7 +810,7 @@ namespace Tgstation.Server.Tests.Live.Instance var features = new PosixProcessFeatures( new Lazy(Mock.Of()), - new DefaultIOManager(), + new DefaultIOManager(new FileSystem()), Mock.Of>()); features.SuspendProcess(proc); @@ -876,7 +877,7 @@ namespace Tgstation.Server.Tests.Live.Instance executor = new ProcessExecutor( RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? new WindowsProcessFeatures(Mock.Of>()) - : new PosixProcessFeatures(new Lazy(() => executor), new DefaultIOManager(), Mock.Of>()), + : new PosixProcessFeatures(new Lazy(() => executor), new DefaultIOManager(new FileSystem()), Mock.Of>()), Mock.Of(), Mock.Of>(), LoggerFactory.Create(x => { })); diff --git a/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs b/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs index ac4f519fa6..09bf058c1d 100644 --- a/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs +++ b/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs @@ -6,6 +6,7 @@ using System; using System.Collections.Generic; using System.Globalization; using System.IO; +using System.IO.Abstractions; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -43,7 +44,7 @@ namespace Tgstation.Server.Tests.Live for (int i = 0; i < 5; ++i) try { - new DefaultIOManager().DeleteDirectory(directory, default).GetAwaiter().GetResult(); + new DefaultIOManager(new FileSystem()).DeleteDirectory(directory, default).GetAwaiter().GetResult(); } catch { diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index 8df9a3d612..c872d9ab29 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; +using System.IO.Abstractions; using System.Linq; using System.Management; using System.Net; @@ -415,7 +416,7 @@ namespace Tgstation.Server.Tests.Live var gitHubToken = Environment.GetEnvironmentVariable("TGS_TEST_GITHUB_TOKEN"); if (String.IsNullOrWhiteSpace(gitHubToken)) gitHubToken = null; - await new Host.IO.DefaultIOManager().DeleteDirectory(server.UpdatePath, cancellationToken); + await new Host.IO.DefaultIOManager(new FileSystem()).DeleteDirectory(server.UpdatePath, cancellationToken); serverTask = server.Run(cancellationToken).AsTask(); await using (var adminClient = await CreateAdminClient(server.ApiUrl, cancellationToken)) @@ -1135,7 +1136,7 @@ namespace Tgstation.Server.Tests.Live ApiValidationSecurityLevel = DreamDaemonSecurity.Trusted, }, cancellationToken); - var ioManager = new Host.IO.DefaultIOManager(); + var ioManager = new Host.IO.DefaultIOManager(new FileSystem()); var repoPath = ioManager.ConcatPath(instance.Path, "Repository"); await using var jobsTest = new JobsRequiredTest(instanceClient.Jobs); var postWriteHandler = (Host.IO.IPostWriteHandler)(new PlatformIdentifier().IsWindows diff --git a/tests/Tgstation.Server.Tests/TestRepository.cs b/tests/Tgstation.Server.Tests/TestRepository.cs index 89cf34daac..96a2349bea 100644 --- a/tests/Tgstation.Server.Tests/TestRepository.cs +++ b/tests/Tgstation.Server.Tests/TestRepository.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using System.IO.Abstractions; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -58,7 +59,7 @@ namespace Tgstation.Server.Tests } finally { - await new DefaultIOManager().DeleteDirectory( + await new DefaultIOManager(new FileSystem()).DeleteDirectory( Path.GetDirectoryName(tempPath), CancellationToken.None); } @@ -75,7 +76,7 @@ namespace Tgstation.Server.Tests using var manager = new RepositoryManager( repoFac, commands, - new DefaultIOManager().CreateResolverForSubdirectory( + new DefaultIOManager(new FileSystem()).CreateResolverForSubdirectory( tempPath), Mock.Of(), new WindowsPostWriteHandler(), @@ -130,7 +131,7 @@ namespace Tgstation.Server.Tests } finally { - await new DefaultIOManager().DeleteDirectory( + await new DefaultIOManager(new FileSystem()).DeleteDirectory( Path.GetDirectoryName(tempPath), CancellationToken.None); } diff --git a/tests/Tgstation.Server.Tests/TestSystemInteraction.cs b/tests/Tgstation.Server.Tests/TestSystemInteraction.cs index e7c8e28962..f2601acdee 100644 --- a/tests/Tgstation.Server.Tests/TestSystemInteraction.cs +++ b/tests/Tgstation.Server.Tests/TestSystemInteraction.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using System.IO.Abstractions; using System.Threading; using System.Threading.Tasks; @@ -24,7 +25,7 @@ namespace Tgstation.Server.Tests var platformIdentifier = new PlatformIdentifier(); var processExecutor = new ProcessExecutor( Mock.Of(), - new DefaultIOManager(), + new DefaultIOManager(new FileSystem()), Mock.Of>(), loggerFactory); @@ -52,7 +53,7 @@ namespace Tgstation.Server.Tests var platformIdentifier = new PlatformIdentifier(); var processExecutor = new ProcessExecutor( Mock.Of(), - new DefaultIOManager(), + new DefaultIOManager(new FileSystem()), loggerFactory.CreateLogger(), loggerFactory); diff --git a/tests/Tgstation.Server.Tests/TestVersions.cs b/tests/Tgstation.Server.Tests/TestVersions.cs index a0809b78f6..047de71987 100644 --- a/tests/Tgstation.Server.Tests/TestVersions.cs +++ b/tests/Tgstation.Server.Tests/TestVersions.cs @@ -33,6 +33,7 @@ using Tgstation.Server.Host.System; using Tgstation.Server.Api.Models; using Tgstation.Server.Tests.Live; using Tgstation.Server.Host.Properties; +using System.IO.Abstractions; namespace Tgstation.Server.Tests { @@ -207,7 +208,7 @@ namespace Tgstation.Server.Tests loggerFactory.CreateLogger()) : new PosixByondInstaller( new PosixPostWriteHandler(loggerFactory.CreateLogger()), - new DefaultIOManager(), + new DefaultIOManager(new FileSystem()), fileDownloader, loggerFactory.CreateLogger()); using var disposable = byondInstaller as IDisposable; @@ -217,13 +218,13 @@ namespace Tgstation.Server.Tests ? new WindowsProcessFeatures(Mock.Of>()) : new PosixProcessFeatures( new Lazy(() => null), - new DefaultIOManager(), + new DefaultIOManager(new FileSystem()), loggerFactory.CreateLogger()), Mock.Of(), loggerFactory.CreateLogger(), loggerFactory); - var ioManager = new DefaultIOManager(); + var ioManager = new DefaultIOManager(new FileSystem()); var tempPath = ioManager.ConcatPath(LiveTestingServer.BaseDirectory, "mapthreads"); await ioManager.CreateDirectory(tempPath, default); try diff --git a/tests/Tgstation.Server.Tests/TestingUtils.cs b/tests/Tgstation.Server.Tests/TestingUtils.cs index 3167d2c3d3..3e1a2a3267 100644 --- a/tests/Tgstation.Server.Tests/TestingUtils.cs +++ b/tests/Tgstation.Server.Tests/TestingUtils.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using System.IO.Abstractions; using System.IO.Compression; using System.Reflection; using System.Threading; @@ -67,7 +68,7 @@ namespace Tgstation.Server.Tests } finally { - await new DefaultIOManager().DeleteDirectory(tempFolder, cancellationToken); + await new DefaultIOManager(new FileSystem()).DeleteDirectory(tempFolder, cancellationToken); } } } From 933b9d9fe84a7af89b4a2ccc20bc840f908298bf Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 21 Jul 2025 18:03:39 -0400 Subject: [PATCH 25/52] Finish properly setting up mirror and caching courtesy of @AffectedArc07 --- .github/CONTRIBUTING.md | 2 ++ .github/workflows/ci-pipeline.yml | 2 ++ .../CachingFileDownloader.cs | 30 ++++++++++++++----- .../Live/TestLiveServer.cs | 14 +++++++-- tests/Tgstation.Server.Tests/TestingUtils.cs | 2 +- 5 files changed, 39 insertions(+), 11 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index c2912e0af5..54178ff953 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -51,6 +51,8 @@ You must also have the following environment variables set. To run them more acc - `TGS_TEST_CONNECTION_STRING`: To a valid database connection string. You can use the setup wizard to create one. - (Optional) `TGS_TEST_GITHUB_TOKEN`: A GitHub personal access token with no scopes used to bypass rate limits. - (Optional) `TGS_TEST_BYOND_ZIP_DOWNLOAD_TEMPLATE`: Template URL for downloading BYOND zip files from a non-official mirror. +- (Optional) `TGS_TEST_BYOND_MIRROR_VERSION_TXT`: version.txt for a BYOND zip mirror. Requires `TGS_TEST_BYOND_ZIP_DOWNLOAD_TEMPLATE`. +- (Optional) `TGS_TEST_BYOND_ZIPS_BASE_PATH`: Directory on disk to cache BYOND zip files. - (Optional) The following variables are all interdependent, so if one is set they all must be. - `TGS_TEST_DISCORD_TOKEN`: To a valid discord bot token. - `TGS_TEST_DISCORD_CHANNEL`: To a valid discord channel ID that the above bot can access. diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 9f9c9a5010..0e098dc503 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -42,6 +42,8 @@ env: TGS_NODE_VERSION: 20.x TGS_TEST_GITHUB_TOKEN: ${{ secrets.LIVE_TESTS_TOKEN }} PACKAGING_PRIVATE_KEY_PASSPHRASE: ${{ secrets.PACKAGING_PRIVATE_KEY_PASSPHRASE }} + TGS_TEST_BYOND_ZIP_DOWNLOAD_TEMPLATE: https://mocha.affectedarc07.co.uk/tgs_byond_mirrors/${Major}.${Minor}_byond${Linux:_linux}.zip + TGS_TEST_BYOND_MIRROR_VERSION_TXT: https://mocha.affectedarc07.co.uk/tgs_byond_mirrors/version.txt concurrency: group: "ci-${{ (github.event_name != 'push' && github.event_name != 'schedule' && github.event.inputs.pull_request_number) || github.run_id }}-${{ github.event_name }}" diff --git a/tests/Tgstation.Server.Tests/CachingFileDownloader.cs b/tests/Tgstation.Server.Tests/CachingFileDownloader.cs index 8fccd1234c..f2897cf808 100644 --- a/tests/Tgstation.Server.Tests/CachingFileDownloader.cs +++ b/tests/Tgstation.Server.Tests/CachingFileDownloader.cs @@ -92,15 +92,21 @@ namespace Tgstation.Server.Tests var url = ByondInstallerBase.GetDownloadZipUrl(byondVersion, urlTemplate, new PlatformIdentifier().IsWindows ? "Windows" : "Linux"); string path = null; - if (TestingUtils.RunningInGitHubActions) + string basePath = Environment.GetEnvironmentVariable("TGS_TEST_BYOND_ZIPS_BASE_PATH"); + if (basePath == null && TestingUtils.RunningInGitHubActions) { - // actions is supposed to cache BYOND for us - - var dir = Path.Combine( + // actions is supposed to cache BYOND for us here + basePath = Path.Combine( Environment.GetFolderPath( Environment.SpecialFolder.UserProfile, Environment.SpecialFolderOption.DoNotVerify), - "byond-zips-cache", + "byond-zips-cache"); + } + + if (basePath != null) + { + var dir = Path.Combine( + basePath, "live", windows ? "windows" : "linux"); path = Path.Combine( @@ -109,15 +115,23 @@ namespace Tgstation.Server.Tests $"{version.Version.Major}.{version.Version.Minor}.zip"); } + Uri overrideUrl = null; + if (urlCacheOverrideTemplate != null) + { + overrideUrl = url; + url = ByondInstallerBase.GetDownloadZipUrl(byondVersion, urlCacheOverrideTemplate, new PlatformIdentifier().IsWindows ? "Windows" : "Linux"); + } + await (await CacheFile( logger, - urlCacheOverrideTemplate != null - ? ByondInstallerBase.GetDownloadZipUrl(byondVersion, urlCacheOverrideTemplate, new PlatformIdentifier().IsWindows ? "Windows" : "Linux") - : url, + url, null, path, cancellationToken)) .DisposeAsync(); + + if (overrideUrl != null) + cachedPaths[overrideUrl.ToString()] = cachedPaths[url.ToString()]; } public static void Cleanup() diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index 7eeec04af3..61013ea17e 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -1626,6 +1626,16 @@ namespace Tgstation.Server.Tests.Live if (openDreamOnly) return; + var windowsMinCompat = new Version(510, 1346); + var linuxMinCompat = new Version(512, 1451); // http://www.byond.com/forum/?forum=5&command=search&scope=local&text=resolved%3a512.1451 + await CachingFileDownloader.InitializeByondVersion( + GetLogger(), + new PlatformIdentifier().IsWindows + ? windowsMinCompat + : linuxMinCompat, + new PlatformIdentifier().IsWindows, + cancellationToken); + var compatTests = FailFast( instanceTest .RunCompatTests( @@ -1633,8 +1643,8 @@ namespace Tgstation.Server.Tests.Live { Engine = EngineType.Byond, Version = new PlatformIdentifier().IsWindows - ? new Version(510, 1346) - : new Version(512, 1451) // http://www.byond.com/forum/?forum=5&command=search&scope=local&text=resolved%3a512.1451 + ? windowsMinCompat + : linuxMinCompat, }, server.OpenDreamUrl, firstAdminRestClient.Instances.CreateClient(compatInstance), diff --git a/tests/Tgstation.Server.Tests/TestingUtils.cs b/tests/Tgstation.Server.Tests/TestingUtils.cs index 9cc11f7966..40925352dd 100644 --- a/tests/Tgstation.Server.Tests/TestingUtils.cs +++ b/tests/Tgstation.Server.Tests/TestingUtils.cs @@ -136,7 +136,7 @@ namespace Tgstation.Server.Tests const string DefaultMirror = "https://www.byond.com/download/version.txt"; edgeVersion = await GetVersionFromResponse(DefaultMirror); - logger.LogInformation("Downloading edge version from BYOND.com"); + logger.LogInformation("Downloading edge version from BYOND.com {edge}", edgeVersion); // if we got the result from byond.com, make sure the cache grabs the zip from there as well await CachingFileDownloader.InitializeByondVersion(logger, Version.Parse(edgeVersion), new PlatformIdentifier().IsWindows, cancellationToken, GeneralConfiguration.DefaultByondZipDownloadTemplate); From cf82c4be7497645887a0428065135a8aaad1055d Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 21 Jul 2025 18:09:01 -0400 Subject: [PATCH 26/52] Fix EDGE version evaluation in scripts --- .github/workflows/ci-pipeline.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 0e098dc503..8a52bba815 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -169,6 +169,7 @@ jobs: sudo apt-get install -y -o APT::Immediate-Configure=0 libc6-i386 libstdc++6:i386 libgcc-s1:i386 - name: Evaluate EDGE BYOND version + if: ${{ matrix.byond == 'EDGE' }} id: edge_version_evaluation run: | FULL_VERSION=${{ matrix.byond }} @@ -185,7 +186,7 @@ jobs: FULL_VERSION=${bad_linux_releases[$FULL_VERSION]} fi fi - run: echo "EVALUATED_EDGE_VERSION=$FULL_VERSION" >> $GITHUB_OUTPUT + echo "EVALUATED_EDGE_VERSION=$FULL_VERSION" >> $GITHUB_OUTPUT - name: Cache BYOND .zips uses: actions/cache@v4 @@ -197,7 +198,10 @@ jobs: - name: Setup BYOND Cache if Necessary and Install run: | echo "Setting up BYOND." - FULL_VERSION=${{ steps.edge_version_evaluation.outputs.EVALUATED_EDGE_VERSION }} + FULL_VERSION=${{ matrix.byond }} + if [[ "$FULL_VERSION" = "EDGE" ]] ; then + FULL_VERSION=${{ steps.edge_version_evaluation.outputs.EVALUATED_EDGE_VERSION }} + fi if [[ ! -f $HOME/byond-zips-cache/linux/$FULL_VERSION/$FULL_VERSION.zip ]] ; then BYOND_MAJOR=${FULL_VERSION%.*} mkdir -p $HOME/byond-zips-cache/linux From 0e86acbaa3f3372da378314777969351c9ec431f Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 21 Jul 2025 18:11:30 -0400 Subject: [PATCH 27/52] Fix version evaluation --- .github/workflows/ci-pipeline.yml | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 8a52bba815..68bec6e0f2 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -168,9 +168,8 @@ jobs: sudo apt-get update sudo apt-get install -y -o APT::Immediate-Configure=0 libc6-i386 libstdc++6:i386 libgcc-s1:i386 - - name: Evaluate EDGE BYOND version - if: ${{ matrix.byond == 'EDGE' }} - id: edge_version_evaluation + - name: Evaluate BYOND version + id: version_evaluation run: | FULL_VERSION=${{ matrix.byond }} if [[ "$FULL_VERSION" = "EDGE" ]] ; then @@ -186,22 +185,19 @@ jobs: FULL_VERSION=${bad_linux_releases[$FULL_VERSION]} fi fi - echo "EVALUATED_EDGE_VERSION=$FULL_VERSION" >> $GITHUB_OUTPUT + echo "EVALUATED_VERSION=$FULL_VERSION" >> $GITHUB_OUTPUT - name: Cache BYOND .zips uses: actions/cache@v4 id: cache-byond with: - path: ~/byond-zips-cache/linux/${{ steps.edge_version_evaluation.outputs.EVALUATED_EDGE_VERSION }} - key: byond-zips-linux-${{ steps.edge_version_evaluation.outputs.EVALUATED_EDGE_VERSION }} + path: ~/byond-zips-cache/linux/${{ steps.version_evaluation.outputs.EVALUATED_VERSION }} + key: byond-zips-linux-${{ steps.version_evaluation.outputs.EVALUATED_VERSION }} - name: Setup BYOND Cache if Necessary and Install run: | echo "Setting up BYOND." - FULL_VERSION=${{ matrix.byond }} - if [[ "$FULL_VERSION" = "EDGE" ]] ; then - FULL_VERSION=${{ steps.edge_version_evaluation.outputs.EVALUATED_EDGE_VERSION }} - fi + FULL_VERSION=${{ steps.version_evaluation.outputs.EVALUATED_VERSION }} if [[ ! -f $HOME/byond-zips-cache/linux/$FULL_VERSION/$FULL_VERSION.zip ]] ; then BYOND_MAJOR=${FULL_VERSION%.*} mkdir -p $HOME/byond-zips-cache/linux From 8ef960a5bff5ec3aa6c9e0671a4f32aa09f2b4ff Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Mon, 21 Jul 2025 18:21:31 -0400 Subject: [PATCH 28/52] Mirror everything except EDGE --- .github/workflows/ci-pipeline.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 68bec6e0f2..dc5060ad24 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -44,6 +44,7 @@ env: PACKAGING_PRIVATE_KEY_PASSPHRASE: ${{ secrets.PACKAGING_PRIVATE_KEY_PASSPHRASE }} TGS_TEST_BYOND_ZIP_DOWNLOAD_TEMPLATE: https://mocha.affectedarc07.co.uk/tgs_byond_mirrors/${Major}.${Minor}_byond${Linux:_linux}.zip TGS_TEST_BYOND_MIRROR_VERSION_TXT: https://mocha.affectedarc07.co.uk/tgs_byond_mirrors/version.txt + DMAPI_BYOND_BUILD_MIRROR_DOWNLOAD_TEMPLATE: https://mocha.affectedarc07.co.uk/tgs_byond_mirrors/${FULL_VERSION}_byond_linux.zip # Available vars are $BYOND_MAJOR (i.e. 516) and $FULL_VERSION (i.e. 516.1666) concurrency: group: "ci-${{ (github.event_name != 'push' && github.event_name != 'schedule' && github.event.inputs.pull_request_number) || github.run_id }}-${{ github.event_name }}" @@ -200,8 +201,12 @@ jobs: FULL_VERSION=${{ steps.version_evaluation.outputs.EVALUATED_VERSION }} if [[ ! -f $HOME/byond-zips-cache/linux/$FULL_VERSION/$FULL_VERSION.zip ]] ; then BYOND_MAJOR=${FULL_VERSION%.*} - mkdir -p $HOME/byond-zips-cache/linux - curl "https://www.byond.com/download/$BYOND_MAJOR/${FULL_VERSION}_byond_linux.zip" -o $HOME/byond-zips-cache/linux/$FULL_VERSION/$FULL_VERSION.zip + mkdir -p $HOME/byond-zips-cache/linux/$FULL_VERSION + DOWNLOAD_URL="${{ env.DMAPI_BYOND_BUILD_MIRROR_DOWNLOAD_TEMPLATE }}" + if [[ "$FULL_VERSION" = "EDGE" ]] ; then + DOWNLOAD_URL="https://www.byond.com/download/$BYOND_MAJOR/${FULL_VERSION}_byond_linux.zip" + fi + curl "$DOWNLOAD_URL" -o $HOME/byond-zips-cache/linux/$FULL_VERSION/$FULL_VERSION.zip fi mkdir -p "$HOME/BYOND" cd "$HOME/BYOND" From 0904014028ba200ef42e05367cd9069bef3e0960 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 22 Jul 2025 05:26:00 +0000 Subject: [PATCH 29/52] Bump form-data Bumps the npm_and_yarn group with 1 update in the /src/Tgstation.Server.Host.Utils.GitLab.GraphQL directory: [form-data](https://github.com/form-data/form-data). Updates `form-data` from 4.0.2 to 4.0.4 - [Release notes](https://github.com/form-data/form-data/releases) - [Changelog](https://github.com/form-data/form-data/blob/master/CHANGELOG.md) - [Commits](https://github.com/form-data/form-data/compare/v4.0.2...v4.0.4) --- updated-dependencies: - dependency-name: form-data dependency-version: 4.0.4 dependency-type: indirect dependency-group: npm_and_yarn ... Signed-off-by: dependabot[bot] --- src/Tgstation.Server.Host.Utils.GitLab.GraphQL/yarn.lock | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Tgstation.Server.Host.Utils.GitLab.GraphQL/yarn.lock b/src/Tgstation.Server.Host.Utils.GitLab.GraphQL/yarn.lock index d1d93cd55d..5e4dca122d 100644 --- a/src/Tgstation.Server.Host.Utils.GitLab.GraphQL/yarn.lock +++ b/src/Tgstation.Server.Host.Utils.GitLab.GraphQL/yarn.lock @@ -1855,14 +1855,15 @@ __metadata: linkType: hard "form-data@npm:^4.0.0": - version: 4.0.2 - resolution: "form-data@npm:4.0.2" + version: 4.0.4 + resolution: "form-data@npm:4.0.4" dependencies: asynckit: "npm:^0.4.0" combined-stream: "npm:^1.0.8" es-set-tostringtag: "npm:^2.1.0" + hasown: "npm:^2.0.2" mime-types: "npm:^2.1.12" - checksum: 10/82c65b426af4a40090e517a1bc9057f76970b4c6043e37aa49859c447d88553e77d4cc5626395079a53d2b0889ba5f2a49f3900db3ad3f3f1bf76613532572fb + checksum: 10/a4b62e21932f48702bc468cc26fb276d186e6b07b557e3dd7cc455872bdbb82db7db066844a64ad3cf40eaf3a753c830538183570462d3649fdfd705601cbcfb languageName: node linkType: hard From 1d0e30bc7fef7102f917c14704f449a169e30e1c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 25 Jul 2025 09:49:31 +0000 Subject: [PATCH 30/52] Bump the mstest group with 2 updates Bumps MSTest.TestAdapter from 3.8.3 to 3.9.3 Bumps MSTest.TestFramework from 3.8.3 to 3.9.3 --- updated-dependencies: - dependency-name: MSTest.TestAdapter dependency-version: 3.9.3 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: mstest - dependency-name: MSTest.TestFramework dependency-version: 3.9.3 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: mstest ... Signed-off-by: dependabot[bot] --- build/TestCommon.props | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/TestCommon.props b/build/TestCommon.props index 52ba145ada..44c2a0c2e4 100644 --- a/build/TestCommon.props +++ b/build/TestCommon.props @@ -18,9 +18,9 @@ - + - + From 23f02e95976858b2a4d4f74e6ab9aa56157d9a35 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 26 Jul 2025 10:23:02 -0400 Subject: [PATCH 31/52] Hopefully use Spacestation13 BYOND builds mirror --- .github/workflows/ci-pipeline.yml | 53 +++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index dc5060ad24..9e1c37201e 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -174,7 +174,7 @@ jobs: run: | FULL_VERSION=${{ matrix.byond }} if [[ "$FULL_VERSION" = "EDGE" ]] ; then - VERSIONS=$(curl https://www.byond.com/download/version.txt) + VERSIONS=$(curl https://spacestation13.github.io/byond-builds/version.txt) FULL_VERSION=$(echo "$VERSIONS" | tail -n1) echo "EDGE version evaluated to $FULL_VERSION" @@ -203,8 +203,8 @@ jobs: BYOND_MAJOR=${FULL_VERSION%.*} mkdir -p $HOME/byond-zips-cache/linux/$FULL_VERSION DOWNLOAD_URL="${{ env.DMAPI_BYOND_BUILD_MIRROR_DOWNLOAD_TEMPLATE }}" - if [[ "$FULL_VERSION" = "EDGE" ]] ; then - DOWNLOAD_URL="https://www.byond.com/download/$BYOND_MAJOR/${FULL_VERSION}_byond_linux.zip" + if [[ "${{ matrix.byond }}" = "EDGE" ]] ; then + DOWNLOAD_URL="https://spacestation13.github.io/byond-builds/$BYOND_MAJOR/${FULL_VERSION}_byond_linux.zip" fi curl "$DOWNLOAD_URL" -o $HOME/byond-zips-cache/linux/$FULL_VERSION/$FULL_VERSION.zip fi @@ -597,9 +597,50 @@ jobs: name: windows-unit-test-coverage-${{ matrix.configuration }} path: ./TestResults/ + prep-edge-versions: + name: Prepare Live Tests Cache of EDGE Versions + needs: start-gate + runs-on: ubuntu-latest + steps: + - name: Cache BYOND .zips (Linux) + uses: actions/cache@v4 + with: + path: ~/byond-zips-cache/live/linux + key: byond-zips-linux-live + + - name: Cache BYOND .zips (Windows) + uses: actions/cache@v4 + with: + path: ~/byond-zips-cache/live/windows + key: byond-zips-windows-live + + - name: Evaluate BYOND version + id: version_evaluation + run: | + VERSIONS=$(curl https://spacestation13.github.io/byond-builds/version.txt) + FULL_VERSION=$(echo "$VERSIONS" | tail -n1) + echo "EDGE version evaluated to $FULL_VERSION" + echo "EVALUATED_VERSION=$FULL_VERSION" >> $GITHUB_OUTPUT + + - name: Setup BYOND Cache if Necessary and Install + run: | + echo "Downloading BYOND." + FULL_VERSION=${{ steps.version_evaluation.outputs.EVALUATED_VERSION }} + BYOND_MAJOR=${FULL_VERSION%.*} + if [[ ! -f $HOME/byond-zips-cache/live/linux/$FULL_VERSION/$FULL_VERSION.zip ]] ; then + mkdir -p $HOME/byond-zips-cache/live/linux/$FULL_VERSION + DOWNLOAD_URL="https://spacestation13.github.io/byond-builds/$BYOND_MAJOR/${FULL_VERSION}_byond_linux.zip" + curl "$DOWNLOAD_URL" -o $HOME/byond-zips-cache/live/linux/$FULL_VERSION/$FULL_VERSION.zip + fi + if [[ ! -f $HOME/byond-zips-cache/live/windows/$FULL_VERSION/$FULL_VERSION.zip ]] ; then + mkdir -p $HOME/byond-zips-cache/live/windows/$FULL_VERSION + DOWNLOAD_URL="https://spacestation13.github.io/byond-builds/$BYOND_MAJOR/${FULL_VERSION}_byond.zip" + curl "$DOWNLOAD_URL" -o $HOME/byond-zips-cache/live/windows/$FULL_VERSION/$FULL_VERSION.zip + fi + windows-integration-tests: name: Windows Live Tests - needs: [dmapi-build, opendream-build] + needs: [dmapi-build, opendream-build, prep-edge-versions] strategy: fail-fast: false matrix: @@ -634,7 +675,6 @@ jobs: - name: Cache BYOND .zips uses: actions/cache@v4 - id: cache-byond with: path: ~/byond-zips-cache/live/windows key: byond-zips-windows-live @@ -815,7 +855,7 @@ jobs: linux-integration-tests: name: Linux Live Tests - needs: [dmapi-build, opendream-build] + needs: [dmapi-build, opendream-build, prep-edge-versions] services: # We start all dbs here so we can just code the stuff once mssql: image: ${{ (matrix.database-type == 'SqlServer') && 'mcr.microsoft.com/mssql/server:2019-latest' || '' }} @@ -892,7 +932,6 @@ jobs: - name: Cache BYOND .zips uses: actions/cache@v4 - id: cache-byond with: path: ~/byond-zips-cache/live/linux key: byond-zips-linux-live From ce35c4e64e500ce3b9700361879d3f3fb76204ac Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 26 Jul 2025 10:27:34 -0400 Subject: [PATCH 32/52] Add libcurl4 to CI --- .github/workflows/ci-pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 9e1c37201e..19794e284a 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -167,7 +167,7 @@ jobs: run: | sudo dpkg --add-architecture i386 sudo apt-get update - sudo apt-get install -y -o APT::Immediate-Configure=0 libc6-i386 libstdc++6:i386 libgcc-s1:i386 + sudo apt-get install -y -o APT::Immediate-Configure=0 libc6-i386 libstdc++6:i386 libgcc-s1:i386 libcurl4:i386 - name: Evaluate BYOND version id: version_evaluation @@ -923,7 +923,7 @@ jobs: run: | sudo dpkg --add-architecture i386 sudo apt-get update - sudo apt-get install -y -o APT::Immediate-Configure=0 libc6-i386 libstdc++6:i386 gdb libgcc-s1:i386 libgdiplus + sudo apt-get install -y -o APT::Immediate-Configure=0 libc6-i386 libstdc++6:i386 gdb libgcc-s1:i386 libgdiplus libcurl4:i386 - name: Setup Node.JS uses: actions/setup-node@v4 From 5026cd31619c009dd39ee1cec78709cd71a2eb84 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 26 Jul 2025 10:47:20 -0400 Subject: [PATCH 33/52] Update nix setup, add flake validation job --- .github/workflows/ci-pipeline.yml | 26 ++++++++++++++++++++++++++ .github/workflows/nix-deployment.yml | 2 +- build/package/nix/flake.nix | 1 + 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 19794e284a..cd6ce47071 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -93,6 +93,31 @@ jobs: name: release_notes_bins path: ./release_notes_bins/ + validate-nix-flake: + name: Validate Nix Flake + needs: start-gate + runs-on: ubuntu-latest + steps: + - name: Setup Nix + uses: cachix/install-nix-action@v31 + with: + nix_path: nixpkgs=channel:nixos-unstable + + - name: Checkout (Branch) + uses: actions/checkout@v4 + if: github.event_name == 'push' || github.event_name == 'schedule' + + - name: Checkout (PR Merge) + uses: actions/checkout@v4 + if: github.event_name != 'push' && github.event_name != 'schedule' + with: + ref: "refs/pull/${{ inputs.pull_request_number }}/merge" + + - name: Check Flake + run: | + cd build/package/nix + nix flake check + code-scanning: name: Run CodeQL needs: start-gate @@ -1695,6 +1720,7 @@ jobs: check-winget-pr-template, efcore-version-match, code-scanning, + validate-nix-flake, ] runs-on: ubuntu-latest steps: diff --git a/.github/workflows/nix-deployment.yml b/.github/workflows/nix-deployment.yml index 16c62fe453..470913346e 100644 --- a/.github/workflows/nix-deployment.yml +++ b/.github/workflows/nix-deployment.yml @@ -15,7 +15,7 @@ jobs: sudo apt-get install -y xmlstarlet - name: Setup Nix - uses: cachix/install-nix-action@v30 + uses: cachix/install-nix-action@v31 with: nix_path: nixpkgs=channel:nixos-unstable diff --git a/build/package/nix/flake.nix b/build/package/nix/flake.nix index f2bf8c14a9..16c7c70a89 100644 --- a/build/package/nix/flake.nix +++ b/build/package/nix/flake.nix @@ -9,5 +9,6 @@ imports = [ ./tgstation-server.nix ]; }; }; + checks.x86_64-linux.flake-build = self.packages.x86_64-linux.default; }; } From e0a96a630cbf3c07c09eb82a092a06036fe05a8a Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 26 Jul 2025 10:50:02 -0400 Subject: [PATCH 34/52] Breaking this temporarily --- .github/workflows/ci-security.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-security.yml b/.github/workflows/ci-security.yml index eb7806f7d1..28efbaf9d5 100644 --- a/.github/workflows/ci-security.yml +++ b/.github/workflows/ci-security.yml @@ -1,4 +1,5 @@ -name: CI Security +na*(@$&UQ!*@)(*$!U)U$! + ASLDKme: CI Security on: pull_request: @@ -21,7 +22,7 @@ concurrency: group: "ci-security-${{ github.head_ref || github.run_id }}-${{ github.event_name }}" cancel-in-progress: true -jobs: +job1231s: security-checkpoint: name: Check CI Clearance if: github.event_name == 'pull_request_target' && (github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id || github.event.pull_request.user.id == 49699333) && github.event.pull_request.state == 'open' From 2c9591a7f4ef378eecd2ee723ebe13f32e062c48 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 26 Jul 2025 11:02:26 -0400 Subject: [PATCH 35/52] Try this --- build/package/nix/flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/package/nix/flake.nix b/build/package/nix/flake.nix index 16c7c70a89..a67c9fcaec 100644 --- a/build/package/nix/flake.nix +++ b/build/package/nix/flake.nix @@ -9,6 +9,6 @@ imports = [ ./tgstation-server.nix ]; }; }; - checks.x86_64-linux.flake-build = self.packages.x86_64-linux.default; + checks.x86_64-linux.package-build = pkgs.callPackage ./package.nix { } }; } From ab8a3f1f5224ea313e279dccc4b7b5bee6b50c6e Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 26 Jul 2025 11:02:43 -0400 Subject: [PATCH 36/52] Add missing semicolon --- build/package/nix/flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/package/nix/flake.nix b/build/package/nix/flake.nix index a67c9fcaec..11bb337751 100644 --- a/build/package/nix/flake.nix +++ b/build/package/nix/flake.nix @@ -9,6 +9,6 @@ imports = [ ./tgstation-server.nix ]; }; }; - checks.x86_64-linux.package-build = pkgs.callPackage ./package.nix { } + checks.x86_64-linux.package-build = pkgs.callPackage ./package.nix { }; }; } From 93c570bd2e3e2e6e200528cae6642ef4dba36229 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 26 Jul 2025 11:03:59 -0400 Subject: [PATCH 37/52] Try with nixpkgs --- build/package/nix/flake.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/package/nix/flake.nix b/build/package/nix/flake.nix index 11bb337751..52ef59cd67 100644 --- a/build/package/nix/flake.nix +++ b/build/package/nix/flake.nix @@ -3,12 +3,12 @@ inputs = {}; - outputs = { ... }: { + outputs = { nixpkgs, ... }: { nixosModules = { default = { ... }: { imports = [ ./tgstation-server.nix ]; }; }; - checks.x86_64-linux.package-build = pkgs.callPackage ./package.nix { }; + checks.x86_64-linux.package-build = nixpkgs.legacyPackages.x86_64-linux.callPackage ./package.nix { }; }; } From cd6619b7d894bde2d7cf5ec595293e29daa60834 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 26 Jul 2025 11:15:06 -0400 Subject: [PATCH 38/52] Should get things where we need them --- .github/workflows/ci-pipeline.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index cd6ce47071..7f342a125b 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -97,6 +97,8 @@ jobs: name: Validate Nix Flake needs: start-gate runs-on: ubuntu-latest + env: + TEST_TGS_VERSION: "6.17.0" # Version we use here doesn't matter as it won't be executed. Just used to download a zip and calc hash steps: - name: Setup Nix uses: cachix/install-nix-action@v31 @@ -113,6 +115,24 @@ jobs: with: ref: "refs/pull/${{ inputs.pull_request_number }}/merge" + - name: Replace current TGS version with test version + run: + CURRENT_TGS_VERSION="$(xmlstarlet sel -N X="http://schemas.microsoft.com/developer/msbuild/2003" --template --value-of /X:Project/X:PropertyGroup/X:TgsCoreVersion build/Version.props)" + sed -i -e "s/${CURRENT_TGS_VERSION}/${{ env.TEST_TGS_VERSION }}" build/Version.props + + - name: Retrieve ServerConsole.zip Artifact + run: | + mkdir release + curl -L https://github.com/tgstation/tgstation-server/releases/download/tgstation-server-v${{ env.TEST_TGS_VERSION }}/ServerConsole.zip -f -o release/ServerConsole.zip + + - name: Regenerate Nix Hash + run: | + nix hash path ./release > build/package/nix/ServerConsole.sha256 + cat build/package/nix/ServerConsole.sha256 + + - name: Cleanup Download + run: rm -rf ./release + - name: Check Flake run: | cd build/package/nix From 9921e26c7c206b416dc085a8b74019bb6ce01a60 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 26 Jul 2025 11:15:14 -0400 Subject: [PATCH 39/52] Revert "Breaking this temporarily" This reverts commit e0a96a630cbf3c07c09eb82a092a06036fe05a8a. --- .github/workflows/ci-security.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-security.yml b/.github/workflows/ci-security.yml index 28efbaf9d5..eb7806f7d1 100644 --- a/.github/workflows/ci-security.yml +++ b/.github/workflows/ci-security.yml @@ -1,5 +1,4 @@ -na*(@$&UQ!*@)(*$!U)U$! - ASLDKme: CI Security +name: CI Security on: pull_request: @@ -22,7 +21,7 @@ concurrency: group: "ci-security-${{ github.head_ref || github.run_id }}-${{ github.event_name }}" cancel-in-progress: true -job1231s: +jobs: security-checkpoint: name: Check CI Clearance if: github.event_name == 'pull_request_target' && (github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id || github.event.pull_request.user.id == 49699333) && github.event.pull_request.state == 'open' From 5ed5d2aafc70533d0273401ec70e9caac2f4b557 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 26 Jul 2025 11:16:52 -0400 Subject: [PATCH 40/52] Add missing `xmlstarlet` dep --- .github/workflows/ci-pipeline.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 7f342a125b..16e237c9d5 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -100,6 +100,11 @@ jobs: env: TEST_TGS_VERSION: "6.17.0" # Version we use here doesn't matter as it won't be executed. Just used to download a zip and calc hash steps: + - name: Install Native Packages # Name checked in rerunFlakyTests.js + run: | + sudo apt-get update + sudo apt-get install -y xmlstarlet + - name: Setup Nix uses: cachix/install-nix-action@v31 with: From bed489af78f9326e0f34cbc81820ef3fee6cc867 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 26 Jul 2025 11:19:17 -0400 Subject: [PATCH 41/52] Add libcurl as a nix package dependency --- build/package/nix/package.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/build/package/nix/package.nix b/build/package/nix/package.nix index 848ebe91f8..23e25072e3 100644 --- a/build/package/nix/package.nix +++ b/build/package/nix/package.nix @@ -86,6 +86,7 @@ stdenv.mkDerivation { gcc_multi glibc bash + libcurl4 ]; nativeBuildInputs = with pkgs; [ makeWrapper From 731b65d9dfcbda0b33ccf1a4fb70efed1865c063 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 26 Jul 2025 11:19:50 -0400 Subject: [PATCH 42/52] Fix sed command --- .github/workflows/ci-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 16e237c9d5..ffc2dc8ff4 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -123,7 +123,7 @@ jobs: - name: Replace current TGS version with test version run: CURRENT_TGS_VERSION="$(xmlstarlet sel -N X="http://schemas.microsoft.com/developer/msbuild/2003" --template --value-of /X:Project/X:PropertyGroup/X:TgsCoreVersion build/Version.props)" - sed -i -e "s/${CURRENT_TGS_VERSION}/${{ env.TEST_TGS_VERSION }}" build/Version.props + sed -i -e "s/${CURRENT_TGS_VERSION}/${{ env.TEST_TGS_VERSION }}/g" build/Version.props - name: Retrieve ServerConsole.zip Artifact run: | From cac2a9d90fd5fea99d9fab15e9c8aa3928ea0aa3 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 26 Jul 2025 11:22:16 -0400 Subject: [PATCH 43/52] Okay, fuck it, just get full curl --- build/package/nix/package.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/package/nix/package.nix b/build/package/nix/package.nix index 23e25072e3..9deeba9fb8 100644 --- a/build/package/nix/package.nix +++ b/build/package/nix/package.nix @@ -86,7 +86,7 @@ stdenv.mkDerivation { gcc_multi glibc bash - libcurl4 + curl ]; nativeBuildInputs = with pkgs; [ makeWrapper From 29b58366a23f462939c3b90ac9490510471227a3 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 26 Jul 2025 12:31:31 -0400 Subject: [PATCH 44/52] Uhh what the hell? --- .github/workflows/ci-pipeline.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index ffc2dc8ff4..b79e92a130 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -124,6 +124,7 @@ jobs: run: CURRENT_TGS_VERSION="$(xmlstarlet sel -N X="http://schemas.microsoft.com/developer/msbuild/2003" --template --value-of /X:Project/X:PropertyGroup/X:TgsCoreVersion build/Version.props)" sed -i -e "s/${CURRENT_TGS_VERSION}/${{ env.TEST_TGS_VERSION }}/g" build/Version.props + cat build/Version.props - name: Retrieve ServerConsole.zip Artifact run: | From dc2656190da74ab3dea9097b536fd8eaaefdee54 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 26 Jul 2025 12:43:36 -0400 Subject: [PATCH 45/52] Excuse me wtf? --- .github/workflows/ci-pipeline.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index b79e92a130..12ea98db29 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -122,6 +122,8 @@ jobs: - name: Replace current TGS version with test version run: + ls -al + ls -al build CURRENT_TGS_VERSION="$(xmlstarlet sel -N X="http://schemas.microsoft.com/developer/msbuild/2003" --template --value-of /X:Project/X:PropertyGroup/X:TgsCoreVersion build/Version.props)" sed -i -e "s/${CURRENT_TGS_VERSION}/${{ env.TEST_TGS_VERSION }}/g" build/Version.props cat build/Version.props From ee7d44a23e2bbd8a87111107283e3ffa7e59eb85 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 26 Jul 2025 14:04:31 -0400 Subject: [PATCH 46/52] Ah --- .github/workflows/ci-pipeline.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 12ea98db29..4651095d0a 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -121,12 +121,9 @@ jobs: ref: "refs/pull/${{ inputs.pull_request_number }}/merge" - name: Replace current TGS version with test version - run: - ls -al - ls -al build + run: | CURRENT_TGS_VERSION="$(xmlstarlet sel -N X="http://schemas.microsoft.com/developer/msbuild/2003" --template --value-of /X:Project/X:PropertyGroup/X:TgsCoreVersion build/Version.props)" sed -i -e "s/${CURRENT_TGS_VERSION}/${{ env.TEST_TGS_VERSION }}/g" build/Version.props - cat build/Version.props - name: Retrieve ServerConsole.zip Artifact run: | From dfb9c199e0a6558d0da4877019aa25594c01aa6f Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 26 Jul 2025 14:12:13 -0400 Subject: [PATCH 47/52] Microsoft has stopped supporting their package repo https://learn.microsoft.com/en-ca/dotnet/core/install/linux-ubuntu-decision#register-the-microsoft-package-repository --- build/package/deb/build_package.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/build/package/deb/build_package.sh b/build/package/deb/build_package.sh index 29d828f1bb..58ee50cdfe 100755 --- a/build/package/deb/build_package.sh +++ b/build/package/deb/build_package.sh @@ -22,11 +22,6 @@ apt-get install -y \ xmlstarlet \ libgdiplus -declare repo_version=$(if command -v lsb_release &> /dev/null; then lsb_release -r -s; else grep -oP '(?<=^VERSION_ID=).+' /etc/os-release | tr -d '"'; fi) -curl -L https://packages.microsoft.com/config/ubuntu/$repo_version/packages-microsoft-prod.deb -o packages-microsoft-prod.deb -dpkg -i ./packages-microsoft-prod.deb -rm packages-microsoft-prod.deb - # https://github.com/nodesource/distributions mkdir -p /etc/apt/keyrings curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg From 1b8994cf2b81b93e5f2780e381c89b6ffe72e02e Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 26 Jul 2025 14:19:58 -0400 Subject: [PATCH 48/52] I'm tired of fighting you BYOND! --- tests/Tgstation.Server.Tests/TestingUtils.cs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/Tgstation.Server.Tests/TestingUtils.cs b/tests/Tgstation.Server.Tests/TestingUtils.cs index 40925352dd..f710141b36 100644 --- a/tests/Tgstation.Server.Tests/TestingUtils.cs +++ b/tests/Tgstation.Server.Tests/TestingUtils.cs @@ -133,13 +133,19 @@ namespace Tgstation.Server.Tests try { // always check byond.com first for latest up-to-date, mirror should ALWAYS have stable versions - const string DefaultMirror = "https://www.byond.com/download/version.txt"; + // except byond hates all CI runners now + const string DefaultMirror = "https://spacestation13.github.io/byond-builds/version.txt"; edgeVersion = await GetVersionFromResponse(DefaultMirror); - logger.LogInformation("Downloading edge version from BYOND.com {edge}", edgeVersion); + logger.LogInformation("Downloading edge version from SS13 mirror {edge}", edgeVersion); // if we got the result from byond.com, make sure the cache grabs the zip from there as well - await CachingFileDownloader.InitializeByondVersion(logger, Version.Parse(edgeVersion), new PlatformIdentifier().IsWindows, cancellationToken, GeneralConfiguration.DefaultByondZipDownloadTemplate); + await CachingFileDownloader.InitializeByondVersion( + logger, + Version.Parse(edgeVersion), + new PlatformIdentifier().IsWindows, + cancellationToken, + "https://spacestation13.github.io/byond-builds/${Major}/${Major}.${Minor}_byond${Linux:_linux}.zip"); } catch (Exception ex) { From 1e0c888da21a8c380a3f3b60286955bed5d728ce Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 26 Jul 2025 19:15:15 -0400 Subject: [PATCH 49/52] Bring all nuget packages up to date and trim the fat --- build/TestCommon.props | 2 +- .../Tgstation.Server.Api.csproj | 2 +- .../Tgstation.Server.Client.GraphQL.csproj | 3 +- .../Tgstation.Server.Client.csproj | 4 +- .../Tgstation.Server.Host.Console.csproj | 4 +- .../Tgstation.Server.Host.Service.csproj | 10 ++--- ...on.Server.Host.Utils.GitLab.GraphQL.csproj | 3 +- .../Tgstation.Server.Host.Watchdog.csproj | 2 +- .../.config/dotnet-tools.json | 2 +- .../Tgstation.Server.Host.csproj | 43 ++++++++++--------- .../Tgstation.Server.Shared.csproj | 2 +- .../Tgstation.Server.Client.Tests.csproj | 2 +- .../Tgstation.Server.Host.Tests.csproj | 3 +- .../Tgstation.Server.Tests.csproj | 4 -- 14 files changed, 42 insertions(+), 44 deletions(-) diff --git a/build/TestCommon.props b/build/TestCommon.props index 44c2a0c2e4..8a6c5d675f 100644 --- a/build/TestCommon.props +++ b/build/TestCommon.props @@ -13,7 +13,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/src/Tgstation.Server.Api/Tgstation.Server.Api.csproj b/src/Tgstation.Server.Api/Tgstation.Server.Api.csproj index 0e83f44424..6f32743090 100644 --- a/src/Tgstation.Server.Api/Tgstation.Server.Api.csproj +++ b/src/Tgstation.Server.Api/Tgstation.Server.Api.csproj @@ -28,7 +28,7 @@ - + diff --git a/src/Tgstation.Server.Client.GraphQL/Tgstation.Server.Client.GraphQL.csproj b/src/Tgstation.Server.Client.GraphQL/Tgstation.Server.Client.GraphQL.csproj index 9a575c625f..67e9d74754 100644 --- a/src/Tgstation.Server.Client.GraphQL/Tgstation.Server.Client.GraphQL.csproj +++ b/src/Tgstation.Server.Client.GraphQL/Tgstation.Server.Client.GraphQL.csproj @@ -9,8 +9,7 @@ - - + diff --git a/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj b/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj index 83d41dfff1..f681ecff8f 100644 --- a/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj +++ b/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj @@ -11,9 +11,9 @@ - + - + diff --git a/src/Tgstation.Server.Host.Console/Tgstation.Server.Host.Console.csproj b/src/Tgstation.Server.Host.Console/Tgstation.Server.Host.Console.csproj index 4942cfd2ff..75f657b562 100644 --- a/src/Tgstation.Server.Host.Console/Tgstation.Server.Host.Console.csproj +++ b/src/Tgstation.Server.Host.Console/Tgstation.Server.Host.Console.csproj @@ -13,9 +13,9 @@ - + - + diff --git a/src/Tgstation.Server.Host.Service/Tgstation.Server.Host.Service.csproj b/src/Tgstation.Server.Host.Service/Tgstation.Server.Host.Service.csproj index 6a31f318db..59ab045d5a 100644 --- a/src/Tgstation.Server.Host.Service/Tgstation.Server.Host.Service.csproj +++ b/src/Tgstation.Server.Host.Service/Tgstation.Server.Host.Service.csproj @@ -21,21 +21,21 @@ - + - + - + - + - + diff --git a/src/Tgstation.Server.Host.Utils.GitLab.GraphQL/Tgstation.Server.Host.Utils.GitLab.GraphQL.csproj b/src/Tgstation.Server.Host.Utils.GitLab.GraphQL/Tgstation.Server.Host.Utils.GitLab.GraphQL.csproj index 41df3014e0..ab2f3f9dbc 100644 --- a/src/Tgstation.Server.Host.Utils.GitLab.GraphQL/Tgstation.Server.Host.Utils.GitLab.GraphQL.csproj +++ b/src/Tgstation.Server.Host.Utils.GitLab.GraphQL/Tgstation.Server.Host.Utils.GitLab.GraphQL.csproj @@ -38,7 +38,8 @@ - + + diff --git a/src/Tgstation.Server.Host.Watchdog/Tgstation.Server.Host.Watchdog.csproj b/src/Tgstation.Server.Host.Watchdog/Tgstation.Server.Host.Watchdog.csproj index af47f499f3..5f31ab855f 100644 --- a/src/Tgstation.Server.Host.Watchdog/Tgstation.Server.Host.Watchdog.csproj +++ b/src/Tgstation.Server.Host.Watchdog/Tgstation.Server.Host.Watchdog.csproj @@ -10,7 +10,7 @@ - + diff --git a/src/Tgstation.Server.Host/.config/dotnet-tools.json b/src/Tgstation.Server.Host/.config/dotnet-tools.json index 76e6bce7b3..7cf60e92f7 100644 --- a/src/Tgstation.Server.Host/.config/dotnet-tools.json +++ b/src/Tgstation.Server.Host/.config/dotnet-tools.json @@ -3,7 +3,7 @@ "isRoot": true, "tools": { "dotnet-ef": { - "version": "9.0.4", + "version": "9.0.7", "commands": [ "dotnet-ef" ] diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index 32fd89607a..91dcae4c65 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -96,41 +96,41 @@ - + - + - + - + - + - + - + - + - + - + - + runtime; build; native; contentfiles; analyzers; buildtransitive - + - + - + - + @@ -144,9 +144,9 @@ - + - + @@ -156,14 +156,15 @@ - + - + - + + - + diff --git a/src/Tgstation.Server.Shared/Tgstation.Server.Shared.csproj b/src/Tgstation.Server.Shared/Tgstation.Server.Shared.csproj index 861618a8a9..e636c80a73 100644 --- a/src/Tgstation.Server.Shared/Tgstation.Server.Shared.csproj +++ b/src/Tgstation.Server.Shared/Tgstation.Server.Shared.csproj @@ -10,7 +10,7 @@ - + diff --git a/tests/Tgstation.Server.Client.Tests/Tgstation.Server.Client.Tests.csproj b/tests/Tgstation.Server.Client.Tests/Tgstation.Server.Client.Tests.csproj index 725b7c8bcd..66888f4430 100644 --- a/tests/Tgstation.Server.Client.Tests/Tgstation.Server.Client.Tests.csproj +++ b/tests/Tgstation.Server.Client.Tests/Tgstation.Server.Client.Tests.csproj @@ -6,7 +6,7 @@ - + diff --git a/tests/Tgstation.Server.Host.Tests/Tgstation.Server.Host.Tests.csproj b/tests/Tgstation.Server.Host.Tests/Tgstation.Server.Host.Tests.csproj index f696ba76b7..2b96d6fd6a 100644 --- a/tests/Tgstation.Server.Host.Tests/Tgstation.Server.Host.Tests.csproj +++ b/tests/Tgstation.Server.Host.Tests/Tgstation.Server.Host.Tests.csproj @@ -7,7 +7,8 @@ - + + diff --git a/tests/Tgstation.Server.Tests/Tgstation.Server.Tests.csproj b/tests/Tgstation.Server.Tests/Tgstation.Server.Tests.csproj index 8b87acdc61..b2e176f1e3 100644 --- a/tests/Tgstation.Server.Tests/Tgstation.Server.Tests.csproj +++ b/tests/Tgstation.Server.Tests/Tgstation.Server.Tests.csproj @@ -5,10 +5,6 @@ $(TgsFrameworkVersion) - - - - From fd050f068d391cf66716fb6d512c94a80f664362 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 26 Jul 2025 19:19:11 -0400 Subject: [PATCH 50/52] Update wix version --- build/package/winget/.config/dotnet-tools.json | 2 +- .../Tgstation.Server.Host.Service.Wix.Bundle.wixproj | 6 +++--- .../Tgstation.Server.Host.Service.Wix.Extensions.csproj | 2 +- .../Tgstation.Server.Host.Service.Wix.wixproj | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/build/package/winget/.config/dotnet-tools.json b/build/package/winget/.config/dotnet-tools.json index 588fcd1095..998101e103 100644 --- a/build/package/winget/.config/dotnet-tools.json +++ b/build/package/winget/.config/dotnet-tools.json @@ -3,7 +3,7 @@ "isRoot": true, "tools": { "wix": { - "version": "5.0.2", + "version": "6.0.1", "commands": [ "wix" ] diff --git a/build/package/winget/Tgstation.Server.Host.Service.Wix.Bundle/Tgstation.Server.Host.Service.Wix.Bundle.wixproj b/build/package/winget/Tgstation.Server.Host.Service.Wix.Bundle/Tgstation.Server.Host.Service.Wix.Bundle.wixproj index df2363b5ac..42304195a2 100644 --- a/build/package/winget/Tgstation.Server.Host.Service.Wix.Bundle/Tgstation.Server.Host.Service.Wix.Bundle.wixproj +++ b/build/package/winget/Tgstation.Server.Host.Service.Wix.Bundle/Tgstation.Server.Host.Service.Wix.Bundle.wixproj @@ -1,4 +1,4 @@ - + ProductVersion=$(TgsCoreVersion);NetMajorVersion=$(TgsNetMajorVersion);DotnetRedistUrl=$(TgsDotnetRedistUrl);MariaDBRedistUrl=https://github.com/tgstation/tgstation-server/releases/download/tgstation-server-v$(TgsCoreVersion)/mariadb-$(TgsMariaDBRedistVersion)-winx64.msi @@ -24,8 +24,8 @@ - - + + diff --git a/build/package/winget/Tgstation.Server.Host.Service.Wix.Extensions/Tgstation.Server.Host.Service.Wix.Extensions.csproj b/build/package/winget/Tgstation.Server.Host.Service.Wix.Extensions/Tgstation.Server.Host.Service.Wix.Extensions.csproj index fcdbd512a6..1e53056e15 100644 --- a/build/package/winget/Tgstation.Server.Host.Service.Wix.Extensions/Tgstation.Server.Host.Service.Wix.Extensions.csproj +++ b/build/package/winget/Tgstation.Server.Host.Service.Wix.Extensions/Tgstation.Server.Host.Service.Wix.Extensions.csproj @@ -7,7 +7,7 @@ - + diff --git a/build/package/winget/Tgstation.Server.Host.Service.Wix/Tgstation.Server.Host.Service.Wix.wixproj b/build/package/winget/Tgstation.Server.Host.Service.Wix/Tgstation.Server.Host.Service.Wix.wixproj index 272694a58e..8e32edfa4b 100644 --- a/build/package/winget/Tgstation.Server.Host.Service.Wix/Tgstation.Server.Host.Service.Wix.wixproj +++ b/build/package/winget/Tgstation.Server.Host.Service.Wix/Tgstation.Server.Host.Service.Wix.wixproj @@ -1,4 +1,4 @@ - + ProductVersion=$(TgsCoreVersion) @@ -20,8 +20,8 @@ - - + + From 5a37c36c6588bb3a0533108acf89bd200017930e Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 26 Jul 2025 19:27:39 -0400 Subject: [PATCH 51/52] Revert "Update wix version" This reverts commit fd050f068d391cf66716fb6d512c94a80f664362. --- build/package/winget/.config/dotnet-tools.json | 2 +- .../Tgstation.Server.Host.Service.Wix.Bundle.wixproj | 6 +++--- .../Tgstation.Server.Host.Service.Wix.Extensions.csproj | 2 +- .../Tgstation.Server.Host.Service.Wix.wixproj | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/build/package/winget/.config/dotnet-tools.json b/build/package/winget/.config/dotnet-tools.json index 998101e103..588fcd1095 100644 --- a/build/package/winget/.config/dotnet-tools.json +++ b/build/package/winget/.config/dotnet-tools.json @@ -3,7 +3,7 @@ "isRoot": true, "tools": { "wix": { - "version": "6.0.1", + "version": "5.0.2", "commands": [ "wix" ] diff --git a/build/package/winget/Tgstation.Server.Host.Service.Wix.Bundle/Tgstation.Server.Host.Service.Wix.Bundle.wixproj b/build/package/winget/Tgstation.Server.Host.Service.Wix.Bundle/Tgstation.Server.Host.Service.Wix.Bundle.wixproj index 42304195a2..df2363b5ac 100644 --- a/build/package/winget/Tgstation.Server.Host.Service.Wix.Bundle/Tgstation.Server.Host.Service.Wix.Bundle.wixproj +++ b/build/package/winget/Tgstation.Server.Host.Service.Wix.Bundle/Tgstation.Server.Host.Service.Wix.Bundle.wixproj @@ -1,4 +1,4 @@ - + ProductVersion=$(TgsCoreVersion);NetMajorVersion=$(TgsNetMajorVersion);DotnetRedistUrl=$(TgsDotnetRedistUrl);MariaDBRedistUrl=https://github.com/tgstation/tgstation-server/releases/download/tgstation-server-v$(TgsCoreVersion)/mariadb-$(TgsMariaDBRedistVersion)-winx64.msi @@ -24,8 +24,8 @@ - - + + diff --git a/build/package/winget/Tgstation.Server.Host.Service.Wix.Extensions/Tgstation.Server.Host.Service.Wix.Extensions.csproj b/build/package/winget/Tgstation.Server.Host.Service.Wix.Extensions/Tgstation.Server.Host.Service.Wix.Extensions.csproj index 1e53056e15..fcdbd512a6 100644 --- a/build/package/winget/Tgstation.Server.Host.Service.Wix.Extensions/Tgstation.Server.Host.Service.Wix.Extensions.csproj +++ b/build/package/winget/Tgstation.Server.Host.Service.Wix.Extensions/Tgstation.Server.Host.Service.Wix.Extensions.csproj @@ -7,7 +7,7 @@ - + diff --git a/build/package/winget/Tgstation.Server.Host.Service.Wix/Tgstation.Server.Host.Service.Wix.wixproj b/build/package/winget/Tgstation.Server.Host.Service.Wix/Tgstation.Server.Host.Service.Wix.wixproj index 8e32edfa4b..272694a58e 100644 --- a/build/package/winget/Tgstation.Server.Host.Service.Wix/Tgstation.Server.Host.Service.Wix.wixproj +++ b/build/package/winget/Tgstation.Server.Host.Service.Wix/Tgstation.Server.Host.Service.Wix.wixproj @@ -1,4 +1,4 @@ - + ProductVersion=$(TgsCoreVersion) @@ -20,8 +20,8 @@ - - + + From 0d4d68d7362d81a8d065045e048346dd1c207c22 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sat, 26 Jul 2025 20:08:31 -0400 Subject: [PATCH 52/52] Fix `DefaultIOManager` bugs - Fix `CreateResolverForSubdirectory` not considering if the input path was rooted. - Fix `DeleteDirectory` not resolving the input path. --- src/Tgstation.Server.Host/IO/DefaultIOManager.cs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/Tgstation.Server.Host/IO/DefaultIOManager.cs b/src/Tgstation.Server.Host/IO/DefaultIOManager.cs index 470f437d7f..eaf2b27736 100644 --- a/src/Tgstation.Server.Host/IO/DefaultIOManager.cs +++ b/src/Tgstation.Server.Host/IO/DefaultIOManager.cs @@ -162,7 +162,8 @@ namespace Tgstation.Server.Host.IO => Task.Factory.StartNew( () => { - var di = fileSystem.DirectoryInfo.New(path); + var di = fileSystem.DirectoryInfo.New( + ResolvePath(path)); if (di.Exists) NormalizeAndDelete(di, cancellationToken); }, @@ -416,11 +417,14 @@ namespace Tgstation.Server.Host.IO { ArgumentNullException.ThrowIfNull(subdirectoryPath); + if (!Path.IsPathRooted(subdirectoryPath)) + subdirectoryPath = ConcatPath( + ResolvePath(), + subdirectoryPath); + return new ResolvingIOManager( fileSystem, - ConcatPath( - ResolvePath(), - subdirectoryPath)); + subdirectoryPath); } ///