From 18d399ce3ad63be9c1a0271bff41d50dd5f22de5 Mon Sep 17 00:00:00 2001 From: Dominion Date: Sat, 3 Jun 2023 23:26:33 -0400 Subject: [PATCH 01/41] Lock down some JobManager operations --- .../Components/InstanceManager.cs | 18 +++++++++--------- src/Tgstation.Server.Host/Core/Application.cs | 3 ++- src/Tgstation.Server.Host/Jobs/IJobManager.cs | 10 +--------- src/Tgstation.Server.Host/Jobs/IJobService.cs | 18 ++++++++++++++++++ .../Jobs/{JobManager.cs => JobService.cs} | 16 ++++++++-------- 5 files changed, 38 insertions(+), 27 deletions(-) create mode 100644 src/Tgstation.Server.Host/Jobs/IJobService.cs rename src/Tgstation.Server.Host/Jobs/{JobManager.cs => JobService.cs} (96%) diff --git a/src/Tgstation.Server.Host/Components/InstanceManager.cs b/src/Tgstation.Server.Host/Components/InstanceManager.cs index ad3ee3de51..a0c7f8ceec 100644 --- a/src/Tgstation.Server.Host/Components/InstanceManager.cs +++ b/src/Tgstation.Server.Host/Components/InstanceManager.cs @@ -58,9 +58,9 @@ namespace Tgstation.Server.Host.Components readonly IAssemblyInformationProvider assemblyInformationProvider; /// - /// The for the . + /// The for the . /// - readonly IJobManager jobManager; + readonly IJobService jobService; /// /// The for the . @@ -149,7 +149,7 @@ namespace Tgstation.Server.Host.Components /// The value of . /// The value of . /// The value of . - /// The value of . + /// The value of . /// The value of . /// The value of . /// The value of . @@ -163,7 +163,7 @@ namespace Tgstation.Server.Host.Components IIOManager ioManager, IDatabaseContextFactory databaseContextFactory, IAssemblyInformationProvider assemblyInformationProvider, - IJobManager jobManager, + IJobService jobService, IServerControl serverControl, ISystemIdentityFactory systemIdentityFactory, IAsyncDelayer asyncDelayer, @@ -177,7 +177,7 @@ namespace Tgstation.Server.Host.Components this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); this.databaseContextFactory = databaseContextFactory ?? throw new ArgumentNullException(nameof(databaseContextFactory)); this.assemblyInformationProvider = assemblyInformationProvider ?? throw new ArgumentNullException(nameof(assemblyInformationProvider)); - this.jobManager = jobManager ?? throw new ArgumentNullException(nameof(jobManager)); + this.jobService = jobService ?? throw new ArgumentNullException(nameof(jobService)); this.serverControl = serverControl ?? throw new ArgumentNullException(nameof(serverControl)); this.systemIdentityFactory = systemIdentityFactory ?? throw new ArgumentNullException(nameof(systemIdentityFactory)); this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer)); @@ -344,7 +344,7 @@ namespace Tgstation.Server.Host.Components }) .ToListAsync(cancellationToken); foreach (var job in jobs) - tasks.Add(jobManager.CancelJob(job, user, true, cancellationToken)); + tasks.Add(jobService.CancelJob(job, user, true, cancellationToken)); }); await Task.WhenAll(tasks); @@ -444,7 +444,7 @@ namespace Tgstation.Server.Host.Components } var instanceFactoryStopTask = instanceFactory.StopAsync(cancellationToken); - await jobManager.StopAsync(cancellationToken); + await jobService.StopAsync(cancellationToken); async Task OfflineInstanceImmediate(IInstance instance, CancellationToken cancellationToken) { @@ -563,7 +563,7 @@ namespace Tgstation.Server.Host.Components .ToListAsync(cancellationToken)); var factoryStartup = instanceFactory.StartAsync(cancellationToken); - var jobManagerStartup = jobManager.StartAsync(cancellationToken); + var jobManagerStartup = jobService.StartAsync(cancellationToken); await Task.WhenAll(instanceEnumeration, factoryStartup, jobManagerStartup); @@ -582,7 +582,7 @@ namespace Tgstation.Server.Host.Components await Task.WhenAll(instanceOnliningTasks); - jobManager.Activate(this); + jobService.Activate(this); logger.LogInformation("Server ready!"); readyTcs.SetResult(); diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index fa5897a9e5..552ab73290 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -373,7 +373,8 @@ namespace Tgstation.Server.Host.Core services.AddSingleton(); // configure root services - services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(x => x.GetRequiredService()); services.AddSingleton(); services.AddSingleton(x => x.GetRequiredService()); diff --git a/src/Tgstation.Server.Host/Jobs/IJobManager.cs b/src/Tgstation.Server.Host/Jobs/IJobManager.cs index 30a0a45cc7..f02f765994 100644 --- a/src/Tgstation.Server.Host/Jobs/IJobManager.cs +++ b/src/Tgstation.Server.Host/Jobs/IJobManager.cs @@ -1,9 +1,7 @@ using System.Threading; using System.Threading.Tasks; -using Microsoft.Extensions.Hosting; using Tgstation.Server.Api.Models.Response; -using Tgstation.Server.Host.Components; using Tgstation.Server.Host.Models; namespace Tgstation.Server.Host.Jobs @@ -11,7 +9,7 @@ namespace Tgstation.Server.Host.Jobs /// /// Manages the runtime of s. /// - public interface IJobManager : IHostedService + public interface IJobManager { /// /// Set the and for a given . @@ -47,11 +45,5 @@ namespace Tgstation.Server.Host.Jobs /// The for the operation. /// A resulting in the updated if it was cancelled, if it couldn't be found. Task CancelJob(Job job, User user, bool blocking, CancellationToken cancellationToken); - - /// - /// Activate the . - /// - /// The for the . - void Activate(IInstanceCoreProvider instanceCoreProvider); } } diff --git a/src/Tgstation.Server.Host/Jobs/IJobService.cs b/src/Tgstation.Server.Host/Jobs/IJobService.cs new file mode 100644 index 0000000000..982a69f79b --- /dev/null +++ b/src/Tgstation.Server.Host/Jobs/IJobService.cs @@ -0,0 +1,18 @@ +using Microsoft.Extensions.Hosting; + +using Tgstation.Server.Host.Components; + +namespace Tgstation.Server.Host.Jobs +{ + /// + /// The service that manages everything to do with jobs. + /// + public interface IJobService : IJobManager, IHostedService + { + /// + /// Activate the . + /// + /// The for the . + void Activate(IInstanceCoreProvider instanceCoreProvider); + } +} diff --git a/src/Tgstation.Server.Host/Jobs/JobManager.cs b/src/Tgstation.Server.Host/Jobs/JobService.cs similarity index 96% rename from src/Tgstation.Server.Host/Jobs/JobManager.cs rename to src/Tgstation.Server.Host/Jobs/JobService.cs index cac6b4fab3..c0d01f2dcf 100644 --- a/src/Tgstation.Server.Host/Jobs/JobManager.cs +++ b/src/Tgstation.Server.Host/Jobs/JobService.cs @@ -17,22 +17,22 @@ using Tgstation.Server.Host.Utils; namespace Tgstation.Server.Host.Jobs { /// - sealed class JobManager : IJobManager, IDisposable + sealed class JobService : IJobService, IDisposable { /// - /// The for the . + /// The for the . /// readonly IDatabaseContextFactory databaseContextFactory; /// - /// The for the . + /// The for the . /// readonly ILoggerFactory loggerFactory; /// - /// The for the . + /// The for the . /// - readonly ILogger logger; + readonly ILogger logger; /// /// of s to running s. @@ -60,15 +60,15 @@ namespace Tgstation.Server.Host.Jobs volatile bool noMoreJobsShouldStart; /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// /// The value of . /// The value of . /// The value of . - public JobManager( + public JobService( IDatabaseContextFactory databaseContextFactory, ILoggerFactory loggerFactory, - ILogger logger) + ILogger logger) { this.databaseContextFactory = databaseContextFactory ?? throw new ArgumentNullException(nameof(databaseContextFactory)); this.loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory)); From 0fb61510b66d21f0b90f91c491947ac9aff1a8fb Mon Sep 17 00:00:00 2001 From: Dominion Date: Sat, 3 Jun 2023 23:35:45 -0400 Subject: [PATCH 02/41] Differentiate between component services and `IHostedService`s --- .../Components/Byond/IByondManager.cs | 4 +--- .../Components/Chat/IChatManager.cs | 4 +--- .../Components/Deployment/DmbFactory.cs | 2 +- .../Components/Deployment/IDmbFactory.cs | 4 +--- .../Components/IComponentService.cs | 11 +++++++++++ src/Tgstation.Server.Host/Components/IInstance.cs | 4 +--- .../Components/IInstanceFactory.cs | 4 +--- .../Components/StaticFiles/IConfiguration.cs | 4 +--- .../Components/Watchdog/IWatchdog.cs | 4 +--- src/Tgstation.Server.Host/Jobs/IJobService.cs | 6 ++---- 10 files changed, 21 insertions(+), 26 deletions(-) create mode 100644 src/Tgstation.Server.Host/Components/IComponentService.cs diff --git a/src/Tgstation.Server.Host/Components/Byond/IByondManager.cs b/src/Tgstation.Server.Host/Components/Byond/IByondManager.cs index 10222483ec..58ab6d7480 100644 --- a/src/Tgstation.Server.Host/Components/Byond/IByondManager.cs +++ b/src/Tgstation.Server.Host/Components/Byond/IByondManager.cs @@ -4,8 +4,6 @@ using System.IO; using System.Threading; using System.Threading.Tasks; -using Microsoft.Extensions.Hosting; - using Tgstation.Server.Host.Jobs; namespace Tgstation.Server.Host.Components.Byond @@ -14,7 +12,7 @@ namespace Tgstation.Server.Host.Components.Byond /// For managing the BYOND installation. /// /// When passing in s, ensure they are BYOND format versions unless referring to a custom version. This means should NEVER be 0. - public interface IByondManager : IHostedService, IDisposable + public interface IByondManager : IComponentService, IDisposable { /// /// The currently active BYOND version. diff --git a/src/Tgstation.Server.Host/Components/Chat/IChatManager.cs b/src/Tgstation.Server.Host/Components/Chat/IChatManager.cs index 59f3e2b1c5..7004bfc592 100644 --- a/src/Tgstation.Server.Host/Components/Chat/IChatManager.cs +++ b/src/Tgstation.Server.Host/Components/Chat/IChatManager.cs @@ -3,8 +3,6 @@ using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; -using Microsoft.Extensions.Hosting; - using Tgstation.Server.Api.Models.Internal; using Tgstation.Server.Host.Components.Interop; @@ -13,7 +11,7 @@ namespace Tgstation.Server.Host.Components.Chat /// /// For managing connected chat services. /// - public interface IChatManager : IHostedService, IAsyncDisposable + public interface IChatManager : IComponentService, IAsyncDisposable { /// /// Registers a to use. diff --git a/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs b/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs index f57222c8b2..7d57085eff 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs @@ -90,7 +90,7 @@ namespace Tgstation.Server.Host.Components.Deployment IDmbProvider nextDmbProvider; /// - /// If the is "started" via . + /// If the is "started" via . /// bool started; diff --git a/src/Tgstation.Server.Host/Components/Deployment/IDmbFactory.cs b/src/Tgstation.Server.Host/Components/Deployment/IDmbFactory.cs index 4c4872a61c..0d10b7dcd1 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/IDmbFactory.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/IDmbFactory.cs @@ -2,8 +2,6 @@ using System.Threading; using System.Threading.Tasks; -using Microsoft.Extensions.Hosting; - using Tgstation.Server.Host.Models; namespace Tgstation.Server.Host.Components.Deployment @@ -11,7 +9,7 @@ namespace Tgstation.Server.Host.Components.Deployment /// /// Factory for s. /// - public interface IDmbFactory : ILatestCompileJobProvider, IHostedService, IDisposable + public interface IDmbFactory : ILatestCompileJobProvider, IComponentService, IDisposable { /// /// Get a that completes when the result of a call to will be different than the previous call if any. diff --git a/src/Tgstation.Server.Host/Components/IComponentService.cs b/src/Tgstation.Server.Host/Components/IComponentService.cs new file mode 100644 index 0000000000..b094e0a092 --- /dev/null +++ b/src/Tgstation.Server.Host/Components/IComponentService.cs @@ -0,0 +1,11 @@ +using Microsoft.Extensions.Hosting; + +namespace Tgstation.Server.Host.Components +{ + /// + /// Represents a component meant to be started and stopped by its parent component. + /// + public interface IComponentService : IHostedService + { + } +} diff --git a/src/Tgstation.Server.Host/Components/IInstance.cs b/src/Tgstation.Server.Host/Components/IInstance.cs index 1d26b9382d..9d8c34988e 100644 --- a/src/Tgstation.Server.Host/Components/IInstance.cs +++ b/src/Tgstation.Server.Host/Components/IInstance.cs @@ -1,13 +1,11 @@ using System; -using Microsoft.Extensions.Hosting; - namespace Tgstation.Server.Host.Components { /// /// Component version of . /// - interface IInstance : IInstanceCore, IHostedService, IAsyncDisposable + interface IInstance : IInstanceCore, IComponentService, IAsyncDisposable { } } diff --git a/src/Tgstation.Server.Host/Components/IInstanceFactory.cs b/src/Tgstation.Server.Host/Components/IInstanceFactory.cs index 0c37b40778..fda8c9afad 100644 --- a/src/Tgstation.Server.Host/Components/IInstanceFactory.cs +++ b/src/Tgstation.Server.Host/Components/IInstanceFactory.cs @@ -1,7 +1,5 @@ using System.Threading.Tasks; -using Microsoft.Extensions.Hosting; - using Tgstation.Server.Host.Components.Interop.Bridge; using Tgstation.Server.Host.IO; @@ -10,7 +8,7 @@ namespace Tgstation.Server.Host.Components /// /// Factory for creating s. /// - interface IInstanceFactory : IHostedService + interface IInstanceFactory : IComponentService { /// /// Create an . diff --git a/src/Tgstation.Server.Host/Components/StaticFiles/IConfiguration.cs b/src/Tgstation.Server.Host/Components/StaticFiles/IConfiguration.cs index 2758ba15fc..bc8b2be171 100644 --- a/src/Tgstation.Server.Host/Components/StaticFiles/IConfiguration.cs +++ b/src/Tgstation.Server.Host/Components/StaticFiles/IConfiguration.cs @@ -3,8 +3,6 @@ using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; -using Microsoft.Extensions.Hosting; - using Tgstation.Server.Api.Models; using Tgstation.Server.Api.Models.Response; using Tgstation.Server.Host.Components.Events; @@ -15,7 +13,7 @@ namespace Tgstation.Server.Host.Components.StaticFiles /// /// For managing the Configuration directory. /// - public interface IConfiguration : IHostedService, IEventConsumer, IDisposable + public interface IConfiguration : IComponentService, IEventConsumer, IDisposable { /// /// Copies all files in the CodeModifications directory to . diff --git a/src/Tgstation.Server.Host/Components/Watchdog/IWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/IWatchdog.cs index 68bebfb81c..1e908eb85f 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/IWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/IWatchdog.cs @@ -2,8 +2,6 @@ using System.Threading; using System.Threading.Tasks; -using Microsoft.Extensions.Hosting; - using Tgstation.Server.Api.Models; using Tgstation.Server.Api.Models.Internal; using Tgstation.Server.Host.Components.Events; @@ -14,7 +12,7 @@ namespace Tgstation.Server.Host.Components.Watchdog /// /// Runs and monitors the twin server controllers. /// - public interface IWatchdog : IHostedService, IAsyncDisposable, IEventConsumer, IRenameNotifyee + public interface IWatchdog : IComponentService, IAsyncDisposable, IEventConsumer, IRenameNotifyee { /// /// The current . diff --git a/src/Tgstation.Server.Host/Jobs/IJobService.cs b/src/Tgstation.Server.Host/Jobs/IJobService.cs index 982a69f79b..e92ead5e6a 100644 --- a/src/Tgstation.Server.Host/Jobs/IJobService.cs +++ b/src/Tgstation.Server.Host/Jobs/IJobService.cs @@ -1,13 +1,11 @@ -using Microsoft.Extensions.Hosting; - -using Tgstation.Server.Host.Components; +using Tgstation.Server.Host.Components; namespace Tgstation.Server.Host.Jobs { /// /// The service that manages everything to do with jobs. /// - public interface IJobService : IJobManager, IHostedService + public interface IJobService : IJobManager, IComponentService { /// /// Activate the . From fe7d91a06b70cef515a35678c5bc5839458f7fc9 Mon Sep 17 00:00:00 2001 From: Dominion Date: Sun, 4 Jun 2023 00:33:05 -0400 Subject: [PATCH 03/41] GitHubService and factory + AdministrationController replacement --- .../Controllers/AdministrationController.cs | 46 ++++------ src/Tgstation.Server.Host/Core/Application.cs | 3 + .../Utils/GitHubClientFactory.cs | 4 +- .../Utils/GitHubService.cs | 89 +++++++++++++++++++ .../Utils/GitHubServiceFactory.cs | 66 ++++++++++++++ .../Utils/IGitHubService.cs | 29 ++++++ .../Utils/IGitHubServiceFactory.cs | 21 +++++ .../Utils/TestGitHubServiceFactory.cs | 57 ++++++++++++ 8 files changed, 283 insertions(+), 32 deletions(-) create mode 100644 src/Tgstation.Server.Host/Utils/GitHubService.cs create mode 100644 src/Tgstation.Server.Host/Utils/GitHubServiceFactory.cs create mode 100644 src/Tgstation.Server.Host/Utils/IGitHubService.cs create mode 100644 src/Tgstation.Server.Host/Utils/IGitHubServiceFactory.cs create mode 100644 tests/Tgstation.Server.Host.Tests/Utils/TestGitHubServiceFactory.cs diff --git a/src/Tgstation.Server.Host/Controllers/AdministrationController.cs b/src/Tgstation.Server.Host/Controllers/AdministrationController.cs index b8e50b458c..63ef457aee 100644 --- a/src/Tgstation.Server.Host/Controllers/AdministrationController.cs +++ b/src/Tgstation.Server.Host/Controllers/AdministrationController.cs @@ -19,7 +19,6 @@ using Tgstation.Server.Api.Rights; using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.Core; using Tgstation.Server.Host.Database; -using Tgstation.Server.Host.Extensions; using Tgstation.Server.Host.IO; using Tgstation.Server.Host.Security; using Tgstation.Server.Host.System; @@ -40,9 +39,9 @@ namespace Tgstation.Server.Host.Controllers const string OctokitException = "Bad GitHub API response, check configuration!"; /// - /// The for the . + /// The for the . /// - readonly IGitHubClientFactory gitHubClientFactory; + readonly IGitHubService gitHubService; /// /// The for the . @@ -74,11 +73,6 @@ namespace Tgstation.Server.Host.Controllers /// readonly IFileTransferTicketProvider fileTransferService; - /// - /// The for the . - /// - readonly UpdatesConfiguration updatesConfiguration; - /// /// The for the . /// @@ -89,7 +83,7 @@ namespace Tgstation.Server.Host.Controllers /// /// The for the . /// The for the . - /// The value of . + /// The value of . /// The value of . /// The value of . /// The value of . @@ -97,12 +91,11 @@ namespace Tgstation.Server.Host.Controllers /// The value of . /// The value of . /// The for the . - /// The containing value of . /// The containing value of . public AdministrationController( IDatabaseContext databaseContext, IAuthenticationContextFactory authenticationContextFactory, - IGitHubClientFactory gitHubClientFactory, + IGitHubService gitHubService, IServerControl serverControl, IServerUpdateInitiator serverUpdateInitiator, IAssemblyInformationProvider assemblyInformationProvider, @@ -110,7 +103,6 @@ namespace Tgstation.Server.Host.Controllers IPlatformIdentifier platformIdentifier, IFileTransferTicketProvider fileTransferService, ILogger logger, - IOptions updatesConfigurationOptions, IOptions fileLoggingConfigurationOptions) : base( databaseContext, @@ -118,14 +110,13 @@ namespace Tgstation.Server.Host.Controllers logger, true) { - this.gitHubClientFactory = gitHubClientFactory ?? throw new ArgumentNullException(nameof(gitHubClientFactory)); + this.gitHubService = gitHubService ?? throw new ArgumentNullException(nameof(gitHubService)); this.serverControl = serverControl ?? throw new ArgumentNullException(nameof(serverControl)); this.serverUpdateInitiator = serverUpdateInitiator ?? throw new ArgumentNullException(nameof(serverUpdateInitiator)); this.assemblyInformationProvider = assemblyInformationProvider ?? throw new ArgumentNullException(nameof(assemblyInformationProvider)); this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); this.platformIdentifier = platformIdentifier ?? throw new ArgumentNullException(nameof(platformIdentifier)); this.fileTransferService = fileTransferService ?? throw new ArgumentNullException(nameof(fileTransferService)); - updatesConfiguration = updatesConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(updatesConfigurationOptions)); fileLoggingConfiguration = fileLoggingConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(fileLoggingConfigurationOptions)); } @@ -150,26 +141,19 @@ namespace Tgstation.Server.Host.Controllers Uri repoUrl = null; try { - var gitHubClient = gitHubClientFactory.CreateClient(); - var repositoryTask = gitHubClient - .Repository - .Get(updatesConfiguration.GitHubRepositoryId) - .WithToken(cancellationToken); - var releases = (await gitHubClient - .Repository - .Release - .GetAll(updatesConfiguration.GitHubRepositoryId) - .WithToken(cancellationToken)) - .Where(x => x.TagName.StartsWith( - updatesConfiguration.GitTagPrefix, - StringComparison.InvariantCulture)); + var repositoryUrlTask = gitHubService.GetUpdatesRepositoryUrl(cancellationToken); + var releases = await gitHubService.GetTgsReleases(cancellationToken); - foreach (var release in releases) - if (Version.TryParse(release.TagName.Replace(updatesConfiguration.GitTagPrefix, String.Empty, StringComparison.Ordinal), out var version) - && version.Major > 3 // Forward/backward compatible but not before TGS4 + foreach (var kvp in releases) + { + var version = kvp.Key; + var release = kvp.Value; + if (version.Major > 3 // Forward/backward compatible but not before TGS4 && (greatestVersion == null || version > greatestVersion)) greatestVersion = version; - repoUrl = new Uri((await repositoryTask).HtmlUrl); + } + + repoUrl = await repositoryUrlTask; } catch (NotFoundException e) { diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index 552ab73290..2bdac39c75 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -370,7 +370,10 @@ namespace Tgstation.Server.Host.Core services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(x => x.GetRequiredService().CreateService()); // configure root services services.AddSingleton(); diff --git a/src/Tgstation.Server.Host/Utils/GitHubClientFactory.cs b/src/Tgstation.Server.Host/Utils/GitHubClientFactory.cs index 5bdb7091e1..da74748e6f 100644 --- a/src/Tgstation.Server.Host/Utils/GitHubClientFactory.cs +++ b/src/Tgstation.Server.Host/Utils/GitHubClientFactory.cs @@ -67,7 +67,9 @@ namespace Tgstation.Server.Host.Utils public IGitHubClient CreateClient() => GetOrCreateClient(generalConfiguration.GitHubAccessToken); /// - public IGitHubClient CreateClient(string accessToken) => GetOrCreateClient(accessToken ?? throw new ArgumentNullException(nameof(accessToken))); + public IGitHubClient CreateClient(string accessToken) + => GetOrCreateClient( + accessToken ?? throw new ArgumentNullException(nameof(accessToken))); /// /// Retrieve a from the or add a new one based on a given . diff --git a/src/Tgstation.Server.Host/Utils/GitHubService.cs b/src/Tgstation.Server.Host/Utils/GitHubService.cs new file mode 100644 index 0000000000..5d1d6ef5d4 --- /dev/null +++ b/src/Tgstation.Server.Host/Utils/GitHubService.cs @@ -0,0 +1,89 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +using Microsoft.Extensions.Logging; + +using Octokit; + +using Tgstation.Server.Host.Configuration; +using Tgstation.Server.Host.Extensions; + +namespace Tgstation.Server.Host.Utils +{ + /// + sealed class GitHubService : IGitHubService + { + /// + /// The for the . + /// + readonly IGitHubClient gitHubClient; + + /// + /// The for the . + /// + readonly ILogger logger; + + /// + /// The for the . + /// + readonly UpdatesConfiguration updatesConfiguration; + + /// + /// Initializes a new instance of the class. + /// + /// The value of . + /// The value of . + /// The value of . + public GitHubService(IGitHubClient gitHubClient, ILogger logger, UpdatesConfiguration updatesConfiguration) + { + this.gitHubClient = gitHubClient ?? throw new ArgumentNullException(nameof(gitHubClient)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + this.updatesConfiguration = updatesConfiguration ?? throw new ArgumentNullException(nameof(updatesConfiguration)); + } + + /// + public async Task> GetTgsReleases(CancellationToken cancellationToken) + { + logger.LogTrace("GetTgsReleases"); + var allReleases = await gitHubClient + .Repository + .Release + .GetAll(updatesConfiguration.GitHubRepositoryId) + .WithToken(cancellationToken); + + logger.LogTrace("{totalReleases} total releases", allReleases.Count); + var releases = allReleases + .Select(release => + { + if (!release.TagName.StartsWith(updatesConfiguration.GitTagPrefix, StringComparison.InvariantCulture) + || !Version.TryParse(release.TagName.Replace(updatesConfiguration.GitTagPrefix, String.Empty, StringComparison.Ordinal), out var version)) + return null; + + return Tuple.Create(version, release); + }) + .Where(tuple => tuple != null) + .ToDictionary(tuple => tuple.Item1, tuple => tuple.Item2); + + logger.LogTrace("{parsedReleases} parsed releases", releases.Count); + return releases; + } + + /// + public async Task GetUpdatesRepositoryUrl(CancellationToken cancellationToken) + { + logger.LogTrace("GetUpdatesRepositoryUrl"); + var repository = await gitHubClient + .Repository + .Get(updatesConfiguration.GitHubRepositoryId) + .WithToken(cancellationToken); + + var repoUrl = new Uri(repository.HtmlUrl); + logger.LogTrace("Maps to {repostioryUrl}", repoUrl); + + return repoUrl; + } + } +} diff --git a/src/Tgstation.Server.Host/Utils/GitHubServiceFactory.cs b/src/Tgstation.Server.Host/Utils/GitHubServiceFactory.cs new file mode 100644 index 0000000000..85332165c8 --- /dev/null +++ b/src/Tgstation.Server.Host/Utils/GitHubServiceFactory.cs @@ -0,0 +1,66 @@ +using System; + +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +using Octokit; + +using Tgstation.Server.Host.Configuration; + +namespace Tgstation.Server.Host.Utils +{ + /// + sealed class GitHubServiceFactory : IGitHubServiceFactory + { + /// + /// The for the . + /// + readonly IGitHubClientFactory gitHubClientFactory; + + /// + /// The for the . + /// + readonly ILoggerFactory loggerFactory; + + /// + /// The for the . + /// + readonly UpdatesConfiguration updatesConfiguration; + + /// + /// Initializes a new instance of the class. + /// + /// The value of . + /// The value of . + /// The containing value of . + public GitHubServiceFactory( + IGitHubClientFactory gitHubClientFactory, + ILoggerFactory loggerFactory, + IOptions updatesConfigurationOptions) + { + this.gitHubClientFactory = gitHubClientFactory ?? throw new ArgumentNullException(nameof(gitHubClientFactory)); + this.loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory)); + updatesConfiguration = updatesConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(updatesConfigurationOptions)); + } + + /// + public IGitHubService CreateService() => CreateServiceImpl(gitHubClientFactory.CreateClient()); + + /// + public IGitHubService CreateService(string accessToken) + => CreateServiceImpl( + gitHubClientFactory.CreateClient( + accessToken ?? throw new ArgumentNullException(nameof(accessToken)))); + + /// + /// Create a . + /// + /// The for the . + /// A new . + GitHubService CreateServiceImpl(IGitHubClient gitHubClient) + => new ( + gitHubClient, + loggerFactory.CreateLogger(), + updatesConfiguration); + } +} diff --git a/src/Tgstation.Server.Host/Utils/IGitHubService.cs b/src/Tgstation.Server.Host/Utils/IGitHubService.cs new file mode 100644 index 0000000000..2a6da1aef4 --- /dev/null +++ b/src/Tgstation.Server.Host/Utils/IGitHubService.cs @@ -0,0 +1,29 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +using Octokit; + +namespace Tgstation.Server.Host.Utils +{ + /// + /// Service for interacting with GitHub. + /// + public interface IGitHubService + { + /// + /// Gets the of the repository designated as the updates repository. + /// + /// The for the operation. + /// A resulting in the of the designated updates repository. + Task GetUpdatesRepositoryUrl(CancellationToken cancellationToken); + + /// + /// Get all valid TGS s from the configured update source. + /// + /// The for the operation. + /// A resulting in a of TGS s keyed by their . + Task> GetTgsReleases(CancellationToken cancellationToken); + } +} diff --git a/src/Tgstation.Server.Host/Utils/IGitHubServiceFactory.cs b/src/Tgstation.Server.Host/Utils/IGitHubServiceFactory.cs new file mode 100644 index 0000000000..c6997f9d5f --- /dev/null +++ b/src/Tgstation.Server.Host/Utils/IGitHubServiceFactory.cs @@ -0,0 +1,21 @@ +namespace Tgstation.Server.Host.Utils +{ + /// + /// Factory for s. + /// + public interface IGitHubServiceFactory + { + /// + /// Create a . + /// + /// A new . + public IGitHubService CreateService(); + + /// + /// Create a . + /// + /// The access token to use for communication with GitHub. + /// A new . + public IGitHubService CreateService(string accessToken); + } +} diff --git a/tests/Tgstation.Server.Host.Tests/Utils/TestGitHubServiceFactory.cs b/tests/Tgstation.Server.Host.Tests/Utils/TestGitHubServiceFactory.cs new file mode 100644 index 0000000000..6287e3b271 --- /dev/null +++ b/tests/Tgstation.Server.Host.Tests/Utils/TestGitHubServiceFactory.cs @@ -0,0 +1,57 @@ +using System; + +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +using Moq; + +using Octokit; + +using Tgstation.Server.Host.Configuration; + +namespace Tgstation.Server.Host.Utils.Tests +{ + [TestClass] + public sealed class TestGitHubServiceFactory + { + [TestMethod] + public void TestConstructor() + { + Assert.ThrowsException(() => new GitHubServiceFactory(null, null, null)); + Assert.ThrowsException(() => new GitHubServiceFactory(Mock.Of(), null, null)); + Assert.ThrowsException(() => new GitHubServiceFactory(Mock.Of(), Mock.Of(), null)); + var mockOptions = new Mock>(); + mockOptions.SetupGet(x => x.Value).Returns(new UpdatesConfiguration()); + + _ = new GitHubServiceFactory(Mock.Of(), Mock.Of(), mockOptions.Object); + } + + [TestMethod] + public void TestCreateService() + { + var mockFactory = new Mock(); + + mockFactory.Setup(x => x.CreateClient()).Returns(Mock.Of()).Verifiable(); + + var mockToken = "asdf"; + mockFactory.Setup(x => x.CreateClient(mockToken)).Returns(Mock.Of()).Verifiable(); + + var mockOptions = new Mock>(); + mockOptions.SetupGet(x => x.Value).Returns(new UpdatesConfiguration()); + + var factory = new GitHubServiceFactory(mockFactory.Object, Mock.Of(), mockOptions.Object); + + Assert.ThrowsException(() => factory.CreateService(null)); + Assert.AreEqual(0, mockFactory.Invocations.Count); + + var result1 = factory.CreateService(); + Assert.IsNotNull(result1); + + var result2 = factory.CreateService(mockToken); + Assert.IsNotNull(result2); + + mockFactory.VerifyAll(); + } + } +} From 95cd4b9015b92f733938d20b2fba046a91199c64 Mon Sep 17 00:00:00 2001 From: Dominion Date: Sun, 4 Jun 2023 00:38:13 -0400 Subject: [PATCH 04/41] Add the "unparsable release tag" log message to GitHubService --- src/Tgstation.Server.Host/Utils/GitHubService.cs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Tgstation.Server.Host/Utils/GitHubService.cs b/src/Tgstation.Server.Host/Utils/GitHubService.cs index 5d1d6ef5d4..4acc35c612 100644 --- a/src/Tgstation.Server.Host/Utils/GitHubService.cs +++ b/src/Tgstation.Server.Host/Utils/GitHubService.cs @@ -58,10 +58,15 @@ namespace Tgstation.Server.Host.Utils var releases = allReleases .Select(release => { - if (!release.TagName.StartsWith(updatesConfiguration.GitTagPrefix, StringComparison.InvariantCulture) - || !Version.TryParse(release.TagName.Replace(updatesConfiguration.GitTagPrefix, String.Empty, StringComparison.Ordinal), out var version)) + if (!release.TagName.StartsWith(updatesConfiguration.GitTagPrefix, StringComparison.InvariantCulture)) return null; + if (!Version.TryParse(release.TagName.Replace(updatesConfiguration.GitTagPrefix, String.Empty, StringComparison.Ordinal), out var version)) + { + logger.LogDebug("Unparsable release tag: {releaseTag}", release.TagName); + return null; + } + return Tuple.Create(version, release); }) .Where(tuple => tuple != null) From 083667d04db86fe6bb997eb7a187e21a8e920ba2 Mon Sep 17 00:00:00 2001 From: Dominion Date: Sun, 4 Jun 2023 00:40:14 -0400 Subject: [PATCH 05/41] Migrate ServerUpdater to use GitHubService --- .../Configuration/UpdatesConfiguration.cs | 6 -- .../Core/ServerUpdater.cs | 89 +++++++------------ .../Live/LiveTestingServer.cs | 5 +- .../Live/TestLiveServer.cs | 2 +- 4 files changed, 33 insertions(+), 69 deletions(-) diff --git a/src/Tgstation.Server.Host/Configuration/UpdatesConfiguration.cs b/src/Tgstation.Server.Host/Configuration/UpdatesConfiguration.cs index fae04d8684..406560f458 100644 --- a/src/Tgstation.Server.Host/Configuration/UpdatesConfiguration.cs +++ b/src/Tgstation.Server.Host/Configuration/UpdatesConfiguration.cs @@ -39,11 +39,5 @@ /// Asset package containing the new assembly in zip form. /// public string UpdatePackageAssetName { get; set; } = DefaultUpdatePackageAssetName; - - /// - /// Dump all retrieved releases from the GitHub API when a requested release is not found. - /// - /// This is an internal config and may be adjusted or removed without a version change. - public bool DumpReleasesOnNotFound { get; set; } } } diff --git a/src/Tgstation.Server.Host/Core/ServerUpdater.cs b/src/Tgstation.Server.Host/Core/ServerUpdater.cs index 86623be571..d03b6ac2f1 100644 --- a/src/Tgstation.Server.Host/Core/ServerUpdater.cs +++ b/src/Tgstation.Server.Host/Core/ServerUpdater.cs @@ -8,7 +8,6 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Tgstation.Server.Host.Configuration; -using Tgstation.Server.Host.Extensions; using Tgstation.Server.Host.IO; using Tgstation.Server.Host.Swarm; using Tgstation.Server.Host.Utils; @@ -19,9 +18,9 @@ namespace Tgstation.Server.Host.Core sealed class ServerUpdater : IServerUpdater, IServerUpdateExecutor { /// - /// The for the . + /// The for the . /// - readonly IGitHubClientFactory gitHubClientFactory; + readonly IGitHubService gitHubService; /// /// The for the . @@ -61,7 +60,7 @@ namespace Tgstation.Server.Host.Core /// /// Initializes a new instance of the class. /// - /// The value of . + /// The value of . /// The value of . /// The value of . /// The value of . @@ -69,7 +68,7 @@ namespace Tgstation.Server.Host.Core /// The containing the value of . /// The containing the value of . public ServerUpdater( - IGitHubClientFactory gitHubClientFactory, + IGitHubService gitHubService, IIOManager ioManager, IFileDownloader fileDownloader, IServerControl serverControl, @@ -77,7 +76,7 @@ namespace Tgstation.Server.Host.Core IOptions generalConfigurationOptions, IOptions updatesConfigurationOptions) { - this.gitHubClientFactory = gitHubClientFactory ?? throw new ArgumentNullException(nameof(gitHubClientFactory)); + this.gitHubService = gitHubService ?? throw new ArgumentNullException(nameof(gitHubService)); this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); this.fileDownloader = fileDownloader ?? throw new ArgumentNullException(nameof(fileDownloader)); this.serverControl = serverControl ?? throw new ArgumentNullException(nameof(serverControl)); @@ -224,64 +223,38 @@ namespace Tgstation.Server.Host.Core async Task BeginUpdateImpl(ISwarmService swarmService, Version newVersion, bool recursed, CancellationToken cancellationToken) { logger.LogDebug("Looking for GitHub releases version {version}...", newVersion); - var gitHubClient = gitHubClientFactory.CreateClient(); - var releases = await gitHubClient - .Repository - .Release - .GetAll(updatesConfiguration.GitHubRepositoryId) - .WithToken(cancellationToken); - logger.LogTrace("Received {releaseCount} total releases from GitHub", releases.Count); - - var filteredReleases = releases - .Where(x => x.TagName.StartsWith(updatesConfiguration.GitTagPrefix, StringComparison.InvariantCulture)) - .ToList(); - - logger.LogTrace( - "Filtered to {releaseCount} releases matching the configured tag prefix of \"{tagPrefix}\"", - filteredReleases.Count, - updatesConfiguration.GitTagPrefix); - - foreach (var release in filteredReleases) - if (Version.TryParse( - release.TagName.Replace( - updatesConfiguration.GitTagPrefix, String.Empty, StringComparison.Ordinal), - out var version)) + var releases = await gitHubService.GetTgsReleases(cancellationToken); + foreach (var kvp in releases) + { + var version = kvp.Key; + var release = kvp.Value; + if (version == newVersion) { - if (version == newVersion) + var asset = release.Assets.Where(x => x.Name.Equals(updatesConfiguration.UpdatePackageAssetName, StringComparison.Ordinal)).FirstOrDefault(); + if (asset == default) + continue; + + serverUpdateOperation = new ServerUpdateOperation { - var asset = release.Assets.Where(x => x.Name.Equals(updatesConfiguration.UpdatePackageAssetName, StringComparison.Ordinal)).FirstOrDefault(); - if (asset == default) - continue; + TargetVersion = version, + UpdateZipUrl = new Uri(asset.BrowserDownloadUrl), + SwarmService = swarmService, + }; - serverUpdateOperation = new ServerUpdateOperation - { - TargetVersion = version, - UpdateZipUrl = new Uri(asset.BrowserDownloadUrl), - SwarmService = swarmService, - }; - - try - { - if (!serverControl.TryStartUpdate(this, version)) - return ServerUpdateResult.UpdateInProgress; - } - finally - { - serverUpdateOperation = null; - } - - return ServerUpdateResult.Started; + try + { + if (!serverControl.TryStartUpdate(this, version)) + return ServerUpdateResult.UpdateInProgress; + } + finally + { + serverUpdateOperation = null; } - } - else - logger.LogDebug("Unparsable release tag: {releaseTag}", release.TagName); - if (updatesConfiguration.DumpReleasesOnNotFound) - logger.LogInformation( - "Found releases:{newline}\t{releases}", - Environment.NewLine, - String.Join($"{Environment.NewLine}\t", releases.Select(x => x.TagName).OrderBy(x => x))); + return ServerUpdateResult.Started; + } + } if (!recursed) { diff --git a/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs b/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs index 6a66269efb..4aa9e70604 100644 --- a/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs +++ b/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs @@ -49,7 +49,7 @@ namespace Tgstation.Server.Tests.Live SerilogContextHelper.AddSwarmNodeIdentifierToTemplate(); } - public LiveTestingServer(SwarmConfiguration swarmConfiguration, bool enableOAuth, ushort port = 5010, bool dumpOnMissingUpdate = true) + public LiveTestingServer(SwarmConfiguration swarmConfiguration, bool enableOAuth, ushort port = 5010) { Directory = Environment.GetEnvironmentVariable("TGS_TEST_TEMP_DIRECTORY"); if (string.IsNullOrWhiteSpace(Directory)) @@ -116,9 +116,6 @@ namespace Tgstation.Server.Tests.Live $"Session:LowPriorityDeploymentProcesses={LowPriorityDeployments}", }; - if (dumpOnMissingUpdate) - args.Add("Updates:DumpReleasesOnNotFound=true"); - swarmArgs = new List(); if (swarmConfiguration != null) { diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index 5b10b5b6d6..d75a40c8ef 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -218,7 +218,7 @@ namespace Tgstation.Server.Tests.Live [TestMethod] public async Task TestUpdateBadVersion() { - using var server = new LiveTestingServer(null, false, dumpOnMissingUpdate: false); + using var server = new LiveTestingServer(null, false); using var serverCts = new CancellationTokenSource(); var cancellationToken = serverCts.Token; var serverTask = server.Run(cancellationToken); From 81700e18b50c746e06aca20efaa91f6a81097675 Mon Sep 17 00:00:00 2001 From: Dominion Date: Sun, 4 Jun 2023 00:40:24 -0400 Subject: [PATCH 06/41] Comment on the flakiness of the releases API --- src/Tgstation.Server.Host/Utils/IGitHubService.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Tgstation.Server.Host/Utils/IGitHubService.cs b/src/Tgstation.Server.Host/Utils/IGitHubService.cs index 2a6da1aef4..d65d93f6e2 100644 --- a/src/Tgstation.Server.Host/Utils/IGitHubService.cs +++ b/src/Tgstation.Server.Host/Utils/IGitHubService.cs @@ -24,6 +24,7 @@ namespace Tgstation.Server.Host.Utils /// /// The for the operation. /// A resulting in a of TGS s keyed by their . + /// GitHub has been known to return incomplete results from the API with this call. Task> GetTgsReleases(CancellationToken cancellationToken); } } From 97d5f58fd50abe7d0486b9b200e46321441844f8 Mon Sep 17 00:00:00 2001 From: Dominion Date: Sun, 4 Jun 2023 00:47:15 -0400 Subject: [PATCH 07/41] Move GitHub related utils to their own namespace --- .../Deployment/Remote/GitHubRemoteDeploymentManager.cs | 2 +- .../Deployment/Remote/RemoteDeploymentManagerFactory.cs | 2 +- .../Components/Repository/GitHubRemoteFeatures.cs | 2 +- .../Components/Repository/GitRemoteFeaturesFactory.cs | 2 +- .../Controllers/AdministrationController.cs | 2 +- src/Tgstation.Server.Host/Core/Application.cs | 1 + src/Tgstation.Server.Host/Core/ServerUpdater.cs | 2 +- .../Security/OAuth/GitHubOAuthValidator.cs | 2 +- src/Tgstation.Server.Host/Security/OAuth/OAuthProviders.cs | 2 +- .../Utils/{ => GitHub}/GitHubClientFactory.cs | 2 +- src/Tgstation.Server.Host/Utils/{ => GitHub}/GitHubService.cs | 2 +- .../Utils/{ => GitHub}/GitHubServiceFactory.cs | 2 +- .../Utils/{ => GitHub}/IGitHubClientFactory.cs | 2 +- src/Tgstation.Server.Host/Utils/{ => GitHub}/IGitHubService.cs | 2 +- .../Utils/{ => GitHub}/IGitHubServiceFactory.cs | 2 +- .../Utils/{ => GitHub}/TestGitHubClientFactory.cs | 2 +- .../Utils/{ => GitHub}/TestGitHubServiceFactory.cs | 2 +- 17 files changed, 17 insertions(+), 16 deletions(-) rename src/Tgstation.Server.Host/Utils/{ => GitHub}/GitHubClientFactory.cs (99%) rename src/Tgstation.Server.Host/Utils/{ => GitHub}/GitHubService.cs (98%) rename src/Tgstation.Server.Host/Utils/{ => GitHub}/GitHubServiceFactory.cs (98%) rename src/Tgstation.Server.Host/Utils/{ => GitHub}/IGitHubClientFactory.cs (93%) rename src/Tgstation.Server.Host/Utils/{ => GitHub}/IGitHubService.cs (96%) rename src/Tgstation.Server.Host/Utils/{ => GitHub}/IGitHubServiceFactory.cs (92%) rename tests/Tgstation.Server.Host.Tests/Utils/{ => GitHub}/TestGitHubClientFactory.cs (98%) rename tests/Tgstation.Server.Host.Tests/Utils/{ => GitHub}/TestGitHubServiceFactory.cs (97%) diff --git a/src/Tgstation.Server.Host/Components/Deployment/Remote/GitHubRemoteDeploymentManager.cs b/src/Tgstation.Server.Host/Components/Deployment/Remote/GitHubRemoteDeploymentManager.cs index 8f9f24105d..fadb7539b9 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/Remote/GitHubRemoteDeploymentManager.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/Remote/GitHubRemoteDeploymentManager.cs @@ -14,7 +14,7 @@ using Tgstation.Server.Host.Components.Repository; using Tgstation.Server.Host.Database; using Tgstation.Server.Host.Extensions; using Tgstation.Server.Host.Models; -using Tgstation.Server.Host.Utils; +using Tgstation.Server.Host.Utils.GitHub; namespace Tgstation.Server.Host.Components.Deployment.Remote { diff --git a/src/Tgstation.Server.Host/Components/Deployment/Remote/RemoteDeploymentManagerFactory.cs b/src/Tgstation.Server.Host/Components/Deployment/Remote/RemoteDeploymentManagerFactory.cs index 21b13b7a4c..f9cdf5a240 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/Remote/RemoteDeploymentManagerFactory.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/Remote/RemoteDeploymentManagerFactory.cs @@ -5,7 +5,7 @@ using Microsoft.Extensions.Logging; using Tgstation.Server.Api.Models; using Tgstation.Server.Host.Components.Repository; using Tgstation.Server.Host.Database; -using Tgstation.Server.Host.Utils; +using Tgstation.Server.Host.Utils.GitHub; namespace Tgstation.Server.Host.Components.Deployment.Remote { diff --git a/src/Tgstation.Server.Host/Components/Repository/GitHubRemoteFeatures.cs b/src/Tgstation.Server.Host/Components/Repository/GitHubRemoteFeatures.cs index 52d6c6efe4..31fc74fb2d 100644 --- a/src/Tgstation.Server.Host/Components/Repository/GitHubRemoteFeatures.cs +++ b/src/Tgstation.Server.Host/Components/Repository/GitHubRemoteFeatures.cs @@ -8,7 +8,7 @@ using Octokit; using Tgstation.Server.Api.Models; using Tgstation.Server.Api.Models.Internal; using Tgstation.Server.Host.Extensions; -using Tgstation.Server.Host.Utils; +using Tgstation.Server.Host.Utils.GitHub; namespace Tgstation.Server.Host.Components.Repository { diff --git a/src/Tgstation.Server.Host/Components/Repository/GitRemoteFeaturesFactory.cs b/src/Tgstation.Server.Host/Components/Repository/GitRemoteFeaturesFactory.cs index d43747fe68..21a3d7263e 100644 --- a/src/Tgstation.Server.Host/Components/Repository/GitRemoteFeaturesFactory.cs +++ b/src/Tgstation.Server.Host/Components/Repository/GitRemoteFeaturesFactory.cs @@ -3,7 +3,7 @@ using Microsoft.Extensions.Logging; using Tgstation.Server.Api.Models; -using Tgstation.Server.Host.Utils; +using Tgstation.Server.Host.Utils.GitHub; namespace Tgstation.Server.Host.Components.Repository { diff --git a/src/Tgstation.Server.Host/Controllers/AdministrationController.cs b/src/Tgstation.Server.Host/Controllers/AdministrationController.cs index 63ef457aee..61f6842e8c 100644 --- a/src/Tgstation.Server.Host/Controllers/AdministrationController.cs +++ b/src/Tgstation.Server.Host/Controllers/AdministrationController.cs @@ -23,7 +23,7 @@ using Tgstation.Server.Host.IO; using Tgstation.Server.Host.Security; using Tgstation.Server.Host.System; using Tgstation.Server.Host.Transfer; -using Tgstation.Server.Host.Utils; +using Tgstation.Server.Host.Utils.GitHub; namespace Tgstation.Server.Host.Controllers { diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index 2bdac39c75..99ea878a1d 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -51,6 +51,7 @@ using Tgstation.Server.Host.Swarm; using Tgstation.Server.Host.System; using Tgstation.Server.Host.Transfer; using Tgstation.Server.Host.Utils; +using Tgstation.Server.Host.Utils.GitHub; namespace Tgstation.Server.Host.Core { diff --git a/src/Tgstation.Server.Host/Core/ServerUpdater.cs b/src/Tgstation.Server.Host/Core/ServerUpdater.cs index d03b6ac2f1..625591e4de 100644 --- a/src/Tgstation.Server.Host/Core/ServerUpdater.cs +++ b/src/Tgstation.Server.Host/Core/ServerUpdater.cs @@ -10,7 +10,7 @@ using Microsoft.Extensions.Options; using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.IO; using Tgstation.Server.Host.Swarm; -using Tgstation.Server.Host.Utils; +using Tgstation.Server.Host.Utils.GitHub; namespace Tgstation.Server.Host.Core { diff --git a/src/Tgstation.Server.Host/Security/OAuth/GitHubOAuthValidator.cs b/src/Tgstation.Server.Host/Security/OAuth/GitHubOAuthValidator.cs index 3991fc33f9..0ee4a21c8f 100644 --- a/src/Tgstation.Server.Host/Security/OAuth/GitHubOAuthValidator.cs +++ b/src/Tgstation.Server.Host/Security/OAuth/GitHubOAuthValidator.cs @@ -8,7 +8,7 @@ using Octokit; using Tgstation.Server.Api.Models; using Tgstation.Server.Host.Configuration; -using Tgstation.Server.Host.Utils; +using Tgstation.Server.Host.Utils.GitHub; namespace Tgstation.Server.Host.Security.OAuth { diff --git a/src/Tgstation.Server.Host/Security/OAuth/OAuthProviders.cs b/src/Tgstation.Server.Host/Security/OAuth/OAuthProviders.cs index a217de3935..033dd97199 100644 --- a/src/Tgstation.Server.Host/Security/OAuth/OAuthProviders.cs +++ b/src/Tgstation.Server.Host/Security/OAuth/OAuthProviders.cs @@ -10,7 +10,7 @@ using Microsoft.Extensions.Options; using Tgstation.Server.Api.Models; using Tgstation.Server.Common; using Tgstation.Server.Host.Configuration; -using Tgstation.Server.Host.Utils; +using Tgstation.Server.Host.Utils.GitHub; namespace Tgstation.Server.Host.Security.OAuth { diff --git a/src/Tgstation.Server.Host/Utils/GitHubClientFactory.cs b/src/Tgstation.Server.Host/Utils/GitHub/GitHubClientFactory.cs similarity index 99% rename from src/Tgstation.Server.Host/Utils/GitHubClientFactory.cs rename to src/Tgstation.Server.Host/Utils/GitHub/GitHubClientFactory.cs index da74748e6f..5fbd22f2de 100644 --- a/src/Tgstation.Server.Host/Utils/GitHubClientFactory.cs +++ b/src/Tgstation.Server.Host/Utils/GitHub/GitHubClientFactory.cs @@ -9,7 +9,7 @@ using Octokit; using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.System; -namespace Tgstation.Server.Host.Utils +namespace Tgstation.Server.Host.Utils.GitHub { /// sealed class GitHubClientFactory : IGitHubClientFactory diff --git a/src/Tgstation.Server.Host/Utils/GitHubService.cs b/src/Tgstation.Server.Host/Utils/GitHub/GitHubService.cs similarity index 98% rename from src/Tgstation.Server.Host/Utils/GitHubService.cs rename to src/Tgstation.Server.Host/Utils/GitHub/GitHubService.cs index 4acc35c612..00bff4973f 100644 --- a/src/Tgstation.Server.Host/Utils/GitHubService.cs +++ b/src/Tgstation.Server.Host/Utils/GitHub/GitHubService.cs @@ -11,7 +11,7 @@ using Octokit; using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.Extensions; -namespace Tgstation.Server.Host.Utils +namespace Tgstation.Server.Host.Utils.GitHub { /// sealed class GitHubService : IGitHubService diff --git a/src/Tgstation.Server.Host/Utils/GitHubServiceFactory.cs b/src/Tgstation.Server.Host/Utils/GitHub/GitHubServiceFactory.cs similarity index 98% rename from src/Tgstation.Server.Host/Utils/GitHubServiceFactory.cs rename to src/Tgstation.Server.Host/Utils/GitHub/GitHubServiceFactory.cs index 85332165c8..3873188659 100644 --- a/src/Tgstation.Server.Host/Utils/GitHubServiceFactory.cs +++ b/src/Tgstation.Server.Host/Utils/GitHub/GitHubServiceFactory.cs @@ -7,7 +7,7 @@ using Octokit; using Tgstation.Server.Host.Configuration; -namespace Tgstation.Server.Host.Utils +namespace Tgstation.Server.Host.Utils.GitHub { /// sealed class GitHubServiceFactory : IGitHubServiceFactory diff --git a/src/Tgstation.Server.Host/Utils/IGitHubClientFactory.cs b/src/Tgstation.Server.Host/Utils/GitHub/IGitHubClientFactory.cs similarity index 93% rename from src/Tgstation.Server.Host/Utils/IGitHubClientFactory.cs rename to src/Tgstation.Server.Host/Utils/GitHub/IGitHubClientFactory.cs index 636796a284..ec95fd16ab 100644 --- a/src/Tgstation.Server.Host/Utils/IGitHubClientFactory.cs +++ b/src/Tgstation.Server.Host/Utils/GitHub/IGitHubClientFactory.cs @@ -1,6 +1,6 @@ using Octokit; -namespace Tgstation.Server.Host.Utils +namespace Tgstation.Server.Host.Utils.GitHub { /// /// For creating s. diff --git a/src/Tgstation.Server.Host/Utils/IGitHubService.cs b/src/Tgstation.Server.Host/Utils/GitHub/IGitHubService.cs similarity index 96% rename from src/Tgstation.Server.Host/Utils/IGitHubService.cs rename to src/Tgstation.Server.Host/Utils/GitHub/IGitHubService.cs index d65d93f6e2..923217e87f 100644 --- a/src/Tgstation.Server.Host/Utils/IGitHubService.cs +++ b/src/Tgstation.Server.Host/Utils/GitHub/IGitHubService.cs @@ -5,7 +5,7 @@ using System.Threading.Tasks; using Octokit; -namespace Tgstation.Server.Host.Utils +namespace Tgstation.Server.Host.Utils.GitHub { /// /// Service for interacting with GitHub. diff --git a/src/Tgstation.Server.Host/Utils/IGitHubServiceFactory.cs b/src/Tgstation.Server.Host/Utils/GitHub/IGitHubServiceFactory.cs similarity index 92% rename from src/Tgstation.Server.Host/Utils/IGitHubServiceFactory.cs rename to src/Tgstation.Server.Host/Utils/GitHub/IGitHubServiceFactory.cs index c6997f9d5f..1bf1f1b850 100644 --- a/src/Tgstation.Server.Host/Utils/IGitHubServiceFactory.cs +++ b/src/Tgstation.Server.Host/Utils/GitHub/IGitHubServiceFactory.cs @@ -1,4 +1,4 @@ -namespace Tgstation.Server.Host.Utils +namespace Tgstation.Server.Host.Utils.GitHub { /// /// Factory for s. diff --git a/tests/Tgstation.Server.Host.Tests/Utils/TestGitHubClientFactory.cs b/tests/Tgstation.Server.Host.Tests/Utils/GitHub/TestGitHubClientFactory.cs similarity index 98% rename from tests/Tgstation.Server.Host.Tests/Utils/TestGitHubClientFactory.cs rename to tests/Tgstation.Server.Host.Tests/Utils/GitHub/TestGitHubClientFactory.cs index 5d7be76cf3..5c9a6f7f31 100644 --- a/tests/Tgstation.Server.Host.Tests/Utils/TestGitHubClientFactory.cs +++ b/tests/Tgstation.Server.Host.Tests/Utils/GitHub/TestGitHubClientFactory.cs @@ -13,7 +13,7 @@ using Octokit; using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.System; -namespace Tgstation.Server.Host.Utils.Tests +namespace Tgstation.Server.Host.Utils.GitHub.Tests { [TestClass] public sealed class TestGitHubClientFactory diff --git a/tests/Tgstation.Server.Host.Tests/Utils/TestGitHubServiceFactory.cs b/tests/Tgstation.Server.Host.Tests/Utils/GitHub/TestGitHubServiceFactory.cs similarity index 97% rename from tests/Tgstation.Server.Host.Tests/Utils/TestGitHubServiceFactory.cs rename to tests/Tgstation.Server.Host.Tests/Utils/GitHub/TestGitHubServiceFactory.cs index 6287e3b271..daa9afdd4c 100644 --- a/tests/Tgstation.Server.Host.Tests/Utils/TestGitHubServiceFactory.cs +++ b/tests/Tgstation.Server.Host.Tests/Utils/GitHub/TestGitHubServiceFactory.cs @@ -10,7 +10,7 @@ using Octokit; using Tgstation.Server.Host.Configuration; -namespace Tgstation.Server.Host.Utils.Tests +namespace Tgstation.Server.Host.Utils.GitHub.Tests { [TestClass] public sealed class TestGitHubServiceFactory From 7f4c30650d8a30006d8bb1e3e0c6ed2be25a9b03 Mon Sep 17 00:00:00 2001 From: Dominion Date: Sun, 4 Jun 2023 01:02:22 -0400 Subject: [PATCH 08/41] Migrate GitHubOAuthValidator to IGitHubService --- .../Security/OAuth/GitHubOAuthValidator.cs | 34 ++++++------------ .../Security/OAuth/OAuthProviders.cs | 6 ++-- .../Utils/GitHub/GitHubService.cs | 36 +++++++++++++++++++ .../Utils/GitHub/IGitHubService.cs | 18 ++++++++++ 4 files changed, 67 insertions(+), 27 deletions(-) diff --git a/src/Tgstation.Server.Host/Security/OAuth/GitHubOAuthValidator.cs b/src/Tgstation.Server.Host/Security/OAuth/GitHubOAuthValidator.cs index 0ee4a21c8f..520baf0484 100644 --- a/src/Tgstation.Server.Host/Security/OAuth/GitHubOAuthValidator.cs +++ b/src/Tgstation.Server.Host/Security/OAuth/GitHubOAuthValidator.cs @@ -21,9 +21,9 @@ namespace Tgstation.Server.Host.Security.OAuth public OAuthProvider Provider => OAuthProvider.GitHub; /// - /// The for the . + /// The for the . /// - readonly IGitHubClientFactory gitHubClientFactory; + readonly IGitHubServiceFactory gitHubServiceFactory; /// /// The for the . @@ -38,15 +38,15 @@ namespace Tgstation.Server.Host.Security.OAuth /// /// Initializes a new instance of the class. /// - /// The value of . + /// The value of . /// The value of . /// The value of . public GitHubOAuthValidator( - IGitHubClientFactory gitHubClientFactory, + IGitHubServiceFactory gitHubServiceFactory, ILogger logger, OAuthConfiguration oAuthConfiguration) { - this.gitHubClientFactory = gitHubClientFactory ?? throw new ArgumentNullException(nameof(gitHubClientFactory)); + this.gitHubServiceFactory = gitHubServiceFactory ?? throw new ArgumentNullException(nameof(gitHubServiceFactory)); this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); this.oAuthConfiguration = oAuthConfiguration ?? throw new ArgumentNullException(nameof(oAuthConfiguration)); } @@ -57,35 +57,21 @@ namespace Tgstation.Server.Host.Security.OAuth if (code == null) throw new ArgumentNullException(nameof(code)); - var client = gitHubClientFactory.CreateClient(); try { logger.LogTrace("Validating response code..."); - var response = await client - .Oauth - .CreateAccessToken( - new OauthTokenRequest( - oAuthConfiguration.ClientId, - oAuthConfiguration.ClientSecret, - code) - { - RedirectUri = oAuthConfiguration.RedirectUrl, - }) - ; - var token = response.AccessToken; + var gitHubService = gitHubServiceFactory.CreateService(); + var token = await gitHubService.CreateOAuthAccessToken(oAuthConfiguration, code, cancellationToken); if (token == null) return null; - var authenticatedClient = gitHubClientFactory.CreateClient(token); + var authenticatedClient = gitHubServiceFactory.CreateService(token); logger.LogTrace("Getting user details..."); - var userDetails = await authenticatedClient - .User - .Current() - ; + var userId = await authenticatedClient.GetCurrentUserId(cancellationToken); - return userDetails.Id.ToString(CultureInfo.InvariantCulture); + return userId.ToString(CultureInfo.InvariantCulture); } catch (RateLimitExceededException) { diff --git a/src/Tgstation.Server.Host/Security/OAuth/OAuthProviders.cs b/src/Tgstation.Server.Host/Security/OAuth/OAuthProviders.cs index 033dd97199..a7f4dc3c0a 100644 --- a/src/Tgstation.Server.Host/Security/OAuth/OAuthProviders.cs +++ b/src/Tgstation.Server.Host/Security/OAuth/OAuthProviders.cs @@ -25,12 +25,12 @@ namespace Tgstation.Server.Host.Security.OAuth /// /// Initializes a new instance of the class. /// - /// The to use. + /// The to use. /// The to use. /// The to use. /// The containing the to use. public OAuthProviders( - IGitHubClientFactory gitHubClientFactory, + IGitHubServiceFactory gitHubServiceFactory, IAbstractHttpClientFactory httpClientFactory, ILoggerFactory loggerFactory, IOptions securityConfigurationOptions) @@ -49,7 +49,7 @@ namespace Tgstation.Server.Host.Security.OAuth if (securityConfiguration.OAuth.TryGetValue(OAuthProvider.GitHub, out var gitHubConfig)) validatorsBuilder.Add( new GitHubOAuthValidator( - gitHubClientFactory, + gitHubServiceFactory, loggerFactory.CreateLogger(), gitHubConfig)); diff --git a/src/Tgstation.Server.Host/Utils/GitHub/GitHubService.cs b/src/Tgstation.Server.Host/Utils/GitHub/GitHubService.cs index 00bff4973f..129f259d6a 100644 --- a/src/Tgstation.Server.Host/Utils/GitHub/GitHubService.cs +++ b/src/Tgstation.Server.Host/Utils/GitHub/GitHubService.cs @@ -44,6 +44,33 @@ namespace Tgstation.Server.Host.Utils.GitHub this.updatesConfiguration = updatesConfiguration ?? throw new ArgumentNullException(nameof(updatesConfiguration)); } + /// + public async Task CreateOAuthAccessToken(OAuthConfiguration oAuthConfiguration, string code, CancellationToken cancellationToken) + { + if (oAuthConfiguration == null) + throw new ArgumentNullException(nameof(oAuthConfiguration)); + + if (code == null) + throw new ArgumentNullException(nameof(code)); + + logger.LogTrace("CreateOAuthAccessToken"); + + var response = await gitHubClient + .Oauth + .CreateAccessToken( + new OauthTokenRequest( + oAuthConfiguration.ClientId, + oAuthConfiguration.ClientSecret, + code) + { + RedirectUri = oAuthConfiguration.RedirectUrl, + }) + .WithToken(cancellationToken); + + var token = response.AccessToken; + return token; + } + /// public async Task> GetTgsReleases(CancellationToken cancellationToken) { @@ -90,5 +117,14 @@ namespace Tgstation.Server.Host.Utils.GitHub return repoUrl; } + + /// + public async Task GetCurrentUserId(CancellationToken cancellationToken) + { + logger.LogTrace("CreateOAuthAccessToken"); + + var userDetails = await gitHubClient.User.Current().WithToken(cancellationToken); + return userDetails.Id; + } } } diff --git a/src/Tgstation.Server.Host/Utils/GitHub/IGitHubService.cs b/src/Tgstation.Server.Host/Utils/GitHub/IGitHubService.cs index 923217e87f..d9c518ab06 100644 --- a/src/Tgstation.Server.Host/Utils/GitHub/IGitHubService.cs +++ b/src/Tgstation.Server.Host/Utils/GitHub/IGitHubService.cs @@ -5,6 +5,8 @@ using System.Threading.Tasks; using Octokit; +using Tgstation.Server.Host.Configuration; + namespace Tgstation.Server.Host.Utils.GitHub { /// @@ -26,5 +28,21 @@ namespace Tgstation.Server.Host.Utils.GitHub /// A resulting in a of TGS s keyed by their . /// GitHub has been known to return incomplete results from the API with this call. Task> GetTgsReleases(CancellationToken cancellationToken); + + /// + /// Attempt to get an OAuth token from a given . + /// + /// The . Must have , and set. + /// The OAuth response code. + /// The for the operation. + /// A resulting in a representing the returned OAuth code from GitHub on success, otherwise. + Task CreateOAuthAccessToken(OAuthConfiguration oAuthConfiguration, string code, CancellationToken cancellationToken); + + /// + /// Get the current user's ID. + /// + /// The for the operation. + /// A resulting in the current user's ID. + Task GetCurrentUserId(CancellationToken cancellationToken); } } From 21f99ad2a08ad70436526361a1f75de5a2875e0d Mon Sep 17 00:00:00 2001 From: Dominion Date: Sun, 4 Jun 2023 01:30:56 -0400 Subject: [PATCH 09/41] Migrate GitHubRemoteDeploymentManager to IGitHubService --- .../Remote/GitHubRemoteDeploymentManager.cs | 128 +++++++---------- .../Remote/RemoteDeploymentManagerFactory.cs | 10 +- .../Utils/GitHub/GitHubService.cs | 135 ++++++++++++++++++ .../Utils/GitHub/IGitHubService.cs | 61 ++++++++ 4 files changed, 252 insertions(+), 82 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Deployment/Remote/GitHubRemoteDeploymentManager.cs b/src/Tgstation.Server.Host/Components/Deployment/Remote/GitHubRemoteDeploymentManager.cs index fadb7539b9..d327a84db2 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/Remote/GitHubRemoteDeploymentManager.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/Remote/GitHubRemoteDeploymentManager.cs @@ -12,7 +12,6 @@ using Octokit; using Tgstation.Server.Host.Components.Repository; using Tgstation.Server.Host.Database; -using Tgstation.Server.Host.Extensions; using Tgstation.Server.Host.Models; using Tgstation.Server.Host.Utils.GitHub; @@ -29,26 +28,26 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote readonly IDatabaseContextFactory databaseContextFactory; /// - /// The for the . + /// The for the . /// - readonly IGitHubClientFactory gitHubClientFactory; + readonly IGitHubServiceFactory gitHubServiceFactory; /// /// Initializes a new instance of the class. /// /// The value of . - /// The value of . + /// The value of . /// The for the . /// The for the . public GitHubRemoteDeploymentManager( IDatabaseContextFactory databaseContextFactory, - IGitHubClientFactory gitHubClientFactory, + IGitHubServiceFactory gitHubServiceFactory, ILogger logger, Api.Models.Instance metadata) : base(logger, metadata) { this.databaseContextFactory = databaseContextFactory ?? throw new ArgumentNullException(nameof(databaseContextFactory)); - this.gitHubClientFactory = gitHubClientFactory ?? throw new ArgumentNullException(nameof(gitHubClientFactory)); + this.gitHubServiceFactory = gitHubServiceFactory ?? throw new ArgumentNullException(nameof(gitHubServiceFactory)); } /// @@ -74,15 +73,14 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote .FirstAsync(cancellationToken)); var instanceAuthenticated = repositorySettings.AccessToken != null; - var gitHubClient = !instanceAuthenticated - ? gitHubClientFactory.CreateClient() - : gitHubClientFactory.CreateClient(repositorySettings.AccessToken); + var gitHubService = !instanceAuthenticated + ? gitHubServiceFactory.CreateService() + : gitHubServiceFactory.CreateService(repositorySettings.AccessToken); - var repositoryTask = gitHubClient - .Repository - .Get( - remoteInformation.RemoteRepositoryOwner, - remoteInformation.RemoteRepositoryName); + var repositoryIdTask = gitHubService.GetRepositoryId( + remoteInformation.RemoteRepositoryOwner, + remoteInformation.RemoteRepositoryName, + cancellationToken); if (!repositorySettings.CreateGitHubDeployments.Value) Logger.LogTrace("Not creating deployment"); @@ -91,44 +89,34 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote else { Logger.LogTrace("Creating deployment..."); - Octokit.Deployment deployment; - try { - deployment = await gitHubClient - .Repository - .Deployment - .Create( - remoteInformation.RemoteRepositoryOwner, - remoteInformation.RemoteRepositoryName, - new NewDeployment(compileJob.RevisionInformation.CommitSha) - { - AutoMerge = false, - Description = "TGS Game Deployment", - Environment = $"TGS: {Metadata.Name}", - ProductionEnvironment = true, - RequiredContexts = new Collection(), - }) - .WithToken(cancellationToken); + compileJob.GitHubDeploymentId = await gitHubService.CreateDeployment( + new NewDeployment(compileJob.RevisionInformation.CommitSha) + { + AutoMerge = false, + Description = "TGS Game Deployment", + Environment = $"TGS: {Metadata.Name}", + ProductionEnvironment = true, + RequiredContexts = new Collection(), + }, + remoteInformation.RemoteRepositoryOwner, + remoteInformation.RemoteRepositoryName, + cancellationToken); - Logger.LogDebug("Created deployment ID {deploymentId}", deployment.Id); + Logger.LogDebug("Created deployment ID {deploymentId}", compileJob.GitHubDeploymentId); - await gitHubClient - .Repository - .Deployment - .Status - .Create( - remoteInformation.RemoteRepositoryOwner, - remoteInformation.RemoteRepositoryName, - deployment.Id, - new NewDeploymentStatus(DeploymentState.InProgress) - { - Description = "The project is being deployed", - AutoInactive = false, - }) - .WithToken(cancellationToken); + await gitHubService.CreateDeploymentStatus( + new NewDeploymentStatus(DeploymentState.InProgress) + { + Description = "The project is being deployed", + AutoInactive = false, + }, + remoteInformation.RemoteRepositoryOwner, + remoteInformation.RemoteRepositoryName, + compileJob.GitHubDeploymentId.Value, + cancellationToken); - compileJob.GitHubDeploymentId = deployment.Id; Logger.LogTrace("In-progress deployment status created"); } catch (ApiException ex) @@ -139,11 +127,7 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote try { - var gitHubRepo = await repositoryTask - .WithToken(cancellationToken) - ; - - compileJob.GitHubRepoId = gitHubRepo.Id; + compileJob.GitHubRepoId = await repositoryIdTask; Logger.LogTrace("Set GitHub ID as {gitHubRepoId}", compileJob.GitHubRepoId); } catch (RateLimitExceededException ex) when (!repositorySettings.CreateGitHubDeployments.Value) @@ -206,16 +190,13 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote return Array.Empty(); } - var gitHubClient = repositorySettings.AccessToken != null - ? gitHubClientFactory.CreateClient(repositorySettings.AccessToken) - : gitHubClientFactory.CreateClient(); + var gitHubService = repositorySettings.AccessToken != null + ? gitHubServiceFactory.CreateService(repositorySettings.AccessToken) + : gitHubServiceFactory.CreateService(); var tasks = revisionInformation .ActiveTestMerges - .Select(x => gitHubClient - .PullRequest - .Get(repository.RemoteRepositoryOwner, repository.RemoteRepositoryName, x.TestMerge.Number) - .WithToken(cancellationToken)); + .Select(x => gitHubService.GetPullRequest(repository.RemoteRepositoryOwner, repository.RemoteRepositoryName, x.TestMerge.Number, cancellationToken)); try { await Task.WhenAll(tasks); @@ -260,13 +241,11 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote int testMergeNumber, CancellationToken cancellationToken) { - var gitHubClient = gitHubClientFactory.CreateClient(repositorySettings.AccessToken); + var gitHubService = gitHubServiceFactory.CreateService(repositorySettings.AccessToken); try { - await gitHubClient.Issue.Comment.Create(remoteRepositoryOwner, remoteRepositoryName, testMergeNumber, comment) - .WithToken(cancellationToken) - ; + await gitHubService.CommentOnIssue(remoteRepositoryOwner, remoteRepositoryName, comment, testMergeNumber, cancellationToken); } catch (ApiException e) { @@ -351,21 +330,16 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote return; } - var gitHubClient = gitHubClientFactory.CreateClient(gitHubAccessToken); + var gitHubService = gitHubServiceFactory.CreateService(gitHubAccessToken); - await gitHubClient - .Repository - .Deployment - .Status - .Create( - compileJob.GitHubRepoId.Value, - compileJob.GitHubDeploymentId.Value, - new NewDeploymentStatus(deploymentState) - { - Description = description, - }) - .WithToken(cancellationToken) - ; + await gitHubService.CreateDeploymentStatus( + new NewDeploymentStatus(deploymentState) + { + Description = description, + }, + compileJob.GitHubRepoId.Value, + compileJob.GitHubDeploymentId.Value, + cancellationToken); } } } diff --git a/src/Tgstation.Server.Host/Components/Deployment/Remote/RemoteDeploymentManagerFactory.cs b/src/Tgstation.Server.Host/Components/Deployment/Remote/RemoteDeploymentManagerFactory.cs index f9cdf5a240..4ba8882137 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/Remote/RemoteDeploymentManagerFactory.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/Remote/RemoteDeploymentManagerFactory.cs @@ -20,7 +20,7 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote /// /// The for the . /// - readonly IGitHubClientFactory gitHubClientFactory; + readonly IGitHubServiceFactory gitHubServiceFactory; /// /// The for the . @@ -41,19 +41,19 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote /// Initializes a new instance of the class. /// /// The value of . - /// The value of . + /// The value of . /// The value of . /// The value of . /// The value of . public RemoteDeploymentManagerFactory( IDatabaseContextFactory databaseContextFactory, - IGitHubClientFactory gitHubClientFactory, + IGitHubServiceFactory gitHubServiceFactory, IGitRemoteFeaturesFactory gitRemoteFeaturesFactory, ILoggerFactory loggerFactory, ILogger logger) { this.databaseContextFactory = databaseContextFactory ?? throw new ArgumentNullException(nameof(databaseContextFactory)); - this.gitHubClientFactory = gitHubClientFactory ?? throw new ArgumentNullException(nameof(gitHubClientFactory)); + this.gitHubServiceFactory = gitHubServiceFactory ?? throw new ArgumentNullException(nameof(gitHubServiceFactory)); this.gitRemoteFeaturesFactory = gitRemoteFeaturesFactory ?? throw new ArgumentNullException(nameof(gitRemoteFeaturesFactory)); this.loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory)); this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); @@ -70,7 +70,7 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote { RemoteGitProvider.GitHub => new GitHubRemoteDeploymentManager( databaseContextFactory, - gitHubClientFactory, + gitHubServiceFactory, loggerFactory.CreateLogger(), metadata), RemoteGitProvider.GitLab => new GitLabRemoteDeploymentManager( diff --git a/src/Tgstation.Server.Host/Utils/GitHub/GitHubService.cs b/src/Tgstation.Server.Host/Utils/GitHub/GitHubService.cs index 129f259d6a..39e62c9fde 100644 --- a/src/Tgstation.Server.Host/Utils/GitHub/GitHubService.cs +++ b/src/Tgstation.Server.Host/Utils/GitHub/GitHubService.cs @@ -126,5 +126,140 @@ namespace Tgstation.Server.Host.Utils.GitHub var userDetails = await gitHubClient.User.Current().WithToken(cancellationToken); return userDetails.Id; } + + /// + public Task CommentOnIssue(string repoOwner, string repoName, string comment, int issueNumber, CancellationToken cancellationToken) + { + if (repoOwner == null) + throw new ArgumentNullException(nameof(repoOwner)); + + if (repoName == null) + throw new ArgumentNullException(nameof(repoName)); + + if (comment == null) + throw new ArgumentNullException(nameof(comment)); + + logger.LogTrace("CommentOnIssue"); + + return gitHubClient + .Issue + .Comment + .Create( + repoOwner, + repoName, + issueNumber, + comment) + .WithToken(cancellationToken); + } + + /// + public async Task GetRepositoryId(string repoOwner, string repoName, CancellationToken cancellationToken) + { + if (repoOwner == null) + throw new ArgumentNullException(nameof(repoOwner)); + + if (repoName == null) + throw new ArgumentNullException(nameof(repoName)); + + logger.LogTrace("GetRepositoryId"); + + var repo = await gitHubClient + .Repository + .Get( + repoOwner, + repoName) + .WithToken(cancellationToken); + + return repo.Id; + } + + /// + public async Task CreateDeployment(NewDeployment newDeployment, string repoOwner, string repoName, CancellationToken cancellationToken) + { + if (newDeployment == null) + throw new ArgumentNullException(nameof(newDeployment)); + + if (repoOwner == null) + throw new ArgumentNullException(nameof(repoOwner)); + + if (repoName == null) + throw new ArgumentNullException(nameof(repoName)); + + logger.LogTrace("CreateDeployment"); + + var deployment = await gitHubClient + .Repository + .Deployment + .Create( + repoOwner, + repoName, + newDeployment) + .WithToken(cancellationToken); + + return deployment.Id; + } + + /// + public Task CreateDeploymentStatus(NewDeploymentStatus newDeploymentStatus, string repoOwner, string repoName, int deploymentId, CancellationToken cancellationToken) + { + if (newDeploymentStatus == null) + throw new ArgumentNullException(nameof(newDeploymentStatus)); + + if (repoOwner == null) + throw new ArgumentNullException(nameof(repoOwner)); + + if (repoName == null) + throw new ArgumentNullException(nameof(repoName)); + + logger.LogTrace("CreateDeploymentStatus"); + return gitHubClient + .Repository + .Deployment + .Status + .Create( + repoOwner, + repoName, + deploymentId, + newDeploymentStatus) + .WithToken(cancellationToken); + } + + /// + public Task CreateDeploymentStatus(NewDeploymentStatus newDeploymentStatus, long repoId, int deploymentId, CancellationToken cancellationToken) + { + if (newDeploymentStatus == null) + throw new ArgumentNullException(nameof(newDeploymentStatus)); + + logger.LogTrace("CreateDeploymentStatus"); + return gitHubClient + .Repository + .Deployment + .Status + .Create( + repoId, + deploymentId, + newDeploymentStatus) + .WithToken(cancellationToken); + } + + /// + public Task GetPullRequest(string repoOwner, string repoName, int pullRequestNumber, CancellationToken cancellationToken) + { + if (repoOwner == null) + throw new ArgumentNullException(nameof(repoOwner)); + + if (repoName == null) + throw new ArgumentNullException(nameof(repoName)); + + logger.LogTrace("GetPullRequest"); + return gitHubClient + .Repository + .PullRequest + .Get( + repoOwner, + repoName, + pullRequestNumber) + .WithToken(cancellationToken); + } } } diff --git a/src/Tgstation.Server.Host/Utils/GitHub/IGitHubService.cs b/src/Tgstation.Server.Host/Utils/GitHub/IGitHubService.cs index d9c518ab06..221b5538bc 100644 --- a/src/Tgstation.Server.Host/Utils/GitHub/IGitHubService.cs +++ b/src/Tgstation.Server.Host/Utils/GitHub/IGitHubService.cs @@ -44,5 +44,66 @@ namespace Tgstation.Server.Host.Utils.GitHub /// The for the operation. /// A resulting in the current user's ID. Task GetCurrentUserId(CancellationToken cancellationToken); + + /// + /// Get a target repostiory's ID. + /// + /// The owner of the target repository. + /// The name of the target repository. + /// The for the operation. + /// A resulting in the target repository's ID. + Task GetRepositoryId(string repoOwner, string repoName, CancellationToken cancellationToken); + + /// + /// Create a comment on a given . + /// + /// The owner of the target repository. + /// The name of the target repository. + /// The text of the comment. + /// The number of the issue to comment on. + /// The for the operation. + /// A representing the running operation. + Task CommentOnIssue(string repoOwner, string repoName, string comment, int issueNumber, CancellationToken cancellationToken); + + /// + /// Create a on a target repostiory. + /// + /// The . + /// The owner of the target repository. + /// The name of the target repository. + /// The for the operation. + /// A resulting in the new deployment's ID. + Task CreateDeployment(NewDeployment newDeployment, string repoOwner, string repoName, CancellationToken cancellationToken); + + /// + /// Create a on a target deployment. + /// + /// The . + /// The owner of the target repository. + /// The name of the target repository. + /// The ID of the parent deployment. + /// The for the operation. + /// A representing the running operation. + Task CreateDeploymentStatus(NewDeploymentStatus newDeploymentStatus, string repoOwner, string repoName, int deploymentId, CancellationToken cancellationToken); + + /// + /// Create a on a target deployment. + /// + /// The . + /// The ID of the target repository. + /// The ID of the parent deployment. + /// The for the operation. + /// A representing the running operation. + Task CreateDeploymentStatus(NewDeploymentStatus newDeploymentStatus, long repoId, int deploymentId, CancellationToken cancellationToken); + + /// + /// Get a given . + /// + /// The owner of the target repository. + /// The name of the target repository. + /// The target . + /// The for the operation. + /// A resulting in the target . + Task GetPullRequest(string repoOwner, string repoName, int pullRequestNumber, CancellationToken cancellationToken); } } From f4571a6ff56d78600de5581d78f16677fa6a6e75 Mon Sep 17 00:00:00 2001 From: Dominion Date: Sun, 4 Jun 2023 01:34:27 -0400 Subject: [PATCH 10/41] Migrate GitHubRemoteFeatures to IGitHubService --- .../Repository/GitHubRemoteFeatures.cs | 23 ++++++++----------- .../Repository/GitRemoteFeaturesFactory.cs | 12 +++++----- 2 files changed, 15 insertions(+), 20 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Repository/GitHubRemoteFeatures.cs b/src/Tgstation.Server.Host/Components/Repository/GitHubRemoteFeatures.cs index 31fc74fb2d..165047dcae 100644 --- a/src/Tgstation.Server.Host/Components/Repository/GitHubRemoteFeatures.cs +++ b/src/Tgstation.Server.Host/Components/Repository/GitHubRemoteFeatures.cs @@ -7,7 +7,6 @@ using Octokit; using Tgstation.Server.Api.Models; using Tgstation.Server.Api.Models.Internal; -using Tgstation.Server.Host.Extensions; using Tgstation.Server.Host.Utils.GitHub; namespace Tgstation.Server.Host.Components.Repository @@ -33,20 +32,20 @@ namespace Tgstation.Server.Host.Components.Repository public override string RemoteRepositoryName { get; } /// - /// The for the . + /// The for the . /// - readonly IGitHubClientFactory gitHubClientFactory; + readonly IGitHubServiceFactory gitHubServiceFactory; /// /// Initializes a new instance of the class. /// - /// The value of . + /// The value of . /// The for the . /// The remote repository . - public GitHubRemoteFeatures(IGitHubClientFactory gitHubClientFactory, ILogger logger, Uri remoteUrl) + public GitHubRemoteFeatures(IGitHubServiceFactory gitHubServiceFactory, ILogger logger, Uri remoteUrl) : base(logger, remoteUrl) { - this.gitHubClientFactory = gitHubClientFactory ?? throw new ArgumentNullException(nameof(gitHubClientFactory)); + this.gitHubServiceFactory = gitHubServiceFactory ?? throw new ArgumentNullException(nameof(gitHubServiceFactory)); if (remoteUrl == null) throw new ArgumentNullException(nameof(remoteUrl)); @@ -63,20 +62,16 @@ namespace Tgstation.Server.Host.Components.Repository RepositorySettings repositorySettings, CancellationToken cancellationToken) { - var gitHubClient = repositorySettings.AccessToken != null - ? gitHubClientFactory.CreateClient(repositorySettings.AccessToken) - : gitHubClientFactory.CreateClient(); + var gitHubService = repositorySettings.AccessToken != null + ? gitHubServiceFactory.CreateService(repositorySettings.AccessToken) + : gitHubServiceFactory.CreateService(); PullRequest pr = null; ApiException exception = null; string errorMessage = null; try { - pr = await gitHubClient - .PullRequest - .Get(RemoteRepositoryOwner, RemoteRepositoryName, parameters.Number) - .WithToken(cancellationToken) - ; + pr = await gitHubService.GetPullRequest(RemoteRepositoryOwner, RemoteRepositoryName, parameters.Number, cancellationToken); } catch (RateLimitExceededException ex) { diff --git a/src/Tgstation.Server.Host/Components/Repository/GitRemoteFeaturesFactory.cs b/src/Tgstation.Server.Host/Components/Repository/GitRemoteFeaturesFactory.cs index 21a3d7263e..e19b4c22f0 100644 --- a/src/Tgstation.Server.Host/Components/Repository/GitRemoteFeaturesFactory.cs +++ b/src/Tgstation.Server.Host/Components/Repository/GitRemoteFeaturesFactory.cs @@ -11,9 +11,9 @@ namespace Tgstation.Server.Host.Components.Repository sealed class GitRemoteFeaturesFactory : IGitRemoteFeaturesFactory { /// - /// The for the . + /// The for the . /// - readonly IGitHubClientFactory gitHubClientFactory; + readonly IGitHubServiceFactory gitHubServiceFactory; /// /// The for the . @@ -28,15 +28,15 @@ namespace Tgstation.Server.Host.Components.Repository /// /// Initializes a new instance of the class. /// - /// The value of . + /// The value of . /// The value of . /// The value of . public GitRemoteFeaturesFactory( - IGitHubClientFactory gitHubClientFactory, + IGitHubServiceFactory gitHubServiceFactory, ILoggerFactory loggerFactory, ILogger logger) { - this.gitHubClientFactory = gitHubClientFactory ?? throw new ArgumentNullException(nameof(gitHubClientFactory)); + this.gitHubServiceFactory = gitHubServiceFactory ?? throw new ArgumentNullException(nameof(gitHubServiceFactory)); this.loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory)); this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); } @@ -52,7 +52,7 @@ namespace Tgstation.Server.Host.Components.Repository return remoteGitProvider switch { RemoteGitProvider.GitHub => new GitHubRemoteFeatures( - gitHubClientFactory, + gitHubServiceFactory, loggerFactory.CreateLogger(), primaryRemote), RemoteGitProvider.GitLab => new GitLabRemoteFeatures( From 078775c8a39e5b929bb63dbfb393f7a392434327 Mon Sep 17 00:00:00 2001 From: Dominion Date: Sun, 4 Jun 2023 09:20:24 -0400 Subject: [PATCH 11/41] Minor timeout fix for repository test --- .../Live/Instance/RepositoryTest.cs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/RepositoryTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/RepositoryTest.cs index d93c50e3d2..4fb827af71 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/RepositoryTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/RepositoryTest.cs @@ -49,7 +49,17 @@ namespace Tgstation.Server.Tests.Live.Instance Assert.IsTrue(activeJobs.Any(x => x.Id == clone.ActiveJob.Id)); Assert.IsTrue(allJobs.Any(x => x.Id == clone.ActiveJob.Id)); - Assert.IsTrue(activeJobs.First(x => x.Id == clone.ActiveJob.Id).Progress.HasValue); + + var targetActiveJob = activeJobs.First(x => x.Id == clone.ActiveJob.Id); + + if (!targetActiveJob.Progress.HasValue) + { + // give it 15 more seconds + targetActiveJob = await WaitForJobProgress(targetActiveJob, 15, cancellationToken); + allJobs = await JobsClient.List(null, cancellationToken); + } + + Assert.IsTrue(targetActiveJob.Progress.HasValue); Assert.IsTrue(allJobs.First(x => x.Id == clone.ActiveJob.Id).Progress.HasValue); return clone.ActiveJob; From 6b2731e5e0aee07825b55d64beca6815dab6840e Mon Sep 17 00:00:00 2001 From: Dominion Date: Sun, 4 Jun 2023 09:30:19 -0400 Subject: [PATCH 12/41] Better guards against the flakiness of the GitHub releases API --- .../Utils/GitHub/GitHubService.cs | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/Tgstation.Server.Host/Utils/GitHub/GitHubService.cs b/src/Tgstation.Server.Host/Utils/GitHub/GitHubService.cs index 39e62c9fde..2000647d73 100644 --- a/src/Tgstation.Server.Host/Utils/GitHub/GitHubService.cs +++ b/src/Tgstation.Server.Host/Utils/GitHub/GitHubService.cs @@ -83,20 +83,33 @@ namespace Tgstation.Server.Host.Utils.GitHub logger.LogTrace("{totalReleases} total releases", allReleases.Count); var releases = allReleases - .Select(release => + .Where(release => { - if (!release.TagName.StartsWith(updatesConfiguration.GitTagPrefix, StringComparison.InvariantCulture)) - return null; + if (!release.PublishedAt.HasValue) + { + logger.LogDebug("Release tag without PublishedAt: {releaseTag}", release.TagName); + return false; + } + if (!release.TagName.StartsWith(updatesConfiguration.GitTagPrefix, StringComparison.InvariantCulture)) + return false; + + return true; + }) + .GroupBy(release => + { if (!Version.TryParse(release.TagName.Replace(updatesConfiguration.GitTagPrefix, String.Empty, StringComparison.Ordinal), out var version)) { logger.LogDebug("Unparsable release tag: {releaseTag}", release.TagName); return null; } - return Tuple.Create(version, release); + return version; }) - .Where(tuple => tuple != null) + .Where(grouping => grouping.Key != null) + + // GitHub can return the same result twice or some other nonsense + .Select(grouping => Tuple.Create(grouping.Key, grouping.OrderBy(x => x.PublishedAt.Value).First())) .ToDictionary(tuple => tuple.Item1, tuple => tuple.Item2); logger.LogTrace("{parsedReleases} parsed releases", releases.Count); From 2da317b9b8aa1052be934b4cf35aaca1efa97aab Mon Sep 17 00:00:00 2001 From: Dominion Date: Sun, 4 Jun 2023 09:34:52 -0400 Subject: [PATCH 13/41] Extend test timeout specifically for GitHub actions --- tests/Tgstation.Server.Tests/Live/LiveTestUtils.cs | 9 +++++++++ tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs | 3 +-- .../Live/RateLimitRetryingApiClient.cs | 4 ++++ tests/Tgstation.Server.Tests/Live/TestLiveServer.cs | 4 ++-- 4 files changed, 16 insertions(+), 4 deletions(-) create mode 100644 tests/Tgstation.Server.Tests/Live/LiveTestUtils.cs diff --git a/tests/Tgstation.Server.Tests/Live/LiveTestUtils.cs b/tests/Tgstation.Server.Tests/Live/LiveTestUtils.cs new file mode 100644 index 0000000000..bc8814ca2f --- /dev/null +++ b/tests/Tgstation.Server.Tests/Live/LiveTestUtils.cs @@ -0,0 +1,9 @@ +using System; + +namespace Tgstation.Server.Tests.Live +{ + static class LiveTestUtils + { + public static bool RunningInGitHubActions { get; } = !String.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("GITHUB_RUN_ID")); + } +} diff --git a/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs b/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs index 4aa9e70604..66e0d46e90 100644 --- a/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs +++ b/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs @@ -89,9 +89,8 @@ namespace Tgstation.Server.Tests.Live // neither of these should really matter but it's better that we test them // high prio DD might help with some topic flakiness actually // github doesn't allow nicing on linux though - var runningInGitHubActions = !String.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("GITHUB_RUN_ID")); var windows = new Host.System.PlatformIdentifier().IsWindows; - var nicingAllowed = windows || !runningInGitHubActions; + var nicingAllowed = windows || !LiveTestUtils.RunningInGitHubActions; HighPriorityDreamDaemon = nicingAllowed; LowPriorityDeployments = nicingAllowed; diff --git a/tests/Tgstation.Server.Tests/Live/RateLimitRetryingApiClient.cs b/tests/Tgstation.Server.Tests/Live/RateLimitRetryingApiClient.cs index c45c12dbf8..9434730bc1 100644 --- a/tests/Tgstation.Server.Tests/Live/RateLimitRetryingApiClient.cs +++ b/tests/Tgstation.Server.Tests/Live/RateLimitRetryingApiClient.cs @@ -3,6 +3,8 @@ using System.Net.Http; using System.Threading; using System.Threading.Tasks; +using Microsoft.VisualStudio.TestTools.UnitTesting; + using Tgstation.Server.Api; using Tgstation.Server.Client; using Tgstation.Server.Common; @@ -29,6 +31,8 @@ namespace Tgstation.Server.Tests.Live var now = DateTimeOffset.UtcNow; Console.WriteLine($"TEST ERROR RATE LIMITED: {ex}"); + if (!LiveTestUtils.RunningInGitHubActions) + Assert.Inconclusive("Rate limited by GitHub!"); var sleepTime = ex.RetryAfter.Value - now; Console.WriteLine($"Sleeping for {sleepTime.TotalMinutes} minutes and retrying..."); diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index d75a40c8ef..726f00b76e 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -746,8 +746,8 @@ namespace Tgstation.Server.Tests.Live Assert.AreEqual(ProcessPriorityClass.Normal, currentProcess.PriorityClass); } - const int MaximumTestMinutes = 30; - using var hardCancellationTokenSource = new CancellationTokenSource(TimeSpan.FromMinutes(MaximumTestMinutes)); + var maximumTestMinutes = LiveTestUtils.RunningInGitHubActions ? 90 : 20; + using var hardCancellationTokenSource = new CancellationTokenSource(TimeSpan.FromMinutes(maximumTestMinutes)); var hardCancellationToken = hardCancellationTokenSource.Token; ServiceCollectionExtensions.UseAdditionalLoggerProvider(); From 3b870711f5362d71c104e9c4112e970ecc4440b6 Mon Sep 17 00:00:00 2001 From: Dominion Date: Sun, 4 Jun 2023 09:39:53 -0400 Subject: [PATCH 14/41] Enable login refresh attempts on test clients --- tests/Tgstation.Server.Tests/Live/TestLiveServer.cs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index 726f00b76e..5a6592eaa2 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -1133,9 +1133,7 @@ namespace Tgstation.Server.Tests.Live url, DefaultCredentials.AdminUserName, DefaultCredentials.DefaultAdminUserPassword, - attemptLoginRefresh: false, - cancellationToken: cancellationToken) - ; + cancellationToken: cancellationToken); } catch (HttpRequestException) { From 63beb10e4e5699d578ce2bdb60efedd4fad0974d Mon Sep 17 00:00:00 2001 From: Dominion Date: Sun, 4 Jun 2023 11:12:30 -0400 Subject: [PATCH 15/41] Do not overlog --- tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index 12aa55e81a..8040fcfb87 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -570,7 +570,7 @@ namespace Tgstation.Server.Tests.Live.Instance while (!cancellationToken.IsCancellationRequested) { var currentSize = baseSize + (int)Math.Pow(2, nextPow); - var topicRequestResult = await TopicClient.SendTopic( + var topicRequestResult = await TopicClientNoLogger.SendTopic( IPAddress.Loopback, $"tgs_integration_test_tactics4={TopicClient.SanitizeString(currentSize.ToString())}", TestLiveServer.DDPort, From 2751abc44dadada919b9f293b16e7531022bdd33 Mon Sep 17 00:00:00 2001 From: Dominion Date: Sun, 4 Jun 2023 11:57:04 -0400 Subject: [PATCH 16/41] Move GitHubService initialization to ServiceCollectionExtensions and make modifiable --- src/Tgstation.Server.Host/Core/Application.cs | 5 +-- .../Extensions/ServiceCollectionExtensions.cs | 35 +++++++++++++++++++ 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index 99ea878a1d..cab990f0e3 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -51,7 +51,6 @@ using Tgstation.Server.Host.Swarm; using Tgstation.Server.Host.System; using Tgstation.Server.Host.Transfer; using Tgstation.Server.Host.Utils; -using Tgstation.Server.Host.Utils.GitHub; namespace Tgstation.Server.Host.Core { @@ -372,9 +371,7 @@ namespace Tgstation.Server.Host.Core services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(x => x.GetRequiredService().CreateService()); + services.AddGitHub(); // configure root services services.AddSingleton(); diff --git a/src/Tgstation.Server.Host/Extensions/ServiceCollectionExtensions.cs b/src/Tgstation.Server.Host/Extensions/ServiceCollectionExtensions.cs index 4980d732d8..0d8e199285 100644 --- a/src/Tgstation.Server.Host/Extensions/ServiceCollectionExtensions.cs +++ b/src/Tgstation.Server.Host/Extensions/ServiceCollectionExtensions.cs @@ -15,6 +15,7 @@ using Serilog.Sinks.Elasticsearch; using Tgstation.Server.Host.Components.Chat.Providers; using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.Utils; +using Tgstation.Server.Host.Utils.GitHub; namespace Tgstation.Server.Host.Extensions { @@ -28,6 +29,11 @@ namespace Tgstation.Server.Host.Extensions /// static Type chatProviderFactoryType = typeof(ProviderFactory); + /// + /// The implementation used in calls to . + /// + static Type gitHubServiceFactoryType = typeof(GitHubServiceFactory); + /// /// A for an additional to use. /// @@ -42,6 +48,32 @@ namespace Tgstation.Server.Host.Extensions chatProviderFactoryType = typeof(TProviderFactory); } + /// + /// Change the used as an implementation for calls to . + /// + /// The implementation to use. + public static void UseGitHubServiceFactory() where TGitHubServiceFactory : IGitHubServiceFactory + { + gitHubServiceFactoryType = typeof(TGitHubServiceFactory); + } + + /// + /// Adds a implementation to the given . + /// + /// The to configure. + /// . + public static IServiceCollection AddGitHub(this IServiceCollection serviceCollection) + { + if (serviceCollection == null) + throw new ArgumentNullException(nameof(serviceCollection)); + + serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(typeof(IGitHubServiceFactory), gitHubServiceFactoryType); + serviceCollection.AddSingleton(x => x.GetRequiredService().CreateService()); + + return serviceCollection; + } + /// /// Add an additional to s that call . /// @@ -60,6 +92,9 @@ namespace Tgstation.Server.Host.Extensions /// . public static IServiceCollection AddChatProviderFactory(this IServiceCollection serviceCollection) { + if (serviceCollection == null) + throw new ArgumentNullException(nameof(serviceCollection)); + return serviceCollection.AddSingleton(typeof(IProviderFactory), chatProviderFactoryType); } From 174d899f70c7dbf8d346b8751e0bc9947cf874c3 Mon Sep 17 00:00:00 2001 From: Dominion Date: Sun, 4 Jun 2023 12:07:10 -0400 Subject: [PATCH 17/41] Split IGitHubService into IAuthenticatedGitHubService --- .../Remote/GitHubRemoteDeploymentManager.cs | 19 +++++-- .../Utils/GitHub/GitHubService.cs | 6 +- .../Utils/GitHub/GitHubServiceFactory.cs | 2 +- .../GitHub/IAuthenticatedGitHubService.cs | 55 +++++++++++++++++++ .../Utils/GitHub/IGitHubService.cs | 48 +--------------- .../Utils/GitHub/IGitHubServiceFactory.cs | 6 +- 6 files changed, 80 insertions(+), 56 deletions(-) create mode 100644 src/Tgstation.Server.Host/Utils/GitHub/IAuthenticatedGitHubService.cs diff --git a/src/Tgstation.Server.Host/Components/Deployment/Remote/GitHubRemoteDeploymentManager.cs b/src/Tgstation.Server.Host/Components/Deployment/Remote/GitHubRemoteDeploymentManager.cs index d327a84db2..17ec7111a1 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/Remote/GitHubRemoteDeploymentManager.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/Remote/GitHubRemoteDeploymentManager.cs @@ -73,9 +73,18 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote .FirstAsync(cancellationToken)); var instanceAuthenticated = repositorySettings.AccessToken != null; - var gitHubService = !instanceAuthenticated - ? gitHubServiceFactory.CreateService() - : gitHubServiceFactory.CreateService(repositorySettings.AccessToken); + IAuthenticatedGitHubService authenticatedGitHubService; + IGitHubService gitHubService; + if (instanceAuthenticated) + { + authenticatedGitHubService = gitHubServiceFactory.CreateService(repositorySettings.AccessToken); + gitHubService = authenticatedGitHubService; + } + else + { + authenticatedGitHubService = null; + gitHubService = gitHubServiceFactory.CreateService(); + } var repositoryIdTask = gitHubService.GetRepositoryId( remoteInformation.RemoteRepositoryOwner, @@ -91,7 +100,7 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote Logger.LogTrace("Creating deployment..."); try { - compileJob.GitHubDeploymentId = await gitHubService.CreateDeployment( + compileJob.GitHubDeploymentId = await authenticatedGitHubService.CreateDeployment( new NewDeployment(compileJob.RevisionInformation.CommitSha) { AutoMerge = false, @@ -106,7 +115,7 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote Logger.LogDebug("Created deployment ID {deploymentId}", compileJob.GitHubDeploymentId); - await gitHubService.CreateDeploymentStatus( + await authenticatedGitHubService.CreateDeploymentStatus( new NewDeploymentStatus(DeploymentState.InProgress) { Description = "The project is being deployed", diff --git a/src/Tgstation.Server.Host/Utils/GitHub/GitHubService.cs b/src/Tgstation.Server.Host/Utils/GitHub/GitHubService.cs index 2000647d73..ba89b52842 100644 --- a/src/Tgstation.Server.Host/Utils/GitHub/GitHubService.cs +++ b/src/Tgstation.Server.Host/Utils/GitHub/GitHubService.cs @@ -13,8 +13,10 @@ using Tgstation.Server.Host.Extensions; namespace Tgstation.Server.Host.Utils.GitHub { - /// - sealed class GitHubService : IGitHubService + /// + /// Service for interacting with GitHub. Authenticated or otherwise. + /// + sealed class GitHubService : IAuthenticatedGitHubService { /// /// The for the . diff --git a/src/Tgstation.Server.Host/Utils/GitHub/GitHubServiceFactory.cs b/src/Tgstation.Server.Host/Utils/GitHub/GitHubServiceFactory.cs index 3873188659..8ff53440a6 100644 --- a/src/Tgstation.Server.Host/Utils/GitHub/GitHubServiceFactory.cs +++ b/src/Tgstation.Server.Host/Utils/GitHub/GitHubServiceFactory.cs @@ -47,7 +47,7 @@ namespace Tgstation.Server.Host.Utils.GitHub public IGitHubService CreateService() => CreateServiceImpl(gitHubClientFactory.CreateClient()); /// - public IGitHubService CreateService(string accessToken) + public IAuthenticatedGitHubService CreateService(string accessToken) => CreateServiceImpl( gitHubClientFactory.CreateClient( accessToken ?? throw new ArgumentNullException(nameof(accessToken)))); diff --git a/src/Tgstation.Server.Host/Utils/GitHub/IAuthenticatedGitHubService.cs b/src/Tgstation.Server.Host/Utils/GitHub/IAuthenticatedGitHubService.cs new file mode 100644 index 0000000000..f920f25912 --- /dev/null +++ b/src/Tgstation.Server.Host/Utils/GitHub/IAuthenticatedGitHubService.cs @@ -0,0 +1,55 @@ +using System.Threading; +using System.Threading.Tasks; + +using Octokit; + +namespace Tgstation.Server.Host.Utils.GitHub +{ + /// + /// that exposes functions that require authentication. + /// + public interface IAuthenticatedGitHubService : IGitHubService + { + /// + /// Create a comment on a given . + /// + /// The owner of the target repository. + /// The name of the target repository. + /// The text of the comment. + /// The number of the issue to comment on. + /// The for the operation. + /// A representing the running operation. + Task CommentOnIssue(string repoOwner, string repoName, string comment, int issueNumber, CancellationToken cancellationToken); + + /// + /// Create a on a target repostiory. + /// + /// The . + /// The owner of the target repository. + /// The name of the target repository. + /// The for the operation. + /// A resulting in the new deployment's ID. + Task CreateDeployment(NewDeployment newDeployment, string repoOwner, string repoName, CancellationToken cancellationToken); + + /// + /// Create a on a target deployment. + /// + /// The . + /// The owner of the target repository. + /// The name of the target repository. + /// The ID of the parent deployment. + /// The for the operation. + /// A representing the running operation. + Task CreateDeploymentStatus(NewDeploymentStatus newDeploymentStatus, string repoOwner, string repoName, int deploymentId, CancellationToken cancellationToken); + + /// + /// Create a on a target deployment. + /// + /// The . + /// The ID of the target repository. + /// The ID of the parent deployment. + /// The for the operation. + /// A representing the running operation. + Task CreateDeploymentStatus(NewDeploymentStatus newDeploymentStatus, long repoId, int deploymentId, CancellationToken cancellationToken); + } +} diff --git a/src/Tgstation.Server.Host/Utils/GitHub/IGitHubService.cs b/src/Tgstation.Server.Host/Utils/GitHub/IGitHubService.cs index 221b5538bc..6c8d479535 100644 --- a/src/Tgstation.Server.Host/Utils/GitHub/IGitHubService.cs +++ b/src/Tgstation.Server.Host/Utils/GitHub/IGitHubService.cs @@ -38,13 +38,6 @@ namespace Tgstation.Server.Host.Utils.GitHub /// A resulting in a representing the returned OAuth code from GitHub on success, otherwise. Task CreateOAuthAccessToken(OAuthConfiguration oAuthConfiguration, string code, CancellationToken cancellationToken); - /// - /// Get the current user's ID. - /// - /// The for the operation. - /// A resulting in the current user's ID. - Task GetCurrentUserId(CancellationToken cancellationToken); - /// /// Get a target repostiory's ID. /// @@ -55,46 +48,11 @@ namespace Tgstation.Server.Host.Utils.GitHub Task GetRepositoryId(string repoOwner, string repoName, CancellationToken cancellationToken); /// - /// Create a comment on a given . + /// Get the current user's ID. /// - /// The owner of the target repository. - /// The name of the target repository. - /// The text of the comment. - /// The number of the issue to comment on. /// The for the operation. - /// A representing the running operation. - Task CommentOnIssue(string repoOwner, string repoName, string comment, int issueNumber, CancellationToken cancellationToken); - - /// - /// Create a on a target repostiory. - /// - /// The . - /// The owner of the target repository. - /// The name of the target repository. - /// The for the operation. - /// A resulting in the new deployment's ID. - Task CreateDeployment(NewDeployment newDeployment, string repoOwner, string repoName, CancellationToken cancellationToken); - - /// - /// Create a on a target deployment. - /// - /// The . - /// The owner of the target repository. - /// The name of the target repository. - /// The ID of the parent deployment. - /// The for the operation. - /// A representing the running operation. - Task CreateDeploymentStatus(NewDeploymentStatus newDeploymentStatus, string repoOwner, string repoName, int deploymentId, CancellationToken cancellationToken); - - /// - /// Create a on a target deployment. - /// - /// The . - /// The ID of the target repository. - /// The ID of the parent deployment. - /// The for the operation. - /// A representing the running operation. - Task CreateDeploymentStatus(NewDeploymentStatus newDeploymentStatus, long repoId, int deploymentId, CancellationToken cancellationToken); + /// A resulting in the current user's ID. + Task GetCurrentUserId(CancellationToken cancellationToken); /// /// Get a given . diff --git a/src/Tgstation.Server.Host/Utils/GitHub/IGitHubServiceFactory.cs b/src/Tgstation.Server.Host/Utils/GitHub/IGitHubServiceFactory.cs index 1bf1f1b850..f4e7be70f5 100644 --- a/src/Tgstation.Server.Host/Utils/GitHub/IGitHubServiceFactory.cs +++ b/src/Tgstation.Server.Host/Utils/GitHub/IGitHubServiceFactory.cs @@ -12,10 +12,10 @@ public IGitHubService CreateService(); /// - /// Create a . + /// Create an . /// /// The access token to use for communication with GitHub. - /// A new . - public IGitHubService CreateService(string accessToken); + /// A new . + public IAuthenticatedGitHubService CreateService(string accessToken); } } From c853276ffb4548689c092dfc8ca5568095c27d59 Mon Sep 17 00:00:00 2001 From: Dominion Date: Sun, 4 Jun 2023 12:07:35 -0400 Subject: [PATCH 18/41] Fix using spacing --- .../Deployment/Remote/GitHubRemoteDeploymentManager.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Tgstation.Server.Host/Components/Deployment/Remote/GitHubRemoteDeploymentManager.cs b/src/Tgstation.Server.Host/Components/Deployment/Remote/GitHubRemoteDeploymentManager.cs index 17ec7111a1..2df99e9bf8 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/Remote/GitHubRemoteDeploymentManager.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/Remote/GitHubRemoteDeploymentManager.cs @@ -8,6 +8,7 @@ using System.Threading.Tasks; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; + using Octokit; using Tgstation.Server.Host.Components.Repository; From 395ee7d31655f1bd30b3e518d96841044b7313e9 Mon Sep 17 00:00:00 2001 From: Dominion Date: Sun, 4 Jun 2023 13:15:09 -0400 Subject: [PATCH 19/41] Add DummyGitHubService for live testing - Also enable repo additional features for testing if possible - Change a bunch of `TestInitialize`rs to `ClassInitialize`rs --- .../Utils/GitHub/GitHubService.cs | 2 +- .../Utils/GitHub/IGitHubService.cs | 2 +- .../Chat/Providers/TestDiscordProvider.cs | 8 +- .../IO/TestSymlinkFactory.cs | 6 +- .../Swarm/TestSwarmProtocol.cs | 14 +- .../System/TestProcessFeatures.cs | 6 +- .../Utils/GitHub/TestGitHubClientFactory.cs | 10 +- .../Live/DummyGitHubService.cs | 134 ++++++++++++++++++ .../Live/DummyGitHubServiceFactory.cs | 33 +++++ .../Live/Instance/RepositoryTest.cs | 17 ++- .../Live/TestLiveServer.cs | 14 +- .../TestDMApiConstants.cs | 6 +- tests/Tgstation.Server.Tests/TestVersions.cs | 8 +- 13 files changed, 220 insertions(+), 40 deletions(-) create mode 100644 tests/Tgstation.Server.Tests/Live/DummyGitHubService.cs create mode 100644 tests/Tgstation.Server.Tests/Live/DummyGitHubServiceFactory.cs diff --git a/src/Tgstation.Server.Host/Utils/GitHub/GitHubService.cs b/src/Tgstation.Server.Host/Utils/GitHub/GitHubService.cs index ba89b52842..354f26f9d0 100644 --- a/src/Tgstation.Server.Host/Utils/GitHub/GitHubService.cs +++ b/src/Tgstation.Server.Host/Utils/GitHub/GitHubService.cs @@ -14,7 +14,7 @@ using Tgstation.Server.Host.Extensions; namespace Tgstation.Server.Host.Utils.GitHub { /// - /// Service for interacting with GitHub. Authenticated or otherwise. + /// Service for interacting with the GitHub API. Authenticated or otherwise. /// sealed class GitHubService : IAuthenticatedGitHubService { diff --git a/src/Tgstation.Server.Host/Utils/GitHub/IGitHubService.cs b/src/Tgstation.Server.Host/Utils/GitHub/IGitHubService.cs index 6c8d479535..1776529dc1 100644 --- a/src/Tgstation.Server.Host/Utils/GitHub/IGitHubService.cs +++ b/src/Tgstation.Server.Host/Utils/GitHub/IGitHubService.cs @@ -10,7 +10,7 @@ using Tgstation.Server.Host.Configuration; namespace Tgstation.Server.Host.Utils.GitHub { /// - /// Service for interacting with GitHub. + /// Service for interacting with the GitHub API. /// public interface IGitHubService { diff --git a/tests/Tgstation.Server.Host.Tests/Components/Chat/Providers/TestDiscordProvider.cs b/tests/Tgstation.Server.Host.Tests/Components/Chat/Providers/TestDiscordProvider.cs index cdc30c888e..7389acbc81 100644 --- a/tests/Tgstation.Server.Host.Tests/Components/Chat/Providers/TestDiscordProvider.cs +++ b/tests/Tgstation.Server.Host.Tests/Components/Chat/Providers/TestDiscordProvider.cs @@ -18,11 +18,11 @@ namespace Tgstation.Server.Host.Components.Chat.Providers.Tests [TestClass] public sealed class TestDiscordProvider { - ChatBot testToken1; - IJobManager mockJobManager; + static ChatBot testToken1; + static IJobManager mockJobManager; - [TestInitialize] - public void Initialize() + [ClassInitialize] + public static void Initialize(TestContext _) { var actualToken = Environment.GetEnvironmentVariable("TGS_TEST_DISCORD_TOKEN"); if (!String.IsNullOrWhiteSpace(actualToken)) diff --git a/tests/Tgstation.Server.Host.Tests/IO/TestSymlinkFactory.cs b/tests/Tgstation.Server.Host.Tests/IO/TestSymlinkFactory.cs index fd5dd76591..27646a9dc7 100644 --- a/tests/Tgstation.Server.Host.Tests/IO/TestSymlinkFactory.cs +++ b/tests/Tgstation.Server.Host.Tests/IO/TestSymlinkFactory.cs @@ -10,10 +10,10 @@ namespace Tgstation.Server.Host.IO.Tests [TestClass] public sealed class TestSymlinkFactory { - ISymlinkFactory symlinkFactory; + static ISymlinkFactory symlinkFactory; - [TestInitialize] - public void SelectFactory() + [ClassInitialize] + public static void SelectFactory(TestContext _) { if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) symlinkFactory = new WindowsSymlinkFactory(); diff --git a/tests/Tgstation.Server.Host.Tests/Swarm/TestSwarmProtocol.cs b/tests/Tgstation.Server.Host.Tests/Swarm/TestSwarmProtocol.cs index 63d348080a..722d967521 100644 --- a/tests/Tgstation.Server.Host.Tests/Swarm/TestSwarmProtocol.cs +++ b/tests/Tgstation.Server.Host.Tests/Swarm/TestSwarmProtocol.cs @@ -13,12 +13,12 @@ namespace Tgstation.Server.Host.Swarm.Tests [TestClass] public sealed class TestSwarmProtocol { - readonly HashSet usedPorts = new (); - ILoggerFactory loggerFactory; - ILogger logger; + static readonly HashSet usedPorts = new (); + static ILoggerFactory loggerFactory; + static ILogger logger; - [TestInitialize] - public void Initialize() + [ClassInitialize] + public static void Initialize(TestContext _) { loggerFactory = LoggerFactory.Create(builder => { @@ -29,8 +29,8 @@ namespace Tgstation.Server.Host.Swarm.Tests logger = loggerFactory.CreateLogger(); } - [TestCleanup] - public void Shutdown() + [ClassCleanup] + public static void Shutdown() { usedPorts.Clear(); loggerFactory.Dispose(); diff --git a/tests/Tgstation.Server.Host.Tests/System/TestProcessFeatures.cs b/tests/Tgstation.Server.Host.Tests/System/TestProcessFeatures.cs index f6f680c50d..066370977c 100644 --- a/tests/Tgstation.Server.Host.Tests/System/TestProcessFeatures.cs +++ b/tests/Tgstation.Server.Host.Tests/System/TestProcessFeatures.cs @@ -15,10 +15,10 @@ namespace Tgstation.Server.Host.System.Tests [TestClass] public sealed class TestProcessFeatures { - IProcessFeatures features; + static IProcessFeatures features; - [TestInitialize] - public void Init() + [ClassInitialize] + public static void Init(TestContext _) { features = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? (IProcessFeatures)new WindowsProcessFeatures(Mock.Of>()) diff --git a/tests/Tgstation.Server.Host.Tests/Utils/GitHub/TestGitHubClientFactory.cs b/tests/Tgstation.Server.Host.Tests/Utils/GitHub/TestGitHubClientFactory.cs index 5c9a6f7f31..6f4fac0ae2 100644 --- a/tests/Tgstation.Server.Host.Tests/Utils/GitHub/TestGitHubClientFactory.cs +++ b/tests/Tgstation.Server.Host.Tests/Utils/GitHub/TestGitHubClientFactory.cs @@ -18,10 +18,10 @@ namespace Tgstation.Server.Host.Utils.GitHub.Tests [TestClass] public sealed class TestGitHubClientFactory { - ILoggerFactory loggerFactory; + static ILoggerFactory loggerFactory; - [TestInitialize] - public void Initialize() + [ClassInitialize] + public static void Initialize(TestContext _) { loggerFactory = LoggerFactory.Create(builder => { @@ -30,8 +30,8 @@ namespace Tgstation.Server.Host.Utils.GitHub.Tests }); } - [TestCleanup] - public void Cleanup() + [ClassCleanup] + public static void Cleanup() { loggerFactory.Dispose(); } diff --git a/tests/Tgstation.Server.Tests/Live/DummyGitHubService.cs b/tests/Tgstation.Server.Tests/Live/DummyGitHubService.cs new file mode 100644 index 0000000000..c1bcd2903e --- /dev/null +++ b/tests/Tgstation.Server.Tests/Live/DummyGitHubService.cs @@ -0,0 +1,134 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +using Moq; + +using Octokit; + +using Tgstation.Server.Host.Configuration; +using Tgstation.Server.Host.Extensions; +using Tgstation.Server.Host.Security; +using Tgstation.Server.Host.System; +using Tgstation.Server.Host.Utils.GitHub; + +namespace Tgstation.Server.Tests.Live +{ + sealed class DummyGitHubService : IAuthenticatedGitHubService + { + static Dictionary releasesDictionary; + static PullRequest testPr; + + readonly ICryptographySuite cryptographySuite; + readonly ILogger logger; + + public static async Task InitializeAndInject(CancellationToken cancellationToken) + { + var mockOptions = new Mock>(); + mockOptions.SetupGet(x => x.Value).Returns(new GeneralConfiguration + { + GitHubAccessToken = Environment.GetEnvironmentVariable("TGS_TEST_GITHUB_TOKEN") + }); + + var gitHubClientFactory = new GitHubClientFactory(new AssemblyInformationProvider(), Mock.Of>(), mockOptions.Object); + var gitHubClient = gitHubClientFactory.CreateClient(); + + Release targetRelease; + do + { + var releases = await gitHubClient + .Repository + .Release + .GetAll("tgstation", "tgstation-server") + .WithToken(cancellationToken); + + targetRelease = releases.FirstOrDefault(release => release.TagName == $"{new UpdatesConfiguration().GitTagPrefix}{TestLiveServer.TestUpdateVersion}"); + } + while (targetRelease == null); + + releasesDictionary = new Dictionary + { + { TestLiveServer.TestUpdateVersion, targetRelease } + }; + + testPr = await gitHubClient + .PullRequest + .Get("Cyberboss", "common_core", 2) + .WithToken(cancellationToken); + + ServiceCollectionExtensions.UseGitHubServiceFactory(); + } + + public DummyGitHubService(ICryptographySuite cryptographySuite, ILogger logger) + { + this.cryptographySuite = cryptographySuite ?? throw new ArgumentNullException(nameof(cryptographySuite)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + logger.LogTrace("Created"); + } + + public Task CommentOnIssue(string repoOwner, string repoName, string comment, int issueNumber, CancellationToken cancellationToken) + { + logger.LogTrace("CommentOnIssue"); + return Task.CompletedTask; + } + + public Task CreateDeployment(NewDeployment newDeployment, string repoOwner, string repoName, CancellationToken cancellationToken) + { + logger.LogTrace("CreateDeployment"); + return Task.FromResult(new Random().Next()); ; + } + + public Task CreateDeploymentStatus(NewDeploymentStatus newDeploymentStatus, string repoOwner, string repoName, int deploymentId, CancellationToken cancellationToken) + { + logger.LogTrace("CreateDeploymentStatus"); + return Task.CompletedTask; + } + + public Task CreateDeploymentStatus(NewDeploymentStatus newDeploymentStatus, long repoId, int deploymentId, CancellationToken cancellationToken) + { + logger.LogTrace("CreateDeploymentStatus"); + return Task.CompletedTask; + } + + public Task CreateOAuthAccessToken(OAuthConfiguration oAuthConfiguration, string code, CancellationToken cancellationToken) + { + logger.LogTrace("CreateOAuthAccessToken"); + return Task.FromResult(cryptographySuite.GetSecureString()); + } + + public Task GetCurrentUserId(CancellationToken cancellationToken) + { + logger.LogTrace("GetCurrentUserId"); + return Task.FromResult(new Random().Next()); + } + + public Task GetRepositoryId(string repoOwner, string repoName, CancellationToken cancellationToken) + { + logger.LogTrace("GetRepositoryId"); + return Task.FromResult(new Random().NextInt64()); + } + + public Task GetUpdatesRepositoryUrl(CancellationToken cancellationToken) + { + logger.LogTrace("GetUpdatesRepositoryUrl"); + return Task.FromResult(new Uri("https://github.com/tgstation/tgstation-server")); + } + + public Task GetPullRequest(string repoOwner, string repoName, int pullRequestNumber, CancellationToken cancellationToken) + { + logger.LogTrace("GetPullRequest"); + return Task.FromResult(testPr); + } + + public Task> GetTgsReleases(CancellationToken cancellationToken) + { + logger.LogTrace("GetTgsReleases"); + return Task.FromResult(releasesDictionary); + } + } +} diff --git a/tests/Tgstation.Server.Tests/Live/DummyGitHubServiceFactory.cs b/tests/Tgstation.Server.Tests/Live/DummyGitHubServiceFactory.cs new file mode 100644 index 0000000000..59bef5fdd0 --- /dev/null +++ b/tests/Tgstation.Server.Tests/Live/DummyGitHubServiceFactory.cs @@ -0,0 +1,33 @@ +using System; + +using Microsoft.Extensions.Logging; + +using Tgstation.Server.Host.Security; +using Tgstation.Server.Host.Utils.GitHub; + +namespace Tgstation.Server.Tests.Live +{ + sealed class DummyGitHubServiceFactory : IGitHubServiceFactory + { + readonly ICryptographySuite cryptographySuite; + readonly ILogger logger; + + public DummyGitHubServiceFactory(ICryptographySuite cryptographySuite, ILogger logger) + { + this.cryptographySuite = cryptographySuite ?? throw new ArgumentNullException(nameof(cryptographySuite)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public IGitHubService CreateService() => CreateDummyService(); + + public IAuthenticatedGitHubService CreateService(string accessToken) + { + if (accessToken == null) + throw new ArgumentNullException(nameof(accessToken)); + + return CreateDummyService(); + } + + DummyGitHubService CreateDummyService() => new DummyGitHubService(cryptographySuite, logger); + } +} diff --git a/tests/Tgstation.Server.Tests/Live/Instance/RepositoryTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/RepositoryTest.cs index 4fb827af71..68e9270e92 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/RepositoryTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/RepositoryTest.cs @@ -80,7 +80,7 @@ namespace Tgstation.Server.Tests.Live.Instance Assert.IsNotNull(secondRead); Assert.IsNull(secondRead.ActiveJob); - const string Origin = "https://github.com/tgstation/common_core"; + const string Origin = "https://github.com/Cyberboss/common_core"; var cloneRequest = new RepositoryCreateRequest { Origin = new Uri(Origin), @@ -118,8 +118,19 @@ namespace Tgstation.Server.Tests.Live.Instance // Back updated = await Checkout(new RepositoryUpdateRequest { Reference = "master" }, false, true, cancellationToken); - var prNumber = 37; + var prNumber = 2; await TestMergeTests(updated, prNumber, cancellationToken); + + // enable the good shit if possible + if (LiveTestUtils.RunningInGitHubActions + || String.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("TGS_TEST_GITHUB_TOKEN")) + || Environment.MachineName.Equals("CYBERSTATIONXVI", StringComparison.OrdinalIgnoreCase)) + await repositoryClient.Update(new RepositoryUpdateRequest + { + CreateGitHubDeployments = true, + PostTestMergeComment = true, + PushTestMergeCommits = true, + }, cancellationToken); } async Task Checkout(RepositoryUpdateRequest updated, bool expectFailure, bool isRef, CancellationToken cancellationToken) @@ -174,7 +185,7 @@ namespace Tgstation.Server.Tests.Live.Instance Assert.IsNotNull(withMerge.RevisionInformation.PrimaryTestMerge.TitleAtMerge); Assert.IsNotNull(withMerge.RevisionInformation.PrimaryTestMerge.BodyAtMerge); if (withMerge.RevisionInformation.PrimaryTestMerge.Url != "GITHUB API ERROR: RATE LIMITED") - Assert.AreEqual($"https://github.com/tgstation/common_core/pull/{prNumber}", withMerge.RevisionInformation.PrimaryTestMerge.Url); + Assert.AreEqual($"https://github.com/Cyberboss/common_core/pull/{prNumber}", withMerge.RevisionInformation.PrimaryTestMerge.Url); Assert.AreEqual(orignCommit, withMerge.RevisionInformation.OriginCommitSha); Assert.AreNotEqual(orignCommit, withMerge.RevisionInformation.CommitSha); diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index 5a6592eaa2..f603a2c6e4 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -44,7 +44,7 @@ namespace Tgstation.Server.Tests.Live public static ushort DDPort { get; } = FreeTcpPort(); public static ushort DMPort { get; } = GetDMPort(); - readonly Version TestUpdateVersion = new(5, 11, 0); + public static readonly Version TestUpdateVersion = new(5, 11, 0); readonly IServerClientFactory clientFactory = new ServerClientFactory(new ProductHeaderValue(Assembly.GetExecutingAssembly().GetName().Name, Assembly.GetExecutingAssembly().GetName().Version.ToString())); @@ -79,9 +79,12 @@ namespace Tgstation.Server.Tests.Live } } - [TestInitialize] - public async Task Initialize() + [ClassInitialize] + public static async Task Initialize(TestContext _) { + if (LiveTestUtils.RunningInGitHubActions || String.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("TGS_TEST_GITHUB_TOKEN"))) + await DummyGitHubService.InitializeAndInject(default); + await DummyChatProvider.RandomDisconnections(true, default); ServerClientFactory.ApiClientFactory = new RateLimitRetryingApiClientFactory(); @@ -155,7 +158,6 @@ namespace Tgstation.Server.Tests.Live var serverTask = server.Run(cancellationToken); try { - var testUpdateVersion = new Version(4, 3, 0); using (var adminClient = await CreateAdminClient(server.Url, cancellationToken)) { // Disabled OAuth test @@ -178,7 +180,7 @@ namespace Tgstation.Server.Tests.Live //attempt to update to stable await adminClient.Administration.Update(new ServerUpdateRequest { - NewVersion = testUpdateVersion + NewVersion = TestUpdateVersion }, cancellationToken); var serverInfo = await adminClient.ServerInformation(cancellationToken); Assert.IsTrue(serverInfo.UpdateInProgress); @@ -195,7 +197,7 @@ namespace Tgstation.Server.Tests.Live Assert.IsTrue(File.Exists(updatedAssemblyPath), "Updated assembly missing!"); var updatedAssemblyVersion = FileVersionInfo.GetVersionInfo(updatedAssemblyPath); - Assert.AreEqual(testUpdateVersion, Version.Parse(updatedAssemblyVersion.FileVersion).Semver()); + Assert.AreEqual(TestUpdateVersion, Version.Parse(updatedAssemblyVersion.FileVersion).Semver()); } finally { diff --git a/tests/Tgstation.Server.Tests/TestDMApiConstants.cs b/tests/Tgstation.Server.Tests/TestDMApiConstants.cs index d248c6df76..fbbcf84319 100644 --- a/tests/Tgstation.Server.Tests/TestDMApiConstants.cs +++ b/tests/Tgstation.Server.Tests/TestDMApiConstants.cs @@ -12,10 +12,10 @@ namespace Tgstation.Server.Tests [TestClass] public sealed class TestDMApiConstants { - string[] definesFileLines; + static string[] definesFileLines; - [TestInitialize] - public async Task Initialize() + [ClassInitialize] + public static async Task Initialize(TestContext _) { definesFileLines = await File.ReadAllLinesAsync("../../../../../src/DMAPI/tgs/v5/_defines.dm"); } diff --git a/tests/Tgstation.Server.Tests/TestVersions.cs b/tests/Tgstation.Server.Tests/TestVersions.cs index 9977f68041..8e206913ba 100644 --- a/tests/Tgstation.Server.Tests/TestVersions.cs +++ b/tests/Tgstation.Server.Tests/TestVersions.cs @@ -20,12 +20,12 @@ namespace Tgstation.Server.Tests [TestCategory("SkipWhenLiveUnitTesting")] public sealed class TestVersions { - XNamespace xmlNamespace; + static XNamespace xmlNamespace; - XElement versionsPropertyGroup; + static XElement versionsPropertyGroup; - [TestInitialize] - public void Init() + [ClassInitialize] + public static void Init(TestContext _) { var doc = XDocument.Load("../../../../../build/Version.props"); var project = doc.Root; From 7b42d442186909b580f40c1751a0ad40c2b41a77 Mon Sep 17 00:00:00 2001 From: Dominion Date: Sun, 4 Jun 2023 13:21:11 -0400 Subject: [PATCH 20/41] Allow IFileDownloader implementation to be changed --- src/Tgstation.Server.Host/Core/Application.cs | 2 +- .../Extensions/ServiceCollectionExtensions.cs | 52 ++++++++++++++++++- 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index cab990f0e3..4e01f9f750 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -367,7 +367,7 @@ namespace Tgstation.Server.Host.Core // configure misc services services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(); + services.AddFileDownloader(); services.AddSingleton(); services.AddSingleton(); diff --git a/src/Tgstation.Server.Host/Extensions/ServiceCollectionExtensions.cs b/src/Tgstation.Server.Host/Extensions/ServiceCollectionExtensions.cs index 0d8e199285..49087b4a50 100644 --- a/src/Tgstation.Server.Host/Extensions/ServiceCollectionExtensions.cs +++ b/src/Tgstation.Server.Host/Extensions/ServiceCollectionExtensions.cs @@ -14,6 +14,7 @@ using Serilog.Sinks.Elasticsearch; using Tgstation.Server.Host.Components.Chat.Providers; using Tgstation.Server.Host.Configuration; +using Tgstation.Server.Host.IO; using Tgstation.Server.Host.Utils; using Tgstation.Server.Host.Utils.GitHub; @@ -27,18 +28,31 @@ namespace Tgstation.Server.Host.Extensions /// /// The implementation used in calls to . /// - static Type chatProviderFactoryType = typeof(ProviderFactory); + static Type chatProviderFactoryType; /// /// The implementation used in calls to . /// - static Type gitHubServiceFactoryType = typeof(GitHubServiceFactory); + static Type gitHubServiceFactoryType; + + /// + /// The implementation used in calls to . + /// + static Type fileDownloaderType; /// /// A for an additional to use. /// static ServiceDescriptor additionalLoggerProvider; + /// + /// Initializes static members of the class. + /// + static ServiceCollectionExtensions() + { + UseDefaultServices(); + } + /// /// Change the used as an implementation for calls to . /// @@ -57,6 +71,30 @@ namespace Tgstation.Server.Host.Extensions gitHubServiceFactoryType = typeof(TGitHubServiceFactory); } + /// + /// Change the used as an implementation for calls to . + /// + /// The implementation to use. + public static void UseFileDownloader() where TFileDownloader : IFileDownloader + { + fileDownloaderType = typeof(TFileDownloader); + } + + /// + /// Adds a implementation to the given . + /// + /// The to configure. + /// . + public static IServiceCollection AddFileDownloader(this IServiceCollection serviceCollection) + { + if (serviceCollection == null) + throw new ArgumentNullException(nameof(serviceCollection)); + + serviceCollection.AddSingleton(typeof(IFileDownloader), fileDownloaderType); + + return serviceCollection; + } + /// /// Adds a implementation to the given . /// @@ -189,5 +227,15 @@ namespace Tgstation.Server.Host.Extensions if (additionalLoggerProvider != null) builder.Services.TryAddEnumerable(additionalLoggerProvider); }); + + /// + /// Set the modifiable services to their default types. + /// + static void UseDefaultServices() + { + UseChatProviderFactory(); + UseGitHubServiceFactory(); + UseFileDownloader(); + } } } From baac58ec92ec24391ef7c470b23240ba964651b9 Mon Sep 17 00:00:00 2001 From: Dominion Date: Sun, 4 Jun 2023 14:03:00 -0400 Subject: [PATCH 21/41] Actually enable the GitHub token for remote git feature testing --- tests/Tgstation.Server.Tests/Live/Instance/RepositoryTest.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/RepositoryTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/RepositoryTest.cs index 68e9270e92..1e974edf6e 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/RepositoryTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/RepositoryTest.cs @@ -130,6 +130,8 @@ namespace Tgstation.Server.Tests.Live.Instance CreateGitHubDeployments = true, PostTestMergeComment = true, PushTestMergeCommits = true, + AccessUser = "Cyberboss", + AccessToken = Environment.GetEnvironmentVariable("TGS_TEST_GITHUB_TOKEN"), }, cancellationToken); } From 0e187a430d6999aae537d336fd6e4048a16f8ae4 Mon Sep 17 00:00:00 2001 From: Dominion Date: Sun, 4 Jun 2023 14:26:21 -0400 Subject: [PATCH 22/41] Add a caching FileDownloader for CI to be nice to Lummox --- .github/workflows/ci-suite.yml | 43 +++++ .../Live/CachingFileDownloader.cs | 155 ++++++++++++++++++ .../Live/DummyChatProvider.cs | 17 +- .../Live/Instance/ByondTest.cs | 8 +- .../Live/Instance/InstanceTest.cs | 7 +- .../Live/Instance/WatchdogTest.cs | 2 +- .../Live/LiveTestUtils.cs | 20 +++ .../Live/TestLiveServer.cs | 14 +- 8 files changed, 241 insertions(+), 25 deletions(-) create mode 100644 tests/Tgstation.Server.Tests/Live/CachingFileDownloader.cs diff --git a/.github/workflows/ci-suite.yml b/.github/workflows/ci-suite.yml index f8b7050ea2..11192797be 100644 --- a/.github/workflows/ci-suite.yml +++ b/.github/workflows/ci-suite.yml @@ -108,6 +108,13 @@ jobs: sudo apt-get update sudo apt-get install -y -o APT::Immediate-Configure=0 libc6-i386 libstdc++6:i386 libgcc-s1:i386 + - name: Restore BYOND cache + uses: actions/cache@v3 + id: cache-byond + with: + path: $HOME/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }} + key: ${{ env.BYOND_MAJOR }}-${{ env.BYOND_MINOR }}-dmapibyond + - name: Install BYOND if: steps.cache-byond.outputs.cache-hit != 'true' run: | @@ -316,12 +323,30 @@ jobs: watchdog-type: [ 'Basic', 'System' ] configuration: [ 'Debug', 'Release' ] runs-on: windows-2019 + env: + BYOND_MAJOR: 514 + BYOND_MINOR: 1588 steps: - name: Setup dotnet uses: actions/setup-dotnet@v2 with: dotnet-version: ${{ env.TGS_DOTNET_VERSION }} + - name: Restore BYOND cache + uses: actions/cache@v3 + id: cache-byond + with: + path: $HOME/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }} + key: ${{ env.BYOND_MAJOR }}-${{ env.BYOND_MINOR }}-livebyond + + - name: Download BYOND + if: steps.cache-byond.outputs.cache-hit != 'true' + run: | + echo "Downloading BYOND..." + mkdir -p "$HOME/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}" + cd "$HOME/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}" + curl "http://www.byond.com/download/build/${{ env.BYOND_MAJOR }}/${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}_byond.zip" -o byond.zip + - name: Upgrade NPM run: npm install -g npm @@ -481,6 +506,9 @@ jobs: watchdog-type: [ 'Basic', 'System' ] configuration: [ 'Debug', 'Release' ] runs-on: ubuntu-latest + env: + BYOND_MAJOR: 514 + BYOND_MINOR: 1588 steps: - name: Disable ptrace_scope run: echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope @@ -491,6 +519,21 @@ jobs: sudo apt-get update sudo apt-get install -y -o APT::Immediate-Configure=0 libc6-i386 libstdc++6:i386 gdb libgcc-s1:i386 + - name: Restore BYOND cache + uses: actions/cache@v3 + id: cache-byond + with: + path: $HOME/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }} + key: ${{ env.BYOND_MAJOR }}-${{ env.BYOND_MINOR }}-livebyond + + - name: Download BYOND + if: steps.cache-byond.outputs.cache-hit != 'true' + run: | + echo "Downloading BYOND..." + mkdir -p "$HOME/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}" + cd "$HOME/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}" + curl "http://www.byond.com/download/build/${{ env.BYOND_MAJOR }}/${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}_byond_linux.zip" -o byond.zip + - name: Install Node 12.X uses: actions/setup-node@v3 with: diff --git a/tests/Tgstation.Server.Tests/Live/CachingFileDownloader.cs b/tests/Tgstation.Server.Tests/Live/CachingFileDownloader.cs new file mode 100644 index 0000000000..da5a6249e5 --- /dev/null +++ b/tests/Tgstation.Server.Tests/Live/CachingFileDownloader.cs @@ -0,0 +1,155 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +using Microsoft.Extensions.Logging; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +using Tgstation.Server.Common; +using Tgstation.Server.Host.Extensions; +using Tgstation.Server.Host.IO; +using Tgstation.Server.Host.System; +using Tgstation.Server.Tests.Live.Instance; + +namespace Tgstation.Server.Tests.Live +{ + sealed class CachingFileDownloader : IFileDownloader, IDisposable + { + static readonly Dictionary> cachedPaths = new Dictionary>(); + + readonly ILogger logger; + + public CachingFileDownloader(ILogger logger) + { + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + logger.LogTrace("Created"); + } + + public static async Task InitializeAndInject(CancellationToken cancellationToken) + { + if (LiveTestUtils.RunningInGitHubActions) + { + // actions caches BYOND for us + var url = new Uri( + $"https://secure.byond.com/download/build/{ByondTest.TestVersion.Major}/{ByondTest.TestVersion.Major}.{ByondTest.TestVersion.Minor}_byond{(!new PlatformIdentifier().IsWindows ? "_linux" : String.Empty)}.zip"); + + var path = $"~/BYOND-{ByondTest.TestVersion.Major}.{ByondTest.TestVersion.Minor}/byond.zip"; + Assert.IsTrue(File.Exists(path)); + System.Console.WriteLine($"CACHE PREWARMED: {url}"); + cachedPaths.Add(url, Tuple.Create(path, false)); + } + + // predownload the target github release update asset + var gitHubToken = Environment.GetEnvironmentVariable("TGS_TEST_GITHUB_TOKEN"); + if (String.IsNullOrWhiteSpace(gitHubToken)) + gitHubToken = null; + + // this can fail, try a few times + var succeeded = false; + using var loggerFactory = LoggerFactory.Create(builder => + { + builder.AddConsole(); + builder.SetMinimumLevel(LogLevel.Trace); + }); + var logger = loggerFactory.CreateLogger("CachingFileDownloader"); + for (var i = 0; i < 10; ++i) + try + { + var url = new Uri($"https://github.com/tgstation/tgstation-server/releases/download/tgstation-server-v{TestLiveServer.TestUpdateVersion}/ServerUpdatePackage.zip"); + await using var stream = await CacheFile(logger, url, gitHubToken, cancellationToken); + succeeded = true; + break; + } + catch (Exception ex) + { + logger.LogWarning(ex, $"TEST: FAILED TO CACHE GITHUB RELEASE."); + } + + Assert.IsTrue(succeeded); + + ServiceCollectionExtensions.UseFileDownloader(); + } + + public void Dispose() + { + logger.LogTrace("Dispose"); + lock (cachedPaths) + { + foreach (var pathAndDelete in cachedPaths.Values) + if (pathAndDelete.Item2) + { + logger.LogTrace("Deleting {path}", pathAndDelete.Item1); + try + { + File.Delete(pathAndDelete.Item1); + } + catch (Exception ex) + { + logger.LogWarning(ex, "Deletion failed"); + } + } + + cachedPaths.Clear(); + } + } + + public async Task DownloadFile(Uri url, string bearerToken, CancellationToken cancellationToken) + { + Tuple tuple; + lock (cachedPaths) + if (!cachedPaths.TryGetValue(url, out tuple)) + { + logger.LogInformation("Cache miss: {url}", url); + tuple = null; + } + + if (tuple == null) + return await CacheFile(logger, url, bearerToken, cancellationToken); + + logger.LogTrace("Cache hit: {url}", url); + var bytes = await new DefaultIOManager().ReadAllBytes(tuple.Item1, cancellationToken); + return new MemoryStream(bytes); + } + + static async Task CacheFile(ILogger logger, Uri url, string bearerToken, CancellationToken cancellationToken) + { + var downloader = new FileDownloader( + new HttpClientFactory( + new AssemblyInformationProvider().ProductInfoHeaderValue), + new Logger( + LiveTestUtils.CreateLoggerFactoryForLogger( + logger, + out _))); + var download = await downloader.DownloadFile(url, bearerToken, cancellationToken); + try + { + var path = Path.GetTempFileName(); + try + { + await using var fs = new DefaultIOManager().CreateAsyncSequentialWriteStream(path); + await download.CopyToAsync(fs, cancellationToken); + + lock (cachedPaths) + cachedPaths.Add(url, Tuple.Create(path, true)); + + logger.LogTrace("Cached to {path}", path); + } + catch + { + File.Delete(path); + throw; + } + + download.Seek(0, SeekOrigin.Begin); + return download; + } + catch + { + await download.DisposeAsync(); + throw; + } + } + } +} diff --git a/tests/Tgstation.Server.Tests/Live/DummyChatProvider.cs b/tests/Tgstation.Server.Tests/Live/DummyChatProvider.cs index 972e7acc75..e8478c794b 100644 --- a/tests/Tgstation.Server.Tests/Live/DummyChatProvider.cs +++ b/tests/Tgstation.Server.Tests/Live/DummyChatProvider.cs @@ -43,21 +43,6 @@ namespace Tgstation.Server.Tests.Live ulong channelIdAllocator; - public static ILoggerFactory CreateLoggerFactoryForLogger(ILogger logger, out Mock mockLoggerFactory) - { - mockLoggerFactory = new Mock(); - mockLoggerFactory.Setup(x => x.CreateLogger(It.IsAny())).Returns(() => - { - var temp = logger; - logger = null; - - Assert.IsNotNull(temp); - return temp; - }) - .Verifiable(); - return mockLoggerFactory.Object; - } - static IAsyncDelayer CreateMockDelayer() { // at time of writing, this is used exclusively for the reconnection interval which works in minutes @@ -79,7 +64,7 @@ namespace Tgstation.Server.Tests.Live ICryptographySuite cryptographySuite, IReadOnlyCollection commands, Random random) - : base(jobManager, CreateMockDelayer(), new Logger(CreateLoggerFactoryForLogger(logger, out var mockLoggerFactory)), chatBot) + : base(jobManager, CreateMockDelayer(), new Logger(LiveTestUtils.CreateLoggerFactoryForLogger(logger, out var mockLoggerFactory)), chatBot) { mockLoggerFactory.VerifyAll(); this.cryptographySuite = cryptographySuite ?? throw new ArgumentNullException(nameof(cryptographySuite)); diff --git a/tests/Tgstation.Server.Tests/Live/Instance/ByondTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/ByondTest.cs index 8eac19aaf9..68e89c0f9f 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/ByondTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/ByondTest.cs @@ -14,7 +14,6 @@ using Tgstation.Server.Api.Models; using Tgstation.Server.Api.Models.Request; using Tgstation.Server.Client; using Tgstation.Server.Client.Components; -using Tgstation.Server.Common; using Tgstation.Server.Host.Components.Byond; using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.IO; @@ -27,13 +26,15 @@ namespace Tgstation.Server.Tests.Live.Instance public static readonly Version TestVersion = new(514, 1588); readonly IByondClient byondClient; + readonly IFileDownloader fileDownloader; readonly Api.Models.Instance metadata; - public ByondTest(IByondClient byondClient, IJobsClient jobsClient, Api.Models.Instance metadata) + public ByondTest(IByondClient byondClient, IJobsClient jobsClient, IFileDownloader fileDownloader, Api.Models.Instance metadata) : base(jobsClient) { this.byondClient = byondClient ?? throw new ArgumentNullException(nameof(byondClient)); + this.fileDownloader = fileDownloader ?? throw new ArgumentNullException(nameof(fileDownloader)); this.metadata = metadata ?? throw new ArgumentNullException(nameof(metadata)); } @@ -159,9 +160,6 @@ namespace Tgstation.Server.Tests.Live.Instance generalConfigOptionsMock.SetupGet(x => x.Value).Returns(new GeneralConfiguration()); var assemblyInformationProvider = new AssemblyInformationProvider(); - var fileDownloader = new FileDownloader( - new HttpClientFactory(assemblyInformationProvider.ProductInfoHeaderValue), - Mock.Of>()); IByondInstaller byondInstaller = new PlatformIdentifier().IsWindows ? new WindowsByondInstaller( diff --git a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs index 0eecae4997..0772133012 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs @@ -5,6 +5,7 @@ using System.Threading.Tasks; using Tgstation.Server.Client; using Tgstation.Server.Client.Components; using Tgstation.Server.Host.Components; +using Tgstation.Server.Host.IO; namespace Tgstation.Server.Tests.Live.Instance { @@ -12,20 +13,22 @@ namespace Tgstation.Server.Tests.Live.Instance { readonly IInstanceClient instanceClient; readonly IInstanceManagerClient instanceManagerClient; + readonly IFileDownloader fileDownloader; readonly InstanceManager instanceManager; readonly ushort serverPort; - public InstanceTest(IInstanceClient instanceClient, IInstanceManagerClient instanceManagerClient, InstanceManager instanceManager, ushort serverPort) + public InstanceTest(IInstanceClient instanceClient, IInstanceManagerClient instanceManagerClient, IFileDownloader fileDownloader, InstanceManager instanceManager, ushort serverPort) { this.instanceClient = instanceClient ?? throw new ArgumentNullException(nameof(instanceClient)); this.instanceManagerClient = instanceManagerClient ?? throw new ArgumentNullException(nameof(instanceManagerClient)); + this.fileDownloader = fileDownloader ?? throw new ArgumentNullException(nameof(fileDownloader)); this.instanceManager = instanceManager ?? throw new ArgumentNullException(nameof(instanceManager)); this.serverPort = serverPort; } public async Task RunTests(CancellationToken cancellationToken, bool highPrioDD, bool lowPrioDeployment) { - var byondTest = new ByondTest(instanceClient.Byond, instanceClient.Jobs, instanceClient.Metadata); + var byondTest = new ByondTest(instanceClient.Byond, instanceClient.Jobs, fileDownloader, instanceClient.Metadata); var chatTest = new ChatTest(instanceClient.ChatBots, instanceManagerClient, instanceClient.Jobs, instanceClient.Metadata); var configTest = new ConfigurationTest(instanceClient.Configuration, instanceClient.Metadata); var repoTest = new RepositoryTest(instanceClient.Repository, instanceClient.Jobs); diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index 8040fcfb87..3b08f87812 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -923,7 +923,7 @@ namespace Tgstation.Server.Tests.Live.Instance ReceiveTimeout = TimeSpan.FromSeconds(30), ConnectTimeout = TimeSpan.FromSeconds(30), DisconnectTimeout = TimeSpan.FromSeconds(30) - }, new Logger(DummyChatProvider.CreateLoggerFactoryForLogger(loggerFactory.CreateLogger("WatchdogTest.TopicClient"), out var mockLoggerFactory))); + }, new Logger(LiveTestUtils.CreateLoggerFactoryForLogger(loggerFactory.CreateLogger("WatchdogTest.TopicClient"), out var mockLoggerFactory))); public static readonly TopicClient TopicClientNoLogger = new(new SocketParameters { diff --git a/tests/Tgstation.Server.Tests/Live/LiveTestUtils.cs b/tests/Tgstation.Server.Tests/Live/LiveTestUtils.cs index bc8814ca2f..b2e21c6353 100644 --- a/tests/Tgstation.Server.Tests/Live/LiveTestUtils.cs +++ b/tests/Tgstation.Server.Tests/Live/LiveTestUtils.cs @@ -1,9 +1,29 @@ using System; +using Microsoft.Extensions.Logging; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +using Moq; + namespace Tgstation.Server.Tests.Live { static class LiveTestUtils { public static bool RunningInGitHubActions { get; } = !String.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("GITHUB_RUN_ID")); + + public static ILoggerFactory CreateLoggerFactoryForLogger(ILogger logger, out Mock mockLoggerFactory) + { + mockLoggerFactory = new Mock(); + mockLoggerFactory.Setup(x => x.CreateLogger(It.IsAny())).Returns(() => + { + var temp = logger; + logger = null; + + Assert.IsNotNull(temp); + return temp; + }) + .Verifiable(); + return mockLoggerFactory.Object; + } } } diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index f603a2c6e4..f24480843b 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -85,6 +85,8 @@ namespace Tgstation.Server.Tests.Live if (LiveTestUtils.RunningInGitHubActions || String.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("TGS_TEST_GITHUB_TOKEN"))) await DummyGitHubService.InitializeAndInject(default); + await CachingFileDownloader.InitializeAndInject(default); + await DummyChatProvider.RandomDisconnections(true, default); ServerClientFactory.ApiClientFactory = new RateLimitRetryingApiClientFactory(); @@ -882,7 +884,17 @@ namespace Tgstation.Server.Tests.Live Assert.IsTrue(Directory.Exists(instanceClient.Metadata.Path)); - var instanceTests = FailFast(new InstanceTest(instanceClient, adminClient.Instances, GetInstanceManager(), (ushort)server.Url.Port).RunTests(cancellationToken, server.HighPriorityDreamDaemon, server.LowPriorityDeployments)); + var instanceTests = FailFast( + new InstanceTest( + instanceClient, + adminClient.Instances, + ((Host.Server)server.RealServer).Host.Services.GetRequiredService(), + GetInstanceManager(), + (ushort)server.Url.Port) + .RunTests( + cancellationToken, + server.HighPriorityDreamDaemon, + server.LowPriorityDeployments)); await Task.WhenAll(rootTest, adminTest, instancesTest, instanceTests, usersTest); From c16d56c990914160ce571ff6ccad45f84ee3a11d Mon Sep 17 00:00:00 2001 From: Dominion Date: Sun, 4 Jun 2023 14:26:29 -0400 Subject: [PATCH 23/41] Downgrade error to warning --- .../Deployment/Remote/GitHubRemoteDeploymentManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Components/Deployment/Remote/GitHubRemoteDeploymentManager.cs b/src/Tgstation.Server.Host/Components/Deployment/Remote/GitHubRemoteDeploymentManager.cs index 2df99e9bf8..d0cf3471ad 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/Remote/GitHubRemoteDeploymentManager.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/Remote/GitHubRemoteDeploymentManager.cs @@ -131,7 +131,7 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote } catch (ApiException ex) { - Logger.LogError(ex, "Unable to create deployment!"); + Logger.LogWarning(ex, "Unable to create deployment!"); } } From 5fcde621253595163756ba5760ef4aa8444adf30 Mon Sep 17 00:00:00 2001 From: Dominion Date: Sun, 4 Jun 2023 14:26:41 -0400 Subject: [PATCH 24/41] Fix a job progress error --- src/Tgstation.Server.Host/Components/Repository/Repository.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Components/Repository/Repository.cs b/src/Tgstation.Server.Host/Components/Repository/Repository.cs index caa3f4381d..6025be0cff 100644 --- a/src/Tgstation.Server.Host/Components/Repository/Repository.cs +++ b/src/Tgstation.Server.Host/Components/Repository/Repository.cs @@ -967,7 +967,8 @@ namespace Tgstation.Server.Host.Components.Repository OnPackBuilderProgress = (stage, current, total) => { var baseProgress = stage == PackBuilderStage.Counting ? 0 : 0.5; - progressReporter.ReportProgress(baseProgress + (0.5 * ((double)current / total))); + var addon = total > 0 && current <= total ? (0.5 * ((double)current / total)) : 0; + progressReporter.ReportProgress(baseProgress + addon); return !cancellationToken.IsCancellationRequested; }, OnNegotiationCompletedBeforePush = (a) => From 937835a5631cf710f8d90e89f8475dc8ec01b5d8 Mon Sep 17 00:00:00 2001 From: Dominion Date: Sun, 4 Jun 2023 14:27:24 -0400 Subject: [PATCH 25/41] Fix ordering for remote GitHub features initialization --- .../Tgstation.Server.Tests/Live/Instance/RepositoryTest.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/RepositoryTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/RepositoryTest.cs index 1e974edf6e..bc644ddeab 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/RepositoryTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/RepositoryTest.cs @@ -118,9 +118,6 @@ namespace Tgstation.Server.Tests.Live.Instance // Back updated = await Checkout(new RepositoryUpdateRequest { Reference = "master" }, false, true, cancellationToken); - var prNumber = 2; - await TestMergeTests(updated, prNumber, cancellationToken); - // enable the good shit if possible if (LiveTestUtils.RunningInGitHubActions || String.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("TGS_TEST_GITHUB_TOKEN")) @@ -133,6 +130,9 @@ namespace Tgstation.Server.Tests.Live.Instance AccessUser = "Cyberboss", AccessToken = Environment.GetEnvironmentVariable("TGS_TEST_GITHUB_TOKEN"), }, cancellationToken); + + var prNumber = 2; + await TestMergeTests(updated, prNumber, cancellationToken); } async Task Checkout(RepositoryUpdateRequest updated, bool expectFailure, bool isRef, CancellationToken cancellationToken) From cf84e1ee7628b72593304a7d9547e223371921e4 Mon Sep 17 00:00:00 2001 From: Dominion Date: Sun, 4 Jun 2023 14:29:14 -0400 Subject: [PATCH 26/41] Adjust ordering of semaphore acquisition to make more sense --- .../Components/Repository/RepositoryManager.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs b/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs index 4ebfd23836..425df06ee7 100644 --- a/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs +++ b/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs @@ -210,9 +210,10 @@ namespace Tgstation.Server.Host.Components.Repository lock (semaphore) if (CloneInProgress) throw new JobException(ErrorCode.RepoCloning); - await semaphore.WaitAsync(cancellationToken); + try { + await semaphore.WaitAsync(cancellationToken); try { var libGit2Repo = await repositoryFactory.CreateFromPath(ioManager.ResolvePath(), cancellationToken); From a9802c9612406d0172b5f33b82443f2c63ba3a25 Mon Sep 17 00:00:00 2001 From: Dominion Date: Sun, 4 Jun 2023 14:45:32 -0400 Subject: [PATCH 28/41] Fix bad semicolon --- .../Components/Repository/RepositoryManager.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs b/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs index 425df06ee7..863487835e 100644 --- a/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs +++ b/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs @@ -167,8 +167,7 @@ namespace Tgstation.Server.Host.Components.Repository url, cloneOptions, repositoryPath, - cancellationToken) - ; + cancellationToken); } catch { From 0fe9ea55e5cb234abd9dff1cd49577298847596c Mon Sep 17 00:00:00 2001 From: Dominion Date: Sun, 4 Jun 2023 14:48:57 -0400 Subject: [PATCH 29/41] Use ~ in place of $HOME --- .github/workflows/ci-suite.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci-suite.yml b/.github/workflows/ci-suite.yml index 11192797be..f51fe93e28 100644 --- a/.github/workflows/ci-suite.yml +++ b/.github/workflows/ci-suite.yml @@ -112,15 +112,15 @@ jobs: uses: actions/cache@v3 id: cache-byond with: - path: $HOME/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }} + path: ~/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }} key: ${{ env.BYOND_MAJOR }}-${{ env.BYOND_MINOR }}-dmapibyond - name: Install BYOND if: steps.cache-byond.outputs.cache-hit != 'true' run: | echo "Setting up BYOND." - mkdir -p "$HOME/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}" - cd "$HOME/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}" + mkdir -p "~/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}" + cd "~/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}" curl "http://www.byond.com/download/build/${{ env.BYOND_MAJOR }}/${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}_byond_linux.zip" -o byond.zip unzip byond.zip cd byond @@ -141,7 +141,7 @@ jobs: run: | set -e retval=1 - source $HOME/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}/byond/bin/byondsetup + source ~/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}/byond/bin/byondsetup if hash DreamMaker 2>/dev/null then @@ -186,12 +186,12 @@ jobs: - name: gh-pages push if: github.event_name == 'push' && github.event.ref == 'refs/heads/dev' run: | - git clone -b gh-pages --single-branch "https://git@github.com/tgstation/tgstation-server" $HOME/tgsdox - pushd $HOME/tgsdox + git clone -b gh-pages --single-branch "https://git@github.com/tgstation/tgstation-server" ~/tgsdox + pushd ~/tgsdox rm -r * popd - echo ./doxout/* | xargs -n 10 sudo mv -t $HOME/tgsdox - cd $HOME/tgsdox + echo ./doxout/* | xargs -n 10 sudo mv -t ~/tgsdox + cd ~/tgsdox git config --global push.default simple git config user.name "tgstation-server" git config user.email "tgstation-server@tgstation13.org" @@ -336,15 +336,15 @@ jobs: uses: actions/cache@v3 id: cache-byond with: - path: $HOME/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }} + path: ~/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }} key: ${{ env.BYOND_MAJOR }}-${{ env.BYOND_MINOR }}-livebyond - name: Download BYOND if: steps.cache-byond.outputs.cache-hit != 'true' run: | echo "Downloading BYOND..." - mkdir -p "$HOME/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}" - cd "$HOME/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}" + mkdir -p "~/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}" + cd "~/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}" curl "http://www.byond.com/download/build/${{ env.BYOND_MAJOR }}/${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}_byond.zip" -o byond.zip - name: Upgrade NPM @@ -523,15 +523,15 @@ jobs: uses: actions/cache@v3 id: cache-byond with: - path: $HOME/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }} + path: ~/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }} key: ${{ env.BYOND_MAJOR }}-${{ env.BYOND_MINOR }}-livebyond - name: Download BYOND if: steps.cache-byond.outputs.cache-hit != 'true' run: | echo "Downloading BYOND..." - mkdir -p "$HOME/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}" - cd "$HOME/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}" + mkdir -p "~/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}" + cd "~/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}" curl "http://www.byond.com/download/build/${{ env.BYOND_MAJOR }}/${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}_byond_linux.zip" -o byond.zip - name: Install Node 12.X From 0158b6330d219d5cc56499fc589de12968dfc959 Mon Sep 17 00:00:00 2001 From: Dominion Date: Sun, 4 Jun 2023 14:51:36 -0400 Subject: [PATCH 30/41] Do not delete release cache between tests in CI --- .../Tgstation.Server.Tests/Live/CachingFileDownloader.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/Tgstation.Server.Tests/Live/CachingFileDownloader.cs b/tests/Tgstation.Server.Tests/Live/CachingFileDownloader.cs index da5a6249e5..ef945dd9b8 100644 --- a/tests/Tgstation.Server.Tests/Live/CachingFileDownloader.cs +++ b/tests/Tgstation.Server.Tests/Live/CachingFileDownloader.cs @@ -58,7 +58,7 @@ namespace Tgstation.Server.Tests.Live try { var url = new Uri($"https://github.com/tgstation/tgstation-server/releases/download/tgstation-server-v{TestLiveServer.TestUpdateVersion}/ServerUpdatePackage.zip"); - await using var stream = await CacheFile(logger, url, gitHubToken, cancellationToken); + await using var stream = await CacheFile(logger, url, gitHubToken, !LiveTestUtils.RunningInGitHubActions, cancellationToken); succeeded = true; break; } @@ -106,14 +106,14 @@ namespace Tgstation.Server.Tests.Live } if (tuple == null) - return await CacheFile(logger, url, bearerToken, cancellationToken); + return await CacheFile(logger, url, bearerToken, true, cancellationToken); logger.LogTrace("Cache hit: {url}", url); var bytes = await new DefaultIOManager().ReadAllBytes(tuple.Item1, cancellationToken); return new MemoryStream(bytes); } - static async Task CacheFile(ILogger logger, Uri url, string bearerToken, CancellationToken cancellationToken) + static async Task CacheFile(ILogger logger, Uri url, string bearerToken, bool temporal, CancellationToken cancellationToken) { var downloader = new FileDownloader( new HttpClientFactory( @@ -132,7 +132,7 @@ namespace Tgstation.Server.Tests.Live await download.CopyToAsync(fs, cancellationToken); lock (cachedPaths) - cachedPaths.Add(url, Tuple.Create(path, true)); + cachedPaths.Add(url, Tuple.Create(path, temporal)); logger.LogTrace("Cached to {path}", path); } From 5f059bda7c8b2a8b7febd199716f0a9a92ba9f64 Mon Sep 17 00:00:00 2001 From: Dominion Date: Sun, 4 Jun 2023 14:55:36 -0400 Subject: [PATCH 31/41] List the directory for debug purposes --- .github/workflows/ci-suite.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci-suite.yml b/.github/workflows/ci-suite.yml index f51fe93e28..02bce9778c 100644 --- a/.github/workflows/ci-suite.yml +++ b/.github/workflows/ci-suite.yml @@ -346,6 +346,7 @@ jobs: mkdir -p "~/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}" cd "~/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}" curl "http://www.byond.com/download/build/${{ env.BYOND_MAJOR }}/${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}_byond.zip" -o byond.zip + ls -l - name: Upgrade NPM run: npm install -g npm @@ -533,6 +534,7 @@ jobs: mkdir -p "~/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}" cd "~/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}" curl "http://www.byond.com/download/build/${{ env.BYOND_MAJOR }}/${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}_byond_linux.zip" -o byond.zip + ls -l - name: Install Node 12.X uses: actions/setup-node@v3 From 2518e7372c31696ed97e3708c617bd9b5c1a6936 Mon Sep 17 00:00:00 2001 From: Dominion Date: Sun, 4 Jun 2023 15:06:33 -0400 Subject: [PATCH 32/41] What the hell is even happening in CI right now? --- .github/workflows/ci-suite.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci-suite.yml b/.github/workflows/ci-suite.yml index 02bce9778c..1e3e9bd800 100644 --- a/.github/workflows/ci-suite.yml +++ b/.github/workflows/ci-suite.yml @@ -141,6 +141,9 @@ jobs: run: | set -e retval=1 + ls -l ~/ + ls -l ~/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}/ + ls -l ~/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}/byond/bin source ~/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}/byond/bin/byondsetup if hash DreamMaker 2>/dev/null From 81b0734dc9fc4ef02f034ca1183b6a0b65cb9ca9 Mon Sep 17 00:00:00 2001 From: Dominion Date: Sun, 4 Jun 2023 15:41:38 -0400 Subject: [PATCH 33/41] Actions hates ~ apparently --- .github/workflows/ci-suite.yml | 33 ++++++++----------- .../Live/CachingFileDownloader.cs | 5 ++- 2 files changed, 18 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci-suite.yml b/.github/workflows/ci-suite.yml index 1e3e9bd800..11192797be 100644 --- a/.github/workflows/ci-suite.yml +++ b/.github/workflows/ci-suite.yml @@ -112,15 +112,15 @@ jobs: uses: actions/cache@v3 id: cache-byond with: - path: ~/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }} + path: $HOME/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }} key: ${{ env.BYOND_MAJOR }}-${{ env.BYOND_MINOR }}-dmapibyond - name: Install BYOND if: steps.cache-byond.outputs.cache-hit != 'true' run: | echo "Setting up BYOND." - mkdir -p "~/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}" - cd "~/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}" + mkdir -p "$HOME/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}" + cd "$HOME/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}" curl "http://www.byond.com/download/build/${{ env.BYOND_MAJOR }}/${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}_byond_linux.zip" -o byond.zip unzip byond.zip cd byond @@ -141,10 +141,7 @@ jobs: run: | set -e retval=1 - ls -l ~/ - ls -l ~/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}/ - ls -l ~/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}/byond/bin - source ~/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}/byond/bin/byondsetup + source $HOME/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}/byond/bin/byondsetup if hash DreamMaker 2>/dev/null then @@ -189,12 +186,12 @@ jobs: - name: gh-pages push if: github.event_name == 'push' && github.event.ref == 'refs/heads/dev' run: | - git clone -b gh-pages --single-branch "https://git@github.com/tgstation/tgstation-server" ~/tgsdox - pushd ~/tgsdox + git clone -b gh-pages --single-branch "https://git@github.com/tgstation/tgstation-server" $HOME/tgsdox + pushd $HOME/tgsdox rm -r * popd - echo ./doxout/* | xargs -n 10 sudo mv -t ~/tgsdox - cd ~/tgsdox + echo ./doxout/* | xargs -n 10 sudo mv -t $HOME/tgsdox + cd $HOME/tgsdox git config --global push.default simple git config user.name "tgstation-server" git config user.email "tgstation-server@tgstation13.org" @@ -339,17 +336,16 @@ jobs: uses: actions/cache@v3 id: cache-byond with: - path: ~/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }} + path: $HOME/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }} key: ${{ env.BYOND_MAJOR }}-${{ env.BYOND_MINOR }}-livebyond - name: Download BYOND if: steps.cache-byond.outputs.cache-hit != 'true' run: | echo "Downloading BYOND..." - mkdir -p "~/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}" - cd "~/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}" + mkdir -p "$HOME/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}" + cd "$HOME/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}" curl "http://www.byond.com/download/build/${{ env.BYOND_MAJOR }}/${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}_byond.zip" -o byond.zip - ls -l - name: Upgrade NPM run: npm install -g npm @@ -527,17 +523,16 @@ jobs: uses: actions/cache@v3 id: cache-byond with: - path: ~/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }} + path: $HOME/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }} key: ${{ env.BYOND_MAJOR }}-${{ env.BYOND_MINOR }}-livebyond - name: Download BYOND if: steps.cache-byond.outputs.cache-hit != 'true' run: | echo "Downloading BYOND..." - mkdir -p "~/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}" - cd "~/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}" + mkdir -p "$HOME/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}" + cd "$HOME/BYOND-${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}" curl "http://www.byond.com/download/build/${{ env.BYOND_MAJOR }}/${{ env.BYOND_MAJOR }}.${{ env.BYOND_MINOR }}_byond_linux.zip" -o byond.zip - ls -l - name: Install Node 12.X uses: actions/setup-node@v3 diff --git a/tests/Tgstation.Server.Tests/Live/CachingFileDownloader.cs b/tests/Tgstation.Server.Tests/Live/CachingFileDownloader.cs index ef945dd9b8..a7c840251f 100644 --- a/tests/Tgstation.Server.Tests/Live/CachingFileDownloader.cs +++ b/tests/Tgstation.Server.Tests/Live/CachingFileDownloader.cs @@ -35,7 +35,10 @@ namespace Tgstation.Server.Tests.Live var url = new Uri( $"https://secure.byond.com/download/build/{ByondTest.TestVersion.Major}/{ByondTest.TestVersion.Major}.{ByondTest.TestVersion.Minor}_byond{(!new PlatformIdentifier().IsWindows ? "_linux" : String.Empty)}.zip"); - var path = $"~/BYOND-{ByondTest.TestVersion.Major}.{ByondTest.TestVersion.Minor}/byond.zip"; + var path = Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), + $"BYOND-{ByondTest.TestVersion.Major}.{ByondTest.TestVersion.Minor}", + "byond.zip"); Assert.IsTrue(File.Exists(path)); System.Console.WriteLine($"CACHE PREWARMED: {url}"); cachedPaths.Add(url, Tuple.Create(path, false)); From 9c5f4bad2655a7ef2b0104ab87edb94022712d95 Mon Sep 17 00:00:00 2001 From: Dominion Date: Sun, 4 Jun 2023 16:07:22 -0400 Subject: [PATCH 34/41] Hashing Uri's doesn't work that way Also fix CachingFileDownloader Cleanup --- .../Live/CachingFileDownloader.cs | 19 +++++++------------ .../Live/TestLiveServer.cs | 6 ++++++ 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/tests/Tgstation.Server.Tests/Live/CachingFileDownloader.cs b/tests/Tgstation.Server.Tests/Live/CachingFileDownloader.cs index a7c840251f..41628f16e9 100644 --- a/tests/Tgstation.Server.Tests/Live/CachingFileDownloader.cs +++ b/tests/Tgstation.Server.Tests/Live/CachingFileDownloader.cs @@ -15,9 +15,9 @@ using Tgstation.Server.Tests.Live.Instance; namespace Tgstation.Server.Tests.Live { - sealed class CachingFileDownloader : IFileDownloader, IDisposable + sealed class CachingFileDownloader : IFileDownloader { - static readonly Dictionary> cachedPaths = new Dictionary>(); + static readonly Dictionary> cachedPaths = new Dictionary>(); readonly ILogger logger; @@ -41,7 +41,7 @@ namespace Tgstation.Server.Tests.Live "byond.zip"); Assert.IsTrue(File.Exists(path)); System.Console.WriteLine($"CACHE PREWARMED: {url}"); - cachedPaths.Add(url, Tuple.Create(path, false)); + cachedPaths.Add(url.ToString(), Tuple.Create(path, false)); } // predownload the target github release update asset @@ -75,24 +75,19 @@ namespace Tgstation.Server.Tests.Live ServiceCollectionExtensions.UseFileDownloader(); } - public void Dispose() + public static void Cleanup() { - logger.LogTrace("Dispose"); lock (cachedPaths) { foreach (var pathAndDelete in cachedPaths.Values) if (pathAndDelete.Item2) - { - logger.LogTrace("Deleting {path}", pathAndDelete.Item1); try { File.Delete(pathAndDelete.Item1); } - catch (Exception ex) + catch { - logger.LogWarning(ex, "Deletion failed"); } - } cachedPaths.Clear(); } @@ -102,7 +97,7 @@ namespace Tgstation.Server.Tests.Live { Tuple tuple; lock (cachedPaths) - if (!cachedPaths.TryGetValue(url, out tuple)) + if (!cachedPaths.TryGetValue(url.ToString(), out tuple)) { logger.LogInformation("Cache miss: {url}", url); tuple = null; @@ -135,7 +130,7 @@ namespace Tgstation.Server.Tests.Live await download.CopyToAsync(fs, cancellationToken); lock (cachedPaths) - cachedPaths.Add(url, Tuple.Create(path, temporal)); + cachedPaths.Add(url.ToString(), Tuple.Create(path, temporal)); logger.LogTrace("Cached to {path}", path); } diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index f24480843b..cba3568f2e 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -151,6 +151,12 @@ namespace Tgstation.Server.Tests.Live } } + [ClassCleanup] + public static void Cleanup() + { + CachingFileDownloader.Cleanup(); + } + [TestMethod] public async Task TestUpdateProtocolAndDisabledOAuth() { From 08c93ecac6f6a4c83ac94ce863698fd6eb895a72 Mon Sep 17 00:00:00 2001 From: Dominion Date: Mon, 5 Jun 2023 09:35:21 -0400 Subject: [PATCH 35/41] Tests are so fast now it's hard to catch this --- .../Tgstation.Server.Tests/Live/TestLiveServer.cs | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index cba3568f2e..bace1d01f8 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -190,8 +190,19 @@ namespace Tgstation.Server.Tests.Live { NewVersion = TestUpdateVersion }, cancellationToken); - var serverInfo = await adminClient.ServerInformation(cancellationToken); - Assert.IsTrue(serverInfo.UpdateInProgress); + + try + { + var serverInfoTask = adminClient.ServerInformation(cancellationToken); + var completedTask = await Task.WhenAny(serverTask, serverInfoTask); + if (completedTask == serverInfoTask) + { + var serverInfo = await serverInfoTask; + Assert.IsTrue(serverInfo.UpdateInProgress); + } + } + catch (ServiceUnavailableException) { } + catch (HttpRequestException) { } } //wait up to 3 minutes for the dl and install From 9c3d7a040245c5d1b7c1a8a14ddbc1a266ee49b9 Mon Sep 17 00:00:00 2001 From: Dominion Date: Mon, 5 Jun 2023 10:03:31 -0400 Subject: [PATCH 36/41] Additional logging to detect the RepositoryManager deadlock --- .../Repository/RepositoryManager.cs | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs b/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs index 863487835e..798bff74f4 100644 --- a/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs +++ b/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs @@ -3,6 +3,7 @@ using System.Threading; using System.Threading.Tasks; using LibGit2Sharp; + using Microsoft.Extensions.Logging; using Tgstation.Server.Api.Models; @@ -142,7 +143,7 @@ namespace Tgstation.Server.Host.Components.Repository { using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken)) { - logger.LogTrace("Semaphore acquired"); + logger.LogTrace("Semaphore acquired for clone"); var repositoryPath = ioManager.ResolvePath(); if (!await ioManager.DirectoryExists(repositoryPath, cancellationToken)) try @@ -191,14 +192,14 @@ namespace Tgstation.Server.Host.Components.Repository return null; } } - - logger.LogInformation("Clone complete!"); } finally { + logger.LogTrace("Semaphore released after clone"); CloneInProgress = false; } + logger.LogInformation("Clone complete!"); return await LoadRepository(cancellationToken); } @@ -215,6 +216,7 @@ namespace Tgstation.Server.Host.Components.Repository await semaphore.WaitAsync(cancellationToken); try { + logger.LogTrace("Semaphore acquired for load"); var libGit2Repo = await repositoryFactory.CreateFromPath(ioManager.ResolvePath(), cancellationToken); return new Repository( @@ -235,6 +237,7 @@ namespace Tgstation.Server.Host.Components.Repository } catch { + logger.LogTrace("Releasing semaphore as load failed"); semaphore.Release(); throw; } @@ -250,10 +253,17 @@ namespace Tgstation.Server.Host.Components.Repository public async Task DeleteRepository(CancellationToken cancellationToken) { logger.LogInformation("Deleting repository..."); - using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken)) + try { - logger.LogTrace("Semaphore acquired, deleting Repository directory..."); - await ioManager.DeleteDirectory(ioManager.ResolvePath(), cancellationToken); + using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken)) + { + logger.LogTrace("Semaphore acquired, deleting Repository directory..."); + await ioManager.DeleteDirectory(ioManager.ResolvePath(), cancellationToken); + } + } + finally + { + logger.LogTrace("Semaphore released after delete attempt"); } } } From 928982e2af30a5a06519fe5ec9721f66df4c5cfb Mon Sep 17 00:00:00 2001 From: Dominion Date: Mon, 5 Jun 2023 10:48:12 -0400 Subject: [PATCH 37/41] Fix a typo --- .../Components/Repository/LibGit2RepositoryFactory.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Components/Repository/LibGit2RepositoryFactory.cs b/src/Tgstation.Server.Host/Components/Repository/LibGit2RepositoryFactory.cs index 55c0f7f445..65af4f9721 100644 --- a/src/Tgstation.Server.Host/Components/Repository/LibGit2RepositoryFactory.cs +++ b/src/Tgstation.Server.Host/Components/Repository/LibGit2RepositoryFactory.cs @@ -46,7 +46,7 @@ namespace Tgstation.Server.Host.Components.Repository var repo = await Task.Factory.StartNew( () => { - logger.LogTrace("Creating libgit2 repostory at {repoPath}...", path); + logger.LogTrace("Creating libgit2 repository at {repoPath}...", path); return new LibGit2Sharp.Repository(path); }, cancellationToken, From 489b7b7e5d6627708d4719abf7e7cf2f28dce76a Mon Sep 17 00:00:00 2001 From: Dominion Date: Mon, 5 Jun 2023 10:52:09 -0400 Subject: [PATCH 38/41] Fix race condition in RepositoryTest --- .../Live/Instance/InstanceTest.cs | 2 +- .../Live/Instance/RepositoryTest.cs | 56 ++++++++++--------- 2 files changed, 32 insertions(+), 26 deletions(-) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs index 0772133012..63b8798223 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs @@ -37,7 +37,7 @@ namespace Tgstation.Server.Tests.Live.Instance var byondTask = byondTest.Run(cancellationToken, out var firstInstall); var chatTask = chatTest.RunPreWatchdog(cancellationToken); - var repoLongJob = repoTest.RunLongClone(cancellationToken); + var repoLongJob = await repoTest.RunLongClone(cancellationToken); await dmTest.RunPreRepoClone(cancellationToken); await repoTest.AbortLongCloneAndCloneSomethingQuick(repoLongJob, cancellationToken); diff --git a/tests/Tgstation.Server.Tests/Live/Instance/RepositoryTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/RepositoryTest.cs index bc644ddeab..c35ccd70dd 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/RepositoryTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/RepositoryTest.cs @@ -23,7 +23,7 @@ namespace Tgstation.Server.Tests.Live.Instance this.repositoryClient = repositoryClient ?? throw new ArgumentNullException(nameof(repositoryClient)); } - public async Task RunLongClone(CancellationToken cancellationToken) + public async Task> RunLongClone(CancellationToken cancellationToken) { var workingBranch = "master"; @@ -35,34 +35,40 @@ namespace Tgstation.Server.Tests.Live.Instance }; var clone = await repositoryClient.Clone(cloneRequest, cancellationToken); - await ApiAssert.ThrowsException(() => repositoryClient.Read(cancellationToken), ErrorCode.RepoCloning); - Assert.IsNotNull(clone); - Assert.AreEqual(cloneRequest.Origin, clone.Origin); - Assert.AreEqual(workingBranch, clone.Reference); - Assert.IsNull(clone.RevisionInformation); - Assert.IsNotNull(clone.ActiveJob); - // throwing this small jobs consistency test in here - await Task.Delay(TimeSpan.FromSeconds(15), cancellationToken); - var activeJobs = await JobsClient.ListActive(null, cancellationToken); - var allJobs = await JobsClient.List(null, cancellationToken); + return Rest(); - Assert.IsTrue(activeJobs.Any(x => x.Id == clone.ActiveJob.Id)); - Assert.IsTrue(allJobs.Any(x => x.Id == clone.ActiveJob.Id)); - - var targetActiveJob = activeJobs.First(x => x.Id == clone.ActiveJob.Id); - - if (!targetActiveJob.Progress.HasValue) + async Task Rest() { - // give it 15 more seconds - targetActiveJob = await WaitForJobProgress(targetActiveJob, 15, cancellationToken); - allJobs = await JobsClient.List(null, cancellationToken); + await ApiAssert.ThrowsException(() => repositoryClient.Read(cancellationToken), ErrorCode.RepoCloning); + Assert.IsNotNull(clone); + Assert.AreEqual(cloneRequest.Origin, clone.Origin); + Assert.AreEqual(workingBranch, clone.Reference); + Assert.IsNull(clone.RevisionInformation); + Assert.IsNotNull(clone.ActiveJob); + + // throwing this small jobs consistency test in here + await Task.Delay(TimeSpan.FromSeconds(15), cancellationToken); + var activeJobs = await JobsClient.ListActive(null, cancellationToken); + var allJobs = await JobsClient.List(null, cancellationToken); + + Assert.IsTrue(activeJobs.Any(x => x.Id == clone.ActiveJob.Id)); + Assert.IsTrue(allJobs.Any(x => x.Id == clone.ActiveJob.Id)); + + var targetActiveJob = activeJobs.First(x => x.Id == clone.ActiveJob.Id); + + if (!targetActiveJob.Progress.HasValue) + { + // give it 15 more seconds + targetActiveJob = await WaitForJobProgress(targetActiveJob, 15, cancellationToken); + allJobs = await JobsClient.List(null, cancellationToken); + } + + Assert.IsTrue(targetActiveJob.Progress.HasValue); + Assert.IsTrue(allJobs.First(x => x.Id == clone.ActiveJob.Id).Progress.HasValue); + + return clone.ActiveJob; } - - Assert.IsTrue(targetActiveJob.Progress.HasValue); - Assert.IsTrue(allJobs.First(x => x.Id == clone.ActiveJob.Id).Progress.HasValue); - - return clone.ActiveJob; } public async Task AbortLongCloneAndCloneSomethingQuick(Task longCloneJob, CancellationToken cancellationToken) From fc236f58203d598c184b5e8d54ef0d7847035bb7 Mon Sep 17 00:00:00 2001 From: Dominion Date: Mon, 5 Jun 2023 12:01:53 -0400 Subject: [PATCH 39/41] Remove host executable from ServerUpdatePackage --- .github/workflows/ci-suite.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-suite.yml b/.github/workflows/ci-suite.yml index 11192797be..118d57eb01 100644 --- a/.github/workflows/ci-suite.yml +++ b/.github/workflows/ci-suite.yml @@ -616,6 +616,7 @@ jobs: cd src/Tgstation.Server.Host dotnet publish -c ${{ matrix.configuration }}NoService --no-build -o ../../Artifacts/ServerUpdate rm ../../Artifacts/ServerUpdate/appsettings.yml + rm ../../Artifacts/ServerUpdate/Tgstation.Server.Host - name: Store Server Console if: ${{ matrix.configuration == 'Release' && matrix.watchdog-type == 'System' && matrix.database-type == 'MariaDB' }} From 85a22861edd14d79bc10d1d287d2fbd5f2d8969f Mon Sep 17 00:00:00 2001 From: Dominion Date: Mon, 5 Jun 2023 12:05:08 -0400 Subject: [PATCH 40/41] Remove executable from Service and Console packages --- .github/workflows/ci-suite.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci-suite.yml b/.github/workflows/ci-suite.yml index 118d57eb01..7f75ab3474 100644 --- a/.github/workflows/ci-suite.yml +++ b/.github/workflows/ci-suite.yml @@ -446,6 +446,7 @@ jobs: cd ../Tgstation.Server.Host dotnet publish -c ${{ matrix.configuration }} --no-build -o ../../Artifacts/Service/lib/Default mv ../../Artifacts/Service/lib/Default/appsettings.yml ../../Artifacts/Service/appsettings.yml + rm ../../Artifacts/Service/lib/Default/Tgstation.Server.Host.exe - name: Store Server Service if: ${{ matrix.configuration == 'Release' && matrix.watchdog-type == 'Basic' }} @@ -609,6 +610,7 @@ jobs: cd ../Tgstation.Server.Host dotnet publish -c ${{ matrix.configuration }}NoService --no-build -o ../../Artifacts/Console/lib/Default mv ../../Artifacts/Console/lib/Default/appsettings.yml ../../Artifacts/Console/appsettings.yml + rm ../../Artifacts/Console/lib/Default/Tgstation.Server.Host - name: Package Server Update Package if: ${{ matrix.configuration == 'Release' && matrix.watchdog-type == 'System' && matrix.database-type == 'PostgresSql' }} From 847094f789813530d623b6cd5c69416ae32adf04 Mon Sep 17 00:00:00 2001 From: Dominion Date: Mon, 5 Jun 2023 15:06:19 -0400 Subject: [PATCH 41/41] Fix Windows coverage upload --- .github/workflows/ci-suite.yml | 122 +++++++++++++++++++++++++++++---- 1 file changed, 109 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci-suite.yml b/.github/workflows/ci-suite.yml index 7f75ab3474..7fd6134029 100644 --- a/.github/workflows/ci-suite.yml +++ b/.github/workflows/ci-suite.yml @@ -428,7 +428,7 @@ jobs: - name: Store Code Coverage uses: actions/upload-artifact@v3 with: - name: windows-integration-test-coverage-${{ matrix.configuration }}-${{ matrix.watchdog-type }} + name: windows-integration-test-coverage-${{ matrix.configuration }}-${{ matrix.watchdog-type }}-${{ matrix.database-type }} path: ./TestResults/ - name: Store OpenAPI Spec @@ -797,29 +797,125 @@ jobs: name: windows-unit-test-coverage-Release path: ./code_coverage/unit_tests/windows_unit_tests_release - - name: Retrieve Windows Integration Test Coverage (Debug, Basic) + - name: Retrieve Windows Integration Test Coverage (Debug, Basic, SqlServer) uses: actions/download-artifact@v3 with: - name: windows-integration-test-coverage-Debug-Basic - path: ./code_coverage/integration_tests/windows_integration_tests_debug_basic + name: windows-integration-test-coverage-Debug-Basic-SqlServer + path: ./code_coverage/integration_tests/windows_integration_tests_debug_basic_sqlserver - - name: Retrieve Windows Integration Test Coverage (Release, Basic) + - name: Retrieve Windows Integration Test Coverage (Release, Basic, SqlServer) uses: actions/download-artifact@v3 with: - name: windows-integration-test-coverage-Release-Basic - path: ./code_coverage/integration_tests/windows_integration_tests_release_basic + name: windows-integration-test-coverage-Release-Basic-SqlServer + path: ./code_coverage/integration_tests/windows_integration_tests_release_basic_sqlserver - - name: Retrieve Windows Integration Test Coverage (Debug, System) + - name: Retrieve Windows Integration Test Coverage (Debug, System, SqlServer) uses: actions/download-artifact@v3 with: - name: windows-integration-test-coverage-Debug-System - path: ./code_coverage/integration_tests/windows_integration_tests_debug_system + name: windows-integration-test-coverage-Debug-System-SqlServer + path: ./code_coverage/integration_tests/windows_integration_tests_debug_system_sqlserver - - name: Retrieve Windows Integration Test Coverage (Release, System) + - name: Retrieve Windows Integration Test Coverage (Release, System, SqlServer) uses: actions/download-artifact@v3 with: - name: windows-integration-test-coverage-Release-System - path: ./code_coverage/integration_tests/windows_integration_tests_release_system + name: windows-integration-test-coverage-Release-System-SqlServer + path: ./code_coverage/integration_tests/windows_integration_tests_release_system_sqlserver + + - name: Retrieve Windows Integration Test Coverage (Debug, Basic, MariaDB) + uses: actions/download-artifact@v3 + with: + name: windows-integration-test-coverage-Debug-Basic-MariaDB + path: ./code_coverage/integration_tests/windows_integration_tests_debug_basic_mariadb + + - name: Retrieve Windows Integration Test Coverage (Release, Basic, MariaDB) + uses: actions/download-artifact@v3 + with: + name: windows-integration-test-coverage-Release-Basic-MariaDB + path: ./code_coverage/integration_tests/windows_integration_tests_release_basic_mariadb + + - name: Retrieve Windows Integration Test Coverage (Debug, System, MariaDB) + uses: actions/download-artifact@v3 + with: + name: windows-integration-test-coverage-Debug-System-MariaDB + path: ./code_coverage/integration_tests/windows_integration_tests_debug_system_mariadb + + - name: Retrieve Windows Integration Test Coverage (Release, System, MariaDB) + uses: actions/download-artifact@v3 + with: + name: windows-integration-test-coverage-Release-System-MariaDB + path: ./code_coverage/integration_tests/windows_integration_tests_release_system_mariadb + + - name: Retrieve Windows Integration Test Coverage (Debug, Basic, MySql) + uses: actions/download-artifact@v3 + with: + name: windows-integration-test-coverage-Debug-Basic-MySql + path: ./code_coverage/integration_tests/windows_integration_tests_debug_basic_mysql + + - name: Retrieve Windows Integration Test Coverage (Release, Basic, MySql) + uses: actions/download-artifact@v3 + with: + name: windows-integration-test-coverage-Release-Basic-MySql + path: ./code_coverage/integration_tests/windows_integration_tests_release_basic_mysql + + - name: Retrieve Windows Integration Test Coverage (Debug, System, MySql) + uses: actions/download-artifact@v3 + with: + name: windows-integration-test-coverage-Debug-System-MySql + path: ./code_coverage/integration_tests/windows_integration_tests_debug_system_mysql + + - name: Retrieve Windows Integration Test Coverage (Release, System, MySql) + uses: actions/download-artifact@v3 + with: + name: windows-integration-test-coverage-Release-System-MySql + path: ./code_coverage/integration_tests/windows_integration_tests_release_system_mysql + + - name: Retrieve Windows Integration Test Coverage (Debug, Basic, PostgresSql) + uses: actions/download-artifact@v3 + with: + name: windows-integration-test-coverage-Debug-Basic-PostgresSql + path: ./code_coverage/integration_tests/windows_integration_tests_debug_basic_postgressql + + - name: Retrieve Windows Integration Test Coverage (Release, Basic, PostgresSql) + uses: actions/download-artifact@v3 + with: + name: windows-integration-test-coverage-Release-Basic-PostgresSql + path: ./code_coverage/integration_tests/windows_integration_tests_release_basic_postgressql + + - name: Retrieve Windows Integration Test Coverage (Debug, System, PostgresSql) + uses: actions/download-artifact@v3 + with: + name: windows-integration-test-coverage-Debug-System-PostgresSql + path: ./code_coverage/integration_tests/windows_integration_tests_debug_system_postgressql + + - name: Retrieve Windows Integration Test Coverage (Release, System, PostgresSql) + uses: actions/download-artifact@v3 + with: + name: windows-integration-test-coverage-Release-System-PostgresSql + path: ./code_coverage/integration_tests/windows_integration_tests_release_system_postgressql + + - name: Retrieve Windows Integration Test Coverage (Debug, Basic, Sqlite) + uses: actions/download-artifact@v3 + with: + name: windows-integration-test-coverage-Debug-Basic-Sqlite + path: ./code_coverage/integration_tests/windows_integration_tests_debug_basic_sqlite + + - name: Retrieve Windows Integration Test Coverage (Release, Basic, Sqlite) + uses: actions/download-artifact@v3 + with: + name: windows-integration-test-coverage-Release-Basic-Sqlite + path: ./code_coverage/integration_tests/windows_integration_tests_release_basic_sqlite + + - name: Retrieve Windows Integration Test Coverage (Debug, System, Sqlite) + uses: actions/download-artifact@v3 + with: + name: windows-integration-test-coverage-Debug-System-Sqlite + path: ./code_coverage/integration_tests/windows_integration_tests_debug_system_sqlite + + - name: Retrieve Windows Integration Test Coverage (Release, System, Sqlite) + uses: actions/download-artifact@v3 + with: + name: windows-integration-test-coverage-Release-System-Sqlite + path: ./code_coverage/integration_tests/windows_integration_tests_release_system_sqlite - name: Upload Coverage to CodeCov uses: codecov/codecov-action@v3