diff --git a/src/Tgstation.Server.Host/Components/Instance.cs b/src/Tgstation.Server.Host/Components/Instance.cs index acd7933f10..59d0becaee 100644 --- a/src/Tgstation.Server.Host/Components/Instance.cs +++ b/src/Tgstation.Server.Host/Components/Instance.cs @@ -382,6 +382,17 @@ namespace Tgstation.Server.Host.Components // build current commit data if it's missing await UpdateRevInfo(repo.Head, false, null); + var preserveTestMerges = repositorySettings.AutoUpdatesKeepTestMerges.Value; + var remoteDeploymentManager = remoteDeploymentManagerFactory.CreateRemoteDeploymentManager( + metadata, + repo.RemoteGitProvider.Value); + + var updatedTestMerges = await remoteDeploymentManager.RemoveMergedTestMerges( + repo, + repositorySettings, + currentRevInfo, + cancellationToken); + var result = await repo.MergeOrigin( NextProgressReporter("Merge Origin"), repositorySettings.CommitterName, @@ -389,21 +400,10 @@ namespace Tgstation.Server.Host.Components true, cancellationToken); - var preserveTestMerges = repositorySettings.AutoUpdatesKeepTestMerges.Value; - var remoteDeploymentManager = remoteDeploymentManagerFactory.CreateRemoteDeploymentManager( - metadata, - repo.RemoteGitProvider.Value); - // take appropriate auto update actions var shouldSyncTracked = false; if (result.HasValue) { - var updatedTestMerges = await remoteDeploymentManager.RemoveMergedTestMerges( - repo, - repositorySettings, - currentRevInfo, - cancellationToken); - if (updatedTestMerges.Count == 0) { logger.LogTrace("All test merges have been merged on remote"); @@ -517,7 +517,12 @@ namespace Tgstation.Server.Host.Components RepositoryAutoUpdateJob, cancellationToken); - await jobManager.WaitForJobCompletion(repositoryUpdateJob, null, cancellationToken, cancellationToken); + var repoUpdateJobResult = await jobManager.WaitForJobCompletion(repositoryUpdateJob, null, cancellationToken, cancellationToken); + if (repoUpdateJobResult == false) + { + logger.LogWarning("Aborting auto-update due to repository update error!"); + continue; + } Job compileProcessJob; using (var repo = await RepositoryManager.LoadRepository(cancellationToken)) diff --git a/src/Tgstation.Server.Host/Jobs/IJobManager.cs b/src/Tgstation.Server.Host/Jobs/IJobManager.cs index 716059f314..80ef200859 100644 --- a/src/Tgstation.Server.Host/Jobs/IJobManager.cs +++ b/src/Tgstation.Server.Host/Jobs/IJobManager.cs @@ -33,8 +33,8 @@ namespace Tgstation.Server.Host.Jobs /// The to cancel the . If the TGS user will be used. /// A that will cancel the . /// The for the operation. - /// A representing the . - ValueTask WaitForJobCompletion(Job job, User canceller, CancellationToken jobCancellationToken, CancellationToken cancellationToken); + /// A representing the . Results in if the completed without errors, if errors occurred, or if the job isn't registered. + ValueTask WaitForJobCompletion(Job job, User canceller, CancellationToken jobCancellationToken, CancellationToken cancellationToken); /// /// Cancels a give . diff --git a/src/Tgstation.Server.Host/Jobs/JobHandler.cs b/src/Tgstation.Server.Host/Jobs/JobHandler.cs index 57aae60fdb..9d9c692c07 100644 --- a/src/Tgstation.Server.Host/Jobs/JobHandler.cs +++ b/src/Tgstation.Server.Host/Jobs/JobHandler.cs @@ -30,20 +30,20 @@ namespace Tgstation.Server.Host.Jobs readonly CancellationTokenSource cancellationTokenSource; /// - /// A taking a and returning a that the will wrap. + /// A taking a and returning a that the will wrap. /// - readonly Func jobActivator; + readonly Func> jobActivator; /// /// The being run. /// - Task task; + Task task; /// /// Initializes a new instance of the class. /// /// The value of . - public JobHandler(Func jobActivator) + public JobHandler(Func> jobActivator) { this.jobActivator = jobActivator ?? throw new ArgumentNullException(nameof(jobActivator)); cancellationTokenSource = new CancellationTokenSource(); @@ -56,8 +56,8 @@ namespace Tgstation.Server.Host.Jobs /// Wait for to complete. /// /// The for the operation. - /// A representing the running operation. - public Task Wait(CancellationToken cancellationToken) + /// A representing the job. Results in if the job completed without errors, otherwise. + public Task Wait(CancellationToken cancellationToken) { if (task == null) throw new InvalidOperationException("Job not started!"); diff --git a/src/Tgstation.Server.Host/Jobs/JobService.cs b/src/Tgstation.Server.Host/Jobs/JobService.cs index 363a8eed28..b5029e2fbe 100644 --- a/src/Tgstation.Server.Host/Jobs/JobService.cs +++ b/src/Tgstation.Server.Host/Jobs/JobService.cs @@ -249,7 +249,7 @@ namespace Tgstation.Server.Host.Jobs } /// - public async ValueTask WaitForJobCompletion(Job job, User canceller, CancellationToken jobCancellationToken, CancellationToken cancellationToken) + public async ValueTask WaitForJobCompletion(Job job, User canceller, CancellationToken jobCancellationToken, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(job); @@ -261,7 +261,7 @@ namespace Tgstation.Server.Host.Jobs lock (synchronizationLock) { if (!jobs.TryGetValue(job.Id.Value, out handler)) - return; + return null; noMoreJobsShouldStart = this.noMoreJobsShouldStart; } @@ -270,11 +270,14 @@ namespace Tgstation.Server.Host.Jobs await Extensions.TaskExtensions.InfiniteTask.WaitAsync(cancellationToken); ValueTask? cancelTask = null; + bool result; using (jobCancellationToken.Register(() => cancelTask = CancelJob(job, canceller, true, cancellationToken))) - await handler.Wait(cancellationToken); + result = await handler.Wait(cancellationToken); if (cancelTask.HasValue) await cancelTask.Value; + + return result; } /// @@ -293,12 +296,14 @@ namespace Tgstation.Server.Host.Jobs /// The for the . /// The for the operation. /// A representing the running operation. - async Task RunJob(Job job, JobEntrypoint operation, CancellationToken cancellationToken) + async Task RunJob(Job job, JobEntrypoint operation, CancellationToken cancellationToken) { using (LogContext.PushProperty(SerilogContextHelper.JobIdContextProperty, job.Id)) try { void LogException(Exception ex) => logger.LogDebug(ex, "Job {jobId} exited with error!", job.Id); + + var result = false; try { var oldJob = job; @@ -336,6 +341,7 @@ namespace Tgstation.Server.Host.Jobs cancellationToken); logger.LogDebug("Job {jobId} completed!", job.Id); + result = true; } catch (OperationCanceledException ex) { @@ -370,6 +376,8 @@ namespace Tgstation.Server.Host.Jobs // DCT: Cancellation token is for job, operation should always run await databaseContext.Save(CancellationToken.None); }); + + return result; } finally { diff --git a/src/Tgstation.Server.Host/Utils/GitHub/GitHubClientFactory.cs b/src/Tgstation.Server.Host/Utils/GitHub/GitHubClientFactory.cs index 2e9abee967..47a67728ab 100644 --- a/src/Tgstation.Server.Host/Utils/GitHub/GitHubClientFactory.cs +++ b/src/Tgstation.Server.Host/Utils/GitHub/GitHubClientFactory.cs @@ -80,6 +80,7 @@ namespace Tgstation.Server.Host.Utils.GitHub { GitHubClient client; bool cacheHit; + DateTimeOffset? lastUsed; lock (clientCache) { string cacheKey; @@ -105,11 +106,13 @@ namespace Tgstation.Server.Host.Utils.GitHub client.Credentials = new Credentials(accessToken); clientCache.Add(cacheKey, (client, now)); + lastUsed = null; } else { logger.LogTrace("Cache hit for GitHubClient"); client = tuple.Item1; + lastUsed = tuple.Item2; tuple.Item2 = now; } @@ -144,13 +147,15 @@ namespace Tgstation.Server.Host.Utils.GitHub rateLimitInfo.Reset.ToString("o")); else if (rateLimitInfo.Remaining < 25) // good luck hitting these lines on codecov logger.LogWarning( - "Requested GitHub client has only {remainingRequests} requests remaining! Limit resets at {resetTime}", + "Requested GitHub client has only {remainingRequests} requests remaining after the usage at {lastUse}! Limit resets at {resetTime}", rateLimitInfo.Remaining, + lastUsed, rateLimitInfo.Reset.ToString("o")); else logger.LogDebug( - "Requested GitHub client has {remainingRequests} requests remaining. Limit resets at {resetTime}", + "Requested GitHub client has {remainingRequests} requests remaining after the usage {lastUse}. Limit resets at {resetTime}", rateLimitInfo.Remaining, + lastUsed, rateLimitInfo.Reset.ToString("o")); return client; diff --git a/tests/Tgstation.Server.Host.Tests/Components/Chat/Providers/TestDiscordProvider.cs b/tests/Tgstation.Server.Host.Tests/Components/Chat/Providers/TestDiscordProvider.cs index 3943d05231..210b31447e 100644 --- a/tests/Tgstation.Server.Host.Tests/Components/Chat/Providers/TestDiscordProvider.cs +++ b/tests/Tgstation.Server.Host.Tests/Components/Chat/Providers/TestDiscordProvider.cs @@ -39,7 +39,7 @@ namespace Tgstation.Server.Host.Components.Chat.Providers.Tests .Returns(ValueTask.CompletedTask); mockSetup .Setup(x => x.WaitForJobCompletion(It.IsNotNull(), It.IsAny(), It.IsAny(), It.IsAny())) - .Returns(ValueTask.CompletedTask); + .Returns(ValueTask.FromResult(true)); mockJobManager = mockSetup.Object; } diff --git a/tests/Tgstation.Server.Host.Tests/Components/Chat/Providers/TestIrcProvider.cs b/tests/Tgstation.Server.Host.Tests/Components/Chat/Providers/TestIrcProvider.cs index 80a5e40b66..9e8565955e 100644 --- a/tests/Tgstation.Server.Host.Tests/Components/Chat/Providers/TestIrcProvider.cs +++ b/tests/Tgstation.Server.Host.Tests/Components/Chat/Providers/TestIrcProvider.cs @@ -76,7 +76,7 @@ namespace Tgstation.Server.Host.Components.Chat.Providers.Tests .Returns(ValueTask.CompletedTask); mockSetup .Setup(x => x.WaitForJobCompletion(It.IsNotNull(), It.IsAny(), It.IsAny(), It.IsAny())) - .Returns(ValueTask.CompletedTask); + .Returns(ValueTask.FromResult(true)); var mockJobManager = mockSetup.Object; await using var provider = new IrcProvider(mockJobManager, new AsyncDelayer(), loggerFactory.CreateLogger(), Mock.Of(), new ChatBot { diff --git a/tests/Tgstation.Server.Host.Tests/Jobs/TestJobHandler.cs b/tests/Tgstation.Server.Host.Tests/Jobs/TestJobHandler.cs index ed8660ec0e..b8a61b4b29 100644 --- a/tests/Tgstation.Server.Host.Tests/Jobs/TestJobHandler.cs +++ b/tests/Tgstation.Server.Host.Tests/Jobs/TestJobHandler.cs @@ -11,10 +11,11 @@ namespace Tgstation.Server.Host.Jobs.Tests Task currentWaitTask; bool cancelled; - async Task TestJob(CancellationToken cancellationToken) + async Task TestJob(CancellationToken cancellationToken) { await currentWaitTask; cancelled = cancellationToken.IsCancellationRequested; + return true; } [TestMethod]