diff --git a/build/Version.props b/build/Version.props index acd526f164..1e333d0271 100644 --- a/build/Version.props +++ b/build/Version.props @@ -3,7 +3,7 @@ - 5.1.1 + 5.1.2 4.4.0 9.6.0 10.0.0 diff --git a/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs b/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs index 9fb041e4a0..ffb2935ac9 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs @@ -717,7 +717,10 @@ namespace Tgstation.Server.Host.Components.Deployment /// A representing the running operation. async Task ProgressTask(JobProgressReporter progressReporter, TimeSpan? estimatedDuration, CancellationToken cancellationToken) { - progressReporter(currentStage, estimatedDuration.HasValue ? (int?)0 : null); + var noEstimate = !estimatedDuration.HasValue; + progressReporter.StageName = currentStage; + progressReporter.ReportProgress(noEstimate ? null : 0); + var sleepInterval = estimatedDuration.HasValue ? estimatedDuration.Value / 100 : TimeSpan.FromMilliseconds(250); logger.LogDebug("Compile is expected to take: {0}", estimatedDuration); @@ -726,7 +729,8 @@ namespace Tgstation.Server.Host.Components.Deployment for (var iteration = 0; iteration < (estimatedDuration.HasValue ? 99 : Int32.MaxValue); ++iteration) { await Task.Delay(sleepInterval, cancellationToken); - progressReporter(currentStage, estimatedDuration.HasValue ? (int?)(iteration + 1) : null); + progressReporter.StageName = currentStage; + progressReporter.ReportProgress(noEstimate ? null : sleepInterval * (iteration + 1) / estimatedDuration.Value); } } catch (OperationCanceledException) diff --git a/src/Tgstation.Server.Host/Components/Deployment/Remote/GitHubRemoteDeploymentManager.cs b/src/Tgstation.Server.Host/Components/Deployment/Remote/GitHubRemoteDeploymentManager.cs index a64a197587..c0d15ed98b 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/Remote/GitHubRemoteDeploymentManager.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/Remote/GitHubRemoteDeploymentManager.cs @@ -91,43 +91,50 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote else { Logger.LogTrace("Creating deployment..."); - var deployment = await gitHubClient - .Repository - .Deployment - .Create( - remoteInformation.RemoteRepositoryOwner, - remoteInformation.RemoteRepositoryName, - new NewDeployment(compileJob.RevisionInformation.CommitSha) - { - AutoMerge = false, - Description = "TGS Game Deployment", - Environment = $"TGS: {Metadata.Name}", - ProductionEnvironment = true, - RequiredContexts = new Collection(), - }) - .WithToken(cancellationToken) - ; + Octokit.Deployment deployment; - compileJob.GitHubDeploymentId = deployment.Id; - Logger.LogDebug("Created deployment ID {deploymentId}", deployment.Id); + try + { + deployment = await gitHubClient + .Repository + .Deployment + .Create( + remoteInformation.RemoteRepositoryOwner, + remoteInformation.RemoteRepositoryName, + new NewDeployment(compileJob.RevisionInformation.CommitSha) + { + AutoMerge = false, + Description = "TGS Game Deployment", + Environment = $"TGS: {Metadata.Name}", + ProductionEnvironment = true, + RequiredContexts = new Collection(), + }) + .WithToken(cancellationToken); - await gitHubClient - .Repository - .Deployment - .Status - .Create( - remoteInformation.RemoteRepositoryOwner, - remoteInformation.RemoteRepositoryName, - deployment.Id, - new NewDeploymentStatus(DeploymentState.InProgress) - { - Description = "The project is being deployed", - AutoInactive = false, - }) - .WithToken(cancellationToken) - ; + Logger.LogDebug("Created deployment ID {deploymentId}", deployment.Id); - Logger.LogTrace("In-progress deployment status created"); + await gitHubClient + .Repository + .Deployment + .Status + .Create( + remoteInformation.RemoteRepositoryOwner, + remoteInformation.RemoteRepositoryName, + deployment.Id, + new NewDeploymentStatus(DeploymentState.InProgress) + { + Description = "The project is being deployed", + AutoInactive = false, + }) + .WithToken(cancellationToken); + + compileJob.GitHubDeploymentId = deployment.Id; + Logger.LogTrace("In-progress deployment status created"); + } + catch (ApiException ex) + { + Logger.LogError(ex, "Unable to create deployment!"); + } } try diff --git a/src/Tgstation.Server.Host/Components/Deployment/Remote/GitLabRemoteDeploymentManager.cs b/src/Tgstation.Server.Host/Components/Deployment/Remote/GitLabRemoteDeploymentManager.cs index af70918b12..8d242b7164 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/Remote/GitLabRemoteDeploymentManager.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/Remote/GitLabRemoteDeploymentManager.cs @@ -67,7 +67,7 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote { await Task.WhenAll(tasks); } - catch (Exception ex) when (!(ex is OperationCanceledException)) + catch (Exception ex) when (ex is not OperationCanceledException) { Logger.LogWarning(ex, "Merge requests update check failed!"); } diff --git a/src/Tgstation.Server.Host/Components/Instance.cs b/src/Tgstation.Server.Host/Components/Instance.cs index 9ab3e243fd..3f6a114571 100644 --- a/src/Tgstation.Server.Host/Components/Instance.cs +++ b/src/Tgstation.Server.Host/Components/Instance.cs @@ -266,23 +266,16 @@ namespace Tgstation.Server.Host.Components throw new InvalidOperationException(DifferentCoreExceptionMessage); // assume 5 steps with synchronize - const int ProgressSections = 7; - const int ProgressStep = 100 / ProgressSections; - var repositorySettingsTask = databaseContext .RepositorySettings .AsQueryable() .Where(x => x.InstanceId == metadata.Id) .FirstAsync(cancellationToken); - const int NumSteps = 3; - var doneSteps = 0; - - JobProgressReporter NextProgressReporter() + const int ProgressSections = 7; + JobProgressReporter NextProgressReporter(string stage) { - var tmpDoneSteps = doneSteps; - ++doneSteps; - return (status, progress) => progressReporter(status, (progress + (100 * tmpDoneSteps)) / NumSteps); + return progressReporter.CreateSection(stage, 1.0 / ProgressSections); } using var repo = await RepositoryManager.LoadRepository(cancellationToken); @@ -305,7 +298,7 @@ namespace Tgstation.Server.Host.Components await repo.FetchOrigin( repositorySettings.AccessUser, repositorySettings.AccessToken, - NextProgressReporter(), + NextProgressReporter("Fetch Origin"), cancellationToken) ; @@ -371,7 +364,7 @@ namespace Tgstation.Server.Host.Components var result = await repo.MergeOrigin( repositorySettings.CommitterName, repositorySettings.CommitterEmail, - NextProgressReporter(), + NextProgressReporter("Merge Origin"), cancellationToken) ; @@ -416,12 +409,13 @@ namespace Tgstation.Server.Host.Components if (!preserveTestMerges) { - logger.LogTrace("Resetting to origin..."); + const string StageName = "Resetting to origin..."; + logger.LogTrace(StageName); await repo.ResetToOrigin( repositorySettings.AccessUser, repositorySettings.AccessToken, repositorySettings.UpdateSubmodules.Value, - NextProgressReporter(), + NextProgressReporter(StageName), cancellationToken) ; @@ -447,7 +441,7 @@ namespace Tgstation.Server.Host.Components repositorySettings.AccessToken, repositorySettings.CommitterName, repositorySettings.CommitterEmail, - NextProgressReporter(), + NextProgressReporter("Synchronize"), shouldSyncTracked, cancellationToken); var currentHead = repo.Head; @@ -466,8 +460,6 @@ namespace Tgstation.Server.Host.Components await repo.ResetToSha(startSha, progressReporter, default); throw; } - - progressReporter(null, 5 * ProgressStep); }); #pragma warning restore CA1502 // Cyclomatic complexity diff --git a/src/Tgstation.Server.Host/Components/Repository/Repository.cs b/src/Tgstation.Server.Host/Components/Repository/Repository.cs index f2d808bfc0..9990d584ce 100644 --- a/src/Tgstation.Server.Host/Components/Repository/Repository.cs +++ b/src/Tgstation.Server.Host/Components/Repository/Repository.cs @@ -214,6 +214,8 @@ namespace Tgstation.Server.Host.Components.Repository MergeResult result = null; + var progressFactor = 1.0 / (updateSubmodules ? 3 : 2); + var sig = new Signature(new Identity(committerName, committerEmail), DateTimeOffset.UtcNow); await Task.Factory.StartNew( () => @@ -225,8 +227,6 @@ namespace Tgstation.Server.Host.Components.Repository logger.LogTrace("Fetching refspec {0}...", refSpec); var remote = libGitRepo.Network.Remotes.First(); - var stage = $"Fetch {refSpec}"; - progressReporter(stage, 0); commands.Fetch( libGitRepo, refSpecList, @@ -236,8 +236,7 @@ namespace Tgstation.Server.Host.Components.Repository Prune = true, OnProgress = (a) => !cancellationToken.IsCancellationRequested, OnTransferProgress = TransferProgressHandler( - (lambdaStage, progress) => progressReporter(lambdaStage, progress / 2), - stage, + progressReporter.CreateSection($"Fetch {refSpec}", progressFactor), cancellationToken), OnUpdateTips = (a, b, c) => !cancellationToken.IsCancellationRequested, CredentialsProvider = credentialsProvider.GenerateCredentialsHandler(username, password), @@ -267,7 +266,7 @@ namespace Tgstation.Server.Host.Components.Repository cancellationToken.ThrowIfCancellationRequested(); - logger.LogTrace("Merging {0} into {1}...", testMergeParameters.TargetCommitSha.Substring(0, 7), Reference); + logger.LogTrace("Merging {0} into {1}...", testMergeParameters.TargetCommitSha[..7], Reference); result = libGitRepo.Merge(testMergeParameters.TargetCommitSha, sig, new MergeOptions { @@ -276,8 +275,7 @@ namespace Tgstation.Server.Host.Components.Repository FastForwardStrategy = FastForwardStrategy.NoFastForward, SkipReuc = true, OnCheckoutProgress = CheckoutProgressHandler( - (lambdaStage, progress) => progressReporter(lambdaStage, progress.HasValue ? (int?)(50 + (progress.Value / 2)) : null), - $"Merge {testMergeParameters.TargetCommitSha}"), + progressReporter.CreateSection($"Merge {testMergeParameters.TargetCommitSha[..7]}", progressFactor)), }); } finally @@ -291,7 +289,8 @@ namespace Tgstation.Server.Host.Components.Repository { var revertTo = originalCommit.CanonicalName ?? originalCommit.Tip.Sha; logger.LogDebug("Merge conflict, aborting and reverting to {0}", revertTo); - RawCheckout(revertTo, progressReporter, cancellationToken); + progressReporter.ReportProgress(0); + RawCheckout(revertTo, progressReporter.CreateSection("Hard Reset to {revertTo}", 1.0), cancellationToken); cancellationToken.ThrowIfCancellationRequested(); } @@ -332,11 +331,13 @@ namespace Tgstation.Server.Host.Components.Repository ; if (updateSubmodules) + { await UpdateSubmodules( - (stage, progress) => progressReporter(stage, 66 + (progress.Value / 3)), + progressReporter.CreateSection("Update Submodules", progressFactor), username, password, cancellationToken); + } } await eventConsumer.HandleEvent( @@ -375,7 +376,7 @@ namespace Tgstation.Server.Host.Components.Repository libGitRepo.RemoveUntrackedFiles(); RawCheckout( committish, - (stage, progress) => progressReporter(stage, progress * (updateSubmodules ? 2 : 3) / 3), + progressReporter.CreateSection(null, updateSubmodules ? 2.0 / 3 : 1.0), cancellationToken); }, cancellationToken, @@ -385,7 +386,7 @@ namespace Tgstation.Server.Host.Components.Repository if (updateSubmodules) await UpdateSubmodules( - (stage, progress) => progressReporter(stage, 66 + (progress / 3)), + progressReporter.CreateSection(null, 1.0 / 3), username, password, cancellationToken); @@ -414,7 +415,7 @@ namespace Tgstation.Server.Host.Components.Repository { Prune = true, OnProgress = (a) => !cancellationToken.IsCancellationRequested, - OnTransferProgress = TransferProgressHandler(progressReporter, "Fetch Origin", cancellationToken), + OnTransferProgress = TransferProgressHandler(progressReporter.CreateSection("Fetch Origin", 1.0), cancellationToken), OnUpdateTips = (a, b, c) => !cancellationToken.IsCancellationRequested, CredentialsProvider = credentialsProvider.GenerateCredentialsHandler(username, password), }, @@ -452,12 +453,16 @@ namespace Tgstation.Server.Host.Components.Repository await eventConsumer.HandleEvent(EventType.RepoResetOrigin, new List { trackedBranch.FriendlyName, trackedBranch.Tip.Sha }, cancellationToken); await ResetToSha( trackedBranch.Tip.Sha, - (stage, progress) => progressReporter(stage, progress / (updateSubmodules ? 2 : 1)), + progressReporter.CreateSection(null, updateSubmodules ? 2.0 / 3 : 1.0), cancellationToken) ; if (updateSubmodules) - await UpdateSubmodules((stage, progress) => progressReporter(stage, 50 + (progress / 2)), username, password, cancellationToken); + await UpdateSubmodules( + progressReporter.CreateSection(null, 1.0 / 3), + username, + password, + cancellationToken); } /// @@ -482,7 +487,7 @@ namespace Tgstation.Server.Host.Components.Repository libGitRepo.Reset(ResetMode.Hard, gitObject.Peel(), new CheckoutOptions { - OnCheckoutProgress = CheckoutProgressHandler(progressReporter, $"Reset to {gitObject.Sha}"), + OnCheckoutProgress = CheckoutProgressHandler(progressReporter.CreateSection($"Reset to {gitObject.Sha}", 1.0)), }); }, cancellationToken, @@ -551,7 +556,7 @@ namespace Tgstation.Server.Host.Components.Repository FailOnConflict = true, FastForwardStrategy = FastForwardStrategy.Default, SkipReuc = true, - OnCheckoutProgress = CheckoutProgressHandler(progressReporter, "Merge Origin"), + OnCheckoutProgress = CheckoutProgressHandler(progressReporter.CreateSection("Merge Origin", 1.0)), }); cancellationToken.ThrowIfCancellationRequested(); @@ -559,9 +564,10 @@ namespace Tgstation.Server.Host.Components.Repository if (result.Status == MergeStatus.Conflicts) { logger.LogDebug("Merge conflict, aborting and reverting to {0}", oldHead.FriendlyName); + progressReporter.ReportProgress(0); libGitRepo.Reset(ResetMode.Hard, oldTip, new CheckoutOptions { - OnCheckoutProgress = CheckoutProgressHandler(progressReporter, $"Hard Reset to {oldHead.FriendlyName}"), + OnCheckoutProgress = CheckoutProgressHandler(progressReporter.CreateSection($"Hard Reset to {oldHead.FriendlyName}", 1.0)), }); cancellationToken.ThrowIfCancellationRequested(); } @@ -646,7 +652,7 @@ namespace Tgstation.Server.Host.Components.Repository { libGitRepo.Reset(ResetMode.Hard, libGitRepo.Head.Tip, new CheckoutOptions { - OnCheckoutProgress = CheckoutProgressHandler((stage, progress) => progressReporter(stage, progress.HasValue ? (int?)(progress.Value / 10) : null), "Hard reset and remove untracked files"), + OnCheckoutProgress = CheckoutProgressHandler(progressReporter.CreateSection("Hard reset and remove untracked files", 0.1)), }); cancellationToken.ThrowIfCancellationRequested(); libGitRepo.RemoveUntrackedFiles(); @@ -657,11 +663,14 @@ namespace Tgstation.Server.Host.Components.Repository ; } - void FinalReporter(string stage, int? progress) => progressReporter(stage, (int)(((float)progress) / 100 * 90)); - + var remainingProgressFactor = 0.9; if (!synchronizeTrackedBranch) { - await PushHeadToTemporaryBranch(username, password, FinalReporter, cancellationToken); + await PushHeadToTemporaryBranch( + username, + password, + progressReporter.CreateSection("Push to temporary branch", remainingProgressFactor), + cancellationToken); return false; } @@ -680,7 +689,13 @@ namespace Tgstation.Server.Host.Components.Repository var remote = libGitRepo.Network.Remotes.First(); try { - libGitRepo.Network.Push(libGitRepo.Head, GeneratePushOptions(FinalReporter, username, password, cancellationToken)); + libGitRepo.Network.Push( + libGitRepo.Head, + GeneratePushOptions( + progressReporter.CreateSection("Push to origin", remainingProgressFactor), + username, + password, + cancellationToken)); return true; } catch (NonFastForwardException) @@ -808,13 +823,14 @@ namespace Tgstation.Server.Host.Components.Repository logger.LogTrace("Checkout: {0}", committish); var stage = $"Checkout {committish}"; - progressReporter(stage, 0); + progressReporter = progressReporter.CreateSection(stage, 1.0); + progressReporter.ReportProgress(0); cancellationToken.ThrowIfCancellationRequested(); var checkoutOptions = new CheckoutOptions { CheckoutModifiers = CheckoutModifiers.Force, - OnCheckoutProgress = CheckoutProgressHandler(progressReporter, stage), + OnCheckoutProgress = CheckoutProgressHandler(progressReporter), }; void RunCheckout() => commands.Checkout( @@ -872,9 +888,9 @@ namespace Tgstation.Server.Host.Components.Repository try { var forcePushString = String.Format(CultureInfo.InvariantCulture, "+{0}:{0}", branch.CanonicalName); - libGitRepo.Network.Push(remote, forcePushString, GeneratePushOptions((stage, progress) => progressReporter(stage, (int)(0.9f * progress)), username, password, cancellationToken)); + libGitRepo.Network.Push(remote, forcePushString, GeneratePushOptions(progressReporter.CreateSection(null, 0.9), username, password, cancellationToken)); var removalString = String.Format(CultureInfo.InvariantCulture, ":{0}", branch.CanonicalName); - libGitRepo.Network.Push(remote, removalString, GeneratePushOptions((stage, progress) => progressReporter(stage, 90 + (int)(0.1f * progress)), username, password, cancellationToken)); + libGitRepo.Network.Push(remote, removalString, GeneratePushOptions(progressReporter.CreateSection(null, 0.1), username, password, cancellationToken)); } catch (UserCancelledException) { @@ -902,22 +918,31 @@ namespace Tgstation.Server.Host.Components.Repository /// The password for the . /// The for the operation. /// A new set of . - PushOptions GeneratePushOptions(JobProgressReporter progressReporter, string username, string password, CancellationToken cancellationToken) => new PushOptions + PushOptions GeneratePushOptions(JobProgressReporter progressReporter, string username, string password, CancellationToken cancellationToken) { - OnPackBuilderProgress = (stage, current, total) => + var subProgressReporter = progressReporter.CreateSection(null, 0.5); + + return new PushOptions { - var baseProgress = stage == PackBuilderStage.Counting ? 0 : 25; - progressReporter("Push", baseProgress + ((int)(25 * ((float)current) / total))); - return !cancellationToken.IsCancellationRequested; - }, - OnNegotiationCompletedBeforePush = (a) => !cancellationToken.IsCancellationRequested, - OnPushTransferProgress = (a, sentBytes, totalBytes) => - { - progressReporter("Push", 50 + ((int)(50 * ((float)sentBytes) / totalBytes))); - return !cancellationToken.IsCancellationRequested; - }, - CredentialsProvider = credentialsProvider.GenerateCredentialsHandler(username, password), - }; + OnPackBuilderProgress = (stage, current, total) => + { + var baseProgress = stage == PackBuilderStage.Counting ? 0 : 0.5; + progressReporter.ReportProgress(baseProgress + (0.5 * ((double)current / total))); + return !cancellationToken.IsCancellationRequested; + }, + OnNegotiationCompletedBeforePush = (a) => + { + subProgressReporter = progressReporter.CreateSection(null, 0.5); + return !cancellationToken.IsCancellationRequested; + }, + OnPushTransferProgress = (a, sentBytes, totalBytes) => + { + progressReporter.ReportProgress((double)sentBytes / totalBytes); + return !cancellationToken.IsCancellationRequested; + }, + CredentialsProvider = credentialsProvider.GenerateCredentialsHandler(username, password), + }; + } /// /// Recusively update all s in the . @@ -938,24 +963,20 @@ namespace Tgstation.Server.Host.Components.Repository logger.LogTrace("Updating submodules with{0} credentials...", username == null ? "out" : String.Empty); - var iteration = 0; - var factor = 100 / submoduleCount; + var factor = 1.0 / submoduleCount / 2; foreach (var submodule in libGitRepo.Submodules) { - void LocalProgressReporter(string stage, int? percentage) => progressReporter(stage, percentage.HasValue ? (int?)((iteration * factor) + (percentage.Value / submoduleCount)) : null); var submoduleUpdateOptions = new SubmoduleUpdateOptions { Init = true, OnTransferProgress = TransferProgressHandler( - (stage, progress) => LocalProgressReporter(stage, progress.Value / 2), - $"Fetch submodule {submodule.Name}", + progressReporter.CreateSection($"Fetch submodule {submodule.Name}", factor), cancellationToken), OnProgress = output => !cancellationToken.IsCancellationRequested, OnUpdateTips = (a, b, c) => !cancellationToken.IsCancellationRequested, CredentialsProvider = credentialsProvider.GenerateCredentialsHandler(username, password), OnCheckoutProgress = CheckoutProgressHandler( - (stage, progress) => LocalProgressReporter(stage, progress.HasValue ? (int?)(50 + (progress.Value / 2)) : null), - $"Checkout submodule {submodule.Name}"), + progressReporter.CreateSection($"Checkout submodule {submodule.Name}", factor)), }; logger.LogDebug("Updating submodule {0}...", submodule.Name); @@ -972,6 +993,7 @@ namespace Tgstation.Server.Host.Components.Repository { // workaround for https://github.com/libgit2/libgit2/issues/3820 // kill off the modules/ folder in .git and try again + progressReporter.ReportProgress(null); credentialsProvider.CheckBadCredentialsException(ex); logger.LogWarning(ex, "Initial update of submodule {0} failed. Deleting submodule directories and re-attempting...", submodule.Name); @@ -1005,11 +1027,10 @@ namespace Tgstation.Server.Host.Components.Repository /// Converts a given to a . /// /// The of the operation. - /// The stage argument for . /// A based on . - CheckoutProgressHandler CheckoutProgressHandler(JobProgressReporter progressReporter, string stage) => (a, completedSteps, totalSteps) => + CheckoutProgressHandler CheckoutProgressHandler(JobProgressReporter progressReporter) => (a, completedSteps, totalSteps) => { - int? percentage; + double? percentage; // short circuit initialization where totalSteps is 0 if (completedSteps == 0) @@ -1018,40 +1039,36 @@ namespace Tgstation.Server.Host.Components.Repository percentage = null; else { - var ratio = ((float)completedSteps) / totalSteps; - percentage = (int)(ratio * 100); + percentage = ((double)completedSteps) / totalSteps; if (percentage < 0) percentage = null; } if (percentage == null) logger.LogDebug( - "Bad checkout progress values (Please tell Cyberboss)! Completeds: {completed}, Total: {total}", + "Bad checkout progress values (Please tell Dominion)! Completeds: {completed}, Total: {total}", completedSteps, totalSteps); - progressReporter( - stage, - percentage); + progressReporter.ReportProgress(percentage); }; /// /// Generate a from a given and . /// /// The of the operation. - /// The stage argument for . /// The for the operation. /// A new based on . - TransferProgressHandler TransferProgressHandler(JobProgressReporter progressReporter, string stage, CancellationToken cancellationToken) => (transferProgress) => + TransferProgressHandler TransferProgressHandler(JobProgressReporter progressReporter, CancellationToken cancellationToken) => (transferProgress) => { - float? percentage; + double? percentage; var totalObjectsToProcess = transferProgress.TotalObjects * 2; var processedObjects = transferProgress.IndexedObjects + transferProgress.ReceivedObjects; if (totalObjectsToProcess < processedObjects || totalObjectsToProcess == 0) percentage = null; else { - percentage = 100 * (((float)processedObjects) / totalObjectsToProcess); + percentage = (double)processedObjects / totalObjectsToProcess; if (percentage < 0) percentage = null; } @@ -1063,7 +1080,7 @@ namespace Tgstation.Server.Host.Components.Repository transferProgress.ReceivedObjects, transferProgress.TotalObjects); - progressReporter(stage, (int?)percentage); + progressReporter.ReportProgress(percentage); return !cancellationToken.IsCancellationRequested; }; } diff --git a/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs b/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs index b272d71d9a..46e3712f62 100644 --- a/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs +++ b/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs @@ -135,8 +135,8 @@ namespace Tgstation.Server.Host.Components.Repository OnProgress = (a) => !cancellationToken.IsCancellationRequested, OnTransferProgress = (a) => { - var percentage = 100 * (((float)a.IndexedObjects + a.ReceivedObjects) / (a.TotalObjects * 2)); - progressReporter("Cloning", (int)percentage); + var percentage = ((double)a.IndexedObjects + a.ReceivedObjects) / (a.TotalObjects * 2); + progressReporter.ReportProgress(percentage); return !cancellationToken.IsCancellationRequested; }, RecurseSubmodules = recurseSubmodules, diff --git a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs index e159daee1b..98d63bf5d8 100644 --- a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs +++ b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs @@ -205,10 +205,9 @@ namespace Tgstation.Server.Host.Components.StaticFiles await EnsureDirectories(cancellationToken); var path = ValidateConfigRelativePath(configurationRelativePath); - if (configurationRelativePath == null) - configurationRelativePath = "/"; + configurationRelativePath ??= "/"; - List result = new List(); + var result = new List(); void ListImpl() { @@ -223,7 +222,7 @@ namespace Tgstation.Server.Host.Components.StaticFiles } catch (IOException e) { - logger.LogDebug(e, "IOException while writing {0}!", path); + logger.LogDebug(e, "IOException while writing {path}!", path); result = null; return; } @@ -376,9 +375,9 @@ namespace Tgstation.Server.Host.Components.StaticFiles task = ioManager.GetDirectories(GameStaticFilesSubdirectory, cancellationToken); var entries = await task; - await Task.WhenAll(entries.Select(async x => + await Task.WhenAll(entries.Select(async file => { - var fileName = ioManager.GetFileName(x); + var fileName = ioManager.GetFileName(file); // need to normalize bool ignored; @@ -389,19 +388,19 @@ namespace Tgstation.Server.Host.Components.StaticFiles if (ignored) { - logger.LogTrace("Ignoring static file {0}...", fileName); + logger.LogTrace("Ignoring static file {fileName}...", fileName); return; } var destPath = ioManager.ConcatPath(destination, fileName); - logger.LogTrace("Symlinking {0} to {1}...", x, destPath); + logger.LogTrace("Symlinking {filePath} to {destPath}...", file, destPath); var fileExistsTask = ioManager.FileExists(destPath, cancellationToken); if (await ioManager.DirectoryExists(destPath, cancellationToken)) await ioManager.DeleteDirectory(destPath, cancellationToken); var fileExists = await fileExistsTask; if (fileExists) await ioManager.DeleteFile(destPath, cancellationToken); - await symlinkFactory.CreateSymbolicLink(ioManager.ResolvePath(x), ioManager.ResolvePath(destPath), cancellationToken); + await symlinkFactory.CreateSymbolicLink(ioManager.ResolvePath(file), ioManager.ResolvePath(destPath), cancellationToken); })); } @@ -440,6 +439,12 @@ namespace Tgstation.Server.Host.Components.StaticFiles success = synchronousIOManager.WriteFileChecked(path, uploadStream, ref fileHash, cancellationToken); } + if (fileTicket == null) + { + logger.LogDebug("File upload ticket for {path} expired!", path); + return; + } + using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken)) if (systemIdentity == null) await Task.Factory.StartNew(WriteCallback, cancellationToken, DefaultIOManager.BlockingTaskCreationOptions, TaskScheduler.Current); @@ -550,13 +555,13 @@ namespace Tgstation.Server.Host.Components.StaticFiles if (!scriptFiles.Any()) { - logger.LogTrace("No event scripts starting with \"{0}\" detected", scriptName); + logger.LogTrace("No event scripts starting with \"{scriptName}\" detected", scriptName); return; } foreach (var scriptFile in scriptFiles) { - logger.LogTrace("Running event script {0}...", scriptFile); + logger.LogTrace("Running event script {scriptFile}...", scriptFile); using (var script = processExecutor.LaunchProcess( ioManager.ConcatPath(resolvedScriptsDir, scriptFile), resolvedScriptsDir, @@ -582,7 +587,7 @@ namespace Tgstation.Server.Host.Components.StaticFiles if (exitCode != 0) throw new JobException($"Script {scriptFile} exited with code {exitCode}:{Environment.NewLine}{scriptOutput}"); else - logger.LogDebug("Script output:{0}{1}", Environment.NewLine, scriptOutput); + logger.LogDebug("Script output:{newLine}{scriptOutput}", Environment.NewLine, scriptOutput); } } } diff --git a/src/Tgstation.Server.Host/Controllers/ByondController.cs b/src/Tgstation.Server.Host/Controllers/ByondController.cs index 40d88ce456..4990d25c6d 100644 --- a/src/Tgstation.Server.Host/Controllers/ByondController.cs +++ b/src/Tgstation.Server.Host/Controllers/ByondController.cs @@ -229,7 +229,7 @@ namespace Tgstation.Server.Host.Controllers } } - return result.InstallJob != null ? (IActionResult)Accepted(result) : Json(result); + return result.InstallJob != null ? Accepted(result) : Json(result); }) ; } diff --git a/src/Tgstation.Server.Host/Controllers/ConfigurationController.cs b/src/Tgstation.Server.Host/Controllers/ConfigurationController.cs index fa76710724..71b132b816 100644 --- a/src/Tgstation.Server.Host/Controllers/ConfigurationController.cs +++ b/src/Tgstation.Server.Host/Controllers/ConfigurationController.cs @@ -87,13 +87,13 @@ namespace Tgstation.Server.Host.Controllers cancellationToken) ; - return model.LastReadHash == null ? (IActionResult)Accepted(newFile) : Json(newFile); + return model.LastReadHash == null ? Accepted(newFile) : Json(newFile); }) ; } catch (IOException e) { - Logger.LogInformation("IOException while updating file {0}: {1}", model.Path, e); + Logger.LogInformation(e, "IOException while updating file {path}!", model.Path); return Conflict(new ErrorMessageResponse(ErrorCode.IOError) { AdditionalData = e.Message, @@ -140,7 +140,7 @@ namespace Tgstation.Server.Host.Controllers } catch (IOException e) { - Logger.LogInformation("IOException while reading file {0}: {1}", filePath, e); + Logger.LogInformation(e, "IOException while reading file {path}!", filePath); return Conflict(new ErrorMessageResponse(ErrorCode.IOError) { AdditionalData = e.Message, @@ -262,7 +262,7 @@ namespace Tgstation.Server.Host.Controllers } catch (IOException e) { - Logger.LogInformation("IOException while creating directory {0}: {1}", model.Path, e); + Logger.LogInformation(e, "IOException while creating directory {path}!", model.Path); return Conflict(new ErrorMessageResponse(ErrorCode.IOError) { Message = e.Message, diff --git a/src/Tgstation.Server.Host/Controllers/RepositoryController.cs b/src/Tgstation.Server.Host/Controllers/RepositoryController.cs index 825c0b9873..38d093fcc8 100644 --- a/src/Tgstation.Server.Host/Controllers/RepositoryController.cs +++ b/src/Tgstation.Server.Host/Controllers/RepositoryController.cs @@ -489,16 +489,14 @@ namespace Tgstation.Server.Host.Controllers var hardResettingToOriginReference = model.UpdateFromOrigin == true && model.Reference != null; var numSteps = (model.NewTestMerges?.Count ?? 0) + (model.UpdateFromOrigin == true ? 1 : 0) + (!modelHasShaOrReference ? 2 : (hardResettingToOriginReference ? 3 : 1)); - var doneSteps = 0; + var progressFactor = 1.0 / numSteps; - JobProgressReporter NextProgressReporter() + JobProgressReporter NextProgressReporter(string stage) { - var tmpDoneSteps = doneSteps; - ++doneSteps; - return (status, progress) => progressReporter(status, (progress + (100 * tmpDoneSteps)) / numSteps); + return progressReporter.CreateSection(stage, progressFactor); } - progressReporter(null, 0); + progressReporter.ReportProgress(0); // get a base line for where we are Models.RevisionInformation lastRevisionInfo = null; @@ -568,11 +566,11 @@ namespace Tgstation.Server.Host.Controllers { if (!repo.Tracking) throw new JobException(ErrorCode.RepoReferenceRequired); - await repo.FetchOrigin(currentModel.AccessUser, currentModel.AccessToken, NextProgressReporter(), ct); - doneSteps = 1; + await repo.FetchOrigin(currentModel.AccessUser, currentModel.AccessToken, NextProgressReporter("Fetch Origin"), ct); + if (!modelHasShaOrReference) { - var fastForward = await repo.MergeOrigin(committerName, currentModel.CommitterEmail, NextProgressReporter(), ct); + var fastForward = await repo.MergeOrigin(committerName, currentModel.CommitterEmail, NextProgressReporter("Merge Origin"), ct); if (!fastForward.HasValue) throw new JobException(ErrorCode.RepoMergeConflict); lastRevisionInfo.OriginCommitSha = await repo.GetOriginSha(cancellationToken); @@ -584,14 +582,14 @@ namespace Tgstation.Server.Host.Controllers currentModel.AccessToken, currentModel.CommitterName, currentModel.CommitterEmail, - NextProgressReporter(), + NextProgressReporter("Sychronize"), true, ct) ; postUpdateSha = repo.Head; } else - NextProgressReporter()(null, 100); + NextProgressReporter(null).ReportProgress(1.0); } } @@ -620,13 +618,13 @@ namespace Tgstation.Server.Host.Controllers currentModel.AccessUser, currentModel.AccessToken, updateSubmodules, - NextProgressReporter(), + NextProgressReporter("Checkout"), ct) ; await CallLoadRevInfo(); // we've either seen origin before or what we're checking out is on origin } else - NextProgressReporter()(null, 100); + NextProgressReporter(null).ReportProgress(1.0); if (hardResettingToOriginReference) { @@ -636,7 +634,7 @@ namespace Tgstation.Server.Host.Controllers currentModel.AccessUser, currentModel.AccessToken, updateSubmodules, - NextProgressReporter(), + NextProgressReporter("Reset to Origin"), ct) ; await repo.Sychronize( @@ -644,7 +642,7 @@ namespace Tgstation.Server.Host.Controllers currentModel.AccessToken, currentModel.CommitterName, currentModel.CommitterEmail, - NextProgressReporter(), + NextProgressReporter("Synchronize"), true, ct) ; @@ -798,7 +796,7 @@ namespace Tgstation.Server.Host.Controllers { // goteem Logger.LogDebug("Reusing existing SHA {0}...", revInfoWereLookingFor.CommitSha); - await repo.ResetToSha(revInfoWereLookingFor.CommitSha, NextProgressReporter(), cancellationToken); + await repo.ResetToSha(revInfoWereLookingFor.CommitSha, NextProgressReporter($"Reset to {revInfoWereLookingFor.CommitSha[..7]}"), cancellationToken); lastRevisionInfo = revInfoWereLookingFor; } @@ -818,14 +816,14 @@ namespace Tgstation.Server.Host.Controllers currentModel.AccessUser, currentModel.AccessToken, updateSubmodules, - NextProgressReporter(), + NextProgressReporter($"Test merge #{newTestMerge.Number}"), ct); if (mergeResult == null) throw new JobException( ErrorCode.RepoTestMergeConflict, new JobException( - $"Test Merge #{newTestMerge.Number} at {newTestMerge.TargetCommitSha.Substring(0, 7)} conflicted!")); + $"Test Merge #{newTestMerge.Number} at {newTestMerge.TargetCommitSha[..7]} conflicted!")); Models.TestMerge fullTestMerge; try @@ -834,7 +832,7 @@ namespace Tgstation.Server.Host.Controllers } catch (Exception ex) { - Logger.LogWarning("Error retrieving metadata for test merge #{0}!", newTestMerge.Number); + Logger.LogWarning("Error retrieving metadata for test merge #{testMergeNumber}!", newTestMerge.Number); fullTestMerge = new Models.TestMerge { @@ -850,9 +848,6 @@ namespace Tgstation.Server.Host.Controllers // Ensure we're getting the full sha from git itself fullTestMerge.TargetCommitSha = newTestMerge.TargetCommitSha; - // MergedBy will be set later - ++doneSteps; - await UpdateRevInfo(fullTestMerge); } } @@ -866,7 +861,7 @@ namespace Tgstation.Server.Host.Controllers currentModel.AccessToken, currentModel.CommitterName, currentModel.CommitterEmail, - NextProgressReporter(), + NextProgressReporter("Synchronize"), false, ct) ; @@ -877,23 +872,25 @@ namespace Tgstation.Server.Host.Controllers } catch { - doneSteps = 0; numSteps = 2; // Forget what we've done and abort + progressReporter.ReportProgress(0.0); + + var secondStep = startReference != null && repo.Head != startSha; + // DCTx2: Cancellation token is for job, operations should always run await repo.CheckoutObject( startReference ?? startSha, currentModel.AccessUser, currentModel.AccessToken, true, - NextProgressReporter(), - default) - ; - if (startReference != null && repo.Head != startSha) - await repo.ResetToSha(startSha, NextProgressReporter(), default); - else - progressReporter(null, 100); + progressReporter.CreateSection($"Checkout {startReference ?? startSha[..7]}", secondStep ? 0.5 : 1.0), + default); + + if (secondStep) + await repo.ResetToSha(startSha, progressReporter.CreateSection($"Hard reset to SHA {startSha[..7]}", 0.5), default); + throw; } } diff --git a/src/Tgstation.Server.Host/IO/SynchronousIOManager.cs b/src/Tgstation.Server.Host/IO/SynchronousIOManager.cs index 9d474a474f..f6ebae6c51 100644 --- a/src/Tgstation.Server.Host/IO/SynchronousIOManager.cs +++ b/src/Tgstation.Server.Host/IO/SynchronousIOManager.cs @@ -85,7 +85,11 @@ namespace Tgstation.Server.Host.IO var directory = Path.GetDirectoryName(path); Directory.CreateDirectory(directory); + + var newFile = !File.Exists(path); + cancellationToken.ThrowIfCancellationRequested(); + using (var file = File.Open(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None)) { cancellationToken.ThrowIfCancellationRequested(); @@ -97,9 +101,20 @@ namespace Tgstation.Server.Host.IO // suppressed due to only using for consistency checks using (var sha1 = SHA1.Create()) { - string GetSha1(Stream dataToHash) => dataToHash != null && dataToHash.Length != 0 ? String.Join(String.Empty, sha1.ComputeHash(dataToHash).Select(b => b.ToString("x2", CultureInfo.InvariantCulture))) : null; + string GetSha1(Stream dataToHash) + { + if (dataToHash == null) + return null; + + byte[] sha1Computed = dataToHash.Length != 0 + ? sha1.ComputeHash(dataToHash) + : sha1.ComputeHash(Array.Empty()); + + return String.Join(String.Empty, sha1Computed.Select(b => b.ToString("x2", CultureInfo.InvariantCulture))); + } + var originalSha1 = GetSha1(file); - if (originalSha1 != sha1InOut) + if (originalSha1 != sha1InOut && !(newFile && sha1InOut == null)) { sha1InOut = originalSha1; return false; diff --git a/src/Tgstation.Server.Host/Jobs/JobManager.cs b/src/Tgstation.Server.Host/Jobs/JobManager.cs index cf9b668cba..77fd1d1211 100644 --- a/src/Tgstation.Server.Host/Jobs/JobManager.cs +++ b/src/Tgstation.Server.Host/Jobs/JobManager.cs @@ -23,6 +23,11 @@ namespace Tgstation.Server.Host.Jobs /// readonly IDatabaseContextFactory databaseContextFactory; + /// + /// The for the . + /// + readonly ILoggerFactory loggerFactory; + /// /// The for the . /// @@ -58,11 +63,17 @@ namespace Tgstation.Server.Host.Jobs /// /// The value of . /// The value of . + /// The value of . /// The value of . - public JobManager(IDatabaseContextFactory databaseContextFactory, Lazy instanceCoreProvider, ILogger logger) + public JobManager( + IDatabaseContextFactory databaseContextFactory, + Lazy instanceCoreProvider, + ILoggerFactory loggerFactory, + ILogger logger) { this.databaseContextFactory = databaseContextFactory ?? throw new ArgumentNullException(nameof(databaseContextFactory)); this.instanceCoreProvider = instanceCoreProvider ?? throw new ArgumentNullException(nameof(instanceCoreProvider)); + this.loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory)); this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); jobs = new Dictionary(); activationTcs = new TaskCompletionSource(); @@ -293,12 +304,12 @@ namespace Tgstation.Server.Host.Jobs var oldJob = job; job = new Job { Id = oldJob.Id }; - void UpdateProgress(string stage, int? progress) + void UpdateProgress(string stage, double? progress) { if (progress.HasValue - && (progress.Value < 0 || progress.Value > 100)) + && (progress.Value < 0 || progress.Value > 1)) { - var exception = new ArgumentOutOfRangeException(nameof(progress), progress, "Progress must be a value from 0-100!"); + var exception = new ArgumentOutOfRangeException(nameof(progress), progress, "Progress must be a value from 0-1!"); logger.LogError(exception, "Invalid progress value!"); return; } @@ -307,7 +318,7 @@ namespace Tgstation.Server.Host.Jobs if (jobs.TryGetValue(oldJob.Id.Value, out var handler)) { handler.Stage = stage; - handler.Progress = progress; + handler.Progress = progress.HasValue ? (int)Math.Floor(progress.Value * 100) : null; } } @@ -318,7 +329,10 @@ namespace Tgstation.Server.Host.Jobs instanceCoreProvider.Value.GetInstance(oldJob.Instance), databaseContextFactory, job, - UpdateProgress, + new JobProgressReporter( + loggerFactory.CreateLogger(), + null, + UpdateProgress), cancellationToken) ; diff --git a/src/Tgstation.Server.Host/Jobs/JobProgressReporter.cs b/src/Tgstation.Server.Host/Jobs/JobProgressReporter.cs index e715659dc9..70ca0b2bb0 100644 --- a/src/Tgstation.Server.Host/Jobs/JobProgressReporter.cs +++ b/src/Tgstation.Server.Host/Jobs/JobProgressReporter.cs @@ -1,13 +1,127 @@ -using Tgstation.Server.Host.Models; +using System; + +using Microsoft.Extensions.Logging; + +using Tgstation.Server.Host.Models; namespace Tgstation.Server.Host.Jobs { /// /// Progress reporter for a . /// - /// A description of what the job is currently doing. - /// The progress of the job on a scale from 0-100. - public delegate void JobProgressReporter( - string stage, - int? progress); + public sealed class JobProgressReporter + { + /// + /// The name of the current stage. + /// + public string StageName { get; set; } + + /// + /// The for the . + /// + readonly ILogger logger; + + /// + /// Progress reporter callback taking a description of what the job is currently doing and the (optional) progress of the job on a scale from 0.0-1.0. + /// + readonly Action callback; + + /// + /// The total progress reported so far in this section. + /// + double sectionProgression; + + /// + /// Initializes a new instance of the class. + /// + /// The value of . + /// The value of . + /// The value of . + public JobProgressReporter(ILogger logger, string stageName, Action callback) + { + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + this.callback = callback ?? throw new ArgumentNullException(nameof(callback)); + StageName = stageName; + + logger.LogDebug("Job progress reporter created. Stage: {stageName}", stageName ?? "(null)"); + } + + /// + /// Report progress. + /// + /// A percentage value from 0.0f-1.0f. + public void ReportProgress(double? progress) + { + var clampedProgress = progress; + if (progress.HasValue) + if (progress > 1 || progress < 0) + { + logger.LogError( + new ArgumentOutOfRangeException(nameof(progress), progress, "Progress must be a value from 0-1!"), + "Invalid progress value for stage {stageName}", + StageName ?? "(null)"); + clampedProgress = null; + } + else + sectionProgression = progress.Value; + + if (clampedProgress.HasValue) + callback(StageName, (int)Math.Floor(clampedProgress.Value * 100)); + else + callback(StageName, null); + } + + /// + /// Create a subsection of the with its optional own stage name. + /// + /// The optional of the new . + /// The 0.0f-1.0f percentage of the current 's percentage should be given to the section. + /// A new that is a subsection of this one. + /// A should only have one active child at a time. + public JobProgressReporter CreateSection(string newStageName, double percentage) + { + if (percentage > 1 || percentage < 0) + { + logger.LogError( + new ArgumentOutOfRangeException(nameof(percentage), percentage, "Percentage must be a value from 0-1!"), + "Invalid percentage value for stage {newStageName}! Clamping...", + newStageName ?? "(null)"); + + percentage = Math.Min(Math.Max(percentage, 0.0), 1.0); + } + + var childBaseProgress = sectionProgression; + if (percentage + childBaseProgress > 1.0) + { + var remainingPercentage = 1.0 - childBaseProgress; + logger.LogError( + "Stage {newStageName} is overbudgeted ({budget}/{remainingPercentage})! Clamping...", + newStageName, + percentage, + remainingPercentage); + percentage = remainingPercentage; + } + + var newReporter = new JobProgressReporter( + logger, + newStageName, + (currentStage, progress) => + { + currentStage ??= StageName; + if (!progress.HasValue) + { + callback(currentStage, null); + return; + } + + var childLocalProgress = progress.Value * percentage; + + sectionProgression = childLocalProgress + childBaseProgress; + callback(currentStage, sectionProgression); + }); + + newReporter.ReportProgress(0); + return newReporter; + } + } } diff --git a/tests/Tgstation.Server.Tests/IntegrationTest.cs b/tests/Tgstation.Server.Tests/IntegrationTest.cs index b821ca125b..7de3b4cdb4 100644 --- a/tests/Tgstation.Server.Tests/IntegrationTest.cs +++ b/tests/Tgstation.Server.Tests/IntegrationTest.cs @@ -32,6 +32,7 @@ using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.Database; using Tgstation.Server.Host.Database.Migrations; using Tgstation.Server.Host.Extensions; +using Tgstation.Server.Host.Jobs; using Tgstation.Server.Host.System; using Tgstation.Server.Tests.Instance; @@ -1066,18 +1067,18 @@ namespace Tgstation.Server.Tests using var testingServer = new TestingServer(null, false); LibGit2Sharp.Repository.Clone("https://github.com/Cyberboss/test", testingServer.Directory); var libGit2Repo = new LibGit2Sharp.Repository(testingServer.Directory); - using var repo = new Host.Components.Repository.Repository( + using var repo = new Repository( libGit2Repo, new LibGit2Commands(), Mock.Of(), Mock.Of(), Mock.Of(), Mock.Of(), - Mock.Of>(), + Mock.Of>(), () => { }); const string StartSha = "af4da8beb9f9b374b04a3cc4d65acca662e8cc1a"; - await repo.CheckoutObject(StartSha, null, null, true, (stage, progress) => { }, default); + await repo.CheckoutObject(StartSha, null, null, true, new JobProgressReporter(Mock.Of>(), null, (stage, progress) => { }), default); var result = await repo.ShaIsParent("2f8588a3ca0f6b027704a2a04381215619de3412", default); Assert.IsTrue(result); Assert.AreEqual(StartSha, repo.Head);