From 4630bdfcc023c95a6a47ccff1a65e1c4a9cfdb78 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Wed, 11 Aug 2021 06:10:04 -0400 Subject: [PATCH] Add submodule updates - Refactor some progress handlers to be ODR compliant - Add `submoduleUpdate` to RepositoryRequest - Add `RepoSubmoduleUpdate` event script --- .../Models/Request/RepositoryUpdateRequest.cs | 7 +- .../Components/Events/EventType.cs | 6 + .../Components/Instance.cs | 8 +- .../Components/Repository/IRepository.cs | 36 ++++- .../Components/Repository/Repository.cs | 132 +++++++++++++++--- .../Controllers/RepositoryController.cs | 29 +++- .../Tgstation.Server.Tests/IntegrationTest.cs | 2 +- 7 files changed, 191 insertions(+), 29 deletions(-) diff --git a/src/Tgstation.Server.Api/Models/Request/RepositoryUpdateRequest.cs b/src/Tgstation.Server.Api/Models/Request/RepositoryUpdateRequest.cs index 4aac9add8b..f7f7ee185e 100644 --- a/src/Tgstation.Server.Api/Models/Request/RepositoryUpdateRequest.cs +++ b/src/Tgstation.Server.Api/Models/Request/RepositoryUpdateRequest.cs @@ -17,10 +17,15 @@ namespace Tgstation.Server.Api.Models.Request public string? CheckoutSha { get; set; } /// - /// Do the equivalent of a git pull. Will attempt to merge unless is also specified in which case a hard reset will be performed after checking out. + /// Do the equivalent of a `git pull`. Will attempt to merge unless is also specified in which case a hard reset will be performed after checking out. /// public bool? UpdateFromOrigin { get; set; } + /// + /// Do the equivalent of a `git submodule update --init --recursive` alongside any resets to origin, checkouts, or test merge additions. + /// + public bool? UpdateSubmodules { get; set; } + /// /// for new s. Note that merges that conflict will not be performed. /// diff --git a/src/Tgstation.Server.Host/Components/Events/EventType.cs b/src/Tgstation.Server.Host/Components/Events/EventType.cs index b7ca4a24c4..bcc4085250 100644 --- a/src/Tgstation.Server.Host/Components/Events/EventType.cs +++ b/src/Tgstation.Server.Host/Components/Events/EventType.cs @@ -143,5 +143,11 @@ /// [EventScript("DreamDaemonLaunch")] DreamDaemonLaunch, + + /// + /// After a single submodule update is performed. Parameters: Updated submodule name + /// + [EventScript("RepoSubmoduleUpdate")] + RepoSubmoduleUpdate, } } diff --git a/src/Tgstation.Server.Host/Components/Instance.cs b/src/Tgstation.Server.Host/Components/Instance.cs index 603602c5f8..d61bfa79e5 100644 --- a/src/Tgstation.Server.Host/Components/Instance.cs +++ b/src/Tgstation.Server.Host/Components/Instance.cs @@ -404,7 +404,13 @@ namespace Tgstation.Server.Host.Components if (!preserveTestMerges) { logger.LogTrace("Resetting to origin..."); - await repo.ResetToOrigin(NextProgressReporter(), cancellationToken).ConfigureAwait(false); + await repo.ResetToOrigin( + repositorySettings.AccessUser, + repositorySettings.AccessToken, + true, + NextProgressReporter(), + cancellationToken) + .ConfigureAwait(false); var currentHead = repo.Head; diff --git a/src/Tgstation.Server.Host/Components/Repository/IRepository.cs b/src/Tgstation.Server.Host/Components/Repository/IRepository.cs index f23e2c8d0c..0ac50e0cf9 100644 --- a/src/Tgstation.Server.Host/Components/Repository/IRepository.cs +++ b/src/Tgstation.Server.Host/Components/Repository/IRepository.cs @@ -43,10 +43,19 @@ namespace Tgstation.Server.Host.Components.Repository /// Checks out a given . /// /// The sha or reference to checkout. + /// The username used for fetching from submodule repositories. + /// The password used for fetching from submodule repositories. + /// If a submodule update should be attempted after the merge. /// to report 0-100 progress of the operation. /// The for the operation. /// A representing the running operation. - Task CheckoutObject(string committish, Action progressReporter, CancellationToken cancellationToken); + Task CheckoutObject( + string committish, + string username, + string password, + bool updateSubmodules, + Action progressReporter, + CancellationToken cancellationToken); /// /// Attempt to merge the revision specified by a given set of into HEAD. @@ -54,12 +63,21 @@ namespace Tgstation.Server.Host.Components.Repository /// The of the pull request. /// The name of the merge committer. /// The e-mail of the merge committer. - /// The username to fetch from the origin repository. - /// The password to fetch from the origin repository. + /// The username used to fetch from the origin and submodule repositories. + /// The password used to fetch from the origin and submodule repositories. + /// If a submodule update should be attempted after the merge. /// to report 0-100 progress of the operation. /// The for the operation. /// A resulting in a representing the merge result that is after a fast forward or up to date, on a non-fast-forward, on a conflict. - Task AddTestMerge(TestMergeParameters testMergeParameters, string committerName, string committerEmail, string username, string password, Action progressReporter, CancellationToken cancellationToken); + Task AddTestMerge( + TestMergeParameters testMergeParameters, + string committerName, + string committerEmail, + string username, + string password, + bool updateSubmodules, + Action progressReporter, + CancellationToken cancellationToken); /// /// Fetch commits from the origin repository. @@ -74,10 +92,18 @@ namespace Tgstation.Server.Host.Components.Repository /// /// Requires the current HEAD to be a tracked reference. Hard resets the reference to what it tracks on the origin repository. /// + /// The username used for fetching from submodule repositories. + /// The password used for fetching from submodule repositories. + /// If a submodule update should be attempted after the merge. /// to report 0-100 progress of the operation. /// The for the operation. /// A resulting in the SHA of the new HEAD. - Task ResetToOrigin(Action progressReporter, CancellationToken cancellationToken); + Task ResetToOrigin( + string username, + string password, + bool updateSubmodules, + Action progressReporter, + CancellationToken cancellationToken); /// /// Requires the current HEAD to be a reference. Hard resets the reference to the given sha. diff --git a/src/Tgstation.Server.Host/Components/Repository/Repository.cs b/src/Tgstation.Server.Host/Components/Repository/Repository.cs index 934c34d2e3..0a5107bc62 100644 --- a/src/Tgstation.Server.Host/Components/Repository/Repository.cs +++ b/src/Tgstation.Server.Host/Components/Repository/Repository.cs @@ -118,6 +118,19 @@ namespace Tgstation.Server.Host.Components.Repository /// A based on . static CheckoutProgressHandler CheckoutProgressHandler(Action progressReporter) => (a, completedSteps, totalSteps) => progressReporter((int)(((float)completedSteps) / totalSteps * 100)); + /// + /// Generate a from a given and . + /// + /// to report 0-100 progress of the operation. + /// The for the operation. + /// A new based on . + static TransferProgressHandler TransferProgressHandler(Action progressReporter, CancellationToken cancellationToken) => (transferProgress) => + { + var percentage = 100 * (((float)transferProgress.IndexedObjects + transferProgress.ReceivedObjects) / (transferProgress.TotalObjects * 2)); + progressReporter((int)percentage); + return !cancellationToken.IsCancellationRequested; + }; + /// /// Rethrow the authentication failure message as a if it is one. /// @@ -187,6 +200,7 @@ namespace Tgstation.Server.Host.Components.Repository string committerEmail, string username, string password, + bool updateSubmodules, Action progressReporter, CancellationToken cancellationToken) { @@ -250,12 +264,7 @@ namespace Tgstation.Server.Host.Components.Repository { Prune = true, OnProgress = (a) => !cancellationToken.IsCancellationRequested, - OnTransferProgress = (a) => - { - var percentage = 50 * (((float)a.IndexedObjects + a.ReceivedObjects) / (a.TotalObjects * 2)); - progressReporter((int)percentage); - return !cancellationToken.IsCancellationRequested; - }, + OnTransferProgress = TransferProgressHandler(percentage => progressReporter(percentage / 2), cancellationToken), OnUpdateTips = (a, b, c) => !cancellationToken.IsCancellationRequested, CredentialsProvider = credentialsProvider.GenerateCredentialsHandler(username, password), }, @@ -287,7 +296,7 @@ namespace Tgstation.Server.Host.Components.Repository FailOnConflict = true, FastForwardStrategy = FastForwardStrategy.NoFastForward, SkipReuc = true, - OnCheckoutProgress = (a, completedSteps, totalSteps) => progressReporter(50 + ((int)(((float)completedSteps) / totalSteps * 50))), + OnCheckoutProgress = CheckoutProgressHandler(percentage => progressReporter(50 + (percentage / 2))), }); } finally @@ -328,7 +337,7 @@ namespace Tgstation.Server.Host.Components.Repository return null; } - if (commitMessage != null && result.Status != MergeStatus.UpToDate) + if (result.Status != MergeStatus.UpToDate) { logger.LogTrace("Committing merge: \"{0}\"...", commitMessage); await Task.Factory.StartNew( @@ -340,6 +349,9 @@ namespace Tgstation.Server.Host.Components.Repository DefaultIOManager.BlockingTaskCreationOptions, TaskScheduler.Current) .ConfigureAwait(false); + + if (updateSubmodules) + await UpdateSubmodules(percentage => progressReporter(66 + (percentage / 3)), username, password, cancellationToken).ConfigureAwait(false); } await eventConsumer.HandleEvent( @@ -358,7 +370,13 @@ namespace Tgstation.Server.Host.Components.Repository #pragma warning restore CA1506 /// - public async Task CheckoutObject(string committish, Action progressReporter, CancellationToken cancellationToken) + public async Task CheckoutObject( + string committish, + string username, + string password, + bool updateSubmodules, + Action progressReporter, + CancellationToken cancellationToken) { if (committish == null) throw new ArgumentNullException(nameof(committish)); @@ -370,12 +388,15 @@ namespace Tgstation.Server.Host.Components.Repository () => { libGitRepo.RemoveUntrackedFiles(); - RawCheckout(committish, progressReporter, cancellationToken); + RawCheckout(committish, percentage => progressReporter(percentage * (updateSubmodules ? 2 : 3) / 3), cancellationToken); }, cancellationToken, DefaultIOManager.BlockingTaskCreationOptions, TaskScheduler.Current) .ConfigureAwait(false); + + if (updateSubmodules) + await UpdateSubmodules(percentage => progressReporter(66 + (percentage / 3)), username, password, cancellationToken).ConfigureAwait(false); } /// @@ -401,12 +422,7 @@ namespace Tgstation.Server.Host.Components.Repository { Prune = true, OnProgress = (a) => !cancellationToken.IsCancellationRequested, - OnTransferProgress = (a) => - { - var percentage = 100 * (((float)a.IndexedObjects + a.ReceivedObjects) / (a.TotalObjects * 2)); - progressReporter((int)percentage); - return !cancellationToken.IsCancellationRequested; - }, + OnTransferProgress = TransferProgressHandler(progressReporter, cancellationToken), OnUpdateTips = (a, b, c) => !cancellationToken.IsCancellationRequested, CredentialsProvider = credentialsProvider.GenerateCredentialsHandler(username, password), }, @@ -428,7 +444,7 @@ namespace Tgstation.Server.Host.Components.Repository } /// - public async Task ResetToOrigin(Action progressReporter, CancellationToken cancellationToken) + public async Task ResetToOrigin(string username, string password, bool updateSubmodules, Action progressReporter, CancellationToken cancellationToken) { if (progressReporter == null) throw new ArgumentNullException(nameof(progressReporter)); @@ -437,7 +453,14 @@ namespace Tgstation.Server.Host.Components.Repository logger.LogTrace("Reset to origin..."); var trackedBranch = libGitRepo.Head.TrackedBranch; await eventConsumer.HandleEvent(EventType.RepoResetOrigin, new List { trackedBranch.FriendlyName, trackedBranch.Tip.Sha }, cancellationToken).ConfigureAwait(false); - await ResetToSha(trackedBranch.Tip.Sha, progressReporter, cancellationToken).ConfigureAwait(false); + await ResetToSha( + trackedBranch.Tip.Sha, + percentage => progressReporter(percentage / (updateSubmodules ? 2 : 1)), + cancellationToken) + .ConfigureAwait(false); + + if (updateSubmodules) + await UpdateSubmodules(percentage => progressReporter(50 + (percentage / 2)), username, password, cancellationToken).ConfigureAwait(false); } /// @@ -890,5 +913,78 @@ namespace Tgstation.Server.Host.Components.Repository }, CredentialsProvider = credentialsProvider.GenerateCredentialsHandler(username, password), }; + + /// + /// Recusively update all s in the . + /// + /// to report 0-100 progress of the operation. + /// The username for the . + /// The password for the . + /// The for the operation. + /// A representing the running operation. + async Task UpdateSubmodules(Action progressReporter, string username, string password, CancellationToken cancellationToken) + { + var submoduleCount = libGitRepo.Submodules.Count(); + if (submoduleCount == 0) + { + logger.LogTrace("No submodules, skipping update"); + return; + } + + logger.LogTrace("Updating submodules with{0} credentials...", username == null ? "out" : String.Empty); + + var iteration = 0; + var factor = 100 / submoduleCount; + foreach (var submodule in libGitRepo.Submodules) + { + void LocalProgressReporter(int percentage) => progressReporter((iteration * factor) + (percentage / submoduleCount)); + var submoduleUpdateOptions = new SubmoduleUpdateOptions + { + Init = true, + OnTransferProgress = TransferProgressHandler(percentage => LocalProgressReporter(percentage / 2), cancellationToken), + OnProgress = output => !cancellationToken.IsCancellationRequested, + OnUpdateTips = (a, b, c) => !cancellationToken.IsCancellationRequested, + CredentialsProvider = credentialsProvider.GenerateCredentialsHandler(username, password), + OnCheckoutProgress = CheckoutProgressHandler(percentage => LocalProgressReporter(50 + (percentage / 2))), + }; + + logger.LogDebug("Updating submodule {0}...", submodule.Name); + Task RawSubModuleUpdate() => Task.Factory.StartNew( + () => libGitRepo.Submodules.Update(submodule.Name, submoduleUpdateOptions), + cancellationToken, + DefaultIOManager.BlockingTaskCreationOptions, + TaskScheduler.Current); + try + { + await RawSubModuleUpdate().ConfigureAwait(false); + } + catch (LibGit2SharpException ex) + { + // workaround for https://github.com/libgit2/libgit2/issues/3820 + // kill off the modules/ folder in .git and try again + CheckBadCredentialsException(ex); + logger.LogWarning(ex, "Initial update of submodule {0} failed. Deleting .git submodule directory and re-attempting...", submodule.Name); + await ioMananger.DeleteDirectory($".git/modules/{submodule.Path}", cancellationToken).ConfigureAwait(false); + + logger.LogTrace("Second update attempt for submodule {0}...", submodule.Name); + try + { + await RawSubModuleUpdate().ConfigureAwait(false); + } + catch (UserCancelledException) + { + cancellationToken.ThrowIfCancellationRequested(); + } + catch (LibGit2SharpException ex2) + { + CheckBadCredentialsException(ex2); + logger.LogTrace(ex2, "Retried update of submodule {0} failed!", submodule.Name); + throw new AggregateException(ex, ex2); + } + } + + await eventConsumer.HandleEvent(EventType.RepoSubmoduleUpdate, new List { submodule.Name }, cancellationToken).ConfigureAwait(false); + } + } } } diff --git a/src/Tgstation.Server.Host/Controllers/RepositoryController.cs b/src/Tgstation.Server.Host/Controllers/RepositoryController.cs index 2652df6067..ba209b5265 100644 --- a/src/Tgstation.Server.Host/Controllers/RepositoryController.cs +++ b/src/Tgstation.Server.Host/Controllers/RepositoryController.cs @@ -314,6 +314,8 @@ namespace Tgstation.Server.Host.Controllers if (model.CommitterEmail?.Length == 0) return BadRequest(new ErrorMessageResponse(ErrorCode.RepoWhitespaceCommitterEmail)); + var updateSubmodules = model?.UpdateSubmodules ?? false; + var newTestMerges = model.NewTestMerges != null && model.NewTestMerges.Count > 0; var userRights = (RepositoryRights)AuthenticationContext.GetRight(RightsType.Repository); if (newTestMerges && !userRights.HasFlag(RepositoryRights.MergePullRequest)) @@ -583,7 +585,14 @@ namespace Tgstation.Server.Host.Controllers if ((isSha && model.Reference != null) || (!isSha && model.CheckoutSha != null)) throw new JobException(ErrorCode.RepoSwappedShaOrReference); - await repo.CheckoutObject(committish, NextProgressReporter(), ct).ConfigureAwait(false); + await repo.CheckoutObject( + committish, + currentModel.AccessUser, + currentModel.AccessToken, + updateSubmodules, + NextProgressReporter(), + ct) + .ConfigureAwait(false); await CallLoadRevInfo().ConfigureAwait(false); // we've either seen origin before or what we're checking out is on origin } else @@ -593,7 +602,13 @@ namespace Tgstation.Server.Host.Controllers { if (!repo.Tracking) throw new JobException(ErrorCode.RepoReferenceNotTracking); - await repo.ResetToOrigin(NextProgressReporter(), ct).ConfigureAwait(false); + await repo.ResetToOrigin( + currentModel.AccessUser, + currentModel.AccessToken, + updateSubmodules, + NextProgressReporter(), + ct) + .ConfigureAwait(false); await repo.Sychronize( currentModel.AccessUser, currentModel.AccessToken, @@ -774,6 +789,7 @@ namespace Tgstation.Server.Host.Controllers currentModel.CommitterEmail, currentModel.AccessUser, currentModel.AccessToken, + updateSubmodules, NextProgressReporter(), ct).ConfigureAwait(false); @@ -839,7 +855,14 @@ namespace Tgstation.Server.Host.Controllers // Forget what we've done and abort // DCTx2: Cancellation token is for job, operations should always run - await repo.CheckoutObject(startReference ?? startSha, NextProgressReporter(), default).ConfigureAwait(false); + await repo.CheckoutObject( + startReference ?? startSha, + currentModel.AccessUser, + currentModel.AccessToken, + true, + NextProgressReporter(), + default) + .ConfigureAwait(false); if (startReference != null && repo.Head != startSha) await repo.ResetToSha(startSha, NextProgressReporter(), default).ConfigureAwait(false); else diff --git a/tests/Tgstation.Server.Tests/IntegrationTest.cs b/tests/Tgstation.Server.Tests/IntegrationTest.cs index 85cf120b56..cdfdfc6072 100644 --- a/tests/Tgstation.Server.Tests/IntegrationTest.cs +++ b/tests/Tgstation.Server.Tests/IntegrationTest.cs @@ -1058,7 +1058,7 @@ namespace Tgstation.Server.Tests () => { }); const string StartSha = "af4da8beb9f9b374b04a3cc4d65acca662e8cc1a"; - await repo.CheckoutObject(StartSha, progress => { }, default); + await repo.CheckoutObject(StartSha, null, null, true, progress => { }, default); var result = await repo.ShaIsParent("2f8588a3ca0f6b027704a2a04381215619de3412", default); Assert.IsTrue(result); Assert.AreEqual(StartSha, repo.Head);