diff --git a/src/Tgstation.Server.Api/Models/Request/RepositoryUpdateRequest.cs b/src/Tgstation.Server.Api/Models/Request/RepositoryUpdateRequest.cs
index 4aac9add8b..f7f7ee185e 100644
--- a/src/Tgstation.Server.Api/Models/Request/RepositoryUpdateRequest.cs
+++ b/src/Tgstation.Server.Api/Models/Request/RepositoryUpdateRequest.cs
@@ -17,10 +17,15 @@ namespace Tgstation.Server.Api.Models.Request
public string? CheckoutSha { get; set; }
///
- /// Do the equivalent of a git pull. Will attempt to merge unless is also specified in which case a hard reset will be performed after checking out.
+ /// Do the equivalent of a `git pull`. Will attempt to merge unless is also specified in which case a hard reset will be performed after checking out.
///
public bool? UpdateFromOrigin { get; set; }
+ ///
+ /// Do the equivalent of a `git submodule update --init --recursive` alongside any resets to origin, checkouts, or test merge additions.
+ ///
+ public bool? UpdateSubmodules { get; set; }
+
///
/// for new s. Note that merges that conflict will not be performed.
///
diff --git a/src/Tgstation.Server.Host/Components/Events/EventType.cs b/src/Tgstation.Server.Host/Components/Events/EventType.cs
index b7ca4a24c4..bcc4085250 100644
--- a/src/Tgstation.Server.Host/Components/Events/EventType.cs
+++ b/src/Tgstation.Server.Host/Components/Events/EventType.cs
@@ -143,5 +143,11 @@
///
[EventScript("DreamDaemonLaunch")]
DreamDaemonLaunch,
+
+ ///
+ /// After a single submodule update is performed. Parameters: Updated submodule name
+ ///
+ [EventScript("RepoSubmoduleUpdate")]
+ RepoSubmoduleUpdate,
}
}
diff --git a/src/Tgstation.Server.Host/Components/Instance.cs b/src/Tgstation.Server.Host/Components/Instance.cs
index 603602c5f8..d61bfa79e5 100644
--- a/src/Tgstation.Server.Host/Components/Instance.cs
+++ b/src/Tgstation.Server.Host/Components/Instance.cs
@@ -404,7 +404,13 @@ namespace Tgstation.Server.Host.Components
if (!preserveTestMerges)
{
logger.LogTrace("Resetting to origin...");
- await repo.ResetToOrigin(NextProgressReporter(), cancellationToken).ConfigureAwait(false);
+ await repo.ResetToOrigin(
+ repositorySettings.AccessUser,
+ repositorySettings.AccessToken,
+ true,
+ NextProgressReporter(),
+ cancellationToken)
+ .ConfigureAwait(false);
var currentHead = repo.Head;
diff --git a/src/Tgstation.Server.Host/Components/Repository/IRepository.cs b/src/Tgstation.Server.Host/Components/Repository/IRepository.cs
index f23e2c8d0c..0ac50e0cf9 100644
--- a/src/Tgstation.Server.Host/Components/Repository/IRepository.cs
+++ b/src/Tgstation.Server.Host/Components/Repository/IRepository.cs
@@ -43,10 +43,19 @@ namespace Tgstation.Server.Host.Components.Repository
/// Checks out a given .
///
/// The sha or reference to checkout.
+ /// The username used for fetching from submodule repositories.
+ /// The password used for fetching from submodule repositories.
+ /// If a submodule update should be attempted after the merge.
/// to report 0-100 progress of the operation.
/// The for the operation.
/// A representing the running operation.
- Task CheckoutObject(string committish, Action progressReporter, CancellationToken cancellationToken);
+ Task CheckoutObject(
+ string committish,
+ string username,
+ string password,
+ bool updateSubmodules,
+ Action progressReporter,
+ CancellationToken cancellationToken);
///
/// Attempt to merge the revision specified by a given set of into HEAD.
@@ -54,12 +63,21 @@ namespace Tgstation.Server.Host.Components.Repository
/// The of the pull request.
/// The name of the merge committer.
/// The e-mail of the merge committer.
- /// The username to fetch from the origin repository.
- /// The password to fetch from the origin repository.
+ /// The username used to fetch from the origin and submodule repositories.
+ /// The password used to fetch from the origin and submodule repositories.
+ /// If a submodule update should be attempted after the merge.
/// to report 0-100 progress of the operation.
/// The for the operation.
/// A resulting in a representing the merge result that is after a fast forward or up to date, on a non-fast-forward, on a conflict.
- Task AddTestMerge(TestMergeParameters testMergeParameters, string committerName, string committerEmail, string username, string password, Action progressReporter, CancellationToken cancellationToken);
+ Task AddTestMerge(
+ TestMergeParameters testMergeParameters,
+ string committerName,
+ string committerEmail,
+ string username,
+ string password,
+ bool updateSubmodules,
+ Action progressReporter,
+ CancellationToken cancellationToken);
///
/// Fetch commits from the origin repository.
@@ -74,10 +92,18 @@ namespace Tgstation.Server.Host.Components.Repository
///
/// Requires the current HEAD to be a tracked reference. Hard resets the reference to what it tracks on the origin repository.
///
+ /// The username used for fetching from submodule repositories.
+ /// The password used for fetching from submodule repositories.
+ /// If a submodule update should be attempted after the merge.
/// to report 0-100 progress of the operation.
/// The for the operation.
/// A resulting in the SHA of the new HEAD.
- Task ResetToOrigin(Action progressReporter, CancellationToken cancellationToken);
+ Task ResetToOrigin(
+ string username,
+ string password,
+ bool updateSubmodules,
+ Action progressReporter,
+ CancellationToken cancellationToken);
///
/// Requires the current HEAD to be a reference. Hard resets the reference to the given sha.
diff --git a/src/Tgstation.Server.Host/Components/Repository/Repository.cs b/src/Tgstation.Server.Host/Components/Repository/Repository.cs
index 934c34d2e3..0a5107bc62 100644
--- a/src/Tgstation.Server.Host/Components/Repository/Repository.cs
+++ b/src/Tgstation.Server.Host/Components/Repository/Repository.cs
@@ -118,6 +118,19 @@ namespace Tgstation.Server.Host.Components.Repository
/// A based on .
static CheckoutProgressHandler CheckoutProgressHandler(Action progressReporter) => (a, completedSteps, totalSteps) => progressReporter((int)(((float)completedSteps) / totalSteps * 100));
+ ///
+ /// Generate a from a given and .
+ ///
+ /// to report 0-100 progress of the operation.
+ /// The for the operation.
+ /// A new based on .
+ static TransferProgressHandler TransferProgressHandler(Action progressReporter, CancellationToken cancellationToken) => (transferProgress) =>
+ {
+ var percentage = 100 * (((float)transferProgress.IndexedObjects + transferProgress.ReceivedObjects) / (transferProgress.TotalObjects * 2));
+ progressReporter((int)percentage);
+ return !cancellationToken.IsCancellationRequested;
+ };
+
///
/// Rethrow the authentication failure message as a if it is one.
///
@@ -187,6 +200,7 @@ namespace Tgstation.Server.Host.Components.Repository
string committerEmail,
string username,
string password,
+ bool updateSubmodules,
Action progressReporter,
CancellationToken cancellationToken)
{
@@ -250,12 +264,7 @@ namespace Tgstation.Server.Host.Components.Repository
{
Prune = true,
OnProgress = (a) => !cancellationToken.IsCancellationRequested,
- OnTransferProgress = (a) =>
- {
- var percentage = 50 * (((float)a.IndexedObjects + a.ReceivedObjects) / (a.TotalObjects * 2));
- progressReporter((int)percentage);
- return !cancellationToken.IsCancellationRequested;
- },
+ OnTransferProgress = TransferProgressHandler(percentage => progressReporter(percentage / 2), cancellationToken),
OnUpdateTips = (a, b, c) => !cancellationToken.IsCancellationRequested,
CredentialsProvider = credentialsProvider.GenerateCredentialsHandler(username, password),
},
@@ -287,7 +296,7 @@ namespace Tgstation.Server.Host.Components.Repository
FailOnConflict = true,
FastForwardStrategy = FastForwardStrategy.NoFastForward,
SkipReuc = true,
- OnCheckoutProgress = (a, completedSteps, totalSteps) => progressReporter(50 + ((int)(((float)completedSteps) / totalSteps * 50))),
+ OnCheckoutProgress = CheckoutProgressHandler(percentage => progressReporter(50 + (percentage / 2))),
});
}
finally
@@ -328,7 +337,7 @@ namespace Tgstation.Server.Host.Components.Repository
return null;
}
- if (commitMessage != null && result.Status != MergeStatus.UpToDate)
+ if (result.Status != MergeStatus.UpToDate)
{
logger.LogTrace("Committing merge: \"{0}\"...", commitMessage);
await Task.Factory.StartNew(
@@ -340,6 +349,9 @@ namespace Tgstation.Server.Host.Components.Repository
DefaultIOManager.BlockingTaskCreationOptions,
TaskScheduler.Current)
.ConfigureAwait(false);
+
+ if (updateSubmodules)
+ await UpdateSubmodules(percentage => progressReporter(66 + (percentage / 3)), username, password, cancellationToken).ConfigureAwait(false);
}
await eventConsumer.HandleEvent(
@@ -358,7 +370,13 @@ namespace Tgstation.Server.Host.Components.Repository
#pragma warning restore CA1506
///
- public async Task CheckoutObject(string committish, Action progressReporter, CancellationToken cancellationToken)
+ public async Task CheckoutObject(
+ string committish,
+ string username,
+ string password,
+ bool updateSubmodules,
+ Action progressReporter,
+ CancellationToken cancellationToken)
{
if (committish == null)
throw new ArgumentNullException(nameof(committish));
@@ -370,12 +388,15 @@ namespace Tgstation.Server.Host.Components.Repository
() =>
{
libGitRepo.RemoveUntrackedFiles();
- RawCheckout(committish, progressReporter, cancellationToken);
+ RawCheckout(committish, percentage => progressReporter(percentage * (updateSubmodules ? 2 : 3) / 3), cancellationToken);
},
cancellationToken,
DefaultIOManager.BlockingTaskCreationOptions,
TaskScheduler.Current)
.ConfigureAwait(false);
+
+ if (updateSubmodules)
+ await UpdateSubmodules(percentage => progressReporter(66 + (percentage / 3)), username, password, cancellationToken).ConfigureAwait(false);
}
///
@@ -401,12 +422,7 @@ namespace Tgstation.Server.Host.Components.Repository
{
Prune = true,
OnProgress = (a) => !cancellationToken.IsCancellationRequested,
- OnTransferProgress = (a) =>
- {
- var percentage = 100 * (((float)a.IndexedObjects + a.ReceivedObjects) / (a.TotalObjects * 2));
- progressReporter((int)percentage);
- return !cancellationToken.IsCancellationRequested;
- },
+ OnTransferProgress = TransferProgressHandler(progressReporter, cancellationToken),
OnUpdateTips = (a, b, c) => !cancellationToken.IsCancellationRequested,
CredentialsProvider = credentialsProvider.GenerateCredentialsHandler(username, password),
},
@@ -428,7 +444,7 @@ namespace Tgstation.Server.Host.Components.Repository
}
///
- public async Task ResetToOrigin(Action progressReporter, CancellationToken cancellationToken)
+ public async Task ResetToOrigin(string username, string password, bool updateSubmodules, Action progressReporter, CancellationToken cancellationToken)
{
if (progressReporter == null)
throw new ArgumentNullException(nameof(progressReporter));
@@ -437,7 +453,14 @@ namespace Tgstation.Server.Host.Components.Repository
logger.LogTrace("Reset to origin...");
var trackedBranch = libGitRepo.Head.TrackedBranch;
await eventConsumer.HandleEvent(EventType.RepoResetOrigin, new List { trackedBranch.FriendlyName, trackedBranch.Tip.Sha }, cancellationToken).ConfigureAwait(false);
- await ResetToSha(trackedBranch.Tip.Sha, progressReporter, cancellationToken).ConfigureAwait(false);
+ await ResetToSha(
+ trackedBranch.Tip.Sha,
+ percentage => progressReporter(percentage / (updateSubmodules ? 2 : 1)),
+ cancellationToken)
+ .ConfigureAwait(false);
+
+ if (updateSubmodules)
+ await UpdateSubmodules(percentage => progressReporter(50 + (percentage / 2)), username, password, cancellationToken).ConfigureAwait(false);
}
///
@@ -890,5 +913,78 @@ namespace Tgstation.Server.Host.Components.Repository
},
CredentialsProvider = credentialsProvider.GenerateCredentialsHandler(username, password),
};
+
+ ///
+ /// Recusively update all s in the .
+ ///
+ /// to report 0-100 progress of the operation.
+ /// The username for the .
+ /// The password for the .
+ /// The for the operation.
+ /// A representing the running operation.
+ async Task UpdateSubmodules(Action progressReporter, string username, string password, CancellationToken cancellationToken)
+ {
+ var submoduleCount = libGitRepo.Submodules.Count();
+ if (submoduleCount == 0)
+ {
+ logger.LogTrace("No submodules, skipping update");
+ return;
+ }
+
+ logger.LogTrace("Updating submodules with{0} credentials...", username == null ? "out" : String.Empty);
+
+ var iteration = 0;
+ var factor = 100 / submoduleCount;
+ foreach (var submodule in libGitRepo.Submodules)
+ {
+ void LocalProgressReporter(int percentage) => progressReporter((iteration * factor) + (percentage / submoduleCount));
+ var submoduleUpdateOptions = new SubmoduleUpdateOptions
+ {
+ Init = true,
+ OnTransferProgress = TransferProgressHandler(percentage => LocalProgressReporter(percentage / 2), cancellationToken),
+ OnProgress = output => !cancellationToken.IsCancellationRequested,
+ OnUpdateTips = (a, b, c) => !cancellationToken.IsCancellationRequested,
+ CredentialsProvider = credentialsProvider.GenerateCredentialsHandler(username, password),
+ OnCheckoutProgress = CheckoutProgressHandler(percentage => LocalProgressReporter(50 + (percentage / 2))),
+ };
+
+ logger.LogDebug("Updating submodule {0}...", submodule.Name);
+ Task RawSubModuleUpdate() => Task.Factory.StartNew(
+ () => libGitRepo.Submodules.Update(submodule.Name, submoduleUpdateOptions),
+ cancellationToken,
+ DefaultIOManager.BlockingTaskCreationOptions,
+ TaskScheduler.Current);
+ try
+ {
+ await RawSubModuleUpdate().ConfigureAwait(false);
+ }
+ catch (LibGit2SharpException ex)
+ {
+ // workaround for https://github.com/libgit2/libgit2/issues/3820
+ // kill off the modules/ folder in .git and try again
+ CheckBadCredentialsException(ex);
+ logger.LogWarning(ex, "Initial update of submodule {0} failed. Deleting .git submodule directory and re-attempting...", submodule.Name);
+ await ioMananger.DeleteDirectory($".git/modules/{submodule.Path}", cancellationToken).ConfigureAwait(false);
+
+ logger.LogTrace("Second update attempt for submodule {0}...", submodule.Name);
+ try
+ {
+ await RawSubModuleUpdate().ConfigureAwait(false);
+ }
+ catch (UserCancelledException)
+ {
+ cancellationToken.ThrowIfCancellationRequested();
+ }
+ catch (LibGit2SharpException ex2)
+ {
+ CheckBadCredentialsException(ex2);
+ logger.LogTrace(ex2, "Retried update of submodule {0} failed!", submodule.Name);
+ throw new AggregateException(ex, ex2);
+ }
+ }
+
+ await eventConsumer.HandleEvent(EventType.RepoSubmoduleUpdate, new List { submodule.Name }, cancellationToken).ConfigureAwait(false);
+ }
+ }
}
}
diff --git a/src/Tgstation.Server.Host/Controllers/RepositoryController.cs b/src/Tgstation.Server.Host/Controllers/RepositoryController.cs
index 2652df6067..ba209b5265 100644
--- a/src/Tgstation.Server.Host/Controllers/RepositoryController.cs
+++ b/src/Tgstation.Server.Host/Controllers/RepositoryController.cs
@@ -314,6 +314,8 @@ namespace Tgstation.Server.Host.Controllers
if (model.CommitterEmail?.Length == 0)
return BadRequest(new ErrorMessageResponse(ErrorCode.RepoWhitespaceCommitterEmail));
+ var updateSubmodules = model?.UpdateSubmodules ?? false;
+
var newTestMerges = model.NewTestMerges != null && model.NewTestMerges.Count > 0;
var userRights = (RepositoryRights)AuthenticationContext.GetRight(RightsType.Repository);
if (newTestMerges && !userRights.HasFlag(RepositoryRights.MergePullRequest))
@@ -583,7 +585,14 @@ namespace Tgstation.Server.Host.Controllers
if ((isSha && model.Reference != null) || (!isSha && model.CheckoutSha != null))
throw new JobException(ErrorCode.RepoSwappedShaOrReference);
- await repo.CheckoutObject(committish, NextProgressReporter(), ct).ConfigureAwait(false);
+ await repo.CheckoutObject(
+ committish,
+ currentModel.AccessUser,
+ currentModel.AccessToken,
+ updateSubmodules,
+ NextProgressReporter(),
+ ct)
+ .ConfigureAwait(false);
await CallLoadRevInfo().ConfigureAwait(false); // we've either seen origin before or what we're checking out is on origin
}
else
@@ -593,7 +602,13 @@ namespace Tgstation.Server.Host.Controllers
{
if (!repo.Tracking)
throw new JobException(ErrorCode.RepoReferenceNotTracking);
- await repo.ResetToOrigin(NextProgressReporter(), ct).ConfigureAwait(false);
+ await repo.ResetToOrigin(
+ currentModel.AccessUser,
+ currentModel.AccessToken,
+ updateSubmodules,
+ NextProgressReporter(),
+ ct)
+ .ConfigureAwait(false);
await repo.Sychronize(
currentModel.AccessUser,
currentModel.AccessToken,
@@ -774,6 +789,7 @@ namespace Tgstation.Server.Host.Controllers
currentModel.CommitterEmail,
currentModel.AccessUser,
currentModel.AccessToken,
+ updateSubmodules,
NextProgressReporter(),
ct).ConfigureAwait(false);
@@ -839,7 +855,14 @@ namespace Tgstation.Server.Host.Controllers
// Forget what we've done and abort
// DCTx2: Cancellation token is for job, operations should always run
- await repo.CheckoutObject(startReference ?? startSha, NextProgressReporter(), default).ConfigureAwait(false);
+ await repo.CheckoutObject(
+ startReference ?? startSha,
+ currentModel.AccessUser,
+ currentModel.AccessToken,
+ true,
+ NextProgressReporter(),
+ default)
+ .ConfigureAwait(false);
if (startReference != null && repo.Head != startSha)
await repo.ResetToSha(startSha, NextProgressReporter(), default).ConfigureAwait(false);
else
diff --git a/tests/Tgstation.Server.Tests/IntegrationTest.cs b/tests/Tgstation.Server.Tests/IntegrationTest.cs
index 85cf120b56..cdfdfc6072 100644
--- a/tests/Tgstation.Server.Tests/IntegrationTest.cs
+++ b/tests/Tgstation.Server.Tests/IntegrationTest.cs
@@ -1058,7 +1058,7 @@ namespace Tgstation.Server.Tests
() => { });
const string StartSha = "af4da8beb9f9b374b04a3cc4d65acca662e8cc1a";
- await repo.CheckoutObject(StartSha, progress => { }, default);
+ await repo.CheckoutObject(StartSha, null, null, true, progress => { }, default);
var result = await repo.ShaIsParent("2f8588a3ca0f6b027704a2a04381215619de3412", default);
Assert.IsTrue(result);
Assert.AreEqual(StartSha, repo.Head);