Merge pull request #1292 from tgstation/1284-AttemptTwo

Add submodule updates

src/Tgstation.Server.Api/Models/Request/RepositoryUpdateRequest.cs

@@ -17,10 +17,15 @@ namespace Tgstation.Server.Api.Models.Request
 		public string? CheckoutSha { get; set; }
 
 		/// <summary>
-		/// Do the equivalent of a git pull. Will attempt to merge unless <see cref="RepositoryApiBase.Reference"/> is also specified in which case a hard reset will be performed after checking out.
+		/// Do the equivalent of a `git pull`. Will attempt to merge unless <see cref="RepositoryApiBase.Reference"/> is also specified in which case a hard reset will be performed after checking out.
 		/// </summary>
 		public bool? UpdateFromOrigin { get; set; }
 
+		/// <summary>
+		/// Do the equivalent of a `git submodule update --init --recursive` alongside any resets to origin, checkouts, or test merge additions.
+		/// </summary>
+		public bool? UpdateSubmodules { get; set; }
+
 		/// <summary>
 		/// <see cref="TestMergeParameters"/> for new <see cref="TestMerge"/>s. Note that merges that conflict will not be performed.
 		/// </summary>

src/Tgstation.Server.Host/Components/Events/EventType.cs

@@ -143,5 +143,11 @@
 		/// </summary>
 		[EventScript("DreamDaemonLaunch")]
 		DreamDaemonLaunch,
+
+		/// <summary>
+		/// After a single submodule update is performed. Parameters: Updated submodule name
+		/// </summary>
+		[EventScript("RepoSubmoduleUpdate")]
+		RepoSubmoduleUpdate,
 	}
 }

src/Tgstation.Server.Host/Components/Instance.cs

@@ -404,7 +404,13 @@ namespace Tgstation.Server.Host.Components
 					if (!preserveTestMerges)
 					{
 						logger.LogTrace("Resetting to origin...");
-						await repo.ResetToOrigin(NextProgressReporter(), cancellationToken).ConfigureAwait(false);
+						await repo.ResetToOrigin(
+							repositorySettings.AccessUser,
+							repositorySettings.AccessToken,
+							true,
+							NextProgressReporter(),
+							cancellationToken)
+						.ConfigureAwait(false);
 
 						var currentHead = repo.Head;
 

src/Tgstation.Server.Host/Components/Repository/IRepository.cs

@@ -43,10 +43,19 @@ namespace Tgstation.Server.Host.Components.Repository
 		/// Checks out a given <paramref name="committish"/>.
 		/// </summary>
 		/// <param name="committish">The sha or reference to checkout.</param>
+		/// <param name="username">The username used for fetching from submodule repositories.</param>
+		/// <param name="password">The password used for fetching from submodule repositories.</param>
+		/// <param name="updateSubmodules">If a submodule update should be attempted after the merge.</param>
 		/// <param name="progressReporter"><see cref="Action{T1}"/> to report 0-100 <see cref="int"/> progress of the operation.</param>
 		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
 		/// <returns>A <see cref="Task"/> representing the running operation.</returns>
-		Task CheckoutObject(string committish, Action<int> progressReporter, CancellationToken cancellationToken);
+		Task CheckoutObject(
+			string committish,
+			string username,
+			string password,
+			bool updateSubmodules,
+			Action<int> progressReporter,
+			CancellationToken cancellationToken);
 
 		/// <summary>
 		/// Attempt to merge the revision specified by a given set of <paramref name="testMergeParameters"/> into HEAD.
@@ -54,12 +63,21 @@ namespace Tgstation.Server.Host.Components.Repository
 		/// <param name="testMergeParameters">The <see cref="TestMergeParameters"/> of the pull request.</param>
 		/// <param name="committerName">The name of the merge committer.</param>
 		/// <param name="committerEmail">The e-mail of the merge committer.</param>
-		/// <param name="username">The username to fetch from the origin repository.</param>
-		/// <param name="password">The password to fetch from the origin repository.</param>
+		/// <param name="username">The username used to fetch from the origin and submodule repositories.</param>
+		/// <param name="password">The password used to fetch from the origin and submodule repositories.</param>
+		/// <param name="updateSubmodules">If a submodule update should be attempted after the merge.</param>
 		/// <param name="progressReporter"><see cref="Action{T1}"/> to report 0-100 <see cref="int"/> progress of the operation.</param>
 		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
 		/// <returns>A <see cref="Task{TResult}"/> resulting in a <see cref="Nullable{T}"/> <see cref="bool"/> representing the merge result that is <see langword="true"/> after a fast forward or up to date, <see langword="false"/> on a non-fast-forward, <see langword="null"/> on a conflict.</returns>
-		Task<bool?> AddTestMerge(TestMergeParameters testMergeParameters, string committerName, string committerEmail, string username, string password, Action<int> progressReporter, CancellationToken cancellationToken);
+		Task<bool?> AddTestMerge(
+			TestMergeParameters testMergeParameters,
+			string committerName,
+			string committerEmail,
+			string username,
+			string password,
+			bool updateSubmodules,
+			Action<int> progressReporter,
+			CancellationToken cancellationToken);
 
 		/// <summary>
 		/// Fetch commits from the origin repository.
@@ -74,10 +92,18 @@ namespace Tgstation.Server.Host.Components.Repository
 		/// <summary>
 		/// Requires the current HEAD to be a tracked reference. Hard resets the reference to what it tracks on the origin repository.
 		/// </summary>
+		/// <param name="username">The username used for fetching from submodule repositories.</param>
+		/// <param name="password">The password used for fetching from submodule repositories.</param>
+		/// <param name="updateSubmodules">If a submodule update should be attempted after the merge.</param>
 		/// <param name="progressReporter"><see cref="Action{T1}"/> to report 0-100 <see cref="int"/> progress of the operation.</param>
 		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
 		/// <returns>A <see cref="Task{TResult}"/> resulting in the SHA of the new HEAD.</returns>
-		Task ResetToOrigin(Action<int> progressReporter, CancellationToken cancellationToken);
+		Task ResetToOrigin(
+			string username,
+			string password,
+			bool updateSubmodules,
+			Action<int> progressReporter,
+			CancellationToken cancellationToken);
 
 		/// <summary>
 		/// Requires the current HEAD to be a reference. Hard resets the reference to the given sha.

src/Tgstation.Server.Host/Components/Repository/Repository.cs

@@ -118,6 +118,19 @@ namespace Tgstation.Server.Host.Components.Repository
 		/// <returns>A <see cref="LibGit2Sharp.Handlers.CheckoutProgressHandler"/> based on <paramref name="progressReporter"/>.</returns>
 		static CheckoutProgressHandler CheckoutProgressHandler(Action<int> progressReporter) => (a, completedSteps, totalSteps) => progressReporter((int)(((float)completedSteps) / totalSteps * 100));
 
+		/// <summary>
+		/// Generate a <see cref="LibGit2Sharp.Handlers.TransferProgressHandler"/> from a given <paramref name="progressReporter"/> and <paramref name="cancellationToken"/>.
+		/// </summary>
+		/// <param name="progressReporter"><see cref="Action{T1}"/> to report 0-100 <see cref="int"/> progress of the operation.</param>
+		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
+		/// <returns>A new <see cref="LibGit2Sharp.Handlers.TransferProgressHandler"/> based on <paramref name="progressReporter"/>.</returns>
+		static TransferProgressHandler TransferProgressHandler(Action<int> progressReporter, CancellationToken cancellationToken) => (transferProgress) =>
+		{
+			var percentage = 100 * (((float)transferProgress.IndexedObjects + transferProgress.ReceivedObjects) / (transferProgress.TotalObjects * 2));
+			progressReporter((int)percentage);
+			return !cancellationToken.IsCancellationRequested;
+		};
+
 		/// <summary>
 		/// Rethrow the authentication failure message as a <see cref="JobException"/> if it is one.
 		/// </summary>
@@ -187,6 +200,7 @@ namespace Tgstation.Server.Host.Components.Repository
 			string committerEmail,
 			string username,
 			string password,
+			bool updateSubmodules,
 			Action<int> progressReporter,
 			CancellationToken cancellationToken)
 		{
@@ -250,12 +264,7 @@ namespace Tgstation.Server.Host.Components.Repository
 								{
 									Prune = true,
 									OnProgress = (a) => !cancellationToken.IsCancellationRequested,
-									OnTransferProgress = (a) =>
-									{
-										var percentage = 50 * (((float)a.IndexedObjects + a.ReceivedObjects) / (a.TotalObjects * 2));
-										progressReporter((int)percentage);
-										return !cancellationToken.IsCancellationRequested;
-									},
+									OnTransferProgress = TransferProgressHandler(percentage => progressReporter(percentage / 2), cancellationToken),
 									OnUpdateTips = (a, b, c) => !cancellationToken.IsCancellationRequested,
 									CredentialsProvider = credentialsProvider.GenerateCredentialsHandler(username, password),
 								},
@@ -287,7 +296,7 @@ namespace Tgstation.Server.Host.Components.Repository
 							FailOnConflict = true,
 							FastForwardStrategy = FastForwardStrategy.NoFastForward,
 							SkipReuc = true,
-							OnCheckoutProgress = (a, completedSteps, totalSteps) => progressReporter(50 + ((int)(((float)completedSteps) / totalSteps * 50))),
+							OnCheckoutProgress = CheckoutProgressHandler(percentage => progressReporter(50 + (percentage / 2))),
 						});
 					}
 					finally
@@ -328,7 +337,7 @@ namespace Tgstation.Server.Host.Components.Repository
 				return null;
 			}
 
-			if (commitMessage != null && result.Status != MergeStatus.UpToDate)
+			if (result.Status != MergeStatus.UpToDate)
 			{
 				logger.LogTrace("Committing merge: \"{0}\"...", commitMessage);
 				await Task.Factory.StartNew(
@@ -340,6 +349,9 @@ namespace Tgstation.Server.Host.Components.Repository
 					DefaultIOManager.BlockingTaskCreationOptions,
 					TaskScheduler.Current)
 					.ConfigureAwait(false);
+
+				if (updateSubmodules)
+					await UpdateSubmodules(percentage => progressReporter(66 + (percentage / 3)), username, password, cancellationToken).ConfigureAwait(false);
 			}
 
 			await eventConsumer.HandleEvent(
@@ -358,7 +370,13 @@ namespace Tgstation.Server.Host.Components.Repository
 #pragma warning restore CA1506
 
 		/// <inheritdoc />
-		public async Task CheckoutObject(string committish, Action<int> progressReporter, CancellationToken cancellationToken)
+		public async Task CheckoutObject(
+			string committish,
+			string username,
+			string password,
+			bool updateSubmodules,
+			Action<int> progressReporter,
+			CancellationToken cancellationToken)
 		{
 			if (committish == null)
 				throw new ArgumentNullException(nameof(committish));
@@ -370,12 +388,15 @@ namespace Tgstation.Server.Host.Components.Repository
 				() =>
 				{
 					libGitRepo.RemoveUntrackedFiles();
-					RawCheckout(committish, progressReporter, cancellationToken);
+					RawCheckout(committish, percentage => progressReporter(percentage * (updateSubmodules ? 2 : 3) / 3), cancellationToken);
 				},
 				cancellationToken,
 				DefaultIOManager.BlockingTaskCreationOptions,
 				TaskScheduler.Current)
 				.ConfigureAwait(false);
+
+			if (updateSubmodules)
+				await UpdateSubmodules(percentage => progressReporter(66 + (percentage / 3)), username, password, cancellationToken).ConfigureAwait(false);
 		}
 
 		/// <inheritdoc />
@@ -401,12 +422,7 @@ namespace Tgstation.Server.Host.Components.Repository
 							{
 								Prune = true,
 								OnProgress = (a) => !cancellationToken.IsCancellationRequested,
-								OnTransferProgress = (a) =>
-								{
-									var percentage = 100 * (((float)a.IndexedObjects + a.ReceivedObjects) / (a.TotalObjects * 2));
-									progressReporter((int)percentage);
-									return !cancellationToken.IsCancellationRequested;
-								},
+								OnTransferProgress = TransferProgressHandler(progressReporter, cancellationToken),
 								OnUpdateTips = (a, b, c) => !cancellationToken.IsCancellationRequested,
 								CredentialsProvider = credentialsProvider.GenerateCredentialsHandler(username, password),
 							},
@@ -428,7 +444,7 @@ namespace Tgstation.Server.Host.Components.Repository
 		}
 
 		/// <inheritdoc />
-		public async Task ResetToOrigin(Action<int> progressReporter, CancellationToken cancellationToken)
+		public async Task ResetToOrigin(string username, string password, bool updateSubmodules, Action<int> progressReporter, CancellationToken cancellationToken)
 		{
 			if (progressReporter == null)
 				throw new ArgumentNullException(nameof(progressReporter));
@@ -437,7 +453,14 @@ namespace Tgstation.Server.Host.Components.Repository
 			logger.LogTrace("Reset to origin...");
 			var trackedBranch = libGitRepo.Head.TrackedBranch;
 			await eventConsumer.HandleEvent(EventType.RepoResetOrigin, new List<string> { trackedBranch.FriendlyName, trackedBranch.Tip.Sha }, cancellationToken).ConfigureAwait(false);
-			await ResetToSha(trackedBranch.Tip.Sha, progressReporter, cancellationToken).ConfigureAwait(false);
+			await ResetToSha(
+				trackedBranch.Tip.Sha,
+				percentage => progressReporter(percentage / (updateSubmodules ? 2 : 1)),
+				cancellationToken)
+				.ConfigureAwait(false);
+
+			if (updateSubmodules)
+				await UpdateSubmodules(percentage => progressReporter(50 + (percentage / 2)), username, password, cancellationToken).ConfigureAwait(false);
 		}
 
 		/// <inheritdoc />
@@ -890,5 +913,78 @@ namespace Tgstation.Server.Host.Components.Repository
 			},
 			CredentialsProvider = credentialsProvider.GenerateCredentialsHandler(username, password),
 		};
+
+		/// <summary>
+		/// Recusively update all <see cref="Submodule"/>s in the <see cref="libGitRepo"/>.
+		/// </summary>
+		/// <param name="progressReporter"><see cref="Action{T1}"/> to report 0-100 <see cref="int"/> progress of the operation.</param>
+		/// <param name="username">The username for the <see cref="credentialsProvider"/>.</param>
+		/// <param name="password">The password for the <see cref="credentialsProvider"/>.</param>
+		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
+		/// <returns>A <see cref="Task"/> representing the running operation.</returns>
+		async Task UpdateSubmodules(Action<int> progressReporter, string username, string password, CancellationToken cancellationToken)
+		{
+			var submoduleCount = libGitRepo.Submodules.Count();
+			if (submoduleCount == 0)
+			{
+				logger.LogTrace("No submodules, skipping update");
+				return;
+			}
+
+			logger.LogTrace("Updating submodules with{0} credentials...", username == null ? "out" : String.Empty);
+
+			var iteration = 0;
+			var factor = 100 / submoduleCount;
+			foreach (var submodule in libGitRepo.Submodules)
+			{
+				void LocalProgressReporter(int percentage) => progressReporter((iteration * factor) + (percentage / submoduleCount));
+				var submoduleUpdateOptions = new SubmoduleUpdateOptions
+				{
+					Init = true,
+					OnTransferProgress = TransferProgressHandler(percentage => LocalProgressReporter(percentage / 2), cancellationToken),
+					OnProgress = output => !cancellationToken.IsCancellationRequested,
+					OnUpdateTips = (a, b, c) => !cancellationToken.IsCancellationRequested,
+					CredentialsProvider = credentialsProvider.GenerateCredentialsHandler(username, password),
+					OnCheckoutProgress = CheckoutProgressHandler(percentage => LocalProgressReporter(50 + (percentage / 2))),
+				};
+
+				logger.LogDebug("Updating submodule {0}...", submodule.Name);
+				Task RawSubModuleUpdate() => Task.Factory.StartNew(
+					() => libGitRepo.Submodules.Update(submodule.Name, submoduleUpdateOptions),
+					cancellationToken,
+					DefaultIOManager.BlockingTaskCreationOptions,
+					TaskScheduler.Current);
+				try
+				{
+					await RawSubModuleUpdate().ConfigureAwait(false);
+				}
+				catch (LibGit2SharpException ex)
+				{
+					// workaround for https://github.com/libgit2/libgit2/issues/3820
+					// kill off the modules/ folder in .git and try again
+					CheckBadCredentialsException(ex);
+					logger.LogWarning(ex, "Initial update of submodule {0} failed. Deleting .git submodule directory and re-attempting...", submodule.Name);
+					await ioMananger.DeleteDirectory($".git/modules/{submodule.Path}", cancellationToken).ConfigureAwait(false);
+
+					logger.LogTrace("Second update attempt for submodule {0}...", submodule.Name);
+					try
+					{
+						await RawSubModuleUpdate().ConfigureAwait(false);
+					}
+					catch (UserCancelledException)
+					{
+						cancellationToken.ThrowIfCancellationRequested();
+					}
+					catch (LibGit2SharpException ex2)
+					{
+						CheckBadCredentialsException(ex2);
+						logger.LogTrace(ex2, "Retried update of submodule {0} failed!", submodule.Name);
+						throw new AggregateException(ex, ex2);
+					}
+				}
+
+				await eventConsumer.HandleEvent(EventType.RepoSubmoduleUpdate, new List<string> { submodule.Name }, cancellationToken).ConfigureAwait(false);
+			}
+		}
 	}
 }

src/Tgstation.Server.Host/Controllers/RepositoryController.cs

@@ -314,6 +314,8 @@ namespace Tgstation.Server.Host.Controllers
 			if (model.CommitterEmail?.Length == 0)
 				return BadRequest(new ErrorMessageResponse(ErrorCode.RepoWhitespaceCommitterEmail));
 
+			var updateSubmodules = model?.UpdateSubmodules ?? false;
+
 			var newTestMerges = model.NewTestMerges != null && model.NewTestMerges.Count > 0;
 			var userRights = (RepositoryRights)AuthenticationContext.GetRight(RightsType.Repository);
 			if (newTestMerges && !userRights.HasFlag(RepositoryRights.MergePullRequest))
@@ -583,7 +585,14 @@ namespace Tgstation.Server.Host.Controllers
 							if ((isSha && model.Reference != null) || (!isSha && model.CheckoutSha != null))
 								throw new JobException(ErrorCode.RepoSwappedShaOrReference);
 
-							await repo.CheckoutObject(committish, NextProgressReporter(), ct).ConfigureAwait(false);
+							await repo.CheckoutObject(
+								committish,
+								currentModel.AccessUser,
+								currentModel.AccessToken,
+								updateSubmodules,
+								NextProgressReporter(),
+								ct)
+								.ConfigureAwait(false);
 							await CallLoadRevInfo().ConfigureAwait(false); // we've either seen origin before or what we're checking out is on origin
 						}
 						else
@@ -593,7 +602,13 @@ namespace Tgstation.Server.Host.Controllers
 						{
 							if (!repo.Tracking)
 								throw new JobException(ErrorCode.RepoReferenceNotTracking);
-							await repo.ResetToOrigin(NextProgressReporter(), ct).ConfigureAwait(false);
+							await repo.ResetToOrigin(
+								currentModel.AccessUser,
+								currentModel.AccessToken,
+								updateSubmodules,
+								NextProgressReporter(),
+								ct)
+								.ConfigureAwait(false);
 							await repo.Sychronize(
 								currentModel.AccessUser,
 								currentModel.AccessToken,
@@ -774,6 +789,7 @@ namespace Tgstation.Server.Host.Controllers
 									currentModel.CommitterEmail,
 									currentModel.AccessUser,
 									currentModel.AccessToken,
+									updateSubmodules,
 									NextProgressReporter(),
 									ct).ConfigureAwait(false);
 
@@ -839,7 +855,14 @@ namespace Tgstation.Server.Host.Controllers
 
 					// Forget what we've done and abort
 					// DCTx2: Cancellation token is for job, operations should always run
-					await repo.CheckoutObject(startReference ?? startSha, NextProgressReporter(), default).ConfigureAwait(false);
+					await repo.CheckoutObject(
+						startReference ?? startSha,
+						currentModel.AccessUser,
+						currentModel.AccessToken,
+						true,
+						NextProgressReporter(),
+						default)
+						.ConfigureAwait(false);
 					if (startReference != null && repo.Head != startSha)
 						await repo.ResetToSha(startSha, NextProgressReporter(), default).ConfigureAwait(false);
 					else

tests/Tgstation.Server.Tests/IntegrationTest.cs

@@ -1058,7 +1058,7 @@ namespace Tgstation.Server.Tests
 				() => { });
 
 			const string StartSha = "af4da8beb9f9b374b04a3cc4d65acca662e8cc1a";
-			await repo.CheckoutObject(StartSha, progress => { }, default);
+			await repo.CheckoutObject(StartSha, null, null, true, progress => { }, default);
 			var result = await repo.ShaIsParent("2f8588a3ca0f6b027704a2a04381215619de3412", default);
 			Assert.IsTrue(result);
 			Assert.AreEqual(StartSha, repo.Head);