Add deployment stage awareness to Discord embeds

- ErrorCode 79 no longer occurs on failing to send a chat message. - Remove unnecessary parameter for previous compile job from IRemoteDeploymentManager.ApplyDeployment. - Clean up DmbFactory. - Fix DmbFactory not doing remote staging if no existing compile job was present.
2026-07-15 10:03:02 +01:00 · 2023-08-12 23:00:50 -04:00
parent 48b7e047c9
commit 0ca44a4fec
20 changed files with 359 additions and 153 deletions
@@ -5,10 +5,10 @@
  <PropertyGroup>
    <TgsCoreVersion>5.14.0</TgsCoreVersion>
    <TgsConfigVersion>4.7.1</TgsConfigVersion>
-    <TgsApiVersion>9.11.0</TgsApiVersion>
+    <TgsApiVersion>9.11.1</TgsApiVersion>
    <TgsCommonLibraryVersion>6.0.0</TgsCommonLibraryVersion>
-    <TgsApiLibraryVersion>11.0.0</TgsApiLibraryVersion>
-    <TgsClientVersion>12.0.0</TgsClientVersion>
+    <TgsApiLibraryVersion>11.0.1</TgsApiLibraryVersion>
+    <TgsClientVersion>12.0.1</TgsClientVersion>
    <TgsDmapiVersion>6.5.2</TgsDmapiVersion>
    <TgsInteropVersion>5.6.1</TgsInteropVersion>
    <TgsHostWatchdogVersion>1.4.0</TgsHostWatchdogVersion>
@@ -486,7 +486,7 @@ namespace Tgstation.Server.Api.Models
 		DreamDaemonPortInUse,

 		/// <summary>
-		/// Failed to post GitHub comments, send chat message, or send TGS event.
+		/// Failed to post GitHub comments, or send TGS event.
 		/// </summary>
 		[Description("The deployment succeeded but one or more notification events failed!")]
 		PostDeployFailure,
@@ -6,7 +6,9 @@ using System.Threading;
 using System.Threading.Tasks;

 using Microsoft.Extensions.Logging;
+
 using Newtonsoft.Json;
+
 using Serilog.Context;

 using Tgstation.Server.Api.Models.Internal;
@@ -354,7 +356,7 @@ namespace Tgstation.Server.Host.Components.Chat
 		}

 		/// <inheritdoc />
-		public Action<string, string> QueueDeploymentMessage(
+		public Func<string, string, Action<bool>> QueueDeploymentMessage(
 			Models.RevisionInformation revisionInformation,
 			Version byondVersion,
 			DateTimeOffset? estimatedCompletionTime,
@@ -368,7 +370,7 @@ namespace Tgstation.Server.Host.Components.Chat

 			logger.LogTrace("Sending deployment message for RevisionInformation: {revisionInfoId}", revisionInformation.Id);

-			var callbacks = new List<Func<string, string, Task>>();
+			var callbacks = new List<Func<string, string, Task<Func<bool, Task>>>>();

 			var task = Task.WhenAll(
 				wdChannels.Select(
@@ -408,17 +410,44 @@ namespace Tgstation.Server.Host.Components.Chat

 			AddMessageTask(task);

-			async Task CollateTasks(string errorMessage, string dreamMakerOutput)
+			Task callbackTask = null;
+			Func<bool, Task> finalUpdateAction = null;
+			async Task CallbackTask(string errorMessage, string dreamMakerOutput)
 			{
 				await task;
-				await Task.WhenAll(
+				var callbackResultTasks =
 					callbacks.Select(
 						x => x(
 							errorMessage,
-							dreamMakerOutput)));
+							dreamMakerOutput))
+					.ToList();
+
+				await Task.WhenAll(callbackResultTasks);
+
+				finalUpdateAction = active => Task.WhenAll(callbackResultTasks.Select(task => task.Result(active)));
 			}

-			return (errorMessage, dreamMakerOutput) => AddMessageTask(CollateTasks(errorMessage, dreamMakerOutput));
+			async Task CompletionTask(bool active)
+			{
+				try
+				{
+					await callbackTask;
+				}
+				catch
+				{
+					// Handled in AddMessageTask
+					return;
+				}
+
+				AddMessageTask(finalUpdateAction(active));
+			}
+
+			return (errorMessage, dreamMakerOutput) =>
+			{
+				callbackTask = CallbackTask(errorMessage, dreamMakerOutput);
+				AddMessageTask(callbackTask);
+				return active => AddMessageTask(CompletionTask(active));
+			};
 		}

 		/// <inheritdoc />
@@ -66,8 +66,8 @@ namespace Tgstation.Server.Host.Components.Chat
 		/// <param name="gitHubOwner">The repository GitHub owner, if any.</param>
 		/// <param name="gitHubRepo">The repository GitHub name, if any.</param>
 		/// <param name="localCommitPushed"><see langword="true"/> if the local deployment commit was pushed to the remote repository.</param>
-		/// <returns>An <see cref="Action{T1, T2}"/> to call to update the message at the deployment's conclusion. Parameters: Error message if any, DreamMaker output if any.</returns>
-		Action<string, string> QueueDeploymentMessage(
+		/// <returns>A <see cref="Func{T1, T2, TResult}"/> to call to update the message at the deployment's conclusion. Parameters: Error message if any, DreamMaker output if any. Returns an <see cref="Action"/> to call to mark the deployment as active/inactive. Parameter: If the deployment is being activated or inactivated.</returns>
+		Func<string, string, Action<bool>> QueueDeploymentMessage(
 			Models.RevisionInformation revisionInformation,
 			Version byondVersion,
 			DateTimeOffset? estimatedCompletionTime,
@@ -326,7 +326,7 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
 		}

 		/// <inheritdoc />
-		public override async Task<Func<string, string, Task>> SendUpdateMessage(
+		public override async Task<Func<string, string, Task<Func<bool, Task>>>> SendUpdateMessage(
 			Models.RevisionInformation revisionInformation,
 			Version byondVersion,
 			DateTimeOffset? estimatedCompletionTime,
@@ -377,13 +377,11 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
 			{
 				var completionString = errorMessage == null ? "Succeeded" : "Failed";

-				embed = new Embed
+				Embed CreateUpdatedEmbed(string message, Color color) => new Embed
 				{
 					Author = embed.Author,
-					Colour = errorMessage == null ? Color.Green : Color.Red,
-					Description = errorMessage == null
-					? "The deployment completed successfully and will be available at the next server reboot."
-					: "The deployment failed.",
+					Colour = color,
+					Description = message,
 					Fields = fields,
 					Title = embed.Title,
 					Footer = new EmbedFooter(
@@ -391,6 +389,15 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
 					Timestamp = DateTimeOffset.UtcNow,
 				};

+				if (errorMessage == null)
+					embed = CreateUpdatedEmbed(
+						"The deployment completed successfully and will be available at the next server reboot.",
+						Color.Blue);
+				else
+					embed = CreateUpdatedEmbed(
+						"The deployment failed.",
+						Color.Red);
+
 				var showDMOutput = outputDisplayType switch
 				{
 					DiscordDMOutputDisplayType.Always => true,
@@ -417,13 +424,14 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
 						errorMessage,
 						false));

-				var updatedMessage = $"DM: Deployment {completionString}!";
+				var updatedMessageText = $"DM: Deployment {completionString}!";

+				IMessage updatedMessage = null;
 				async Task CreateUpdatedMessage()
 				{
 					var createUpdatedMessageResponse = await channelsClient.CreateMessageAsync(
 						new Snowflake(channelId),
-						updatedMessage,
+						updatedMessageText,
 						embeds: new List<IEmbed> { embed },
 						ct: cancellationToken);

@@ -431,6 +439,8 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
 						Logger.LogWarning(
 							"Creating updated deploy embed failed: {result}",
 							createUpdatedMessageResponse.LogFormat());
+					else
+						updatedMessage = createUpdatedMessageResponse.Entity;
 				}

 				if (!messageResponse.IsSuccess)
@@ -440,7 +450,7 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
 					var editResponse = await channelsClient.EditMessageAsync(
 						new Snowflake(channelId),
 						messageResponse.Entity.ID,
-						updatedMessage,
+						updatedMessageText,
 						embeds: new List<IEmbed> { embed },
 						ct: cancellationToken);

@@ -452,7 +462,37 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
 							editResponse.LogFormat());
 						await CreateUpdatedMessage();
 					}
+					else
+						updatedMessage = editResponse.Entity;
 				}
+
+				return async (active) =>
+				{
+					if (updatedMessage == null || errorMessage != null)
+						return;
+
+					if (active)
+						embed = CreateUpdatedEmbed(
+							"The deployment completed successfully and was applied to server.",
+							Color.Green);
+					else
+						embed = CreateUpdatedEmbed(
+							"This deployment has been superceeded by a new one.",
+							Color.Gray);
+
+					var editResponse = await channelsClient.EditMessageAsync(
+						new Snowflake(channelId),
+						updatedMessage.ID,
+						updatedMessageText,
+						embeds: new List<IEmbed> { embed },
+						ct: cancellationToken);
+
+					if (!editResponse.IsSuccess)
+						Logger.LogWarning(
+							"Finalizing deploy embed {messageId} failed: {result}",
+							messageResponse.Entity.ID,
+							editResponse.LogFormat());
+				};
 			};
 		}

@@ -90,8 +90,8 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
 		/// <param name="channelId">The <see cref="ChannelRepresentation.RealId"/> to send to.</param>
 		/// <param name="localCommitPushed"><see langword="true"/> if the local deployment commit was pushed to the remote repository.</param>
 		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
-		/// <returns>A <see cref="Task{TResult}"/> resulting in a <see cref="Func{T1, T2, TResult}"/> to call to update the message at the deployment's conclusion. Parameters: Error message if any, DreamMaker output if any.</returns>
-		Task<Func<string, string, Task>> SendUpdateMessage(
+		/// <returns>A <see cref="Task{TResult}"/> resulting in a <see cref="Func{T1, T2, TResult}"/> to call to update the message at the deployment's conclusion. Parameters: Error message if any, DreamMaker output if any. Returns another callback which should be called to mark the deployment as active.</returns>
+		Task<Func<string, string, Task<Func<bool, Task>>>> SendUpdateMessage(
 			RevisionInformation revisionInformation,
 			Version byondVersion,
 			DateTimeOffset? estimatedCompletionTime,
@@ -218,7 +218,7 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
 		}

 		/// <inheritdoc />
-		public override async Task<Func<string, string, Task>> SendUpdateMessage(
+		public override async Task<Func<string, string, Task<Func<bool, Task>>>> SendUpdateMessage(
 			Models.RevisionInformation revisionInformation,
 			Version byondVersion,
 			DateTimeOffset? estimatedCompletionTime,
@@ -281,14 +281,19 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
 				channelId,
 				cancellationToken);

-			return (errorMessage, dreamMakerOutput) => SendMessage(
-				null,
-				new MessageContent
-				{
-					Text = $"DM: Deployment {(errorMessage == null ? "complete" : "failed")}!",
-				},
-				channelId,
-				cancellationToken);
+			return async (errorMessage, dreamMakerOutput) =>
+			{
+				await SendMessage(
+					null,
+					new MessageContent
+					{
+						Text = $"DM: Deployment {(errorMessage == null ? "complete" : "failed")}!",
+					},
+					channelId,
+					cancellationToken);
+
+				return active => Task.CompletedTask;
+			};
 		}

 		/// <inheritdoc />
@@ -181,7 +181,7 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
 		public abstract Task SendMessage(Message replyTo, MessageContent message, ulong channelId, CancellationToken cancellationToken);

 		/// <inheritdoc />
-		public abstract Task<Func<string, string, Task>> SendUpdateMessage(
+		public abstract Task<Func<string, string, Task<Func<bool, Task>>>> SendUpdateMessage(
 			RevisionInformation revisionInformation,
 			Version byondVersion,
 			DateTimeOffset? estimatedCompletionTime,
@@ -74,16 +74,16 @@ namespace Tgstation.Server.Host.Components.Deployment
 		/// </summary>
 		readonly IDictionary<long, int> jobLockCounts;

+		/// <summary>
+		/// <see cref="TaskCompletionSource"/> resulting in the latest <see cref="DmbProvider"/> yet to exist.
+		/// </summary>
+		volatile TaskCompletionSource newerDmbTcs;
+
 		/// <summary>
 		/// <see cref="Task"/> representing calls to <see cref="CleanRegisteredCompileJob(CompileJob)"/>.
 		/// </summary>
 		Task cleanupTask;

-		/// <summary>
-		/// <see cref="TaskCompletionSource"/> resulting in the latest <see cref="DmbProvider"/> yet to exist.
-		/// </summary>
-		TaskCompletionSource newerDmbTcs;
-
 		/// <summary>
 		/// The latest <see cref="DmbProvider"/>.
 		/// </summary>
@@ -128,7 +128,7 @@ namespace Tgstation.Server.Host.Components.Deployment
 		public void Dispose() => cleanupCts.Dispose(); // we don't dispose nextDmbProvider here, since it might be the only thing we have

 		/// <inheritdoc />
-		public async Task LoadCompileJob(CompileJob job, CancellationToken cancellationToken)
+		public async Task LoadCompileJob(CompileJob job, Action<bool> activationAction, CancellationToken cancellationToken)
 		{
 			ArgumentNullException.ThrowIfNull(job);

@@ -143,8 +143,9 @@ namespace Tgstation.Server.Host.Components.Deployment
 					metadata,
 					job);
 				await remoteDeploymentManager.StageDeployment(
-						newProvider.CompileJob,
-						cancellationToken);
+					newProvider.CompileJob,
+					activationAction,
+					cancellationToken);
 			}

 			lock (jobLockCounts)
@@ -153,8 +154,7 @@ namespace Tgstation.Server.Host.Components.Deployment
 				nextDmbProvider = newProvider;

 				// Oh god dammit
-				var temp = newerDmbTcs;
-				newerDmbTcs = new TaskCompletionSource();
+				var temp = Interlocked.Exchange(ref newerDmbTcs, new TaskCompletionSource());
 				temp.SetResult();
 			}
 		}
@@ -170,7 +170,7 @@ namespace Tgstation.Server.Host.Components.Deployment
 			{
 				var jobId = nextDmbProvider.CompileJob.Id;
 				var incremented = jobLockCounts[jobId.Value] += lockCount;
-				logger.LogTrace("Compile job {0} lock count now: {1}", jobId, incremented);
+				logger.LogTrace("Compile job {jobId} lock count now: {lockCount}", jobId, incremented);
 				return nextDmbProvider;
 			}
 		}
@@ -189,10 +189,16 @@ namespace Tgstation.Server.Host.Components.Deployment
 					.FirstOrDefaultAsync(cancellationToken);
 			});

-			if (cj == default(CompileJob))
-				return;
-			await LoadCompileJob(cj, cancellationToken);
-			started = true;
+			try
+			{
+				if (cj == default(CompileJob))
+					return;
+				await LoadCompileJob(cj, null, cancellationToken);
+			}
+			finally
+			{
+				started = true;
+			}

 			// we dont do CleanUnusedCompileJobs here because the watchdog may have plans for them yet
 		}
@@ -202,6 +208,9 @@ namespace Tgstation.Server.Host.Components.Deployment
 		{
 			try
 			{
+				lock (jobLockCounts)
+					remoteDeploymentManagerFactory.ForgetLocalStateForCompileJobs(jobLockCounts.Keys);
+
 				using (cancellationToken.Register(() => cleanupCts.Cancel()))
 					await cleanupTask;
 			}
@@ -218,7 +227,7 @@ namespace Tgstation.Server.Host.Components.Deployment
 			ArgumentNullException.ThrowIfNull(compileJob);

 			// ensure we have the entire metadata tree
-			logger.LogTrace("Loading compile job {0}...", compileJob.Id);
+			logger.LogTrace("Loading compile job {id}...", compileJob.Id);
 			await databaseContextFactory.UseContext(
 				async db => compileJob = await db
 					.CompileJobs
@@ -240,7 +249,7 @@ namespace Tgstation.Server.Host.Components.Deployment
 				// This happens when we're told to load the compile job that is currently finished up
 				// It constitutes an API violation if it's returned by the DreamDaemonController so just set it here
 				// Bit of a hack, but it works out to be nearly if not the same value that's put in the DB
-				logger.LogTrace("Setting missing StoppedAt for CompileJob.Job #{0}...", compileJob.Job.Id);
+				logger.LogTrace("Setting missing StoppedAt for CompileJob.Job #{id}...", compileJob.Job.Id);
 				compileJob.Job.StoppedAt = DateTimeOffset.UtcNow;
 			}

@@ -288,7 +297,7 @@ namespace Tgstation.Server.Host.Components.Deployment

 					// rebuild the provider because it's using the legacy style directories
 					// Don't dispose it
-					logger.LogDebug("Creating legacy two folder .dmb provider targeting {0} directory...", LegacyADirectoryName);
+					logger.LogDebug("Creating legacy two folder .dmb provider targeting {aDirName} directory...", LegacyADirectoryName);
 					newProvider = new DmbProvider(compileJob, ioManager, CleanupAction, Path.DirectorySeparatorChar + LegacyADirectoryName);
 				}

@@ -304,7 +313,7 @@ namespace Tgstation.Server.Host.Components.Deployment

 					providerSubmitted = true;

-					logger.LogTrace("Compile job {0} lock count now: {1}", compileJob.Id, value);
+					logger.LogTrace("Compile job {id} lock count now: {lockCount}", compileJob.Id, value);
 					return newProvider;
 				}
 			}
@@ -324,7 +333,7 @@ namespace Tgstation.Server.Host.Components.Deployment

 			// don't clean locked directories
 			lock (jobLockCounts)
-				jobIdsToSkip = jobLockCounts.Select(x => x.Key).ToList();
+				jobIdsToSkip = jobLockCounts.Keys.ToList();

 			List<string> jobUidsToNotErase = null;

@@ -350,7 +359,7 @@ namespace Tgstation.Server.Host.Components.Deployment

 			jobUidsToNotErase.Add(SwappableDmbProvider.LiveGameDirectory);

-			logger.LogTrace("We will not clean the following directories: {0}", String.Join(", ", jobUidsToNotErase));
+			logger.LogTrace("We will not clean the following directories: {directoriesToNotClean}", String.Join(", ", jobUidsToNotErase));

 			// cleanup
 			var gameDirectory = ioManager.ResolvePath();
@@ -362,7 +371,7 @@ namespace Tgstation.Server.Host.Components.Deployment
 				var nameOnly = ioManager.GetFileName(x);
 				if (jobUidsToNotErase.Contains(nameOnly))
 					return;
-				logger.LogDebug("Cleaning unused game folder: {0}...", nameOnly);
+				logger.LogDebug("Cleaning unused game folder: {dirName}...", nameOnly);
 				try
 				{
 					++deleting;
@@ -374,7 +383,7 @@ namespace Tgstation.Server.Host.Components.Deployment
 				}
 				catch (Exception e)
 				{
-					logger.LogWarning(e, "Error deleting directory {0}!", x);
+					logger.LogWarning(e, "Error deleting directory {dirName}!", x);
 				}
 			}).ToList();
 			if (deleting > 0)
@@ -427,13 +436,13 @@ namespace Tgstation.Server.Host.Components.Deployment
 					if (currentVal == 1)
 					{
 						jobLockCounts.Remove(job.Id.Value);
-						logger.LogDebug("Cleaning lock-free compile job {0} => {1}", job.Id, job.DirectoryName);
+						logger.LogDebug("Cleaning lock-free compile job {id} => {dirName}", job.Id, job.DirectoryName);
 						cleanupTask = HandleCleanup();
 					}
 					else
 					{
 						var decremented = --jobLockCounts[job.Id.Value];
-						logger.LogTrace("Compile job {0} lock count now: {1}", job.Id, decremented);
+						logger.LogTrace("Compile job {id} lock count now: {lockCount}", job.Id, decremented);
 					}
 				else
 					logger.LogError("Extra Dispose of DmbProvider for CompileJob {compileJobId}!", job.Id);
@@ -117,7 +117,7 @@ namespace Tgstation.Server.Host.Components.Deployment
 		/// <summary>
 		/// The active callback from <see cref="IChatManager.QueueDeploymentMessage"/>.
 		/// </summary>
-		Action<string, string> currentChatCallback;
+		Func<string, string, Action<bool>> currentChatCallback;

 		/// <summary>
 		/// Cached for <see cref="currentChatCallback"/>.
@@ -355,7 +355,8 @@ namespace Tgstation.Server.Host.Components.Deployment
 							logger.LogTrace("Created CompileJob {compileJobId}", compileJob.Id);
 							try
 							{
-								await compileJobConsumer.LoadCompileJob(compileJob, cancellationToken);
+								var chatNotificationAction = currentChatCallback(null, compileJob.Output);
+								await compileJobConsumer.LoadCompileJob(compileJob, chatNotificationAction, cancellationToken);
 							}
 							catch
 							{
@@ -389,8 +390,6 @@ namespace Tgstation.Server.Host.Components.Deployment

 				try
 				{
-					currentChatCallback(null, compileJob.Output);
-
 					await Task.WhenAll(commentsTask, eventTask);
 				}
 				catch (Exception ex)
@@ -1,4 +1,5 @@
-using System.Threading;
+using System;
+using System.Threading;
 using System.Threading.Tasks;

 using Tgstation.Server.Host.Models;
@@ -14,8 +15,9 @@ namespace Tgstation.Server.Host.Components.Deployment
 		/// Load a new <paramref name="job"/> into the <see cref="ICompileJobSink"/>.
 		/// </summary>
 		/// <param name="job">The <see cref="CompileJob"/> to load.</param>
+		/// <param name="activationAction">An <see cref="Action{T1}"/> to be called when the <see cref="CompileJob"/> becomes active or is discarded with <see langword="true"/> or <see langword="false"/> respectively.</param>
 		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
 		/// <returns>A <see cref="Task"/> representing the running operation.</returns>
-		Task LoadCompileJob(CompileJob job, CancellationToken cancellationToken);
+		Task LoadCompileJob(CompileJob job, Action<bool> activationAction, CancellationToken cancellationToken);
 	}
 }
@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
@@ -26,15 +27,25 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote
 		/// </summary>
 		protected ILogger<BaseRemoteDeploymentManager> Logger { get; }

+		/// <summary>
+		/// A map of <see cref="CompileJob"/> <see cref="Api.Models.EntityId.Id"/>s to activation callback <see cref="Action{T1}"/>s.
+		/// </summary>
+		readonly ConcurrentDictionary<long, Action<bool>> activationCallbacks;
+
 		/// <summary>
 		/// Initializes a new instance of the <see cref="BaseRemoteDeploymentManager"/> class.
 		/// </summary>
 		/// <param name="logger">The value of <see cref="Logger"/>.</param>
 		/// <param name="metadata">The value of <see cref="Metadata"/>.</param>
-		protected BaseRemoteDeploymentManager(ILogger<BaseRemoteDeploymentManager> logger, Api.Models.Instance metadata)
+		/// <param name="activationCallbacks">The value of <see cref="activationCallbacks"/>.</param>
+		protected BaseRemoteDeploymentManager(
+			ILogger<BaseRemoteDeploymentManager> logger,
+			Api.Models.Instance metadata,
+			ConcurrentDictionary<long, Action<bool>> activationCallbacks)
 		{
 			Logger = logger ?? throw new ArgumentNullException(nameof(logger));
 			Metadata = metadata ?? throw new ArgumentNullException(nameof(metadata));
+			this.activationCallbacks = activationCallbacks ?? throw new ArgumentNullException(nameof(activationCallbacks));
 		}

 		/// <inheritdoc />
@@ -58,7 +69,6 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote
 			previousRevisionInformation.ActiveTestMerges ??= new List<RevInfoTestMerge>();

 			deployedRevisionInformation.ActiveTestMerges ??= new List<RevInfoTestMerge>();
-			var tasks = new List<Task>();

 			// added prs
 			var addedTestMerges = deployedRevisionInformation
@@ -87,10 +97,12 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote
 				return;

 			Logger.LogTrace(
-				"Commenting on {0} added, {1} removed, and {2} updated test merge sources...",
+				"Commenting on {addedCount} added, {removedCount} removed, and {updatedCount} updated test merge sources...",
 				addedTestMerges.Count,
 				removedTestMerges.Count,
 				updatedTestMerges.Count);
+
+			var tasks = new List<Task>(addedTestMerges.Count + updatedTestMerges.Count + removedTestMerges.Count);
 			foreach (var addedTestMerge in addedTestMerges)
 				tasks.Add(
 					CommentOnTestMergeSource(
@@ -138,13 +150,29 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote
 		}

 		/// <inheritdoc />
-		public abstract Task ApplyDeployment(CompileJob compileJob, CompileJob oldCompileJob, CancellationToken cancellationToken);
+		public Task ApplyDeployment(CompileJob compileJob, CancellationToken cancellationToken)
+		{
+			ArgumentNullException.ThrowIfNull(compileJob);
+
+			if (activationCallbacks.TryGetValue(compileJob.Id.Value, out var activationCallback))
+				activationCallback(true);
+
+			return ApplyDeploymentImpl(compileJob, cancellationToken);
+		}

 		/// <inheritdoc />
 		public abstract Task FailDeployment(CompileJob compileJob, string errorMessage, CancellationToken cancellationToken);

 		/// <inheritdoc />
-		public abstract Task MarkInactive(CompileJob compileJob, CancellationToken cancellationToken);
+		public Task MarkInactive(CompileJob compileJob, CancellationToken cancellationToken)
+		{
+			ArgumentNullException.ThrowIfNull(compileJob);
+
+			if (activationCallbacks.TryRemove(compileJob.Id.Value, out var activationCallback))
+				activationCallback(false);
+
+			return MarkInactiveImpl(compileJob, cancellationToken);
+		}

 		/// <inheritdoc />
 		public abstract Task<IReadOnlyCollection<TestMerge>> RemoveMergedTestMerges(
@@ -154,7 +182,15 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote
 			CancellationToken cancellationToken);

 		/// <inheritdoc />
-		public abstract Task StageDeployment(CompileJob compileJob, CancellationToken cancellationToken);
+		public Task StageDeployment(CompileJob compileJob, Action<bool> activationCallback, CancellationToken cancellationToken)
+		{
+			ArgumentNullException.ThrowIfNull(compileJob);
+
+			if (activationCallback != null && !activationCallbacks.TryAdd(compileJob.Id.Value, activationCallback))
+				Logger.LogError("activationCallbacks conflicted on CompileJob #{id}!", compileJob.Id.Value);
+
+			return StageDeploymentImpl(compileJob, cancellationToken);
+		}

 		/// <inheritdoc />
 		public abstract Task StartDeployment(
@@ -162,6 +198,30 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote
 			CompileJob compileJob,
 			CancellationToken cancellationToken);

+		/// <summary>
+		/// Implementation of <see cref="StageDeployment(CompileJob, Action{bool}, CancellationToken)"/>.
+		/// </summary>
+		/// <param name="compileJob">The staged <see cref="CompileJob"/>.</param>
+		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
+		/// <returns>A <see cref="Task"/> representing the running operation.</returns>
+		protected abstract Task StageDeploymentImpl(CompileJob compileJob, CancellationToken cancellationToken);
+
+		/// <summary>
+		/// Implementation of <see cref="ApplyDeployment(CompileJob, CancellationToken)"/>.
+		/// </summary>
+		/// <param name="compileJob">The <see cref="CompileJob"/> being applied.</param>
+		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
+		/// <returns>A <see cref="Task"/> representing the running operation.</returns>
+		protected abstract Task ApplyDeploymentImpl(CompileJob compileJob, CancellationToken cancellationToken);
+
+		/// <summary>
+		/// Implementation of <see cref="MarkInactive(CompileJob, CancellationToken)"/>.
+		/// </summary>
+		/// <param name="compileJob">The inactive <see cref="CompileJob"/>.</param>
+		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
+		/// <returns>A <see cref="Task"/> representing the running operation.</returns>
+		protected abstract Task MarkInactiveImpl(CompileJob compileJob, CancellationToken cancellationToken);
+
 		/// <summary>
 		/// Formats a comment for a given <paramref name="testMerge"/>.
 		/// </summary>
@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Collections.ObjectModel;
 using System.Globalization;
@@ -40,12 +41,14 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote
 		/// <param name="gitHubServiceFactory">The value of <see cref="gitHubServiceFactory"/>.</param>
 		/// <param name="logger">The <see cref="ILogger"/> for the <see cref="BaseRemoteDeploymentManager"/>.</param>
 		/// <param name="metadata">The <see cref="Api.Models.Instance"/> for the <see cref="BaseRemoteDeploymentManager"/>.</param>
+		/// <param name="activationCallbacks">The activation callback <see cref="ConcurrentDictionary{TKey, TValue}"/> for the <see cref="BaseRemoteDeploymentManager"/>.</param>
 		public GitHubRemoteDeploymentManager(
 			IDatabaseContextFactory databaseContextFactory,
 			IGitHubServiceFactory gitHubServiceFactory,
 			ILogger<GitHubRemoteDeploymentManager> logger,
-			Api.Models.Instance metadata)
-			: base(logger, metadata)
+			Api.Models.Instance metadata,
+			ConcurrentDictionary<long, Action<bool>> activationCallbacks)
+			: base(logger, metadata, activationCallbacks)
 		{
 			this.databaseContextFactory = databaseContextFactory ?? throw new ArgumentNullException(nameof(databaseContextFactory));
 			this.gitHubServiceFactory = gitHubServiceFactory ?? throw new ArgumentNullException(nameof(gitHubServiceFactory));
@@ -144,24 +147,6 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote
 			}
 		}

-		/// <inheritdoc />
-		public override Task StageDeployment(
-			CompileJob compileJob,
-			CancellationToken cancellationToken)
-			=> UpdateDeployment(
-				compileJob,
-				"The deployment succeeded and will be applied a the next server reboot.",
-				DeploymentState.Pending,
-				cancellationToken);
-
-		/// <inheritdoc />
-		public override Task ApplyDeployment(CompileJob compileJob, CompileJob oldCompileJob, CancellationToken cancellationToken)
-			=> UpdateDeployment(
-				compileJob,
-				"The deployment is now live on the server.",
-				DeploymentState.Success,
-				cancellationToken);
-
 		/// <inheritdoc />
 		public override Task FailDeployment(CompileJob compileJob, string errorMessage, CancellationToken cancellationToken)
 			=> UpdateDeployment(
@@ -170,14 +155,6 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote
 				DeploymentState.Error,
 				cancellationToken);

-		/// <inheritdoc />
-		public override Task MarkInactive(CompileJob compileJob, CancellationToken cancellationToken)
-			=> UpdateDeployment(
-				compileJob,
-				"The deployment has been superceeded.",
-				DeploymentState.Inactive,
-				cancellationToken);
-
 		/// <inheritdoc />
 		public override async Task<IReadOnlyCollection<TestMerge>> RemoveMergedTestMerges(
 			IRepository repository,
@@ -237,6 +214,32 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote
 			return newList;
 		}

+		/// <inheritdoc />
+		protected override Task StageDeploymentImpl(
+			CompileJob compileJob,
+			CancellationToken cancellationToken)
+			=> UpdateDeployment(
+				compileJob,
+				"The deployment succeeded and will be applied a the next server reboot.",
+				DeploymentState.Pending,
+				cancellationToken);
+
+		/// <inheritdoc />
+		protected override Task ApplyDeploymentImpl(CompileJob compileJob, CancellationToken cancellationToken)
+			=> UpdateDeployment(
+				compileJob,
+				"The deployment is now live on the server.",
+				DeploymentState.Success,
+				cancellationToken);
+
+		/// <inheritdoc />
+		protected override Task MarkInactiveImpl(CompileJob compileJob, CancellationToken cancellationToken)
+			=> UpdateDeployment(
+				compileJob,
+				"The deployment has been superceeded.",
+				DeploymentState.Inactive,
+				cancellationToken);
+
 		/// <inheritdoc />
 		protected override async Task CommentOnTestMergeSource(
 			RepositorySettings repositorySettings,
@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Globalization;
 using System.Linq;
@@ -25,8 +26,12 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote
 		/// </summary>
 		/// <param name="logger">The <see cref="ILogger"/> for the <see cref="BaseRemoteDeploymentManager"/>.</param>
 		/// <param name="metadata">The <see cref="Api.Models.Instance"/> for the <see cref="BaseRemoteDeploymentManager"/>.</param>
-		public GitLabRemoteDeploymentManager(ILogger<GitLabRemoteDeploymentManager> logger, Api.Models.Instance metadata)
-			: base(logger, metadata)
+		/// <param name="activationCallbacks">The activation callback <see cref="ConcurrentDictionary{TKey, TValue}"/> for the <see cref="BaseRemoteDeploymentManager"/>.</param>
+		public GitLabRemoteDeploymentManager(
+			ILogger<GitLabRemoteDeploymentManager> logger,
+			Api.Models.Instance metadata,
+			ConcurrentDictionary<long, Action<bool>> activationCallbacks)
+			: base(logger, metadata, activationCallbacks)
 		{
 		}

@@ -94,30 +99,29 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote
 			return newList;
 		}

-		/// <inheritdoc />
-		public override Task ApplyDeployment(
-			CompileJob compileJob,
-			CompileJob oldCompileJob,
-			CancellationToken cancellationToken) => Task.CompletedTask;
-
 		/// <inheritdoc />
 		public override Task FailDeployment(
 			CompileJob compileJob,
 			string errorMessage,
 			CancellationToken cancellationToken) => Task.CompletedTask;

-		/// <inheritdoc />
-		public override Task MarkInactive(CompileJob compileJob, CancellationToken cancellationToken) => Task.CompletedTask;
-
-		/// <inheritdoc />
-		public override Task StageDeployment(CompileJob compileJob, CancellationToken cancellationToken) => Task.CompletedTask;
-
 		/// <inheritdoc />
 		public override Task StartDeployment(
 			Api.Models.Internal.IGitRemoteInformation remoteInformation,
 			CompileJob compileJob,
 			CancellationToken cancellationToken) => Task.CompletedTask;

+		/// <inheritdoc />
+		protected override Task ApplyDeploymentImpl(
+			CompileJob compileJob,
+			CancellationToken cancellationToken) => Task.CompletedTask;
+
+		/// <inheritdoc />
+		protected override Task StageDeploymentImpl(CompileJob compileJob, CancellationToken cancellationToken) => Task.CompletedTask;
+
+		/// <inheritdoc />
+		protected override Task MarkInactiveImpl(CompileJob compileJob, CancellationToken cancellationToken) => Task.CompletedTask;
+
 		/// <inheritdoc />
 		protected override async Task CommentOnTestMergeSource(
 			RepositorySettings repositorySettings,
@@ -1,4 +1,5 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 using System.Threading;
 using System.Threading.Tasks;

@@ -28,20 +29,21 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote
 		/// Stage a given <paramref name="compileJob"/>'s deployment.
 		/// </summary>
 		/// <param name="compileJob">The staged <see cref="CompileJob"/>.</param>
+		/// <param name="activationCallback">An optional <see cref="Action{T1}"/> to be called when the <see cref="CompileJob"/> becomes active or is discarded with <see langword="true"/> or <see langword="false"/> respectively.</param>
 		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
 		/// <returns>A <see cref="Task"/> representing the running operation.</returns>
 		Task StageDeployment(
 			CompileJob compileJob,
+			Action<bool> activationCallback,
 			CancellationToken cancellationToken);

 		/// <summary>
 		/// Stage a given <paramref name="compileJob"/>'s deployment.
 		/// </summary>
 		/// <param name="compileJob">The <see cref="CompileJob"/> being applied.</param>
-		/// <param name="oldCompileJob">The currently active <see cref="CompileJob"/>, if any.</param>
 		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
 		/// <returns>A <see cref="Task"/> representing the running operation.</returns>
-		Task ApplyDeployment(CompileJob compileJob, CompileJob oldCompileJob, CancellationToken cancellationToken);
+		Task ApplyDeployment(CompileJob compileJob, CancellationToken cancellationToken);

 		/// <summary>
 		/// Fail a deployment for a given <paramref name="compileJob"/>.
@@ -1,4 +1,6 @@
-using Tgstation.Server.Api.Models;
+using System.Collections.Generic;
+
+using Tgstation.Server.Api.Models;

 namespace Tgstation.Server.Host.Components.Deployment.Remote
 {
@@ -22,5 +24,11 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote
 		/// <param name="compileJob">The <see cref="Models.CompileJob"/> containing the <see cref="Api.Models.Response.CompileJobResponse.RepositoryOrigin"/> to use.</param>
 		/// <returns>A new <see cref="IRemoteDeploymentManager"/>.</returns>
 		IRemoteDeploymentManager CreateRemoteDeploymentManager(Api.Models.Instance metadata, Models.CompileJob compileJob);
+
+		/// <summary>
+		/// Cause the <see cref="IRemoteDeploymentManagerFactory"/> to drop any local state is has for the given <paramref name="compileJobsIds"/>.
+		/// </summary>
+		/// <param name="compileJobsIds">An <see cref="IEnumerable{T}"/> of <see cref="Models.CompileJob"/> <see cref="EntityId.Id"/>s.</param>
+		void ForgetLocalStateForCompileJobs(IEnumerable<long> compileJobsIds);
 	}
 }
@@ -1,8 +1,12 @@
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Threading;
 using System.Threading.Tasks;

+using Microsoft.Extensions.Logging;
+
+using Tgstation.Server.Api.Models.Internal;
 using Tgstation.Server.Host.Components.Repository;
 using Tgstation.Server.Host.Models;

@@ -11,48 +15,66 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote
 	/// <summary>
 	/// No-op implementation of <see cref="IRemoteDeploymentManager"/>.
 	/// </summary>
-	sealed class NoOpRemoteDeploymentManager : IRemoteDeploymentManager
+	sealed class NoOpRemoteDeploymentManager : BaseRemoteDeploymentManager
 	{
-		/// <inheritdoc />
-		public Task ApplyDeployment(
-			CompileJob compileJob,
-			CompileJob oldCompileJob,
-			CancellationToken cancellationToken) => Task.CompletedTask;
+		/// <summary>
+		/// Initializes a new instance of the <see cref="NoOpRemoteDeploymentManager"/> class.
+		/// </summary>
+		/// <param name="logger">The value of <see cref="BaseRemoteDeploymentManager.Logger"/>.</param>
+		/// <param name="metadata">The value of <see cref="BaseRemoteDeploymentManager.Metadata"/>.</param>
+		/// <param name="activationCallbacks">The activation callback <see cref="ConcurrentDictionary{TKey, TValue}"/> for the <see cref="BaseRemoteDeploymentManager"/>.</param>
+		public NoOpRemoteDeploymentManager(
+			ILogger<NoOpRemoteDeploymentManager> logger,
+			Api.Models.Instance metadata,
+			ConcurrentDictionary<long, Action<bool>> activationCallbacks)
+			: base(logger, metadata, activationCallbacks)
+		{
+		}

 		/// <inheritdoc />
-		public Task FailDeployment(
-			CompileJob compileJob,
-			string errorMessage,
-			CancellationToken cancellationToken) => Task.CompletedTask;
+		public override Task FailDeployment(Models.CompileJob compileJob, string errorMessage, CancellationToken cancellationToken)
+		{
+			throw new NotImplementedException();
+		}

 		/// <inheritdoc />
-		public Task MarkInactive(
-			CompileJob compileJob,
-			CancellationToken cancellationToken) => Task.CompletedTask;
+		public override Task<IReadOnlyCollection<TestMerge>> RemoveMergedTestMerges(IRepository repository, Models.RepositorySettings repositorySettings, Models.RevisionInformation revisionInformation, CancellationToken cancellationToken)
+			=> Task.FromResult<IReadOnlyCollection<TestMerge>>(Array.Empty<TestMerge>());

 		/// <inheritdoc />
-		public Task PostDeploymentComments(
-			CompileJob compileJob,
-			RevisionInformation previousRevisionInformation,
-			RepositorySettings repositorySettings,
-			string repoOwner,
-			string repoName,
-			CancellationToken cancellationToken) => Task.CompletedTask;
+		public override Task StartDeployment(IGitRemoteInformation remoteInformation, Models.CompileJob compileJob, CancellationToken cancellationToken)
+			=> Task.CompletedTask;

 		/// <inheritdoc />
-		public Task<IReadOnlyCollection<TestMerge>> RemoveMergedTestMerges(
-			IRepository repository,
-			RepositorySettings repositorySettings,
-			RevisionInformation revisionInformation,
-			CancellationToken cancellationToken) => Task.FromResult<IReadOnlyCollection<TestMerge>>(Array.Empty<TestMerge>());
+		protected override Task ApplyDeploymentImpl(Models.CompileJob compileJob, CancellationToken cancellationToken) => Task.CompletedTask;

 		/// <inheritdoc />
-		public Task StageDeployment(CompileJob compileJob, CancellationToken cancellationToken) => Task.CompletedTask;
+		protected override Task CommentOnTestMergeSource(
+			Models.RepositorySettings repositorySettings,
+			string remoteRepositoryOwner,
+			string remoteRepositoryName,
+			string comment,
+			int testMergeNumber,
+			CancellationToken cancellationToken)
+			=> Task.CompletedTask;

 		/// <inheritdoc />
-		public Task StartDeployment(
-			Api.Models.Internal.IGitRemoteInformation remoteInformation,
-			CompileJob compileJob,
-			CancellationToken cancellationToken) => Task.CompletedTask;
+		protected override string FormatTestMerge(
+			Models.RepositorySettings repositorySettings,
+			Models.CompileJob compileJob,
+			TestMerge testMerge,
+			string remoteRepositoryOwner,
+			string remoteRepositoryName,
+			bool updated)
+			=> String.Empty;
+
+		/// <inheritdoc />
+		protected override Task MarkInactiveImpl(Models.CompileJob compileJob, CancellationToken cancellationToken)
+		{
+			throw new NotImplementedException();
+		}
+
+		/// <inheritdoc />
+		protected override Task StageDeploymentImpl(Models.CompileJob compileJob, CancellationToken cancellationToken) => Task.CompletedTask;
 	}
 }
@@ -1,4 +1,6 @@
 using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;

 using Microsoft.Extensions.Logging;

@@ -37,6 +39,11 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote
 		/// </summary>
 		readonly ILogger<RemoteDeploymentManagerFactory> logger;

+		/// <summary>
+		/// A map of <see cref="Models.CompileJob"/> <see cref="EntityId.Id"/>s to activation callback <see cref="Action{T1}"/>s.
+		/// </summary>
+		readonly ConcurrentDictionary<long, Action<bool>> activationCallbacks;
+
 		/// <summary>
 		/// Initializes a new instance of the <see cref="RemoteDeploymentManagerFactory"/> class.
 		/// </summary>
@@ -57,6 +64,8 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote
 			this.gitRemoteFeaturesFactory = gitRemoteFeaturesFactory ?? throw new ArgumentNullException(nameof(gitRemoteFeaturesFactory));
 			this.loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory));
 			this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
+
+			activationCallbacks = new ConcurrentDictionary<long, Action<bool>>();
 		}

 		/// <inheritdoc />
@@ -71,11 +80,16 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote
 					databaseContextFactory,
 					gitHubServiceFactory,
 					loggerFactory.CreateLogger<GitHubRemoteDeploymentManager>(),
-					metadata),
+					metadata,
+					activationCallbacks),
 				RemoteGitProvider.GitLab => new GitLabRemoteDeploymentManager(
 					loggerFactory.CreateLogger<GitLabRemoteDeploymentManager>(),
-					metadata),
-				RemoteGitProvider.Unknown => new NoOpRemoteDeploymentManager(),
+					metadata,
+					activationCallbacks),
+				RemoteGitProvider.Unknown => new NoOpRemoteDeploymentManager(
+					loggerFactory.CreateLogger<NoOpRemoteDeploymentManager>(),
+					metadata,
+					activationCallbacks),
 				_ => throw new InvalidOperationException($"Invalid RemoteGitProvider: {remoteGitProvider}!"),
 			};
 		}
@@ -97,5 +111,14 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote

 			return CreateRemoteDeploymentManager(metadata, remoteGitProvider);
 		}
+
+		/// <inheritdoc />
+		public void ForgetLocalStateForCompileJobs(IEnumerable<long> compileJobIds)
+		{
+			ArgumentNullException.ThrowIfNull(compileJobIds);
+
+			foreach (var compileJobId in compileJobIds)
+				activationCallbacks.Remove(compileJobId, out _);
+		}
 	}
 }
@@ -680,7 +680,7 @@ namespace Tgstation.Server.Host.Components.Watchdog

 			try
 			{
-				await remoteDeploymentManager.ApplyDeployment(newCompileJob, ActiveCompileJob, cancellationToken);
+				await remoteDeploymentManager.ApplyDeployment(newCompileJob, cancellationToken);
 			}
 			catch (Exception ex)
 			{
@@ -100,7 +100,7 @@ namespace Tgstation.Server.Tests.Live
 			return Task.CompletedTask;
 		}

-		public override Task<Func<string, string, Task>> SendUpdateMessage(RevisionInformation revisionInformation, Version byondVersion, DateTimeOffset? estimatedCompletionTime, string gitHubOwner, string gitHubRepo, ulong channelId, bool localCommitPushed, CancellationToken cancellationToken)
+		public override Task<Func<string, string, Task<Func<bool, Task>>>> SendUpdateMessage(RevisionInformation revisionInformation, Version byondVersion, DateTimeOffset? estimatedCompletionTime, string gitHubOwner, string gitHubRepo, ulong channelId, bool localCommitPushed, CancellationToken cancellationToken)
 		{
 			ArgumentNullException.ThrowIfNull(revisionInformation);
 			ArgumentNullException.ThrowIfNull(byondVersion);
@@ -115,7 +115,7 @@ namespace Tgstation.Server.Tests.Live
 			if (random.Next(0, 100) > 70)
 				throw new Exception("Random SendUpdateMessage failure!"); */

-			return Task.FromResult<Func<string, string, Task>>((_, _) =>
+			return Task.FromResult<Func<string, string, Task<Func<bool, Task>>>>((_, _) =>
 			{
 				cancellationToken.ThrowIfCancellationRequested();

@@ -123,7 +123,7 @@ namespace Tgstation.Server.Tests.Live
 				if (random.Next(0, 100) > 70)
 					throw new Exception("Random SendUpdateMessage failure!"); */

-				return Task.CompletedTask;
+				return Task.FromResult<Func<bool, Task>>(_ => Task.CompletedTask);
 			});
 		}