Merge pull request #1493 from tgstation/1490-FixChatReattach [DMDeploy][TGSDeploy]

Fix potentially sending DMAPI messages to the wrong channels after a TGS restart

.github/CONTRIBUTING.md

@@ -38,15 +38,16 @@ You need the Dotnet 6.0 SDK and npm>=v5.7 (in your PATH) to compile the server.
 
 The recommended IDE is Visual Studio 2019 which has installation options for both of these.
 
-In order to run the integration tests you must have the following environment variables set:
+In order to run the integration tests you must have the following environment variables set. To run them more accurately, include the optional ones.
 - `TGS_TEST_DATABASE_TYPE`: `MySql`, `MariaDB`, `PostgresSql`, or `SqlServer`.
 - `TGS_TEST_CONNECTION_STRING`: To a valid database connection string. You can use the setup wizard to create one.
-- `TSG_TEST_DISCORD_TOKEN`: To a valid discord bot token.
-- `TGS_TEST_DISCORD_CHANNEL`: To a valid discord channel ID that the above bot can access.
-- `TGS_TEST_IRC_CONNECTION_STRING`: To a valid IRC connection string. See the code for [IrcConnectionStringBuilder](../src/Tgstation.Server.Api/Models/IrcConnectionStringBuilder.cs) for details.
-- `TGS_TEST_IRC_CHANNEL`: To a valid IRC channel accessible with the above connection.
 - `TGS_TEST_BRANCH`: Should be either `dev` or `master` depending on what you are working off of. Used for repository tests.
 - (Optional) `TGS_TEST_GITHUB_TOKEN`: A GitHub personal access token with no scopes used to bypass rate limits.
+- (Optional) The following variables are all interdependent, so if one is set they all must be.
+	- `TSG_TEST_DISCORD_TOKEN`: To a valid discord bot token.
+	- `TGS_TEST_DISCORD_CHANNEL`: To a valid discord channel ID that the above bot can access.
+	- `TGS_TEST_IRC_CONNECTION_STRING`: To a valid IRC connection string. See the code for [IrcConnectionStringBuilder](../src/Tgstation.Server.Api/Models/IrcConnectionStringBuilder.cs) for details.
+	- `TGS_TEST_IRC_CHANNEL`: To a valid IRC channel accessible with the above connection.
 
 ### Know your Code
 

.github/workflows/ci-suite.yml

@@ -17,10 +17,6 @@ on:
 
 env:
   TGS_DOTNET_VERSION: 6.0.x
-  TGS_TEST_DISCORD_CHANNEL: ${{ secrets.DISCORD_CHANNEL_ID }}
-  TGS_TEST_DISCORD_TOKEN: ${{ secrets.DISCORD_TOKEN }}
-  TGS_TEST_IRC_CHANNEL: ${{ secrets.IRC_CHANNEL }}
-  TGS_TEST_IRC_CONNECTION_STRING: ${{ secrets.IRC_CONNECTION_STRING }}
   TGS_TEST_GITHUB_TOKEN: ${{ secrets.LIVE_TESTS_TOKEN }}
   TGS_RELEASE_NOTES_TOKEN: ${{ secrets.DEV_PUSH_TOKEN }}
 
@@ -231,6 +227,9 @@ jobs:
       fail-fast: false
       matrix:
         configuration: [ 'Debug', 'Release' ]
+    env:
+      TGS_TEST_DISCORD_TOKEN: ${{ secrets.DISCORD_TOKEN }}
+      TGS_TEST_IRC_CONNECTION_STRING: ${{ secrets.IRC_CONNECTION_STRING }}
     runs-on: ubuntu-latest
     steps:
     - name: Setup dotnet
@@ -271,6 +270,9 @@ jobs:
       fail-fast: false
       matrix:
         configuration: [ 'Debug', 'Release' ]
+    env:
+      TGS_TEST_DISCORD_TOKEN: ${{ secrets.DISCORD_TOKEN }}
+      TGS_TEST_IRC_CONNECTION_STRING: ${{ secrets.IRC_CONNECTION_STRING }}
     runs-on: windows-latest
     steps:
     - name: Setup dotnet
@@ -308,11 +310,11 @@ jobs:
     needs: dmapi-build
     if: "!(cancelled() || failure()) && needs.dmapi-build.result == 'success'"
     env:
-      TGS_TEST_DATABASE_TYPE: SqlServer
       TGS_TEST_DUMP_API_SPEC: yes
     strategy:
       fail-fast: false
       matrix:
+        database-type: [ 'SqlServer' ]
         watchdog-type: [ 'Basic', 'System' ]
         configuration: [ 'Debug', 'Release' ]
     runs-on: windows-2019
@@ -329,11 +331,13 @@ jobs:
       if: ${{ matrix.watchdog-type == 'Basic' }}
       run: echo "General__UseBasicWatchdog=true" >> $Env:GITHUB_ENV
 
-    - name: Set TGS_TEST_CONNECTION_STRING
+    - name: Set SqlServer Connection Info
+      if: ${{ matrix.database-type == 'SqlServer' }}
       shell: bash
       run: |
         TGS_CONNSTRING_VALUE="Server=(localdb)\MSSQLLocalDB;Integrated Security=true;Initial Catalog=TGS_${{ matrix.watchdog-type }}_${{ matrix.configuration }};Application Name=tgstation-server"
         echo "TGS_TEST_CONNECTION_STRING=$(echo $TGS_CONNSTRING_VALUE)" >> $GITHUB_ENV
+        echo "TGS_TEST_DATABASE_TYPE=SqlServer" >> $GITHUB_ENV
 
     - name: Checkout (Branch Push)
       uses: actions/checkout@v3
@@ -361,7 +365,7 @@ jobs:
         path: ./TestResults/
 
     - name: Store OpenAPI Spec
-      if: ${{ matrix.configuration == 'Release' && matrix.watchdog-type == 'System' }}
+      if: ${{ matrix.configuration == 'Release' && matrix.watchdog-type == 'System' && matrix.database-type == 'SqlServer' }}
       uses: actions/upload-artifact@v3
       with:
         name: openapi-spec

build/Version.props

@@ -3,12 +3,12 @@
   <!-- Integration tests will ensure they match across the board -->
   <Import Project="ControlPanelVersion.props" />
   <PropertyGroup>
-    <TgsCoreVersion>5.12.1</TgsCoreVersion>
+    <TgsCoreVersion>5.12.2</TgsCoreVersion>
     <TgsConfigVersion>4.6.0</TgsConfigVersion>
     <TgsApiVersion>9.10.2</TgsApiVersion>
     <TgsApiLibraryVersion>10.4.1</TgsApiLibraryVersion>
     <TgsClientVersion>11.4.2</TgsClientVersion>
-    <TgsDmapiVersion>6.4.3</TgsDmapiVersion>
+    <TgsDmapiVersion>6.4.4</TgsDmapiVersion>
     <TgsInteropVersion>5.6.0</TgsInteropVersion>
     <TgsHostWatchdogVersion>1.2.2</TgsHostWatchdogVersion>
     <TgsContainerScriptVersion>1.2.1</TgsContainerScriptVersion>

src/DMAPI/tgs.dm

@@ -1,6 +1,6 @@
 // tgstation-server DMAPI
 
-#define TGS_DMAPI_VERSION "6.4.3"
+#define TGS_DMAPI_VERSION "6.4.4"
 
 // All functions and datums outside this document are subject to change with any version and should not be relied on.
 

src/DMAPI/tgs/v4/api.dm

@@ -263,7 +263,12 @@
 		for(var/I in channels)
 			var/datum/tgs_chat_channel/channel = I
 			ids += channel.id
+
 	message = UpgradeDeprecatedChatMessage(message)
+
+	if (!length(channels))
+		return
+
 	message = list("message" = message.text, "channelIds" = ids)
 	if(intercepted_message_queue)
 		intercepted_message_queue += list(message)
@@ -276,7 +281,12 @@
 		var/datum/tgs_chat_channel/channel = I
 		if (!channel.is_private_channel && ((channel.is_admin_channel && admin_only) || (!channel.is_admin_channel && !admin_only)))
 			channels += channel.id
+
 	message = UpgradeDeprecatedChatMessage(message)
+
+	if (!length(channels))
+		return
+
 	message = list("message" = message.text, "channelIds" = channels)
 	if(intercepted_message_queue)
 		intercepted_message_queue += list(message)

src/DMAPI/tgs/v5/api.dm

@@ -166,6 +166,10 @@
 		ids += channel.id
 
 	message = UpgradeDeprecatedChatMessage(message)
+
+	if (!length(channels))
+		return
+
 	message = message._interop_serialize()
 	message[DMAPI5_CHAT_MESSAGE_CHANNEL_IDS] = ids
 	if(intercepted_message_queue)
@@ -181,6 +185,10 @@
 			channels += channel.id
 
 	message = UpgradeDeprecatedChatMessage(message)
+
+	if (!length(channels))
+		return
+
 	message = message._interop_serialize()
 	message[DMAPI5_CHAT_MESSAGE_CHANNEL_IDS] = channels
 	if(intercepted_message_queue)
@@ -199,6 +207,7 @@
 
 /datum/tgs_api/v5/ChatChannelInfo()
 	RequireInitialBridgeResponse()
+	WaitForReattach(TRUE)
 	return chat_channels.Copy()
 
 /datum/tgs_api/v5/proc/DecodeChannels(chat_update_json)

src/DMAPI/tgs/v5/bridge.dm

@@ -59,18 +59,22 @@
 	var/json = json_encode(data)
 	return json
 
-/datum/tgs_api/v5/proc/PerformBridgeRequest(bridge_request)
+/datum/tgs_api/v5/proc/WaitForReattach(require_channels = FALSE)
 	if(detached)
 		// Wait up to one minute
 		for(var/i in 1 to 600)
 			sleep(1)
-			if(!detached)
+			if(!detached && (!require_channels || length(chat_channels)))
 				break
 
-			// dad went out for milk cigarettes 20 years ago...
+			// dad went out for milk and cigarettes 20 years ago...
+			// yes, this affects all other waiters, intentional
 			if(i == 600)
 				detached = FALSE
 
+/datum/tgs_api/v5/proc/PerformBridgeRequest(bridge_request)
+	WaitForReattach(FALSE)
+
 	// This is an infinite sleep until we get a response
 	var/export_response = world.Export(bridge_request)
 	if(!export_response)

src/DMAPI/tgs/v5/topic.dm

@@ -71,6 +71,7 @@
 			var/list/event_call = list(event_type)
 			if (event_type == TGS_EVENT_WATCHDOG_DETACH)
 				detached = TRUE
+				chat_channels.Cut() // https://github.com/tgstation/tgstation-server/issues/1490
 
 			if(event_parameters)
 				event_call += event_parameters

src/Tgstation.Server.Host/Components/Chat/ChannelRepresentation.cs

@@ -19,7 +19,7 @@ namespace Tgstation.Server.Host.Components.Chat
 		/// <summary>
 		/// The <see cref="Providers.IProvider"/> channel Id.
 		/// </summary>
-		/// <remarks><see cref="Chat"/> remaps this to an internal id using <see cref="ChannelMapping"/>. Not sent over the DMAPI.</remarks>
+		/// <remarks><see cref="ChatManager"/> remaps this to an internal id using <see cref="ChannelMapping"/>. Not sent over the DMAPI.</remarks>
 		[JsonIgnore]
 		public ulong RealId
 		{

src/Tgstation.Server.Host/Components/Chat/ChatManager.cs

@@ -232,7 +232,6 @@ namespace Tgstation.Server.Host.Components.Chat
 					channelIdCounter += (ulong)results.Count;
 				}
 
-				Task trackingContextUpdateTask;
 				lock (mappedChannels)
 				{
 					lock (providers)
@@ -245,16 +244,13 @@ namespace Tgstation.Server.Host.Components.Chat
 						mappedChannels.Add(newId, newMapping);
 						newMapping.Channel.RealId = newId;
 					}
-
-					lock (trackingContexts)
-						trackingContextUpdateTask = Task.WhenAll(
-							trackingContexts.Select(
-								x => x.UpdateChannels(
-									mappedChannels.Select(y => y.Value.Channel).ToList(),
-									cancellationToken)));
 				}
 
-				await trackingContextUpdateTask;
+				// we only want to update contexts if everything at startup has connected once already
+				// otherwise we could send an incomplete channel set to the DMAPI, which will then spout all its queued messages into it instead of all relevant chatbots
+				// The watchdog can call this if it needs to after starting up
+				if (initialProviderConnectionsTask.IsCompleted)
+					await UpdateTrackingContexts(cancellationToken);
 			}
 			finally
 			{
@@ -333,35 +329,33 @@ namespace Tgstation.Server.Host.Components.Chat
 			if (channelIds == null)
 				throw new ArgumentNullException(nameof(channelIds));
 
-			var task = SendMessage(
-				channelIds,
-				null,
-				message,
-				handlerCts.Token);
-			AddMessageTask(task);
+			QueueMessageInternal(message, () => channelIds, false);
 		}
 
 		/// <inheritdoc />
-		public async Task QueueWatchdogMessage(string message, CancellationToken cancellationToken)
+		public void QueueWatchdogMessage(string message)
 		{
-			List<ulong> wdChannels = null;
+			if (message == null)
+				throw new ArgumentNullException(nameof(message));
+
 			message = String.Format(CultureInfo.InvariantCulture, "WD: {0}", message);
 
 			if (!initialProviderConnectionsTask.IsCompleted)
 				logger.LogTrace("Waiting for initial provider connections before sending watchdog message...");
 
-			await initialProviderConnectionsTask.WithToken(cancellationToken);
-
-			// so it doesn't change while we're using it
-			lock (mappedChannels)
-				wdChannels = mappedChannels.Where(x => x.Value.IsWatchdogChannel).Select(x => x.Key).ToList();
-
-			QueueMessage(
+			// Reimplementing QueueMessage
+			QueueMessageInternal(
 				new MessageContent
 				{
 					Text = message,
 				},
-				wdChannels);
+				() =>
+				{
+					// so it doesn't change while we're using it
+					lock (mappedChannels)
+						return mappedChannels.Where(x => x.Value.IsWatchdogChannel).Select(x => x.Key).ToList();
+				},
+				true);
 		}
 
 		/// <inheritdoc />
@@ -403,10 +397,10 @@ namespace Tgstation.Server.Host.Components.Chat
 								gitHubRepo,
 								channelMapping.ProviderChannelId,
 								localCommitPushed,
-								handlerCts.Token)
-								;
+								handlerCts.Token);
 
-							callbacks.Add(callback);
+							lock (callbacks)
+								callbacks.Add(callback);
 						}
 						catch (Exception ex)
 						{
@@ -419,12 +413,17 @@ namespace Tgstation.Server.Host.Components.Chat
 
 			AddMessageTask(task);
 
-			return (errorMessage, dreamMakerOutput) => AddMessageTask(
-				Task.WhenAll(
+			async Task CollateTasks(string errorMessage, string dreamMakerOutput)
+			{
+				await task;
+				await Task.WhenAll(
 					callbacks.Select(
 						x => x(
 							errorMessage,
-							dreamMakerOutput))));
+							dreamMakerOutput)));
+			}
+
+			return (errorMessage, dreamMakerOutput) => AddMessageTask(CollateTasks(errorMessage, dreamMakerOutput));
 		}
 
 		/// <inheritdoc />
@@ -472,6 +471,38 @@ namespace Tgstation.Server.Host.Components.Chat
 			return context;
 		}
 
+		/// <inheritdoc />
+		public async Task UpdateTrackingContexts(CancellationToken cancellationToken)
+		{
+			var logMessageSent = 0;
+			async Task UpdateTrackingContext(IChatTrackingContext channelSink, IEnumerable<ChannelRepresentation> channels)
+			{
+				if (Interlocked.Exchange(ref logMessageSent, 1) == 0)
+
+				await channelSink.UpdateChannels(channels, cancellationToken);
+			}
+
+			var waitingForInitialConnection = !initialProviderConnectionsTask.IsCompleted;
+			if (waitingForInitialConnection)
+			{
+				logger.LogTrace("Waiting for initial chat bot connections before updating tracking contexts...");
+				await initialProviderConnectionsTask.WithToken(cancellationToken);
+			}
+
+			List<Task> tasks;
+			lock (mappedChannels)
+				lock (trackingContexts)
+					tasks = trackingContexts.Select(x => UpdateTrackingContext(x, mappedChannels.Select(y => y.Value.Channel))).ToList();
+
+			if (waitingForInitialConnection)
+				if (tasks.Count > 0)
+					logger.LogTrace("Updating chat tracking contexts...");
+				else
+					logger.LogTrace("No chat tracking contexts to update");
+
+			await Task.WhenAll(tasks);
+		}
+
 		/// <inheritdoc />
 		public void RegisterCommandHandler(ICustomCommandHandler customCommandHandler)
 		{
@@ -492,13 +523,15 @@ namespace Tgstation.Server.Host.Components.Chat
 				{
 					await provider.Disconnect(cancellationToken);
 				}
-				finally
+				catch (Exception ex)
 				{
-					await provider.DisposeAsync();
-					var duration = DateTimeOffset.UtcNow - startTime;
-					if (duration.TotalSeconds > 3)
-						logger.LogWarning("Disconnecting a {providerType} took {totalSeconds}s!", provider.GetType().Name, duration.TotalSeconds);
+					logger.LogError(ex, "Error disconnecting connection {connectionId}!", connectionId);
 				}
+
+				await provider.DisposeAsync();
+				var duration = DateTimeOffset.UtcNow - startTime;
+				if (duration.TotalSeconds > 3)
+					logger.LogWarning("Disconnecting a {providerType} took {totalSeconds}s!", provider.GetType().Name, duration.TotalSeconds);
 			}
 			else
 				logger.LogTrace("DeleteConnection: ID {connectionId} doesn't exist!", connectionId);
@@ -906,6 +939,12 @@ namespace Tgstation.Server.Host.Components.Chat
 					// process completed ones
 					foreach (var completedMessageTaskKvp in messageTasks.Where(x => x.Value.IsCompleted).ToList())
 					{
+						var provider = completedMessageTaskKvp.Key;
+						messageTasks.Remove(provider);
+
+						if (provider.Disposed) // valid to receive one, but don't process it
+							continue;
+
 						var message = await completedMessageTaskKvp.Value;
 						var messageNumber = Interlocked.Increment(ref messagesProcessed);
 
@@ -915,7 +954,7 @@ namespace Tgstation.Server.Host.Components.Chat
 							using (LogContext.PushProperty(SerilogContextHelper.ChatMessageIterationContextProperty, messageNumber))
 								try
 								{
-									await ProcessMessage(completedMessageTaskKvp.Key, message, false, cancellationToken);
+									await ProcessMessage(provider, message, false, cancellationToken);
 								}
 								catch (Exception ex)
 								{
@@ -926,8 +965,6 @@ namespace Tgstation.Server.Host.Components.Chat
 						}
 
 						activeProcessingTask = WrapProcessMessage();
-
-						messageTasks.Remove(completedMessageTaskKvp.Key);
 					}
 				}
 			}
@@ -957,12 +994,17 @@ namespace Tgstation.Server.Host.Components.Chat
 		/// <returns>A <see cref="Task"/> representing the running operation.</returns>
 		Task SendMessage(IEnumerable<ulong> channelIds, Message replyTo, MessageContent message, CancellationToken cancellationToken)
 		{
+			channelIds = channelIds.ToList();
+
 			logger.LogTrace(
-				"Chat send \"{message}\"{embed} to channels: {channelIdsCommaSeperated}",
+				"Chat send \"{message}\"{embed} to channels: [{channelIdsCommaSeperated}]",
 				message.Text,
 				message.Embed != null ? " (with embed)" : String.Empty,
 				String.Join(", ", channelIds));
 
+			if (!channelIds.Any())
+				return Task.CompletedTask;
+
 			return Task.WhenAll(
 				channelIds.Select(x =>
 				{
@@ -1001,14 +1043,42 @@ namespace Tgstation.Server.Host.Components.Chat
 				{
 					await task;
 				}
+				catch (OperationCanceledException ex)
+				{
+					logger.LogDebug(ex, "Async chat message cancelled!");
+				}
 				catch (Exception ex)
 				{
-					logger.LogWarning(ex, "Error in asynchronous chat message!");
+					logger.LogError(ex, "Error in asynchronous chat message!");
 				}
 			}
 
 			lock (handlerCts)
 				messageSendTask = Wrap(messageSendTask);
 		}
+
+		/// <summary>
+		/// Adds a given <paramref name="message"/> to the send queue.
+		/// </summary>
+		/// <param name="message">The <see cref="MessageContent"/> being sent.</param>
+		/// <param name="channelIdsFactory">A <see cref="Func{TResult}"/> to retrieve he <see cref="Models.ChatChannel.Id"/>s of the <see cref="Models.ChatChannel"/>s to send to.</param>
+		/// <param name="waitForConnections">If <see langword="true"/>, the message send will wait for <see cref="initialProviderConnectionsTask"/> to complete before running.</param>
+		void QueueMessageInternal(MessageContent message, Func<IEnumerable<ulong>> channelIdsFactory, bool waitForConnections)
+		{
+			async Task SendMessageTask()
+			{
+				var cancellationToken = handlerCts.Token;
+				if (waitForConnections)
+					await initialProviderConnectionsTask.WithToken(cancellationToken);
+
+				await SendMessage(
+					channelIdsFactory(),
+					null,
+					message,
+					cancellationToken);
+			}
+
+			AddMessageTask(SendMessageTask());
+		}
 	}
 }

src/Tgstation.Server.Host/Components/Chat/ChatTrackingContext.cs

@@ -21,7 +21,11 @@ namespace Tgstation.Server.Host.Components.Chat
 			{
 				if (active == value)
 					return;
-				logger.LogTrace(value ? "Activated" : "Deactivated");
+				if (value)
+					logger.LogTrace("Activated");
+				else
+					logger.LogTrace("Deactivated");
+
 				active = value;
 			}
 		}

src/Tgstation.Server.Host/Components/Chat/IChatManager.cs

@@ -57,9 +57,7 @@ namespace Tgstation.Server.Host.Components.Chat
 		/// Queue a chat <paramref name="message"/> to configured watchdog channels.
 		/// </summary>
 		/// <param name="message">The message being sent.</param>
-		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
-		/// <returns>A <see cref="Task"/> representing the running operation.</returns>
-		Task QueueWatchdogMessage(string message, CancellationToken cancellationToken);
+		void QueueWatchdogMessage(string message);
 
 		/// <summary>
 		/// Send the message for a deployment to configured deployment channels.
@@ -84,5 +82,12 @@ namespace Tgstation.Server.Host.Components.Chat
 		/// </summary>
 		/// <returns>A new <see cref="IChatTrackingContext"/>.</returns>
 		IChatTrackingContext CreateTrackingContext();
+
+		/// <summary>
+		/// Force an update with the active channels on all active <see cref="IChatTrackingContext"/>s.
+		/// </summary>
+		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
+		/// <returns>A <see cref="Task"/> representing the running operation.</returns>
+		Task UpdateTrackingContexts(CancellationToken cancellationToken);
 	}
 }

src/Tgstation.Server.Host/Components/Chat/Providers/DiscordProvider.cs

@@ -27,6 +27,7 @@ using Tgstation.Server.Host.Extensions;
 using Tgstation.Server.Host.Jobs;
 using Tgstation.Server.Host.Models;
 using Tgstation.Server.Host.System;
+using Tgstation.Server.Host.Utils;
 
 namespace Tgstation.Server.Host.Components.Chat.Providers
 {
@@ -179,15 +180,17 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
 		/// Initializes a new instance of the <see cref="DiscordProvider"/> class.
 		/// </summary>
 		/// <param name="jobManager">The <see cref="IJobManager"/> for the <see cref="Provider"/>.</param>
-		/// <param name="assemblyInformationProvider">The value of <see cref="assemblyInformationProvider"/>.</param>
+		/// <param name="asyncDelayer">The <see cref="IAsyncDelayer"/> for the <see cref="Provider"/>.</param>
 		/// <param name="logger">The <see cref="ILogger"/> for the <see cref="Provider"/>.</param>
+		/// <param name="assemblyInformationProvider">The value of <see cref="assemblyInformationProvider"/>.</param>
 		/// <param name="chatBot">The <see cref="ChatBot"/> for the <see cref="Provider"/>.</param>
 		public DiscordProvider(
 			IJobManager jobManager,
-			IAssemblyInformationProvider assemblyInformationProvider,
+			IAsyncDelayer asyncDelayer,
 			ILogger<DiscordProvider> logger,
+			IAssemblyInformationProvider assemblyInformationProvider,
 			ChatBot chatBot)
-			: base(jobManager, logger, chatBot)
+			: base(jobManager, asyncDelayer, logger, chatBot)
 		{
 			this.assemblyInformationProvider = assemblyInformationProvider ?? throw new ArgumentNullException(nameof(assemblyInformationProvider));
 
@@ -232,6 +235,9 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
 		/// <inheritdoc />
 		public override async Task SendMessage(Message replyTo, MessageContent message, ulong channelId, CancellationToken cancellationToken)
 		{
+			if (message == null)
+				throw new ArgumentNullException(nameof(message));
+
 			Optional<IMessageReference> replyToReference = default;
 			Optional<IAllowedMentions> allowedMentions = default;
 			if (replyTo != null && replyTo is DiscordMessage discordMessage)
@@ -329,6 +335,15 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
 			bool localCommitPushed,
 			CancellationToken cancellationToken)
 		{
+			if (revisionInformation == null)
+				throw new ArgumentNullException(nameof(revisionInformation));
+			if (byondVersion == null)
+				throw new ArgumentNullException(nameof(byondVersion));
+			if (gitHubOwner == null)
+				throw new ArgumentNullException(nameof(gitHubOwner));
+			if (gitHubRepo == null)
+				throw new ArgumentNullException(nameof(gitHubRepo));
+
 			localCommitPushed |= revisionInformation.CommitSha == revisionInformation.OriginCommitSha;
 
 			var fields = BuildUpdateEmbedFields(revisionInformation, byondVersion, gitHubOwner, gitHubRepo, localCommitPushed);
@@ -356,8 +371,7 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
 				new Snowflake(channelId),
 				"DM: Deployment in Progress...",
 				embeds: new List<IEmbed> { embed },
-				ct: cancellationToken)
-				;
+				ct: cancellationToken);
 
 			if (!messageResponse.IsSuccess)
 				Logger.LogWarning("Failed to post deploy embed to channel {channelId}: {result}", channelId, messageResponse.LogFormat());
@@ -414,8 +428,7 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
 						new Snowflake(channelId),
 						updatedMessage,
 						embeds: new List<IEmbed> { embed },
-						ct: cancellationToken)
-						;
+						ct: cancellationToken);
 
 					if (!createUpdatedMessageResponse.IsSuccess)
 						Logger.LogWarning(
@@ -432,8 +445,7 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
 						messageResponse.Entity.ID,
 						updatedMessage,
 						embeds: new List<IEmbed> { embed },
-						ct: cancellationToken)
-						;
+						ct: cancellationToken);
 
 					if (!editResponse.IsSuccess)
 					{

src/Tgstation.Server.Host/Components/Chat/Providers/IrcProvider.cs

@@ -42,11 +42,6 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
 		/// <inheritdoc />
 		public override string BotMention => client.Nickname;
 
-		/// <summary>
-		/// The <see cref="IAsyncDelayer"/> for the <see cref="IrcProvider"/>.
-		/// </summary>
-		readonly IAsyncDelayer asyncDelayer;
-
 		/// <summary>
 		/// The <see cref="IrcFeatures"/> client.
 		/// </summary>
@@ -105,24 +100,22 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
 		/// <summary>
 		/// Initializes a new instance of the <see cref="IrcProvider"/> class.
 		/// </summary>
-		/// <param name="jobManager">The <see cref="IJobManager"/> for the provider.</param>
-		/// <param name="assemblyInformationProvider">The <see cref="IAssemblyInformationProvider"/> to get the <see cref="IAssemblyInformationProvider.VersionString"/> from.</param>
-		/// <param name="asyncDelayer">The value of <see cref="asyncDelayer"/>.</param>
+		/// <param name="jobManager">The <see cref="IJobManager"/> for the <see cref="Provider"/>.</param>
+		/// <param name="asyncDelayer">The <see cref="IAsyncDelayer"/> for the <see cref="Provider"/>.</param>
 		/// <param name="logger">The <see cref="ILogger"/> for the <see cref="Provider"/>.</param>
+		/// <param name="assemblyInformationProvider">The <see cref="IAssemblyInformationProvider"/> to get the <see cref="IAssemblyInformationProvider.VersionString"/> from.</param>
 		/// <param name="chatBot">The <see cref="Models.ChatBot"/> for the <see cref="Provider"/>.</param>
 		public IrcProvider(
 			IJobManager jobManager,
-			IAssemblyInformationProvider assemblyInformationProvider,
 			IAsyncDelayer asyncDelayer,
 			ILogger<IrcProvider> logger,
+			IAssemblyInformationProvider assemblyInformationProvider,
 			Models.ChatBot chatBot)
-			: base(jobManager, logger, chatBot)
+			: base(jobManager, asyncDelayer, logger, chatBot)
 		{
 			if (assemblyInformationProvider == null)
 				throw new ArgumentNullException(nameof(assemblyInformationProvider));
 
-			this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer));
-
 			var builder = chatBot.CreateConnectionStringBuilder();
 			if (builder == null || !builder.Valid || builder is not IrcConnectionStringBuilder ircBuilder)
 				throw new InvalidOperationException("Invalid ChatConnectionStringBuilder!");
@@ -172,53 +165,59 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
 		}
 
 		/// <inheritdoc />
-		public override Task SendMessage(Message replyTo, MessageContent message, ulong channelId, CancellationToken cancellationToken) => Task.Factory.StartNew(
-			() =>
-			{
-				// IRC doesn't allow newlines
-				// Explicitly ignore embeds
-				var messageText = message.Text;
-				messageText ??= $"Embed Only: {JsonConvert.SerializeObject(message.Embed)}";
+		public override Task SendMessage(Message replyTo, MessageContent message, ulong channelId, CancellationToken cancellationToken)
+		{
+			if (message == null)
+				throw new ArgumentNullException(nameof(message));
 
-				messageText = String.Concat(
-					messageText
-						.Where(x => x != '\r')
-						.Select(x => x == '\n' ? '|' : x));
-
-				var channelName = channelIdMap[channelId];
-				SendType sendType;
-				if (channelName == null)
+			return Task.Factory.StartNew(
+				() =>
 				{
-					channelName = queryChannelIdMap[channelId];
-					sendType = SendType.Notice;
-				}
-				else
-					sendType = SendType.Message;
+					// IRC doesn't allow newlines
+					// Explicitly ignore embeds
+					var messageText = message.Text;
+					messageText ??= $"Embed Only: {JsonConvert.SerializeObject(message.Embed)}";
 
-				var messageSize = Encoding.UTF8.GetByteCount(messageText) + Encoding.UTF8.GetByteCount(channelName) + PreambleMessageLength;
-				var messageTooLong = messageSize > MessageBytesLimit;
-				if (messageTooLong)
-					messageText = $"TGS: Could not send message to IRC. Line write exceeded protocol limit of {MessageBytesLimit}B.";
+					messageText = String.Concat(
+						messageText
+							.Where(x => x != '\r')
+							.Select(x => x == '\n' ? '|' : x));
 
-				try
-				{
-					client.SendMessage(sendType, channelName, messageText);
-				}
-				catch (Exception e)
-				{
-					Logger.LogWarning(e, "Unable to send to channel {channelName}!", channelName);
-					return;
-				}
+					var channelName = channelIdMap[channelId];
+					SendType sendType;
+					if (channelName == null)
+					{
+						channelName = queryChannelIdMap[channelId];
+						sendType = SendType.Notice;
+					}
+					else
+						sendType = SendType.Message;
 
-				if (messageTooLong)
-					Logger.LogWarning(
-						"Failed to send to channel {channelId}: Message size ({messageSize}B) exceeds IRC limit of 512B",
-						channelId,
-						messageSize);
-			},
-			cancellationToken,
-			DefaultIOManager.BlockingTaskCreationOptions,
-			TaskScheduler.Current);
+					var messageSize = Encoding.UTF8.GetByteCount(messageText) + Encoding.UTF8.GetByteCount(channelName) + PreambleMessageLength;
+					var messageTooLong = messageSize > MessageBytesLimit;
+					if (messageTooLong)
+						messageText = $"TGS: Could not send message to IRC. Line write exceeded protocol limit of {MessageBytesLimit}B.";
+
+					try
+					{
+						client.SendMessage(sendType, channelName, messageText);
+					}
+					catch (Exception e)
+					{
+						Logger.LogWarning(e, "Unable to send to channel {channelName}!", channelName);
+						return;
+					}
+
+					if (messageTooLong)
+						Logger.LogWarning(
+							"Failed to send to channel {channelId}: Message size ({messageSize}B) exceeds IRC limit of 512B",
+							channelId,
+							messageSize);
+				},
+				cancellationToken,
+				DefaultIOManager.BlockingTaskCreationOptions,
+				TaskScheduler.Current);
+		}
 
 		/// <inheritdoc />
 		public override async Task<Func<string, string, Task>> SendUpdateMessage(
@@ -231,6 +230,15 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
 			bool localCommitPushed,
 			CancellationToken cancellationToken)
 		{
+			if (revisionInformation == null)
+				throw new ArgumentNullException(nameof(revisionInformation));
+			if (byondVersion == null)
+				throw new ArgumentNullException(nameof(byondVersion));
+			if (gitHubOwner == null)
+				throw new ArgumentNullException(nameof(gitHubOwner));
+			if (gitHubRepo == null)
+				throw new ArgumentNullException(nameof(gitHubRepo));
+
 			var commitInsert = revisionInformation.CommitSha[..7];
 			string remoteCommitInsert;
 			if (revisionInformation.CommitSha == revisionInformation.OriginCommitSha)
@@ -626,14 +634,14 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
 
 				var listenTimeSpan = TimeSpan.FromMilliseconds(10);
 				for (; !recievedAck;
-					await asyncDelayer.Delay(listenTimeSpan, timeoutToken))
+					await AsyncDelayer.Delay(listenTimeSpan, timeoutToken))
 					await NonBlockingListen(cancellationToken);
 
 				client.WriteLine("AUTHENTICATE PLAIN", Priority.Critical);
 				timeoutToken.ThrowIfCancellationRequested();
 
 				for (; !recievedPlus;
-					await asyncDelayer.Delay(listenTimeSpan, timeoutToken))
+					await AsyncDelayer.Delay(listenTimeSpan, timeoutToken))
 					await NonBlockingListen(cancellationToken);
 			}
 			finally
@@ -699,8 +707,7 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
 				Task.WhenAll(
 					disconnectTask,
 					listenTask ?? Task.CompletedTask),
-				asyncDelayer.Delay(TimeSpan.FromSeconds(5), cancellationToken))
-				;
+				AsyncDelayer.Delay(TimeSpan.FromSeconds(5), cancellationToken));
 		}
 	}
 }

src/Tgstation.Server.Host/Components/Chat/Providers/Provider.cs

@@ -10,6 +10,7 @@ using Tgstation.Server.Host.Components.Interop;
 using Tgstation.Server.Host.Extensions;
 using Tgstation.Server.Host.Jobs;
 using Tgstation.Server.Host.Models;
+using Tgstation.Server.Host.Utils;
 
 namespace Tgstation.Server.Host.Components.Chat.Providers
 {
@@ -24,6 +25,11 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
 		/// </summary>
 		protected ChatBot ChatBot { get; }
 
+		/// <summary>
+		/// The <see cref="IAsyncDelayer"/> for the <see cref="Provider"/>.
+		/// </summary>
+		protected IAsyncDelayer AsyncDelayer { get; }
+
 		/// <summary>
 		/// The <see cref="ILogger"/> for the <see cref="Provider"/>.
 		/// </summary>
@@ -68,11 +74,13 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
 		/// Initializes a new instance of the <see cref="Provider"/> class.
 		/// </summary>
 		/// <param name="jobManager">The value of <see cref="jobManager"/>.</param>
+		/// <param name="asyncDelayer">The value of <see cref="AsyncDelayer"/>.</param>
 		/// <param name="logger">The value of <see cref="Logger"/>.</param>
 		/// <param name="chatBot">The value of <paramref name="chatBot"/>.</param>
-		protected Provider(IJobManager jobManager, ILogger<Provider> logger, ChatBot chatBot)
+		protected Provider(IJobManager jobManager, IAsyncDelayer asyncDelayer, ILogger<Provider> logger, ChatBot chatBot)
 		{
 			this.jobManager = jobManager ?? throw new ArgumentNullException(nameof(jobManager));
+			AsyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer));
 			Logger = logger ?? throw new ArgumentNullException(nameof(logger));
 			ChatBot = chatBot ?? throw new ArgumentNullException(nameof(chatBot));
 
@@ -98,6 +106,9 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
 		{
 			Disposed = true;
 			await StopReconnectionTimer();
+
+			// queue a final message to shutdown the NextMessage Task
+			EnqueueMessage(null);
 			Logger.LogTrace("Disposed");
 		}
 
@@ -206,7 +217,7 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
 		/// <summary>
 		/// Queues a <paramref name="message"/> for <see cref="NextMessage(CancellationToken)"/>.
 		/// </summary>
-		/// <param name="message">The <see cref="Message"/> to queue. A value of <see langword="null"/> indicates the channel mappings a out of date.</param>
+		/// <param name="message">The <see cref="Message"/> to queue. A value of <see langword="null"/> indicates the channel mappings are out of date.</param>
 		protected void EnqueueMessage(Message message)
 		{
 			if (message == null)
@@ -256,7 +267,7 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
 				try
 				{
 					if (!connectNow)
-						await Task.Delay(TimeSpan.FromMinutes(reconnectInterval), cancellationToken);
+						await AsyncDelayer.Delay(TimeSpan.FromMinutes(reconnectInterval), cancellationToken);
 					else
 						connectNow = false;
 					if (!Connected)
@@ -290,6 +301,7 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
 								}
 								catch
 								{
+									// we set this here because otherwise there could be stuff waiting on to connect us forever
 									initialConnectionTcs.TrySetResult();
 									throw;
 								}

src/Tgstation.Server.Host/Components/Chat/Providers/ProviderFactory.cs

@@ -61,14 +61,15 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
 			{
 				ChatProvider.Irc => new IrcProvider(
 					jobManager,
-					assemblyInformationProvider,
 					asyncDelayer,
 					loggerFactory.CreateLogger<IrcProvider>(),
+					assemblyInformationProvider,
 					settings),
 				ChatProvider.Discord => new DiscordProvider(
 					jobManager,
-					assemblyInformationProvider,
+					asyncDelayer,
 					loggerFactory.CreateLogger<DiscordProvider>(),
+					assemblyInformationProvider,
 					settings),
 				_ => throw new InvalidOperationException(String.Format(CultureInfo.InvariantCulture, "Invalid ChatProvider: {0}", settings.Provider)),
 			};

src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs

@@ -23,6 +23,7 @@ using Tgstation.Server.Host.IO;
 using Tgstation.Server.Host.Jobs;
 using Tgstation.Server.Host.Models;
 using Tgstation.Server.Host.System;
+using Tgstation.Server.Host.Utils;
 
 namespace Tgstation.Server.Host.Components.Deployment
 {
@@ -89,6 +90,11 @@ namespace Tgstation.Server.Host.Components.Deployment
 		/// </summary>
 		readonly IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory;
 
+		/// <summary>
+		/// The <see cref="IAsyncDelayer"/> for <see cref="DreamMaker"/>.
+		/// </summary>
+		readonly IAsyncDelayer asyncDelayer;
+
 		/// <summary>
 		/// The <see cref="ILogger"/> for <see cref="DreamMaker"/>.
 		/// </summary>
@@ -147,6 +153,7 @@ namespace Tgstation.Server.Host.Components.Deployment
 		/// <param name="compileJobConsumer">The value of <see cref="compileJobConsumer"/>.</param>
 		/// <param name="repositoryManager">The value of <see cref="repositoryManager"/>.</param>
 		/// <param name="remoteDeploymentManagerFactory">The value of <see cref="remoteDeploymentManagerFactory"/>.</param>
+		/// <param name="asyncDelayer">The value of <see cref="asyncDelayer"/>.</param>
 		/// <param name="logger">The value of <see cref="logger"/>.</param>
 		/// <param name="sessionConfiguration">The value of <see cref="sessionConfiguration"/>.</param>
 		/// <param name="metadata">The value of <see cref="metadata"/>.</param>
@@ -161,6 +168,7 @@ namespace Tgstation.Server.Host.Components.Deployment
 			ICompileJobSink compileJobConsumer,
 			IRepositoryManager repositoryManager,
 			IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory,
+			IAsyncDelayer asyncDelayer,
 			ILogger<DreamMaker> logger,
 			SessionConfiguration sessionConfiguration,
 			Api.Models.Instance metadata)
@@ -174,6 +182,7 @@ namespace Tgstation.Server.Host.Components.Deployment
 			this.processExecutor = processExecutor ?? throw new ArgumentNullException(nameof(processExecutor));
 			this.compileJobConsumer = compileJobConsumer ?? throw new ArgumentNullException(nameof(compileJobConsumer));
 			this.repositoryManager = repositoryManager ?? throw new ArgumentNullException(nameof(repositoryManager));
+			this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer));
 			this.remoteDeploymentManagerFactory = remoteDeploymentManagerFactory ?? throw new ArgumentNullException(nameof(remoteDeploymentManagerFactory));
 			this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
 			this.sessionConfiguration = sessionConfiguration ?? throw new ArgumentNullException(nameof(sessionConfiguration));
@@ -741,13 +750,13 @@ namespace Tgstation.Server.Host.Components.Deployment
 							var remainingSleepThisInterval = nextInterval - DateTimeOffset.UtcNow;
 							var nextSleepSpan = remainingSleepThisInterval < minimumSleepInterval ? minimumSleepInterval : remainingSleepThisInterval;
 
-							await Task.Delay(nextSleepSpan, cancellationToken);
+							await asyncDelayer.Delay(nextSleepSpan, cancellationToken);
 							progressReporter.ReportProgress(lastReport);
 						}
 						while (DateTimeOffset.UtcNow < nextInterval);
 					}
 					else
-						await Task.Delay(minimumSleepInterval, cancellationToken);
+						await asyncDelayer.Delay(minimumSleepInterval, cancellationToken);
 
 					lastReport = estimatedDuration.HasValue ? sleepInterval * (iteration + 1) / estimatedDuration.Value : null;
 					progressReporter.ReportProgress(lastReport);

src/Tgstation.Server.Host/Components/Instance.cs

@@ -70,6 +70,11 @@ namespace Tgstation.Server.Host.Components
 		/// </summary>
 		readonly IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory;
 
+		/// <summary>
+		/// The <see cref="IAsyncDelayer"/> for the <see cref="Instance"/>.
+		/// </summary>
+		readonly IAsyncDelayer asyncDelayer;
+
 		/// <summary>
 		/// The <see cref="ILogger"/> for the <see cref="Instance"/>.
 		/// </summary>
@@ -109,6 +114,7 @@ namespace Tgstation.Server.Host.Components
 		/// <param name="jobManager">The value of <see cref="jobManager"/>.</param>
 		/// <param name="eventConsumer">The value of <see cref="eventConsumer"/>.</param>
 		/// <param name="remoteDeploymentManagerFactory">The value of <see cref="remoteDeploymentManagerFactory"/>.</param>
+		/// <param name="asyncDelayer">The value of <see cref="asyncDelayer"/>.</param>
 		/// <param name="logger">The value of <see cref="logger"/>.</param>
 		public Instance(
 			Api.Models.Instance metadata,
@@ -123,6 +129,7 @@ namespace Tgstation.Server.Host.Components
 			IJobManager jobManager,
 			IEventConsumer eventConsumer,
 			IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory,
+			IAsyncDelayer asyncDelayer,
 			ILogger<Instance> logger)
 		{
 			this.metadata = metadata ?? throw new ArgumentNullException(nameof(metadata));
@@ -136,6 +143,7 @@ namespace Tgstation.Server.Host.Components
 			this.jobManager = jobManager ?? throw new ArgumentNullException(nameof(jobManager));
 			this.eventConsumer = eventConsumer ?? throw new ArgumentNullException(nameof(eventConsumer));
 			this.remoteDeploymentManagerFactory = remoteDeploymentManagerFactory ?? throw new ArgumentNullException(nameof(remoteDeploymentManagerFactory));
+			this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer));
 			this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
 
 			timerLock = new object();
@@ -488,7 +496,7 @@ namespace Tgstation.Server.Host.Components
 			while (true)
 				try
 				{
-					await Task.Delay(TimeSpan.FromMinutes(minutes > Int32.MaxValue ? Int32.MaxValue : minutes), cancellationToken);
+					await asyncDelayer.Delay(TimeSpan.FromMinutes(minutes > Int32.MaxValue ? Int32.MaxValue : minutes), cancellationToken);
 					logger.LogInformation("Beginning auto update...");
 					await eventConsumer.HandleEvent(EventType.InstanceAutoUpdateStart, Enumerable.Empty<string>(), cancellationToken);
 					try

src/Tgstation.Server.Host/Components/InstanceFactory.cs

@@ -23,6 +23,7 @@ using Tgstation.Server.Host.Jobs;
 using Tgstation.Server.Host.Security;
 using Tgstation.Server.Host.System;
 using Tgstation.Server.Host.Transfer;
+using Tgstation.Server.Host.Utils;
 
 namespace Tgstation.Server.Host.Components
 {
@@ -139,6 +140,11 @@ namespace Tgstation.Server.Host.Components
 		/// </summary>
 		readonly IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory;
 
+		/// <summary>
+		/// The <see cref="IAsyncDelayer"/> for the <see cref="InstanceFactory"/>.
+		/// </summary>
+		readonly IAsyncDelayer asyncDelayer;
+
 		/// <summary>
 		/// The <see cref="GeneralConfiguration"/> for the <see cref="InstanceFactory"/>.
 		/// </summary>
@@ -182,6 +188,7 @@ namespace Tgstation.Server.Host.Components
 		/// <param name="fileTransferService">The value of <see cref="fileTransferService"/>.</param>
 		/// <param name="gitRemoteFeaturesFactory">The value of <see cref="gitRemoteFeaturesFactory"/>.</param>
 		/// <param name="remoteDeploymentManagerFactory">The value of <see cref="remoteDeploymentManagerFactory"/>.</param>
+		/// <param name="asyncDelayer">The value of <see cref="asyncDelayer"/>.</param>
 		/// <param name="generalConfigurationOptions">The <see cref="IOptions{TOptions}"/> containing the value of <see cref="generalConfiguration"/>.</param>
 		/// <param name="sessionConfigurationOptions">The <see cref="IOptions{TOptions}"/> containing the value of <see cref="sessionConfiguration"/>.</param>
 		public InstanceFactory(
@@ -207,6 +214,7 @@ namespace Tgstation.Server.Host.Components
 			IFileTransferTicketProvider fileTransferService,
 			IGitRemoteFeaturesFactory gitRemoteFeaturesFactory,
 			IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory,
+			IAsyncDelayer asyncDelayer,
 			IOptions<GeneralConfiguration> generalConfigurationOptions,
 			IOptions<SessionConfiguration> sessionConfigurationOptions)
 		{
@@ -232,6 +240,7 @@ namespace Tgstation.Server.Host.Components
 			this.fileTransferService = fileTransferService ?? throw new ArgumentNullException(nameof(fileTransferService));
 			this.gitRemoteFeaturesFactory = gitRemoteFeaturesFactory ?? throw new ArgumentNullException(nameof(gitRemoteFeaturesFactory));
 			this.remoteDeploymentManagerFactory = remoteDeploymentManagerFactory ?? throw new ArgumentNullException(nameof(remoteDeploymentManagerFactory));
+			this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer));
 			generalConfiguration = generalConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(generalConfigurationOptions));
 			sessionConfiguration = sessionConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(sessionConfigurationOptions));
 		}
@@ -310,6 +319,7 @@ namespace Tgstation.Server.Host.Components
 						bridgeRegistrar,
 						serverPortProvider,
 						eventConsumer,
+						asyncDelayer,
 						loggerFactory,
 						loggerFactory.CreateLogger<SessionControllerFactory>(),
 						sessionConfiguration,
@@ -358,6 +368,7 @@ namespace Tgstation.Server.Host.Components
 								dmbFactory,
 								repoManager,
 								remoteDeploymentManagerFactory,
+								asyncDelayer,
 								loggerFactory.CreateLogger<DreamMaker>(),
 								sessionConfiguration,
 								metadata);
@@ -374,6 +385,7 @@ namespace Tgstation.Server.Host.Components
 								jobManager,
 								eventConsumer,
 								remoteDeploymentManagerFactory,
+								asyncDelayer,
 								loggerFactory.CreateLogger<Instance>());
 
 							return instance;

src/Tgstation.Server.Host/Components/Session/SessionController.cs

@@ -180,6 +180,7 @@ namespace Tgstation.Server.Host.Components.Session
 		/// <param name="chat">The value of <see cref="chat"/>.</param>
 		/// <param name="chatTrackingContext">The value of <see cref="chatTrackingContext"/>.</param>
 		/// <param name="assemblyInformationProvider">The <see cref="IAssemblyInformationProvider"/> for the <see cref="SessionController"/>.</param>
+		/// <param name="asyncDelayer">The <see cref="IAsyncDelayer"/> for the <see cref="SessionController"/>.</param>
 		/// <param name="logger">The value of <see cref="Chunker.Logger"/>.</param>
 		/// <param name="postLifetimeCallback">The <see cref="Func{TResult}"/> returning a <see cref="Task"/> to be run after the <paramref name="process"/> ends.</param>
 		/// <param name="startupTimeout">The optional time to wait before failing the <see cref="LaunchResult"/>.</param>
@@ -195,6 +196,7 @@ namespace Tgstation.Server.Host.Components.Session
 			IBridgeRegistrar bridgeRegistrar,
 			IChatManager chat,
 			IAssemblyInformationProvider assemblyInformationProvider,
+			IAsyncDelayer asyncDelayer,
 			ILogger<SessionController> logger,
 			Func<Task> postLifetimeCallback,
 			uint? startupTimeout,
@@ -219,7 +221,11 @@ namespace Tgstation.Server.Host.Components.Session
 
 			rebootTcs = new TaskCompletionSource();
 			primeTcs = new TaskCompletionSource();
-			initialBridgeRequestTcs = new TaskCompletionSource();
+
+			// Run this asynchronously because we want to try to avoid any effects sending topics to the server while the initial bridge request is processing
+			// It MAY be the source of a DD crash. See this gist https://gist.github.com/Cyberboss/7776bbeff3a957d76affe0eae95c9f14
+			// Worth further investigation as to if that sequence of events is a reliable crash vector and opening a BYOND bug if it is
+			initialBridgeRequestTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
 			reattachTopicCts = new CancellationTokenSource();
 			synchronizationLock = new object();
 
@@ -246,6 +252,7 @@ namespace Tgstation.Server.Host.Components.Session
 
 			LaunchResult = GetLaunchResult(
 				assemblyInformationProvider,
+				asyncDelayer,
 				startupTimeout,
 				reattached,
 				apiValidate);
@@ -508,12 +515,14 @@ namespace Tgstation.Server.Host.Components.Session
 		/// The <see cref="Task{TResult}"/> for <see cref="LaunchResult"/>.
 		/// </summary>
 		/// <param name="assemblyInformationProvider">The <see cref="IAssemblyInformationProvider"/>.</param>
+		/// <param name="asyncDelayer">The <see cref="IAsyncDelayer"/>.</param>
 		/// <param name="startupTimeout">The, optional, startup timeout in seconds.</param>
 		/// <param name="reattached">If DreamDaemon was reattached.</param>
 		/// <param name="apiValidate">If this is a DMAPI validation session.</param>
 		/// <returns>A <see cref="Task{TResult}"/> resulting in the <see cref="Session.LaunchResult"/> for the operation.</returns>
 		async Task<LaunchResult> GetLaunchResult(
 			IAssemblyInformationProvider assemblyInformationProvider,
+			IAsyncDelayer asyncDelayer,
 			uint? startupTimeout,
 			bool reattached,
 			bool apiValidate)
@@ -526,7 +535,7 @@ namespace Tgstation.Server.Host.Components.Session
 			var toAwait = Task.WhenAny(startupTask, process.Lifetime);
 
 			if (startupTimeout.HasValue)
-				toAwait = Task.WhenAny(toAwait, Task.Delay(TimeSpan.FromSeconds(startupTimeout.Value)));
+				toAwait = Task.WhenAny(toAwait, asyncDelayer.Delay(TimeSpan.FromSeconds(startupTimeout.Value), default)); // DCT: None available, task will clean up after delay
 
 			Logger.LogTrace(
 				"Waiting for LaunchResult based on {launchResultCompletionCause}{possibleTimeout}...",

src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs

@@ -26,6 +26,7 @@ using Tgstation.Server.Host.IO;
 using Tgstation.Server.Host.Jobs;
 using Tgstation.Server.Host.Security;
 using Tgstation.Server.Host.System;
+using Tgstation.Server.Host.Utils;
 
 namespace Tgstation.Server.Host.Components.Session
 {
@@ -102,6 +103,11 @@ namespace Tgstation.Server.Host.Components.Session
 		/// </summary>
 		readonly IEventConsumer eventConsumer;
 
+		/// <summary>
+		/// The <see cref="IAsyncDelayer"/> for the <see cref="SessionControllerFactory"/>.
+		/// </summary>
+		readonly IAsyncDelayer asyncDelayer;
+
 		/// <summary>
 		/// The <see cref="ILoggerFactory"/> for the <see cref="SessionControllerFactory"/>.
 		/// </summary>
@@ -187,10 +193,11 @@ namespace Tgstation.Server.Host.Components.Session
 		/// <param name="platformIdentifier">The value of <see cref="platformIdentifier"/>.</param>
 		/// <param name="bridgeRegistrar">The value of <see cref="bridgeRegistrar"/>.</param>
 		/// <param name="serverPortProvider">The value of <see cref="serverPortProvider"/>.</param>
+		/// <param name="eventConsumer">The value of <see cref="eventConsumer"/>.</param>
+		/// <param name="asyncDelayer">The value of <see cref="asyncDelayer"/>.</param>
 		/// <param name="loggerFactory">The value of <see cref="loggerFactory"/>.</param>
 		/// <param name="logger">The value of <see cref="logger"/>.</param>
 		/// <param name="sessionConfiguration">The value of <see cref="sessionConfiguration"/>.</param>
-		/// <param name="eventConsumer">The value of <see cref="eventConsumer"/>.</param>
 		public SessionControllerFactory(
 			IProcessExecutor processExecutor,
 			IByondManager byond,
@@ -205,6 +212,7 @@ namespace Tgstation.Server.Host.Components.Session
 			IBridgeRegistrar bridgeRegistrar,
 			IServerPortProvider serverPortProvider,
 			IEventConsumer eventConsumer,
+			IAsyncDelayer asyncDelayer,
 			ILoggerFactory loggerFactory,
 			ILogger<SessionControllerFactory> logger,
 			SessionConfiguration sessionConfiguration,
@@ -223,6 +231,7 @@ namespace Tgstation.Server.Host.Components.Session
 			this.bridgeRegistrar = bridgeRegistrar ?? throw new ArgumentNullException(nameof(bridgeRegistrar));
 			this.serverPortProvider = serverPortProvider ?? throw new ArgumentNullException(nameof(serverPortProvider));
 			this.eventConsumer = eventConsumer ?? throw new ArgumentNullException(nameof(eventConsumer));
+			this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer));
 			this.loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory));
 			this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
 			this.sessionConfiguration = sessionConfiguration ?? throw new ArgumentNullException(nameof(sessionConfiguration));
@@ -341,6 +350,7 @@ namespace Tgstation.Server.Host.Components.Session
 							bridgeRegistrar,
 							chat,
 							assemblyInformationProvider,
+							asyncDelayer,
 							loggerFactory.CreateLogger<SessionController>(),
 							() => !launchParameters.LogOutput.Value
 								? LogDDOutput(process, outputFilePath, byondLock.SupportsCli, default) // DCT: None available
@@ -425,6 +435,7 @@ namespace Tgstation.Server.Host.Components.Session
 							bridgeRegistrar,
 							chat,
 							assemblyInformationProvider,
+							asyncDelayer,
 							loggerFactory.CreateLogger<SessionController>(),
 							() => Task.CompletedTask,
 							null,

src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs

@@ -118,22 +118,19 @@ namespace Tgstation.Server.Host.Components.Watchdog
 					if (Server.RebootState == Session.RebootState.Shutdown)
 					{
 						// the time for graceful shutdown is now
-						await Chat.QueueWatchdogMessage(
+						Chat.QueueWatchdogMessage(
 							String.Format(
 								CultureInfo.InvariantCulture,
 								"Server {0}! Shutting down due to graceful termination request...",
-								exitWord),
-							cancellationToken)
-							;
+								exitWord));
 						return MonitorAction.Exit;
 					}
 
-					await Chat.QueueWatchdogMessage(
+					Chat.QueueWatchdogMessage(
 						String.Format(
 							CultureInfo.InvariantCulture,
 							"Server {0}! Rebooting...",
-							exitWord),
-						cancellationToken);
+							exitWord));
 					return MonitorAction.Restart;
 				case MonitorActivationReason.ActiveServerRebooted:
 					var rebootState = Server.RebootState;
@@ -156,10 +153,8 @@ namespace Tgstation.Server.Host.Components.Watchdog
 							return MonitorAction.Restart;
 						case Session.RebootState.Shutdown:
 							// graceful shutdown time
-							await Chat.QueueWatchdogMessage(
-								"Active server rebooted! Shutting down due to graceful termination request...",
-								cancellationToken)
-								;
+							Chat.QueueWatchdogMessage(
+								"Active server rebooted! Shutting down due to graceful termination request...");
 							return MonitorAction.Exit;
 						default:
 							throw new InvalidOperationException($"Invalid reboot state: {rebootState}");
@@ -200,7 +195,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 
 		/// <inheritdoc />
 		protected override async Task InitController(
-			Task chatTask,
+			Task eventTask,
 			ReattachInformation reattachInfo,
 			CancellationToken cancellationToken)
 		{
@@ -221,7 +216,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 					await BeforeApplyDmb(dmbToUse.CompileJob, cancellationToken);
 					dmbToUse = await PrepServerForLaunch(dmbToUse, cancellationToken);
 
-					await chatTask;
+					await eventTask;
 					serverLaunchTask = SessionControllerFactory.LaunchNew(
 						dmbToUse,
 						null,
@@ -231,7 +226,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 				}
 				else
 				{
-					await chatTask;
+					await eventTask;
 					serverLaunchTask = SessionControllerFactory.Reattach(reattachInfo, cancellationToken);
 				}
 
@@ -246,7 +241,6 @@ namespace Tgstation.Server.Host.Components.Watchdog
 					return;
 				}
 
-				// Server.AdjustPriority(true);
 				if (!reattachInProgress)
 					await SessionStartupPersist(cancellationToken);
 
@@ -299,14 +293,17 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// </summary>
 		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
 		/// <returns>A <see cref="Task"/> representing the running operation.</returns>
-		protected virtual Task HandleNewDmbAvailable(CancellationToken cancellationToken)
+		protected virtual async Task HandleNewDmbAvailable(CancellationToken cancellationToken)
 		{
 			gracefulRebootRequired = true;
 			if (Server.CompileJob.DMApiVersion == null)
-				return Chat.QueueWatchdogMessage(
-					"A new deployment has been made but cannot be applied automatically as the currently running server has no DMAPI. Please manually reboot the server to apply the update.",
-					cancellationToken);
-			return Server.SetRebootState(Session.RebootState.Restart, cancellationToken);
+			{
+				Chat.QueueWatchdogMessage(
+					"A new deployment has been made but cannot be applied automatically as the currently running server has no DMAPI. Please manually reboot the server to apply the update.");
+				return;
+			}
+
+			await Server.SetRebootState(Session.RebootState.Restart, cancellationToken);
 		}
 
 		/// <summary>

src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs

@@ -347,10 +347,9 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			{
 				if (!graceful)
 				{
-					var chatTask = Chat.QueueWatchdogMessage("Manual restart triggered...", cancellationToken);
+					Chat.QueueWatchdogMessage("Manual restart triggered...");
 					await TerminateNoLock(false, false, cancellationToken);
 					await LaunchNoLock(true, false, true, null, cancellationToken);
-					await chatTask;
 					return;
 				}
 
@@ -365,7 +364,8 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		public async Task StartAsync(CancellationToken cancellationToken)
 		{
 			var reattachInfo = await SessionPersistor.Load(cancellationToken);
-			if (!autoStart && reattachInfo == null)
+			var reattaching = reattachInfo != null;
+			if (!autoStart && !reattaching)
 				return;
 
 			var job = new Models.Job
@@ -374,7 +374,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 				{
 					Id = metadata.Id,
 				},
-				Description = $"Instance startup watchdog {(reattachInfo != null ? "reattach" : "launch")}",
+				Description = $"Instance startup watchdog {(reattaching ? "reattach" : "launch")}",
 				CancelRight = (ulong)DreamDaemonRights.Shutdown,
 				CancelRightsType = RightsType.DreamDaemon,
 			};
@@ -384,8 +384,11 @@ namespace Tgstation.Server.Host.Components.Watchdog
 				{
 					if (core.Watchdog != this)
 						throw new InvalidOperationException(Instance.DifferentCoreExceptionMessage);
+
 					using (await SemaphoreSlimContext.Lock(synchronizationSemaphore, ct))
 						await LaunchNoLock(true, true, true, reattachInfo, ct);
+
+					await Chat.UpdateTrackingContexts(ct);
 				},
 				cancellationToken)
 				;
@@ -422,7 +425,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 
 			releaseServers = true;
 			if (Status == WatchdogStatus.Online)
-				await Chat.QueueWatchdogMessage("Detaching...", cancellationToken);
+				Chat.QueueWatchdogMessage("Detaching...");
 			else
 				Logger.LogTrace("Not sending detach chat message as status is: {status}", Status);
 		}
@@ -474,11 +477,11 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// <summary>
 		/// Starts all <see cref="ISessionController"/>s.
 		/// </summary>
-		/// <param name="chatTask">A, possibly active, <see cref="Task"/> for an outgoing chat message.</param>
+		/// <param name="eventTask">A, possibly active, <see cref="Task"/> for an event that's running.</param>
 		/// <param name="reattachInfo"><see cref="ReattachInformation"/> to use, if any.</param>
 		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
 		/// <returns>A <see cref="Task"/> representing the running operation.</returns>
-		protected abstract Task InitController(Task chatTask, ReattachInformation reattachInfo, CancellationToken cancellationToken);
+		protected abstract Task InitController(Task eventTask, ReattachInformation reattachInfo, CancellationToken cancellationToken);
 
 		/// <summary>
 		/// Launches the watchdog.
@@ -504,21 +507,16 @@ namespace Tgstation.Server.Host.Components.Watchdog
 				throw new JobException(ErrorCode.WatchdogCompileJobCorrupted);
 
 			// this is necessary, the monitor could be in it's sleep loop trying to restart, if so cancel THAT monitor and start our own with blackjack and hookers
-			Task announceTask;
+			var eventTask = Task.CompletedTask;
 			if (announce)
 			{
-				announceTask = Chat.QueueWatchdogMessage(
+				Chat.QueueWatchdogMessage(
 					reattachInfo == null
 						? "Launching..."
-						: "Reattaching...",
-					cancellationToken); // simple announce
+						: "Reattaching..."); // simple announce
 				if (reattachInfo == null)
-					announceTask = Task.WhenAll(
-						HandleEvent(EventType.WatchdogLaunch, Enumerable.Empty<string>(), false, cancellationToken),
-						announceTask);
+					eventTask = HandleEvent(EventType.WatchdogLaunch, Enumerable.Empty<string>(), false, cancellationToken);
 			}
-			else
-				announceTask = Task.CompletedTask; // no announce
 
 			// since neither server is running, this is safe to do
 			LastLaunchParameters = ActiveLaunchParameters;
@@ -526,7 +524,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 
 			try
 			{
-				await InitController(announceTask, reattachInfo, cancellationToken);
+				await InitController(eventTask, reattachInfo, cancellationToken);
 			}
 			catch (OperationCanceledException ex)
 			{
@@ -536,15 +534,15 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			catch (Exception e)
 			{
 				Logger.LogWarning(e, "Failed to start watchdog!");
-				var originalChatTask = announceTask;
-				async Task ChainChatTaskWithErrorMessage()
+				var originalChatTask = eventTask;
+				async Task ChainEventTaskWithErrorMessage()
 				{
 					await originalChatTask;
 					if (announceFailure)
-						await Chat.QueueWatchdogMessage("Startup failed!", cancellationToken);
+						Chat.QueueWatchdogMessage("Startup failed!");
 				}
 
-				announceTask = ChainChatTaskWithErrorMessage();
+				eventTask = ChainEventTaskWithErrorMessage();
 				throw;
 			}
 			finally
@@ -552,7 +550,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 				// finish the chat task that's in flight
 				try
 				{
-					await announceTask;
+					await eventTask;
 				}
 				catch (OperationCanceledException ex)
 				{
@@ -623,8 +621,8 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			const string FailReattachMessage = "Unable to properly reattach to server! Restarting watchdog...";
 			Logger.LogWarning(FailReattachMessage);
 
-			var chatTask = Chat.QueueWatchdogMessage(FailReattachMessage, cancellationToken);
-			await InitController(chatTask, null, cancellationToken);
+			Chat.QueueWatchdogMessage(FailReattachMessage);
+			await InitController(Task.CompletedTask, null, cancellationToken);
 		}
 
 		/// <summary>
@@ -727,7 +725,6 @@ namespace Tgstation.Server.Host.Components.Watchdog
 
 			await DisposeAndNullControllers(cancellationToken);
 
-			var chatTask = Task.CompletedTask;
 			for (var retryAttempts = 1; ; ++retryAttempts)
 			{
 				Status = WatchdogStatus.Restoring;
@@ -745,10 +742,6 @@ namespace Tgstation.Server.Host.Components.Watchdog
 					{
 						launchException = e;
 					}
-					finally
-					{
-						await chatTask;
-					}
 
 				Logger.LogWarning(launchException, "Failed to automatically restart the watchdog! Attempt: {attemptNumber}", retryAttempts);
 				Status = WatchdogStatus.DelayedRestart;
@@ -758,16 +751,12 @@ namespace Tgstation.Server.Host.Components.Watchdog
 						Math.Pow(2, retryAttempts)),
 					TimeSpan.FromHours(1).TotalSeconds); // max of one hour, increasing by a power of 2 each time
 
-				chatTask = Chat.QueueWatchdogMessage(
-					$"Failed to restart (Attempt: {retryAttempts}), retrying in {retryDelay}s...",
-					cancellationToken);
+				Chat.QueueWatchdogMessage(
+					$"Failed to restart (Attempt: {retryAttempts}), retrying in {retryDelay}s...");
 
-				await Task.WhenAll(
-					AsyncDelayer.Delay(
-						TimeSpan.FromSeconds(retryDelay),
-						cancellationToken),
-					chatTask)
-					;
+				await AsyncDelayer.Delay(
+					TimeSpan.FromSeconds(retryDelay),
+					cancellationToken);
 			}
 		}
 
@@ -963,9 +952,8 @@ namespace Tgstation.Server.Host.Components.Watchdog
 							var nextActionMessage = nextAction != MonitorAction.Exit
 								? "Recovering"
 								: "Shutting down";
-							var chatTask = Chat.QueueWatchdogMessage(
-								$"Monitor crashed, this should NEVER happen! Please report this, full details in logs! {nextActionMessage}. Error: {e.Message}",
-								cancellationToken);
+							Chat.QueueWatchdogMessage(
+								$"Monitor crashed, this should NEVER happen! Please report this, full details in logs! {nextActionMessage}. Error: {e.Message}");
 
 							if (disposed)
 								nextAction = MonitorAction.Exit;
@@ -977,8 +965,6 @@ namespace Tgstation.Server.Host.Components.Watchdog
 									Logger.LogDebug("Server seems to be okay, not restarting");
 								nextAction = MonitorAction.Continue;
 							}
-
-							await chatTask;
 						}
 			}
 			catch (OperationCanceledException)
@@ -1023,15 +1009,14 @@ namespace Tgstation.Server.Host.Components.Watchdog
 					releaseServers,
 					cancellationToken);
 
-				var chatTask = announce ? Chat.QueueWatchdogMessage("Shutting down...", cancellationToken) : Task.CompletedTask;
+				if (announce)
+					Chat.QueueWatchdogMessage("Shutting down...");
 
 				await eventTask;
 
 				await StopMonitor();
 
 				LastLaunchParameters = null;
-
-				await chatTask;
 				return;
 			}
 
@@ -1068,7 +1053,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 					case 2:
 						const string message2 = "DEFCON 3: DreamDaemon has missed 2 heartbeats!";
 						Logger.LogInformation(message2);
-						await Chat.QueueWatchdogMessage(message2, cancellationToken);
+						Chat.QueueWatchdogMessage(message2);
 						break;
 					case 3:
 						var actionToTake = shouldShutdown
@@ -1076,12 +1061,11 @@ namespace Tgstation.Server.Host.Components.Watchdog
 							: "be restarted";
 						const string logTemplate1 = "DEFCON 2: DreamDaemon has missed 3 heartbeats! If it does not respond to the next one, the watchdog will {actionToTake}!";
 						Logger.LogWarning(logTemplate1, actionToTake);
-						await Chat.QueueWatchdogMessage(
+						Chat.QueueWatchdogMessage(
 							logTemplate1.Replace(
 								"{actionToTake}",
 								actionToTake,
-								StringComparison.Ordinal),
-							cancellationToken);
+								StringComparison.Ordinal));
 						break;
 					case 4:
 						var actionTaken = shouldShutdown
@@ -1089,12 +1073,11 @@ namespace Tgstation.Server.Host.Components.Watchdog
 							: "Restarting";
 						const string logTemplate2 = "DEFCON 1: Four heartbeats have been missed! {actionTaken}...";
 						Logger.LogWarning(logTemplate2, actionTaken);
-						await Chat.QueueWatchdogMessage(
+						Chat.QueueWatchdogMessage(
 							logTemplate2.Replace(
 								"{actionTaken}",
 								actionTaken,
-								StringComparison.Ordinal),
-							cancellationToken);
+								StringComparison.Ordinal));
 
 						if (ActiveLaunchParameters.DumpOnHeartbeatRestart.Value)
 						{

src/Tgstation.Server.Host/Controllers/ChatController.cs

@@ -115,8 +115,7 @@ namespace Tgstation.Server.Host.Controllers
 				.ChatBots
 				.AsQueryable()
 				.Where(x => x.InstanceId == Instance.Id)
-				.CountAsync(cancellationToken)
-				;
+				.CountAsync(cancellationToken);
 
 			if (countOfExistingBotsInInstance >= Instance.ChatBotLimit.Value)
 				return Conflict(new ErrorMessageResponse(ErrorCode.ChatBotMax));

src/Tgstation.Server.Host/Core/Application.cs

@@ -30,7 +30,6 @@ using Tgstation.Server.Common;
 using Tgstation.Server.Host.Components;
 using Tgstation.Server.Host.Components.Byond;
 using Tgstation.Server.Host.Components.Chat;
-using Tgstation.Server.Host.Components.Chat.Providers;
 using Tgstation.Server.Host.Components.Deployment.Remote;
 using Tgstation.Server.Host.Components.Interop;
 using Tgstation.Server.Host.Components.Interop.Bridge;
@@ -360,7 +359,7 @@ namespace Tgstation.Server.Host.Core
 			services.AddSingleton<ILibGit2RepositoryFactory, LibGit2RepositoryFactory>();
 			services.AddSingleton<ILibGit2Commands, LibGit2Commands>();
 			services.AddSingleton<IRemoteDeploymentManagerFactory, RemoteDeploymentManagerFactory>();
-			services.AddSingleton<IProviderFactory, ProviderFactory>();
+			services.AddChatProviderFactory();
 			services.AddSingleton<IChatManagerFactory, ChatManagerFactory>();
 			services.AddSingleton<IServerUpdater, ServerUpdater>();
 			services.AddSingleton<IServerUpdateInitiator, ServerUpdateInitiator>();

src/Tgstation.Server.Host/Extensions/ServiceCollectionExtensions.cs

@@ -11,6 +11,7 @@ using Serilog;
 using Serilog.Configuration;
 using Serilog.Sinks.Elasticsearch;
 
+using Tgstation.Server.Host.Components.Chat.Providers;
 using Tgstation.Server.Host.Configuration;
 using Tgstation.Server.Host.Utils;
 
@@ -21,6 +22,30 @@ namespace Tgstation.Server.Host.Extensions
 	/// </summary>
 	static class ServiceCollectionExtensions
 	{
+		/// <summary>
+		/// The <see cref="IProviderFactory"/> implementation used in calls to <see cref="AddChatProviderFactory(IServiceCollection)"/>.
+		/// </summary>
+		static Type chatProviderFactoryType = typeof(ProviderFactory);
+
+		/// <summary>
+		/// Change the <see cref="Type"/> used as an implementation for calls to <see cref="AddChatProviderFactory(IServiceCollection)"/>.
+		/// </summary>
+		/// <typeparam name="TProviderFactory">The <see cref="IProviderFactory"/> implementation to use.</typeparam>
+		public static void UseChatProviderFactory<TProviderFactory>() where TProviderFactory : IProviderFactory
+		{
+			chatProviderFactoryType = typeof(TProviderFactory);
+		}
+
+		/// <summary>
+		/// Adds a <see cref="IProviderFactory"/> implementation to the given <paramref name="serviceCollection"/>.
+		/// </summary>
+		/// <param name="serviceCollection">The <see cref="IServiceCollection"/> to configure.</param>
+		/// <returns><paramref name="serviceCollection"/>.</returns>
+		public static IServiceCollection AddChatProviderFactory(this IServiceCollection serviceCollection)
+		{
+			return serviceCollection.AddSingleton(typeof(IProviderFactory), chatProviderFactoryType);
+		}
+
 		/// <summary>
 		/// Add a standard <typeparamref name="TConfig"/> binding.
 		/// </summary>

src/Tgstation.Server.Host/Server.cs

@@ -314,14 +314,14 @@ namespace Tgstation.Server.Host
 							? generalConfiguration.ShutdownTimeoutMinutes
 							: generalConfiguration.RestartTimeoutMinutes));
 				var cancellationToken = cts.Token;
-				var eventsTask = Task.WhenAll(
-					restartHandlers.Select(
-						x => x.HandleRestart(newVersion, isGracefulShutdown, cancellationToken))
-					.ToList());
-
-				logger.LogTrace("Joining restart handlers...");
 				try
 				{
+					var eventsTask = Task.WhenAll(
+						restartHandlers.Select(
+							x => x.HandleRestart(newVersion, isGracefulShutdown, cancellationToken))
+						.ToList());
+
+					logger.LogTrace("Joining restart handlers...");
 					await eventsTask;
 				}
 				catch (OperationCanceledException ex)

tests/DMAPI/ApiFree/Test.dm

@@ -1,7 +1,2 @@
-/world/New()
-	text2file("SUCCESS", "test_success.txt")
+/world/proc/RunTest()
 	log << "Hello world!"
-
-/world/Error(exception)
-	fdel("test_success.txt")
-	text2file("Runtime Error: [exception]", "test_fail_reason.txt")

tests/DMAPI/ApiFree/api_free.dme

@@ -11,5 +11,6 @@
 // END_PREFERENCES
 
 // BEGIN_INCLUDE
+#include "../test_setup.dm"
 #include "Test.dm"
 // END_INCLUDE

tests/DMAPI/BasicOperation/Test.dm

@@ -1,4 +1,4 @@
-/world/New()
+/world/proc/RunTest()
 	text2file("SUCCESS", "test_success.txt")
 	log << "About to call TgsNew()"
 	sleep_offline = FALSE
@@ -6,10 +6,6 @@
 	log << "About to call StartAsync()"
 	StartAsync()
 
-/world/Error(exception)
-	fdel("test_success.txt")
-	text2file("Runtime Error: [exception]", "test_fail_reason.txt")
-
 /proc/StartAsync()
 	set waitfor = FALSE
 	Run()
@@ -21,7 +17,7 @@
 
 	var/list/world_params = world.params
 	if(!("test" in world_params) || world_params["test"] != "bababooey")
-		text2file("Expected parameter test=bababooey but did not receive", "test_fail_reason.txt")
+		FailTest("Expected parameter test=bababooey but did not receive", "test_fail_reason.txt")
 
 	world.log << "sleep2"
 	sleep(150)

tests/DMAPI/BasicOperation/basic_operation_test.dme

@@ -12,6 +12,6 @@
 
 // BEGIN_INCLUDE
 #include "Config.dm"
-#include "..\tgs_include.dm"
+#include "../test_prelude.dm"
 #include "Test.dm"
 // END_INCLUDE

tests/DMAPI/LongRunning/Test.dm

@@ -2,32 +2,7 @@
 	sleep_offline = FALSE
 	loop_checks = FALSE
 
-/world/Error(exception/E, datum/e_src)
-	var/list/usrinfo = null
-	var/list/splitlines = splittext(E.desc, "\n")
-	var/list/desclines = list()
-	for(var/line in splitlines)
-		if(length(line) < 3 || findtext(line, "source file:") || findtext(line, "usr.loc:"))
-			continue
-		if(findtext(line, "usr:"))
-			if(usrinfo)
-				desclines.Add(usrinfo)
-				usrinfo = null
-			continue // Our usr info is better, replace it
-
-		if(copytext(line, 1, 3) != "  ")//3 == length("  ") + 1
-			desclines += ("  " + line) // Pad any unpadded lines, so they look pretty
-		else
-			desclines += line
-
-	if(usrinfo) //If this info isn't null, it hasn't been added yet
-		desclines.Add(usrinfo)
-
-	fdel("test_success.txt")
-	text2file("Runtime Error: [E]", "test_fail_reason.txt")
-
-/world/New()
-	text2file("SUCCESS", "test_success.txt")
+/world/proc/RunTest()
 	log << "Initial value of sleep_offline: [sleep_offline]"
 	sleep_offline = FALSE
 
@@ -35,6 +10,11 @@
 	for(var/i in 1 to 10000000)
 		dab()
 	TgsNew(new /datum/tgs_event_handler/impl, TGS_SECURITY_SAFE)
+
+	var/list/channels = TgsChatChannelInfo()
+	if(!length(channels))
+		FailTest("Expected some chat channels!")
+
 	StartAsync()
 
 /proc/dab()
@@ -118,8 +98,7 @@ var/run_bridge_test
 	if(tactics4)
 		var/size = isnum(tactics4) ? tactics4 : text2num(tactics4)
 		if(!isnum(size))
-			text2file("tgs_integration_test_tactics4 wasn't a number!", "test_fail_reason.txt")
-			del(world)
+			FailTest("tgs_integration_test_tactics4 wasn't a number!")
 
 		var/payload = create_payload(size)
 		return payload
@@ -133,9 +112,19 @@ var/run_bridge_test
 	// Bridge response queuing
 	var/tactics6 = data["tgs_integration_test_tactics6"]
 	if(tactics6)
+		// hack hack, calling world.TgsChatChannelInfo() will try to delay until the channels come back
+		var/datum/tgs_api/v5/api = TGS_READ_GLOBAL(tgs)
+		if (length(api.chat_channels))
+			return "channels_present!"
+
 		DetachedChatMessageQueuing()
 		return "queued"
 
+	var/tactics7 = data["tgs_integration_test_tactics7"]
+	if(tactics7)
+		var/list/channels = TgsChatChannelInfo()
+		return "[length(channels)]"
+
 	TgsChatBroadcast(new /datum/tgs_message_content("Recieved non-tgs topic: `[T]`"))
 
 	return "feck"
@@ -162,6 +151,12 @@ var/run_bridge_test
 /datum/tgs_event_handler/impl/HandleEvent(event_code, ...)
 	set waitfor = FALSE
 
+	if(event_code == TGS_EVENT_WATCHDOG_DETACH)
+		// hack hack, calling world.TgsChatChannelInfo() will try to delay until the channels come back
+		var/datum/tgs_api/v5/api = TGS_READ_GLOBAL(tgs)
+		if(length(api.chat_channels))
+			FailTest("Expected no chat channels after detach!")
+
 	world.TgsChatBroadcast(new /datum/tgs_message_content("Recieved event: `[json_encode(args)]`"))
 
 /world/Export(url)
@@ -276,7 +271,6 @@ var/lastTgsError
 	// this actually gets doubled because it's in two fields for backwards compatibility, but that's fine
 	var/list/final_result = api.Bridge(0, list("chatMessage" = list("text" = "done:[create_payload(limit * 3)]")))
 	if(!final_result || lastTgsError || final_result["integrationHack"] != "ok")
-		text2file("Failed to end bridge limit test! [(istype(final_result) ? json_encode(final_result): (final_result || "null"))]", "test_fail_reason.txt")
-		del(world)
+		FailTest("Failed to end bridge limit test! [(istype(final_result) ? json_encode(final_result): (final_result || "null"))]")
 
 	api.access_identifier = old_ai

tests/DMAPI/LongRunning/long_running_test.dme

@@ -12,6 +12,6 @@
 
 // BEGIN_INCLUDE
 #include "Config.dm"
-#include "..\tgs_include.dm"
+#include "../test_prelude.dm"
 #include "Test.dm"
 // END_INCLUDE

tests/DMAPI/LongRunning/long_running_test_copy.dme

@@ -11,7 +11,5 @@
 // END_PREFERENCES
 
 // BEGIN_INCLUDE
-#include "Config.dm"
-#include "..\tgs_include.dm"
-#include "Test.dm"
+#include "long_running_test.dme"
 // END_INCLUDE

tests/DMAPI/test_prelude.dm

@@ -1,2 +1,3 @@
 #include "..\..\src\DMAPI\tgs.dm"
 #include "..\..\src\DMAPI\tgs\includes.dm"
+#include "test_setup.dm"

tests/DMAPI/test_setup.dm

@@ -0,0 +1,32 @@
+/world/New()
+	text2file("SUCCESS", "test_success.txt")
+	world.RunTest()
+
+/world/Error(exception/E, datum/e_src)
+	var/list/usrinfo = null
+	var/list/splitlines = splittext(E.desc, "\n")
+	var/list/desclines = list()
+	for(var/line in splitlines)
+		if(length(line) < 3 || findtext(line, "source file:") || findtext(line, "usr.loc:"))
+			continue
+		if(findtext(line, "usr:"))
+			if(usrinfo)
+				desclines.Add(usrinfo)
+				usrinfo = null
+			continue // Our usr info is better, replace it
+
+		if(copytext(line, 1, 3) != "  ")//3 == length("  ") + 1
+			desclines += ("  " + line) // Pad any unpadded lines, so they look pretty
+		else
+			desclines += line
+
+	if(usrinfo) //If this info isn't null, it hasn't been added yet
+		desclines.Add(usrinfo)
+
+	FailTest("Runtime Error: [E]")
+
+/proc/FailTest(reason)
+	world.log << "TEST ERROR DM-SIDE: [reason]"
+	fdel("test_success.txt")
+	text2file(reason, "test_fail_reason.txt")
+	del(world)

tests/Tgstation.Server.Host.Tests/Components/Chat/Providers/TestDiscordProvider.cs

@@ -7,9 +7,11 @@ using Microsoft.Extensions.Logging;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 using Moq;
 
+using Tgstation.Server.Api.Models;
 using Tgstation.Server.Host.Jobs;
 using Tgstation.Server.Host.Models;
 using Tgstation.Server.Host.System;
+using Tgstation.Server.Host.Utils;
 
 namespace Tgstation.Server.Host.Components.Chat.Providers.Tests
 {
@@ -44,16 +46,21 @@ namespace Tgstation.Server.Host.Components.Chat.Providers.Tests
 		[TestMethod]
 		public async Task TestConstructionAndDisposal()
 		{
-			if (testToken1 == null)
-				Assert.Inconclusive("Required environment variable TGS_TEST_DISCORD_TOKEN isn't set!");
+			var bot = new ChatBot
+			{
+				ConnectionString = "fake_token",
+				ReconnectionInterval = 1,
+			};
 
-			Assert.ThrowsException<ArgumentNullException>(() => new DiscordProvider(null, null, null, null));
-			Assert.ThrowsException<ArgumentNullException>(() => new DiscordProvider(mockJobManager, null, null, null));
-			var mockAss = new Mock<IAssemblyInformationProvider>();
-			Assert.ThrowsException<ArgumentNullException>(() => new DiscordProvider(mockJobManager, mockAss.Object, null, null));
-			var mockLogger = new Mock<ILogger<DiscordProvider>>();
-			Assert.ThrowsException<ArgumentNullException>(() => new DiscordProvider(mockJobManager, null, mockLogger.Object, null));
-			await new DiscordProvider(mockJobManager, mockAss.Object, mockLogger.Object, testToken1).DisposeAsync();
+			Assert.ThrowsException<ArgumentNullException>(() => new DiscordProvider(null, null, null, null, null));
+			Assert.ThrowsException<ArgumentNullException>(() => new DiscordProvider(mockJobManager, null, null, null, null));
+			var mockDel = Mock.Of<IAsyncDelayer>();
+			Assert.ThrowsException<ArgumentNullException>(() => new DiscordProvider(mockJobManager, mockDel, null, null, null));
+			var mockLogger = Mock.Of<ILogger<DiscordProvider>>();
+			Assert.ThrowsException<ArgumentNullException>(() => new DiscordProvider(mockJobManager, mockDel, mockLogger, null, null));
+			var mockAss = Mock.Of<IAssemblyInformationProvider>();
+			Assert.ThrowsException<ArgumentNullException>(() => new DiscordProvider(mockJobManager, mockDel, mockLogger, mockAss, null));
+			await new DiscordProvider(mockJobManager, mockDel, mockLogger, mockAss, bot).DisposeAsync();
 		}
 
 		static Task InvokeConnect(IProvider provider, CancellationToken cancellationToken = default) => (Task)provider.GetType().GetMethod("Connect", BindingFlags.Instance | BindingFlags.NonPublic).Invoke(provider, new object[] { cancellationToken });
@@ -62,7 +69,7 @@ namespace Tgstation.Server.Host.Components.Chat.Providers.Tests
 		public async Task TestConnectWithFakeTokenFails()
 		{
 			var mockLogger = new Mock<ILogger<DiscordProvider>>();
-			await using var provider = new DiscordProvider(mockJobManager, Mock.Of<IAssemblyInformationProvider>(), mockLogger.Object, new ChatBot
+			await using var provider = new DiscordProvider(mockJobManager, Mock.Of<IAsyncDelayer>(), mockLogger.Object, Mock.Of<IAssemblyInformationProvider>(), new ChatBot
 			{
 				ReconnectionInterval = 1,
 				ConnectionString = "asdf"
@@ -77,8 +84,11 @@ namespace Tgstation.Server.Host.Components.Chat.Providers.Tests
 			if (testToken1 == null)
 				Assert.Inconclusive("Required environment variable TGS_TEST_DISCORD_TOKEN isn't set!");
 
+			if (!new DiscordConnectionStringBuilder(testToken1.ConnectionString).Valid)
+				Assert.Fail("TGS_TEST_DISCORD_TOKEN is not a valid Discord connection string!");
+
 			var mockLogger = new Mock<ILogger<DiscordProvider>>();
-			await using var provider = new DiscordProvider(mockJobManager, Mock.Of<IAssemblyInformationProvider>(), mockLogger.Object, testToken1);
+			await using var provider = new DiscordProvider(mockJobManager, Mock.Of<IAsyncDelayer>(), mockLogger.Object, Mock.Of<IAssemblyInformationProvider>(), testToken1);
 			Assert.IsFalse(provider.Connected);
 			await InvokeConnect(provider);
 			Assert.IsTrue(provider.Connected);

tests/Tgstation.Server.Host.Tests/Components/Chat/Providers/TestIrcProvider.cs

@@ -1,8 +1,14 @@
+using System;
+using System.Reflection;
+using System.Threading;
+using System.Threading.Tasks;
+using System.Xml.Linq;
+
 using Microsoft.Extensions.Logging;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 using Moq;
-using System;
-using System.Threading.Tasks;
+using Serilog.Parsing;
+
 using Tgstation.Server.Api.Models;
 using Tgstation.Server.Host.Jobs;
 using Tgstation.Server.Host.Models;
@@ -20,12 +26,12 @@ namespace Tgstation.Server.Host.Components.Chat.Providers.Tests
 			Assert.ThrowsException<ArgumentNullException>(() => new IrcProvider(null, null, null, null, null));
 			var mockJobManager = new Mock<IJobManager>();
 			Assert.ThrowsException<ArgumentNullException>(() => new IrcProvider(mockJobManager.Object, null, null, null, null));
-			var mockAss = new Mock<IAssemblyInformationProvider>();
-			Assert.ThrowsException<ArgumentNullException>(() => new IrcProvider(mockJobManager.Object, mockAss.Object, null, null, null));
 			var mockAsyncDelayer = new Mock<IAsyncDelayer>();
-			Assert.ThrowsException<ArgumentNullException>(() => new IrcProvider(mockJobManager.Object, mockAss.Object, mockAsyncDelayer.Object, null, null));
+			Assert.ThrowsException<ArgumentNullException>(() => new IrcProvider(mockJobManager.Object, mockAsyncDelayer.Object, null, null, null));
 			var mockLogger = new Mock<ILogger<IrcProvider>>();
-			Assert.ThrowsException<ArgumentNullException>(() => new IrcProvider(mockJobManager.Object, mockAss.Object, mockAsyncDelayer.Object, mockLogger.Object, null));
+			Assert.ThrowsException<ArgumentNullException>(() => new IrcProvider(mockJobManager.Object, mockAsyncDelayer.Object, mockLogger.Object, null, null));
+			var mockAss = new Mock<IAssemblyInformationProvider>();
+			Assert.ThrowsException<ArgumentNullException>(() => new IrcProvider(mockJobManager.Object, mockAsyncDelayer.Object, mockLogger.Object, mockAss.Object, null));
 
 			var mockBot = new ChatBot
 			{
@@ -33,7 +39,7 @@ namespace Tgstation.Server.Host.Components.Chat.Providers.Tests
 				Provider = ChatProvider.Irc
 			};
 
-			Assert.ThrowsException<InvalidOperationException>(() => new IrcProvider(mockJobManager.Object, mockAss.Object, mockAsyncDelayer.Object, mockLogger.Object, mockBot));
+			Assert.ThrowsException<InvalidOperationException>(() => new IrcProvider(mockJobManager.Object, mockAsyncDelayer.Object, mockLogger.Object, mockAss.Object, mockBot));
 
 			mockBot.ConnectionString = new IrcConnectionStringBuilder
 			{
@@ -43,7 +49,46 @@ namespace Tgstation.Server.Host.Components.Chat.Providers.Tests
 				Port = 6667
 			}.ToString();
 
-			await new IrcProvider(mockJobManager.Object, mockAss.Object, mockAsyncDelayer.Object, mockLogger.Object, mockBot).DisposeAsync();
+			await new IrcProvider(mockJobManager.Object, mockAsyncDelayer.Object, mockLogger.Object, mockAss.Object, mockBot).DisposeAsync();
+		}
+
+		static Task InvokeConnect(IProvider provider, CancellationToken cancellationToken = default) => (Task)provider.GetType().GetMethod("Connect", BindingFlags.Instance | BindingFlags.NonPublic).Invoke(provider, new object[] { cancellationToken });
+
+		[TestMethod]
+		public async Task TestConnectAndDisconnect()
+		{
+			var actualToken = Environment.GetEnvironmentVariable("TGS_TEST_IRC_CONNECTION_STRING");
+			if (actualToken == null)
+				Assert.Inconclusive("Required environment variable TGS_TEST_IRC_CONNECTION_STRING isn't set!");
+
+			if (!new IrcConnectionStringBuilder(actualToken).Valid)
+				Assert.Fail("TGS_TEST_IRC_CONNECTION_STRING is not a valid IRC connection string!");
+
+			using var loggerFactory = LoggerFactory.Create(builder =>
+			{
+				builder.AddConsole();
+				builder.SetMinimumLevel(LogLevel.Trace);
+			});
+			var mockSetup = new Mock<IJobManager>();
+			mockSetup
+				.Setup(x => x.RegisterOperation(It.IsNotNull<Job>(), It.IsNotNull<JobEntrypoint>(), It.IsAny<CancellationToken>()))
+				.Callback<Job, JobEntrypoint, CancellationToken>((job, entrypoint, cancellationToken) => job.StartedBy ??= new User { })
+				.Returns(Task.CompletedTask);
+			mockSetup
+				.Setup(x => x.WaitForJobCompletion(It.IsNotNull<Job>(), It.IsAny<User>(), It.IsAny<CancellationToken>(), It.IsAny<CancellationToken>()))
+				.Returns(Task.CompletedTask);
+			var mockJobManager = mockSetup.Object;
+			await using var provider = new IrcProvider(mockJobManager, new AsyncDelayer(), loggerFactory.CreateLogger<IrcProvider>(), Mock.Of<IAssemblyInformationProvider>(), new ChatBot
+			{
+				ConnectionString = actualToken,
+				Provider = ChatProvider.Irc,
+			});
+			Assert.IsFalse(provider.Connected);
+			await InvokeConnect(provider);
+			Assert.IsTrue(provider.Connected);
+
+			await provider.Disconnect(default);
+			Assert.IsFalse(provider.Connected);
 		}
 	}
 }

tests/Tgstation.Server.Tests/Live/DummyChatProvider.cs

@@ -0,0 +1,303 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+
+using Microsoft.Extensions.Logging;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+using Moq;
+
+using Tgstation.Server.Host.Components.Chat;
+using Tgstation.Server.Host.Components.Chat.Commands;
+using Tgstation.Server.Host.Components.Chat.Providers;
+using Tgstation.Server.Host.Components.Interop;
+using Tgstation.Server.Host.Jobs;
+using Tgstation.Server.Host.Models;
+using Tgstation.Server.Host.Security;
+using Tgstation.Server.Host.Utils;
+
+namespace Tgstation.Server.Tests.Live
+{
+	sealed class DummyChatProvider : Provider
+	{
+		public override bool Connected => connected;
+
+		public override string BotMention => $"Dummy{ChatBot.Provider}-I-{ChatBot.InstanceId}-N-{ChatBot.Name}";
+
+		static int enableRandomDisconnections = 1;
+
+		readonly Random random; // this RNG isn't perfect as calls into this class can theoretically happen in a random order due to async
+
+		readonly IReadOnlyCollection<ICommand> commands;
+		readonly ICryptographySuite cryptographySuite;
+		readonly CancellationTokenSource randomMessageCts;
+		readonly Task randomMessageTask;
+
+		bool connectedOnce;
+		bool connected;
+
+		ulong channelIdAllocator;
+
+		static ILoggerFactory CreateLoggerFactoryForLogger(ILogger logger, out Mock<ILoggerFactory> mockLoggerFactory)
+		{
+			mockLoggerFactory = new Mock<ILoggerFactory>();
+			mockLoggerFactory.Setup(x => x.CreateLogger(It.IsAny<string>())).Returns(() =>
+			{
+				var temp = logger;
+				logger = null;
+
+				Assert.IsNotNull(temp);
+				return temp;
+			})
+			.Verifiable();
+			return mockLoggerFactory.Object;
+		}
+
+		static IAsyncDelayer CreateMockDelayer()
+		{
+			// at time of writing, this is used exclusively for the reconnection interval which works in minutes
+			// shorten it to 3s
+			var mock = new Mock<IAsyncDelayer>();
+			mock.Setup(x => x.Delay(It.IsAny<TimeSpan>(), It.IsAny<CancellationToken>())).Returns<TimeSpan, CancellationToken>((delay, cancellationToken) => Task.Delay(TimeSpan.FromSeconds(3), cancellationToken));
+			return mock.Object;
+		}
+		public static async Task RandomDisconnections(bool enabled, CancellationToken cancellationToken)
+		{
+			if (Interlocked.Exchange(ref enableRandomDisconnections, enabled ? 1 : 0) != 0 && !enabled)
+				await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken);
+		}
+
+		public DummyChatProvider(
+			IJobManager jobManager,
+			ILogger logger,
+			ChatBot chatBot,
+			ICryptographySuite cryptographySuite,
+			IReadOnlyCollection<ICommand> commands,
+			Random random)
+			: base(jobManager, CreateMockDelayer(), new Logger<DummyChatProvider>(CreateLoggerFactoryForLogger(logger, out var mockLoggerFactory)), chatBot)
+		{
+			mockLoggerFactory.VerifyAll();
+			this.cryptographySuite = cryptographySuite ?? throw new ArgumentNullException(nameof(cryptographySuite));
+			this.commands = commands ?? throw new ArgumentNullException(nameof(commands));
+			this.random = random ?? throw new ArgumentNullException(nameof(random));
+
+			// this could be random but there's no point
+			channelIdAllocator = 100000;
+			logger.LogTrace("Base channel ID {baseChannelId}", channelIdAllocator);
+
+			this.randomMessageCts = new CancellationTokenSource();
+			this.randomMessageTask = RandomMessageLoop(this.randomMessageCts.Token);
+		}
+
+		public override async ValueTask DisposeAsync()
+		{
+			Logger.LogTrace("DisposeAsync Child");
+			this.randomMessageCts.Cancel();
+			this.randomMessageCts.Dispose();
+			await this.randomMessageTask;
+			await base.DisposeAsync();
+		}
+
+		public override Task SendMessage(Message replyTo, MessageContent message, ulong channelId, CancellationToken cancellationToken)
+		{
+			if (message == null)
+				throw new ArgumentNullException(nameof(message));
+
+			Logger.LogTrace("SendMessage");
+
+			Assert.AreNotEqual(0UL, channelId);
+			Assert.IsTrue(channelId <= channelIdAllocator);
+
+			cancellationToken.ThrowIfCancellationRequested();
+
+			/* SendMessage is no-throw
+			if (random.Next(0, 100) > 70)
+				throw new Exception("Random SendMessage failure!"); */
+
+			return Task.CompletedTask;
+		}
+
+		public override Task<Func<string, string, Task>> SendUpdateMessage(RevisionInformation revisionInformation, Version byondVersion, DateTimeOffset? estimatedCompletionTime, string gitHubOwner, string gitHubRepo, ulong channelId, bool localCommitPushed, CancellationToken cancellationToken)
+		{
+			if (revisionInformation == null)
+				throw new ArgumentNullException(nameof(revisionInformation));
+			if (byondVersion == null)
+				throw new ArgumentNullException(nameof(byondVersion));
+			if (gitHubOwner == null)
+				throw new ArgumentNullException(nameof(gitHubOwner));
+			if (gitHubRepo == null)
+				throw new ArgumentNullException(nameof(gitHubRepo));
+
+			Logger.LogTrace("SendUpdateMessage");
+
+			Assert.AreNotEqual(0UL, channelId);
+			Assert.IsTrue(channelId <= channelIdAllocator);
+
+			cancellationToken.ThrowIfCancellationRequested();
+
+			/* SendUpdateMessage is no-throw
+			if (random.Next(0, 100) > 70)
+				throw new Exception("Random SendUpdateMessage failure!"); */
+
+			return Task.FromResult<Func<string, string, Task>>((_, _) =>
+			{
+				cancellationToken.ThrowIfCancellationRequested();
+
+				/* SendUpdateMessage callbacks are no-throw
+				if (random.Next(0, 100) > 70)
+					throw new Exception("Random SendUpdateMessage failure!"); */
+
+				return Task.CompletedTask;
+			});
+		}
+
+		protected override Task Connect(CancellationToken cancellationToken)
+		{
+			Logger.LogTrace("Connect");
+			cancellationToken.ThrowIfCancellationRequested();
+
+			// 30% chance to fail AFTER initial connection
+			if (connectedOnce && enableRandomDisconnections != 0 && random.Next(0, 100) > 70)
+				throw new Exception("Random connection failure!");
+
+			connected = true;
+			connectedOnce = true;
+			return Task.CompletedTask;
+		}
+
+		protected override Task DisconnectImpl(CancellationToken cancellationToken)
+		{
+			Logger.LogTrace("DisconnectImpl");
+			cancellationToken.ThrowIfCancellationRequested();
+			connected = false;
+
+			if (random.Next(0, 100) > 70)
+				throw new Exception("Random disconnection failure!");
+			return Task.CompletedTask;
+		}
+
+		protected override Task<Dictionary<ChatChannel, IEnumerable<ChannelRepresentation>>> MapChannelsImpl(IEnumerable<ChatChannel> channels, CancellationToken cancellationToken)
+		{
+			channels = channels.ToList();
+			Logger.LogTrace("MapChannels: [{channels}]", String.Join(", ", channels.Select(channel => channel.IrcChannel ?? channel.DiscordChannelId?.ToString() ?? throw new InvalidOperationException("BAD CHANNEL"))));
+
+			cancellationToken.ThrowIfCancellationRequested();
+
+			/* MapChannelsImpl is no-throw
+			if (random.Next(0, 100) > 70)
+				throw new Exception("Random MapChannelsImpl failure!"); */
+
+			return Task.FromResult(
+				new Dictionary<ChatChannel, IEnumerable<ChannelRepresentation>>(
+					channels.Select(
+						channel => new KeyValuePair<ChatChannel, IEnumerable<ChannelRepresentation>>(
+							channel,
+							new List<ChannelRepresentation>
+							{
+								new ChannelRepresentation
+								{
+									IsAdminChannel = channel.IsAdminChannel.Value,
+									ConnectionName = $"Connection_{channelIdAllocator + 1}",
+									EmbedsSupported = ChatBot.Provider.Value != Api.Models.ChatProvider.Irc,
+									FriendlyName = $"(Friendly) Channel_ID_{channelIdAllocator + 1}",
+									IsPrivateChannel = false,
+									RealId = ++channelIdAllocator,
+									Tag = channel.Tag,
+								}
+							}))));
+		}
+
+		async Task RandomMessageLoop(CancellationToken cancellationToken)
+		{
+			Logger.LogTrace("RandomMessageLoop");
+			try
+			{
+				for (var i = 0UL; !cancellationToken.IsCancellationRequested; ++i)
+				{
+					// random intervals under 10s
+					var delay = random.Next(0, 10000);
+					await Task.Delay(delay, cancellationToken);
+
+					if (!connected)
+						continue;
+
+					// %5 chance to disconnect randomly
+					if (enableRandomDisconnections != 0 && random.Next(0, 100) > 95)
+						connected = false;
+
+					if (channelIdAllocator >= Int32.MaxValue / 2)
+						Assert.Fail("Too many channels have been allocated!");
+
+					var isPm = channelIdAllocator == 0 || random.Next(0, 100) > 20;
+					var realId = (ulong)random.Next(1, (int)channelIdAllocator);
+
+					if (isPm)
+						realId += Int32.MaxValue / 2;
+
+					var username = $"RandomUser{i}";
+					var sender = new ChatUser
+					{
+						Channel = new ChannelRepresentation
+						{
+							RealId = realId,
+							IsPrivateChannel = isPm,
+							ConnectionName = isPm ? $"{username}_Connection" : $"Connection_{realId}",
+							FriendlyName = isPm ? $"{username}_Channel" : $"(Friendly) Channel_ID_{realId}",
+							EmbedsSupported = ChatBot.Provider.Value != Api.Models.ChatProvider.Irc,
+
+							// isAdmin and Tag populated by manager
+						},
+						FriendlyName = username,
+						RealId = i + 50000,
+						Mention = $"@{username}",
+					};
+
+					var dice = random.Next(0, 100);
+					string content;
+					// 70% chance to be random chat
+					if (dice < 70)
+						content = cryptographySuite.GetSecureString();
+					// 15% chance to be a !tgs
+					else if (dice < 85)
+						content = "!tgs";
+					// 15% chance to be a strict mention
+					else
+						content = BotMention;
+
+					// 30% chance to request help
+					if (random.Next(0, 100) > 70)
+						content = $"{content} help";
+
+					dice = random.Next(0, 100);
+
+					// 20% chance to whiff
+					if (dice > 20)
+						// 40% chance to attempt a built-in TGS command
+						if (dice < 68)
+							// equal chance for each
+							content = $"{content} {commands.ElementAt(random.Next(0, commands.Count)).Name}";
+						// 40% chance to attempt a custom chat command in long_running_test
+						else
+							// equal chance for each
+							if (random.Next(0, 100) > 50)
+								content = $"{content} embeds_test";
+							else
+								content = $"{content} response_overload_test";
+
+					EnqueueMessage(new Message
+					{
+						Content = content,
+						User = sender,
+					});
+				}
+
+			}
+			catch (OperationCanceledException)
+			{
+				Logger.LogTrace("RandomMessageLoop cancelled");
+			}
+		}
+	}
+}

tests/Tgstation.Server.Tests/Live/DummyChatProviderFactory.cs

@@ -0,0 +1,84 @@
+using System;
+using System.Collections.Generic;
+
+using Microsoft.Extensions.Logging;
+
+using Moq;
+
+using Tgstation.Server.Api.Models;
+using Tgstation.Server.Host.Components.Byond;
+using Tgstation.Server.Host.Components.Chat.Commands;
+using Tgstation.Server.Host.Components.Chat.Providers;
+using Tgstation.Server.Host.Components.Repository;
+using Tgstation.Server.Host.Components.Watchdog;
+using Tgstation.Server.Host.Database;
+using Tgstation.Server.Host.Jobs;
+using Tgstation.Server.Host.Models;
+using Tgstation.Server.Host.Security;
+using Tgstation.Server.Host.System;
+
+namespace Tgstation.Server.Tests.Live
+{
+	sealed class DummyChatProviderFactory : IProviderFactory
+	{
+		readonly IJobManager jobManager;
+		readonly ICryptographySuite cryptographySuite;
+		readonly ILoggerFactory loggerFactory;
+		readonly ILogger<DummyChatProviderFactory> logger;
+
+		readonly IReadOnlyList<ICommand> commands;
+
+		readonly Dictionary<ChatProvider, Random> seededRng;
+
+		public DummyChatProviderFactory(IJobManager jobManager, ICryptographySuite cryptographySuite, ILoggerFactory loggerFactory, ILogger<DummyChatProviderFactory> logger)
+		{
+			this.jobManager = jobManager ?? throw new ArgumentNullException(nameof(jobManager));
+			this.cryptographySuite = cryptographySuite ?? throw new ArgumentNullException(nameof(cryptographySuite));
+			this.loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory));
+			this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
+
+			var commandFactory = new CommandFactory(
+				Mock.Of<IAssemblyInformationProvider>(),
+				Mock.Of<IByondManager>(),
+				Mock.Of<IRepositoryManager>(),
+				Mock.Of<IDatabaseContextFactory>(),
+				new Host.Models.Instance());
+
+			commandFactory.SetWatchdog(Mock.Of<IWatchdog>());
+			commands = commandFactory.GenerateCommands();
+
+			var baseRng = new Random(22475);
+			seededRng = new Dictionary<ChatProvider, Random>{
+				{ ChatProvider.Irc, new Random(baseRng.Next()) },
+				{ ChatProvider.Discord, new Random(baseRng.Next()) },
+			}; // hope you get the reference
+		}
+
+		public IProvider CreateProvider(ChatBot settings)
+		{
+			logger.LogTrace("CreateProvider");
+			if (settings == null)
+				throw new ArgumentNullException(nameof(settings));
+
+			var provider = settings.Provider;
+			switch (provider)
+			{
+				case ChatProvider.Irc:
+				case ChatProvider.Discord:
+					logger.LogTrace("Creating DummyChatProvider in place of requested {providerType}Provider", settings.Provider);
+
+					// for RNG to work, chat bots need to get created in a certain order
+					// the ChatTest creates one of each provider type
+					return new DummyChatProvider(
+						jobManager,
+						loggerFactory.CreateLogger($"Dummy{settings.Provider}Provider"),
+						settings,
+						cryptographySuite,
+						commands,
+						new Random(seededRng[provider.Value].Next()));
+				default:
+					throw new InvalidOperationException($"Invalid ChatProvider: {provider}");
+			}
+		}
+	}
+}

tests/Tgstation.Server.Tests/Live/Instance/ChatTest.cs

@@ -13,13 +13,14 @@ using Tgstation.Server.Client.Components;
 
 namespace Tgstation.Server.Tests.Live.Instance
 {
-	sealed class ChatTest
+	sealed class ChatTest : JobsRequiredTest
 	{
 		readonly IChatBotsClient chatClient;
 		readonly IInstanceManagerClient instanceClient;
 		readonly Api.Models.Instance metadata;
 
-		public ChatTest(IChatBotsClient chatClient, IInstanceManagerClient instanceClient, Api.Models.Instance metadata)
+		public ChatTest(IChatBotsClient chatClient, IInstanceManagerClient instanceClient, IJobsClient jobsClient, Api.Models.Instance metadata)
+			: base(jobsClient)
 		{
 			this.chatClient = chatClient ?? throw new ArgumentNullException(nameof(chatClient));
 			this.instanceClient = instanceClient ?? throw new ArgumentNullException(nameof(instanceClient));
@@ -40,9 +41,25 @@ namespace Tgstation.Server.Tests.Live.Instance
 
 		async Task RunIrc(CancellationToken cancellationToken)
 		{
+			var connectionString = Environment.GetEnvironmentVariable("TGS_TEST_IRC_CONNECTION_STRING");
+			if (String.IsNullOrWhiteSpace(connectionString))
+				// needs to just be valid
+				connectionString = new IrcConnectionStringBuilder
+				{
+					Address = "irc.fake.com",
+					Nickname = "irc_nick",
+					Password = "some_pw",
+					PasswordType = IrcPasswordType.Server,
+					Port = 6668,
+					UseSsl = true,
+				}.ToString();
+			else
+				// standardize
+				connectionString = new IrcConnectionStringBuilder(connectionString).ToString();
+
 			var firstBotReq = new ChatBotCreateRequest
 			{
-				ConnectionString = Environment.GetEnvironmentVariable("TGS_TEST_IRC_CONNECTION_STRING"),
+				ConnectionString = connectionString,
 				Enabled = false,
 				Name = "tgs_integration_test",
 				Provider = ChatProvider.Irc,
@@ -65,6 +82,8 @@ namespace Tgstation.Server.Tests.Live.Instance
 			var retrievedBot = await chatClient.GetId(firstBot, cancellationToken);
 			Assert.AreEqual(firstBot.Id, retrievedBot.Id);
 
+			var beforeChatBotEnabled = DateTimeOffset.UtcNow;
+
 			var updatedBot = await chatClient.Update(new ChatBotUpdateRequest
 			{
 				Id = firstBot.Id,
@@ -73,7 +92,20 @@ namespace Tgstation.Server.Tests.Live.Instance
 
 			Assert.AreEqual(true, updatedBot.Enabled);
 
-			var channelId = Environment.GetEnvironmentVariable("TGS_TEST_IRC_CHANNEL"); ;
+			await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken);
+
+			var jobs = await JobsClient.List(null, cancellationToken);
+			var reconnectJob = jobs
+				.Where(x => x.StartedAt >= beforeChatBotEnabled && x.Description.Contains(updatedBot.Name))
+				.OrderByDescending(x => x.StartedAt)
+				.FirstOrDefault();
+
+			Assert.IsNotNull(reconnectJob);
+			await WaitForJob(reconnectJob, 60, false, null, cancellationToken);
+
+			var channelId = Environment.GetEnvironmentVariable("TGS_TEST_IRC_CHANNEL");
+			if (String.IsNullOrWhiteSpace(channelId))
+				channelId = "#botbus";
 
 			updatedBot = await chatClient.Update(new ChatBotUpdateRequest
 			{
@@ -110,14 +142,23 @@ namespace Tgstation.Server.Tests.Live.Instance
 
 		async Task RunDiscord(CancellationToken cancellationToken)
 		{
+			var connectionString = Environment.GetEnvironmentVariable("TGS_TEST_DISCORD_TOKEN");
+			if (String.IsNullOrWhiteSpace(connectionString))
+				// needs to just be valid
+				connectionString = new DiscordConnectionStringBuilder
+				{
+					BasedMeme = true,
+					BotToken = "some_token",
+					DeploymentBranding = true,
+					DMOutputDisplay = DiscordDMOutputDisplayType.Never,
+				}.ToString();
+			else
+				// standardize
+				connectionString = new DiscordConnectionStringBuilder(connectionString).ToString();
+
 			var firstBotReq = new ChatBotCreateRequest
 			{
-				ConnectionString =
-					new DiscordConnectionStringBuilder
-					{
-						BotToken = Environment.GetEnvironmentVariable("TGS_TEST_DISCORD_TOKEN"),
-						DMOutputDisplay = DiscordDMOutputDisplayType.OnError
-					}.ToString(),
+				ConnectionString = connectionString,
 				Enabled = false,
 				Name = "r4407",
 				Provider = ChatProvider.Discord,
@@ -140,6 +181,8 @@ namespace Tgstation.Server.Tests.Live.Instance
 			var retrievedBot = await chatClient.GetId(firstBot, cancellationToken);
 			Assert.AreEqual(firstBot.Id, retrievedBot.Id);
 
+			var beforeChatBotEnabled = DateTimeOffset.UtcNow;
+
 			var updatedBot = await chatClient.Update(new ChatBotUpdateRequest
 			{
 				Id = firstBot.Id,
@@ -148,7 +191,22 @@ namespace Tgstation.Server.Tests.Live.Instance
 
 			Assert.AreEqual(true, updatedBot.Enabled);
 
-			var channelId = ulong.Parse(Environment.GetEnvironmentVariable("TGS_TEST_DISCORD_CHANNEL"));
+			await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken);
+
+			var jobs = await JobsClient.List(null, cancellationToken);
+			var reconnectJob = jobs
+				.Where(x => x.StartedAt >= beforeChatBotEnabled && x.Description.Contains(updatedBot.Name))
+				.OrderByDescending(x => x.StartedAt)
+				.FirstOrDefault();
+
+			Assert.IsNotNull(reconnectJob);
+			await WaitForJob(reconnectJob, 60, false, null, cancellationToken);
+
+			var channelIdStr = Environment.GetEnvironmentVariable("TGS_TEST_DISCORD_CHANNEL");
+			if (String.IsNullOrWhiteSpace(channelIdStr))
+				channelIdStr = "487268744419344384";
+
+			var channelId = ulong.Parse(channelIdStr);
 
 			updatedBot = await chatClient.Update(new ChatBotUpdateRequest
 			{

tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs

@@ -26,7 +26,7 @@ namespace Tgstation.Server.Tests.Live.Instance
 		public async Task RunTests(CancellationToken cancellationToken)
 		{
 			var byondTest = new ByondTest(instanceClient.Byond, instanceClient.Jobs, instanceClient.Metadata);
-			var chatTest = new ChatTest(instanceClient.ChatBots, instanceManagerClient, instanceClient.Metadata);
+			var chatTest = new ChatTest(instanceClient.ChatBots, instanceManagerClient, instanceClient.Jobs, instanceClient.Metadata);
 			var configTest = new ConfigurationTest(instanceClient.Configuration, instanceClient.Metadata);
 			var repoTest = new RepositoryTest(instanceClient.Repository, instanceClient.Jobs);
 			var dmTest = new DeploymentTest(instanceClient, instanceClient.Jobs);

tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs

@@ -79,6 +79,9 @@ namespace Tgstation.Server.Tests.Live.Instance
 
 			await TestDMApiFreeDeploy(cancellationToken);
 
+			// long running test likes consistency with the channels
+			await DummyChatProvider.RandomDisconnections(false, cancellationToken);
+
 			await RunLongRunningTestThenUpdate(cancellationToken);
 
 			await RunLongRunningTestThenUpdateWithNewDme(cancellationToken);
@@ -717,9 +720,8 @@ namespace Tgstation.Server.Tests.Live.Instance
 		async Task RunLongRunningTestThenUpdateWithNewDme(CancellationToken cancellationToken)
 		{
 			System.Console.WriteLine("TEST: WATCHDOG LONG RUNNING WITH NEW DME TEST");
-			const string DmeName = "LongRunning/long_running_test";
 
-			var daemonStatus = await DeployTestDme(DmeName, DreamDaemonSecurity.Trusted, true, cancellationToken);
+			var daemonStatus = await DeployTestDme("LongRunning/long_running_test", DreamDaemonSecurity.Trusted, true, cancellationToken);
 
 			var initialCompileJob = daemonStatus.ActiveCompileJob;
 			Assert.AreEqual(WatchdogStatus.Offline, daemonStatus.Status.Value);
@@ -732,7 +734,7 @@ namespace Tgstation.Server.Tests.Live.Instance
 
 			await WaitForJob(startJob, 40, false, null, cancellationToken);
 
-			daemonStatus = await DeployTestDme(DmeName + "_copy", DreamDaemonSecurity.Safe, true, cancellationToken);
+			daemonStatus = await DeployTestDme("LongRunning/long_running_test_copy", DreamDaemonSecurity.Safe, true, cancellationToken);
 
 			Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value);
 

tests/Tgstation.Server.Tests/Live/TestLiveServer.cs

@@ -44,7 +44,7 @@ namespace Tgstation.Server.Tests.Live
 		public static ushort DDPort { get; } = FreeTcpPort();
 		public static ushort DMPort { get; } = GetDMPort();
 
-		readonly Version TestUpdateVersion = new Version(5, 11, 0);
+		readonly Version TestUpdateVersion = new(5, 11, 0);
 
 		readonly IServerClientFactory clientFactory = new ServerClientFactory(new ProductHeaderValue(Assembly.GetExecutingAssembly().GetName().Name, Assembly.GetExecutingAssembly().GetName().Version.ToString()));
 
@@ -82,6 +82,7 @@ namespace Tgstation.Server.Tests.Live
 		[TestInitialize]
 		public async Task Initialize()
 		{
+			await DummyChatProvider.RandomDisconnections(true, default);
 			ServerClientFactory.ApiClientFactory = new RateLimitRetryingApiClientFactory();
 
 			var connectionString = Environment.GetEnvironmentVariable("TGS_TEST_CONNECTION_STRING");
@@ -697,6 +698,32 @@ namespace Tgstation.Server.Tests.Live
 		[TestMethod]
 		public async Task TestStandardTgsOperation()
 		{
+			var discordConnectionString = Environment.GetEnvironmentVariable("TGS_TEST_DISCORD_TOKEN");
+			var ircConnectionString = Environment.GetEnvironmentVariable("TGS_TEST_IRC_CONNECTION_STRING");
+			var missingChatVarsCount = Convert.ToInt32(String.IsNullOrWhiteSpace(discordConnectionString))
+				+ Convert.ToInt32(String.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("TGS_TEST_DISCORD_CHANNEL")))
+				+ Convert.ToInt32(String.IsNullOrWhiteSpace(ircConnectionString))
+				+ Convert.ToInt32(String.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("TGS_TEST_IRC_CHANNEL")));
+
+			const int TotalChatVars = 4;
+
+			// uncomment to force this test to run with DummyChatProviders
+			// missingChatVarsCount = TotalChatVars;
+
+			if (missingChatVarsCount != 0)
+			{
+				if (missingChatVarsCount != TotalChatVars)
+					Assert.Fail("All TGS_TEST_* chat environment variables must be present or none at all!");
+
+				ServiceCollectionExtensions.UseChatProviderFactory<DummyChatProviderFactory>();
+			}
+			else
+			{
+				// prevalidate
+				Assert.IsTrue(new DiscordConnectionStringBuilder(discordConnectionString).Valid);
+				Assert.IsTrue(new IrcConnectionStringBuilder(ircConnectionString).Valid);
+			}
+
 			var procs = System.Diagnostics.Process.GetProcessesByName("byond");
 			if (procs.Any())
 			{
@@ -732,6 +759,7 @@ namespace Tgstation.Server.Tests.Live
 						using var httpClient = new HttpClient();
 						var webRequestTask = httpClient.GetAsync(server.Url.ToString() + "swagger/v1/swagger.json");
 						using var response = await webRequestTask;
+						response.EnsureSuccessStatusCode();
 						using var content = await response.Content.ReadAsStreamAsync();
 						using var output = new FileStream(@"C:\swagger.json", FileMode.Create);
 						await content.CopyToAsync(output);
@@ -840,6 +868,26 @@ namespace Tgstation.Server.Tests.Live
 					var dd = await instanceClient.DreamDaemon.Read(cancellationToken);
 					Assert.AreEqual(WatchdogStatus.Online, dd.Status.Value);
 
+					var chatReadTask = instanceClient.ChatBots.List(null, cancellationToken);
+
+					// Check the DMAPI got the channels again https://github.com/tgstation/tgstation-server/issues/1490
+					topicRequestResult = await WatchdogTest.TopicClient.SendTopic(
+						IPAddress.Loopback,
+						$"tgs_integration_test_tactics7=1",
+						DDPort,
+						cancellationToken);
+
+					Assert.IsNotNull(topicRequestResult);
+					if(!Int32.TryParse(topicRequestResult.StringData, out var channelsPresent))
+					{
+						Assert.Fail("Expected DD to send us an int!");
+					}
+
+					var currentChatBots = await chatReadTask;
+					var connectedChannelCount = currentChatBots.Where(x => x.Enabled.Value).SelectMany(x => x.Channels).Count();
+
+					Assert.AreEqual(connectedChannelCount, channelsPresent);
+
 					await instanceClient.DreamDaemon.Shutdown(cancellationToken);
 					dd = await instanceClient.DreamDaemon.Update(new DreamDaemonRequest
 					{
@@ -948,7 +996,7 @@ namespace Tgstation.Server.Tests.Live
 					Assert.IsNull(currentDD.StagedCompileJob);
 
 					var repoTest = new RepositoryTest(instanceClient.Repository, instanceClient.Jobs).RunPostTest(cancellationToken);
-					await new ChatTest(instanceClient.ChatBots, adminClient.Instances, instance).RunPostTest(cancellationToken);
+					await new ChatTest(instanceClient.ChatBots, adminClient.Instances, instanceClient.Jobs, instance).RunPostTest(cancellationToken);
 					await repoTest;
 
 					await new InstanceManagerTest(adminClient, server.Directory).RunPostTest(cancellationToken);

tgstation-server.sln

@@ -93,7 +93,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{3BB10856-AA0
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{82066812-6C73-4360-943B-B23F2F491261}"
 	ProjectSection(SolutionItems) = preProject
-		tests\DMAPI\tgs_include.dm = tests\DMAPI\tgs_include.dm
+		tests\DMAPI\test_prelude.dm = tests\DMAPI\test_prelude.dm
+		tests\DMAPI\test_setup.dm = tests\DMAPI\test_setup.dm
 	EndProjectSection
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "v4", "v4", "{057FAC33-CC31-4948-91C6-B0977C335890}"
@@ -193,7 +194,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tgstation.Server.Migrator.C
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tgstation.Server.Host.Common", "src\Tgstation.Server.Host.Common\Tgstation.Server.Host.Common.csproj", "{CF3968A0-EA81-4464-B2D4-C7D40F6B5BCB}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tgstation.Server.Common", "src\Tgstation.Server.Common\Tgstation.Server.Common.csproj", "{70CD9A98-D31A-44A4-81D1-D02764CEEEFD}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tgstation.Server.Common", "src\Tgstation.Server.Common\Tgstation.Server.Common.csproj", "{70CD9A98-D31A-44A4-81D1-D02764CEEEFD}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution