Merge branch 'dev' of https://github.com/tgstation/tgstation-server into Nightlies

.github/workflows/ci-suite.yml

@@ -311,8 +311,6 @@ jobs:
     name: Windows Live Tests
     needs: dmapi-build
     if: "!(cancelled() || failure()) && needs.dmapi-build.result == 'success'"
-    env:
-      TGS_TEST_DUMP_API_SPEC: yes
     strategy:
       fail-fast: false
       matrix:
@@ -329,6 +327,10 @@ jobs:
     - name: Upgrade NPM
       run: npm install -g npm
 
+    - name: Set TGS_TEST_DUMP_API_SPEC
+      if: ${{ matrix.configuration == 'Release' && matrix.watchdog-type == 'System' && matrix.database-type == 'SqlServer' }}
+      run: echo "TGS_TEST_DUMP_API_SPEC=yes" >> $Env:GITHUB_ENV
+
     - name: Set General__UseBasicWatchdog
       if: ${{ matrix.watchdog-type == 'Basic' }}
       run: echo "General__UseBasicWatchdog=true" >> $Env:GITHUB_ENV

build/Version.props

@@ -9,7 +9,7 @@
     <TgsApiLibraryVersion>10.4.1</TgsApiLibraryVersion>
     <TgsClientVersion>11.4.2</TgsClientVersion>
     <TgsDmapiVersion>6.4.4</TgsDmapiVersion>
-    <TgsInteropVersion>5.6.0</TgsInteropVersion>
+    <TgsInteropVersion>5.6.1</TgsInteropVersion>
     <TgsHostWatchdogVersion>1.2.2</TgsHostWatchdogVersion>
     <TgsContainerScriptVersion>1.2.1</TgsContainerScriptVersion>
     <TgsMigratorVersion>1.0.1</TgsMigratorVersion>

src/DMAPI/tgs/v5/__interop_version.dm

@@ -1 +1 @@
-"5.6.0"
+"5.6.1"

src/Tgstation.Server.Host/Components/Chat/ChatManager.cs

@@ -662,10 +662,19 @@ namespace Tgstation.Server.Host.Components.Chat
 				// important, otherwise we could end up processing during shutdown
 				cancellationToken.ThrowIfCancellationRequested();
 
-				providerId = providers
+				var providerIdNullable = providers
 					.Where(x => x.Value == provider)
-					.Select(x => x.Key)
-					.First();
+					.Select(x => (long?)x.Key)
+					.FirstOrDefault();
+
+				if (!providerIdNullable.HasValue)
+				{
+					// possible to have a message queued and then the provider immediately disconnects
+					logger.LogDebug("Unable to process command \"{command}\" due to provider disconnecting", message.Content);
+					return;
+				}
+
+				providerId = providerIdNullable.Value;
 				mappedChannel = mappedChannels
 					.Where(x => x.Value.ProviderId == providerId && x.Value.ProviderChannelId == providerChannelId)
 					.Select(x => (KeyValuePair<ulong, ChannelMapping>?)x)
@@ -675,7 +684,7 @@ namespace Tgstation.Server.Host.Components.Chat
 					.Any();
 			}
 
-			if (!recursed && !mappedChannel.HasValue && hasChannelZero)
+			if (!recursed && !mappedChannel.HasValue && !message.User.Channel.IsPrivateChannel && hasChannelZero)
 			{
 				logger.LogInformation("Receieved message from unmapped channel whose provider contains ID 0. Remapping...");
 				await RemapProvider(provider, cancellationToken);
@@ -721,6 +730,9 @@ namespace Tgstation.Server.Host.Components.Chat
 						"Error mapping message: Provider ID: {providerId}, Channel Real ID: {realId}",
 						providerId,
 						message.User.Channel.RealId);
+					logger.LogTrace("message: {messageJson}", JsonConvert.SerializeObject(message));
+					lock (mappedChannels)
+						logger.LogTrace("mappedChannels: {mappedChannelsJson}", JsonConvert.SerializeObject(mappedChannels));
 					await SendMessage(
 						new List<ulong>
 						{
@@ -731,8 +743,7 @@ namespace Tgstation.Server.Host.Components.Chat
 						{
 							Text = "TGS: Processing error, check logs!",
 						},
-						cancellationToken)
-						;
+						cancellationToken);
 					return;
 				}
 

src/Tgstation.Server.Host/Components/Chat/Commands/ByondCommand.cs

@@ -61,7 +61,7 @@ namespace Tgstation.Server.Host.Components.Chat.Commands
 				});
 			return Task.FromResult(new MessageContent
 			{
-				Text = watchdog.ActiveCompileJob.ByondVersion,
+				Text = watchdog.ActiveCompileJob?.ByondVersion ?? "None!",
 			});
 		}
 	}

src/Tgstation.Server.Host/Components/Chat/Providers/DiscordProvider.cs

@@ -611,15 +611,18 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
 				{
 					await Task.WhenAny(gatewayReadyTcs.Task, localGatewayTask);
 
-					if (localGatewayTask.IsCompleted || cancellationToken.IsCancellationRequested)
+					cancellationToken.ThrowIfCancellationRequested();
+					if (localGatewayTask.IsCompleted)
 						throw new JobException(ErrorCode.ChatCannotConnectProvider);
 
 					var userClient = serviceProvider.GetRequiredService<IDiscordRestUserAPI>();
 
 					using var localCombinedCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, gatewayCancellationToken);
-					var currentUserResult = await userClient.GetCurrentUserAsync(localCombinedCts.Token);
+					var localCombinedCancellationToken = localCombinedCts.Token;
+					var currentUserResult = await userClient.GetCurrentUserAsync(localCombinedCancellationToken);
 					if (!currentUserResult.IsSuccess)
 					{
+						localCombinedCancellationToken.ThrowIfCancellationRequested();
 						Logger.LogWarning("Unable to retrieve current user: {result}", currentUserResult.LogFormat());
 						throw new JobException(ErrorCode.ChatCannotConnectProvider);
 					}

src/Tgstation.Server.Host/Components/Chat/Providers/IrcProvider.cs

@@ -455,11 +455,7 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
 
 				Logger.LogTrace("Connection established!");
 			}
-			catch (OperationCanceledException)
-			{
-				throw;
-			}
-			catch (Exception e)
+			catch (Exception e) when (e is not OperationCanceledException)
 			{
 				throw new JobException(ErrorCode.ChatCannotConnectProvider, e);
 			}

src/Tgstation.Server.Host/Components/Chat/Providers/Provider.cs

@@ -312,16 +312,18 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
 						await jobManager.WaitForJobCompletion(job, null, cancellationToken, default);
 					}
 				}
-				catch (OperationCanceledException)
+				catch (OperationCanceledException e)
 				{
-					break;
+					Logger.LogTrace(e, "ReconnectionLoop cancelled");
 				}
 				catch (Exception e)
 				{
 					Logger.LogError(e, "Error reconnecting!");
 				}
 			}
-			while (true);
+			while (!cancellationToken.IsCancellationRequested);
+
+			Logger.LogTrace("ReconnectionLoop exiting...");
 		}
 	}
 }

src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs

@@ -412,7 +412,20 @@ namespace Tgstation.Server.Host.Components.Deployment
 
 				var deleteTask = DeleteCompileJobContent(job.DirectoryName.ToString(), cleanupCts.Token);
 				var otherTask = cleanupTask;
-				await Task.WhenAll(otherTask, deleteTask, deploymentJob);
+
+				async Task WrapThrowableTasks()
+				{
+					try
+					{
+						await Task.WhenAll(deleteTask, deploymentJob);
+					}
+					catch (Exception ex)
+					{
+						logger.LogWarning(ex, "Error cleaning up compile job {jobGuid}!", job.DirectoryName);
+					}
+				}
+
+				await Task.WhenAll(otherTask, WrapThrowableTasks());
 			}
 
 			lock (jobLockCounts)

src/Tgstation.Server.Host/Components/Instance.cs

@@ -207,8 +207,7 @@ namespace Tgstation.Server.Host.Components
 					Configuration.StopAsync(cancellationToken),
 					ByondManager.StopAsync(cancellationToken),
 					Chat.StopAsync(cancellationToken),
-					dmbFactory.StopAsync(cancellationToken))
-					;
+					dmbFactory.StopAsync(cancellationToken));
 			}
 		}
 

src/Tgstation.Server.Host/Components/Interop/Topic/TopicParameters.cs

@@ -1,5 +1,7 @@
 using System;
 
+using Newtonsoft.Json;
+
 using Tgstation.Server.Host.Components.Session;
 
 namespace Tgstation.Server.Host.Components.Interop.Topic
@@ -55,6 +57,25 @@ namespace Tgstation.Server.Host.Components.Interop.Topic
 		/// </summary>
 		public ChunkData Chunk { get; }
 
+		/// <summary>
+		/// Whether or not the <see cref="TopicParameters"/> constitute a priority request.
+		/// </summary>
+		[JsonIgnore]
+		public bool IsPriority => CommandType switch
+		{
+			TopicCommandType.EventNotification
+			or TopicCommandType.ChangePort
+			or TopicCommandType.ChangeRebootState
+			or TopicCommandType.InstanceRenamed
+			or TopicCommandType.ChatChannelsUpdate
+			or TopicCommandType.ServerRestarted => true,
+			TopicCommandType.ChatCommand
+			or TopicCommandType.Heartbeat
+			or TopicCommandType.ReceiveChunk => false,
+			TopicCommandType.SendChunk => throw new InvalidOperationException("SendChunk topic priority should be based on the original TopicParameters!"),
+			_ => throw new InvalidOperationException($"Invalid value for {nameof(CommandType)}: {CommandType}"),
+		};
+
 		/// <summary>
 		/// Initializes a new instance of the <see cref="TopicParameters"/> class.
 		/// </summary>

src/Tgstation.Server.Host/Components/Session/SessionController.cs

@@ -123,6 +123,11 @@ namespace Tgstation.Server.Host.Components.Session
 		/// </summary>
 		readonly IChatManager chat;
 
+		/// <summary>
+		/// The <see cref="IAsyncDelayer"/> for the <see cref="SessionController"/>.
+		/// </summary>
+		readonly IAsyncDelayer asyncDelayer;
+
 		/// <summary>
 		/// <see langword="lock"/> <see cref="object"/> for port updates and <see cref="disposed"/>.
 		/// </summary>
@@ -212,7 +217,12 @@ namespace Tgstation.Server.Host.Components.Session
 			this.chatTrackingContext = chatTrackingContext ?? throw new ArgumentNullException(nameof(chatTrackingContext));
 			if (bridgeRegistrar == null)
 				throw new ArgumentNullException(nameof(bridgeRegistrar));
+
 			this.chat = chat ?? throw new ArgumentNullException(nameof(chat));
+			if (assemblyInformationProvider == null)
+				throw new ArgumentNullException(nameof(assemblyInformationProvider));
+
+			this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer));
 
 			portClosedForReboot = false;
 			disposed = false;
@@ -760,8 +770,9 @@ namespace Tgstation.Server.Host.Components.Session
 			var fullCommandString = GenerateQueryString(parameters, out var json);
 			Logger.LogTrace("Topic request: {json}", json);
 			var fullCommandByteCount = Encoding.UTF8.GetByteCount(fullCommandString);
+			var topicPriority = parameters.IsPriority;
 			if (fullCommandByteCount <= DMApiConstants.MaximumTopicRequestLength)
-				return await SendRawTopic(fullCommandString, cancellationToken);
+				return await SendRawTopic(fullCommandString, topicPriority, cancellationToken);
 
 			var interopChunkingVersion = new Version(5, 6, 0);
 			if (ReattachInformation.Dmb.CompileJob.DMApiVersion < interopChunkingVersion)
@@ -849,7 +860,7 @@ namespace Tgstation.Server.Host.Components.Session
 
 			foreach (var chunkCommandString in chunkQueryStrings)
 			{
-				combinedResponse = await SendRawTopic(chunkCommandString, cancellationToken);
+				combinedResponse = await SendRawTopic(chunkCommandString, topicPriority, cancellationToken);
 				if (LogRequestIssue(chunkCommandString == chunkQueryStrings.Last()))
 					return null;
 			}
@@ -861,7 +872,7 @@ namespace Tgstation.Server.Host.Components.Session
 				foreach (var missingChunkIndex in combinedResponse.InteropResponse.MissingChunks)
 				{
 					var chunkCommandString = chunkQueryStrings[(int)missingChunkIndex];
-					combinedResponse = await SendRawTopic(chunkCommandString, cancellationToken);
+					combinedResponse = await SendRawTopic(chunkCommandString, topicPriority, cancellationToken);
 					if (LogRequestIssue(missingChunkIndex == lastIndex))
 						return null;
 				}
@@ -891,22 +902,53 @@ namespace Tgstation.Server.Host.Components.Session
 		/// Send a given <paramref name="queryString"/> to DreamDaemon's /world/Topic.
 		/// </summary>
 		/// <param name="queryString">The sanitized topic query string to send.</param>
+		/// <param name="priority">If this is a priority message. If so, the topic will make 5 attempts to send unless BYOND reboots or exits.</param>
 		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
 		/// <returns>A <see cref="Task{TResult}"/> resulting in the <see cref="CombinedTopicResponse"/> of the topic request.</returns>
-		async Task<CombinedTopicResponse> SendRawTopic(string queryString, CancellationToken cancellationToken)
+		async Task<CombinedTopicResponse> SendRawTopic(string queryString, bool priority, CancellationToken cancellationToken)
 		{
 			var targetPort = ReattachInformation.Port;
-			global::Byond.TopicSender.TopicResponse byondResponse;
-			try
+			var killedOrRebootedTask = Task.WhenAny(Lifetime, OnReboot);
+			global::Byond.TopicSender.TopicResponse byondResponse = null;
+			var firstSend = true;
+
+			const int PrioritySendAttempts = 5;
+			for (var i = PrioritySendAttempts - 1; i >= 0 && (priority || firstSend); --i)
+				try
+				{
+					firstSend = false;
+					if (!killedOrRebootedTask.IsCompleted)
+						byondResponse = await byondTopicSender.SendTopic(
+							new IPEndPoint(IPAddress.Loopback, targetPort),
+							queryString,
+							cancellationToken);
+
+					break;
+				}
+				catch (Exception ex)
+				{
+					Logger.LogWarning(ex, "SendTopic exception!{retryDetails}", priority ? $" {i} attempts remaining." : String.Empty);
+
+					if (priority && i > 0)
+					{
+						var delayTask = asyncDelayer.Delay(TimeSpan.FromSeconds(2), cancellationToken);
+						await Task.WhenAny(killedOrRebootedTask, delayTask);
+					}
+				}
+
+			if (byondResponse == null)
 			{
-				byondResponse = await byondTopicSender.SendTopic(
-					new IPEndPoint(IPAddress.Loopback, targetPort),
-					queryString,
-					cancellationToken);
-			}
-			catch (Exception ex)
-			{
-				Logger.LogWarning(ex, "SendTopic exception!");
+				if (priority)
+					if (killedOrRebootedTask.IsCompleted)
+						Logger.LogWarning(
+							"Unable to send priority topic \"{queryString}\" DreamDaemon {stateClearAction}!",
+							queryString,
+							Lifetime.IsCompleted ? "process ended" : "rebooted");
+					else
+						Logger.LogError(
+							"Unable to send priority topic \"{queryString}\"!",
+							queryString);
+
 				return null;
 			}
 

src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs

@@ -272,17 +272,15 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		{
 			using (await SemaphoreSlimContext.Lock(synchronizationSemaphore, cancellationToken))
 			{
-				if (Status == WatchdogStatus.Offline)
+				var commandObject = new ChatCommand(sender, commandName, arguments);
+				var command = new TopicParameters(commandObject);
+				var activeServer = GetActiveController();
+				if (Status != WatchdogStatus.Online || activeServer == null)
 					return new MessageContent
 					{
 						Text = "TGS: Server offline!",
 					};
 
-				var commandObject = new ChatCommand(sender, commandName, arguments);
-
-				var command = new TopicParameters(commandObject);
-
-				var activeServer = GetActiveController();
 				var commandResult = await activeServer.SendCommand(command, cancellationToken);
 
 				if (commandResult == null)

src/Tgstation.Server.Host/Configuration/UpdatesConfiguration.cs

@@ -39,5 +39,11 @@
 		/// Asset package containing the new <see cref="Host"/> assembly in zip form.
 		/// </summary>
 		public string UpdatePackageAssetName { get; set; } = DefaultUpdatePackageAssetName;
+
+		/// <summary>
+		/// Dump all retrieved releases from the GitHub API when a requested release is not found.
+		/// </summary>
+		/// <remarks>This is an internal config and may be adjusted or removed without a version change.</remarks>
+		public bool DumpReleasesOnNotFound { get; set; }
 	}
 }

src/Tgstation.Server.Host/Core/ServerUpdater.cs

@@ -108,13 +108,16 @@ namespace Tgstation.Server.Host.Core
 
 			logger.LogTrace("Received {releaseCount} total releases from GitHub", releases.Count);
 
-			releases = releases
+			var filteredReleases = releases
 				.Where(x => x.TagName.StartsWith(updatesConfiguration.GitTagPrefix, StringComparison.InvariantCulture))
 				.ToList();
 
-			logger.LogTrace("Filtered to {releaseCount} releases matching a TGS version", releases.Count);
+			logger.LogTrace(
+				"Filtered to {releaseCount} releases matching the configured tag prefix of \"{tagPrefix}\"",
+				filteredReleases.Count,
+				updatesConfiguration.GitTagPrefix);
 
-			foreach (var release in releases)
+			foreach (var release in filteredReleases)
 				if (Version.TryParse(
 					release.TagName.Replace(
 						updatesConfiguration.GitTagPrefix, String.Empty, StringComparison.Ordinal),
@@ -149,6 +152,12 @@ namespace Tgstation.Server.Host.Core
 				else
 					logger.LogDebug("Unparsable release tag: {releaseTag}", release.TagName);
 
+			if (updatesConfiguration.DumpReleasesOnNotFound)
+				logger.LogInformation(
+					"Found releases:{newline}\t{releases}",
+					Environment.NewLine,
+					String.Join($"{Environment.NewLine}\t", releases.Select(x => x.TagName).OrderBy(x => x)));
+
 			return ServerUpdateResult.ReleaseMissing;
 		}
 

src/Tgstation.Server.Host/Extensions/ServiceCollectionExtensions.cs

@@ -5,6 +5,7 @@ using System.Globalization;
 using Elastic.CommonSchema.Serilog;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.DependencyInjection.Extensions;
 using Microsoft.Extensions.Logging;
 
 using Serilog;
@@ -27,6 +28,11 @@ namespace Tgstation.Server.Host.Extensions
 		/// </summary>
 		static Type chatProviderFactoryType = typeof(ProviderFactory);
 
+		/// <summary>
+		/// A <see cref="ServiceDescriptor"/> for an additional <see cref="ILoggerProvider"/> to use.
+		/// </summary>
+		static ServiceDescriptor additionalLoggerProvider;
+
 		/// <summary>
 		/// Change the <see cref="Type"/> used as an implementation for calls to <see cref="AddChatProviderFactory(IServiceCollection)"/>.
 		/// </summary>
@@ -36,6 +42,17 @@ namespace Tgstation.Server.Host.Extensions
 			chatProviderFactoryType = typeof(TProviderFactory);
 		}
 
+		/// <summary>
+		/// Add an additional <see cref="ILoggerProvider"/> to <see cref="IServiceCollection"/>s that call <see cref="SetupLogging(IServiceCollection, Action{LoggerConfiguration}, Action{LoggerSinkConfiguration}, ElasticsearchConfiguration)"/>.
+		/// </summary>
+		/// <typeparam name="TLoggerProvider">The <see cref="Type"/> of <see cref="ILoggerProvider"/> to add.</typeparam>
+		public static void UseAdditionalLoggerProvider<TLoggerProvider>() where TLoggerProvider : class, ILoggerProvider
+		{
+			if (additionalLoggerProvider != null)
+				throw new InvalidOperationException("Cannot have multiple additionalLoggerProviders!");
+			additionalLoggerProvider = ServiceDescriptor.Singleton<ILoggerProvider, TLoggerProvider>();
+		}
+
 		/// <summary>
 		/// Adds a <see cref="IProviderFactory"/> implementation to the given <paramref name="serviceCollection"/>.
 		/// </summary>
@@ -133,6 +150,9 @@ namespace Tgstation.Server.Host.Extensions
 
 				if (Debugger.IsAttached)
 					builder.AddDebug();
+
+				if (additionalLoggerProvider != null)
+					builder.Services.TryAddEnumerable(additionalLoggerProvider);
 			});
 	}
 }

src/Tgstation.Server.Host/IO/DefaultIOManager.cs

@@ -52,8 +52,15 @@ namespace Tgstation.Server.Host.IO
 			foreach (var file in dir.EnumerateFiles())
 			{
 				cancellationToken.ThrowIfCancellationRequested();
-				file.Attributes = FileAttributes.Normal;
-				file.Delete();
+				try
+				{
+					file.Attributes = FileAttributes.Normal;
+					file.Delete();
+				}
+				catch (FileNotFoundException)
+				{
+					// has happened before with .dyn.rsc.lk
+				}
 			}
 
 			cancellationToken.ThrowIfCancellationRequested();

src/Tgstation.Server.Host/Jobs/JobManager.cs

@@ -185,16 +185,11 @@ namespace Tgstation.Server.Host.Jobs
 			JobHandler handler;
 			lock (addCancelLock)
 			{
-				try
-				{
-					handler = CheckGetJob(job);
-				}
-				catch (InvalidOperationException)
-				{
-					// this is fine
-					return null;
-				}
+				lock (synchronizationLock)
+					if (!jobs.TryGetValue(job.Id.Value, out handler))
+						return null;
 
+				logger.LogDebug("Cancelling job ID {jobId}...", job.Id.Value);
 				handler.Cancel(); // this will ensure the db update is only done once
 			}
 
@@ -267,21 +262,6 @@ namespace Tgstation.Server.Host.Jobs
 			activationTcs.SetResult(instanceCoreProvider);
 		}
 
-		/// <summary>
-		/// Gets the <see cref="JobHandler"/> for a given <paramref name="job"/> if it exists.
-		/// </summary>
-		/// <param name="job">The <see cref="Job"/> to get the <see cref="JobHandler"/> for.</param>
-		/// <returns>The <see cref="JobHandler"/>.</returns>
-		JobHandler CheckGetJob(Job job)
-		{
-			lock (synchronizationLock)
-			{
-				if (!jobs.TryGetValue(job.Id.Value, out JobHandler jobHandler))
-					throw new InvalidOperationException("Job not running!");
-				return jobHandler;
-			}
-		}
-
 		/// <summary>
 		/// Runner for <see cref="JobHandler"/>s.
 		/// </summary>

src/Tgstation.Server.Host/Swarm/SwarmService.cs

@@ -826,7 +826,7 @@ namespace Tgstation.Server.Host.Swarm
 					updateCommitTcs = new TaskCompletionSource<bool>();
 				}
 
-				logger.LogDebug("Prepared for update to version {version}", version);
+				logger.LogDebug("Local node prepared for update to version {version}", version);
 			}
 			catch (Exception ex)
 			{

tests/DMAPI/LongRunning/Config.dm

@@ -4,7 +4,7 @@
 #define TGS_WRITE_GLOBAL(Name, Value) global.##Name = ##Value
 #define TGS_PROTECT_DATUM(Path)
 #define TGS_WORLD_ANNOUNCE(message) world << ##message
-#define TGS_INFO_LOG(message) world.log << "Info: [##message]"
+#define TGS_INFO_LOG(message) TgsInfo(##message)
 #define TGS_WARNING_LOG(message) world.log << "Warn: [##message]"
 #define TGS_ERROR_LOG(message) TgsError(##message)
 #define TGS_NOTIFY_ADMINS(event)

tests/DMAPI/LongRunning/Test.dm

@@ -54,7 +54,10 @@
 		CheckBridgeLimits()
 
 /world/Topic(T, Addr, Master, Keys)
-	log << "Topic: [T]"
+	if(findtext(T, "tgs_integration_test_tactics3") == 0)
+		log << "Topic: [T]"
+	else
+		log << "tgs_integration_test_tactics3 <TOPIC SUPPRESSED>"
 	. =  HandleTopic(T)
 	log << "Response: [.]"
 
@@ -210,10 +213,18 @@ var/run_bridge_test
 	return response
 
 var/lastTgsError
+var/suppress_bridge_spam = FALSE
+
+/proc/TgsInfo(message)
+	if(suppress_bridge_spam && findtext(message, "Export: http://127.0.0.1:") != 0)
+		return
+	world.log << "Info: [message]"
 
 /proc/TgsError(message)
-	world.log << "Err: [message]"
 	lastTgsError = message
+	if(suppress_bridge_spam && findtext(message, "Failed bridge request: http://127.0.0.1:") != 0)
+		return
+	world.log << "Err: [message]"
 
 /proc/create_payload(size)
 	var/builder = list()
@@ -230,7 +241,9 @@ var/lastTgsError
 /proc/BridgeWithoutChunking(command, list/data)
 	var/datum/tgs_api/v5/api = TGS_READ_GLOBAL(tgs)
 	var/bridge_request = api.CreateBridgeRequest(command, data)
-	return api.PerformBridgeRequest(bridge_request)
+	suppress_bridge_spam = TRUE
+	. = api.PerformBridgeRequest(bridge_request)
+	suppress_bridge_spam = FALSE
 
 /proc/CheckBridgeLimitsImpl()
 	sleep(30)

tests/Tgstation.Server.Tests/Live/DummyChatProvider.cs

@@ -9,6 +9,8 @@ using Microsoft.VisualStudio.TestTools.UnitTesting;
 
 using Moq;
 
+using Newtonsoft.Json;
+
 using Tgstation.Server.Host.Components.Chat;
 using Tgstation.Server.Host.Components.Chat.Commands;
 using Tgstation.Server.Host.Components.Chat.Providers;
@@ -34,13 +36,14 @@ namespace Tgstation.Server.Tests.Live
 		readonly ICryptographySuite cryptographySuite;
 		readonly CancellationTokenSource randomMessageCts;
 		readonly Task randomMessageTask;
+		readonly Dictionary<ulong, ChannelRepresentation> knownChannels;
 
 		bool connectedOnce;
 		bool connected;
 
 		ulong channelIdAllocator;
 
-		static ILoggerFactory CreateLoggerFactoryForLogger(ILogger logger, out Mock<ILoggerFactory> mockLoggerFactory)
+		public static ILoggerFactory CreateLoggerFactoryForLogger(ILogger logger, out Mock<ILoggerFactory> mockLoggerFactory)
 		{
 			mockLoggerFactory = new Mock<ILoggerFactory>();
 			mockLoggerFactory.Setup(x => x.CreateLogger(It.IsAny<string>())).Returns(() =>
@@ -83,12 +86,11 @@ namespace Tgstation.Server.Tests.Live
 			this.commands = commands ?? throw new ArgumentNullException(nameof(commands));
 			this.random = random ?? throw new ArgumentNullException(nameof(random));
 
-			// this could be random but there's no point
-			channelIdAllocator = 100000;
 			logger.LogTrace("Base channel ID {baseChannelId}", channelIdAllocator);
 
-			this.randomMessageCts = new CancellationTokenSource();
-			this.randomMessageTask = RandomMessageLoop(this.randomMessageCts.Token);
+			knownChannels = new Dictionary<ulong, ChannelRepresentation>();
+			randomMessageCts = new CancellationTokenSource();
+			randomMessageTask = RandomMessageLoop(this.randomMessageCts.Token);
 		}
 
 		public override async ValueTask DisposeAsync()
@@ -107,8 +109,7 @@ namespace Tgstation.Server.Tests.Live
 
 			Logger.LogTrace("SendMessage");
 
-			Assert.AreNotEqual(0UL, channelId);
-			Assert.IsTrue(channelId <= channelIdAllocator);
+			Assert.IsTrue(knownChannels.ContainsKey(channelId));
 
 			cancellationToken.ThrowIfCancellationRequested();
 
@@ -132,8 +133,7 @@ namespace Tgstation.Server.Tests.Live
 
 			Logger.LogTrace("SendUpdateMessage");
 
-			Assert.AreNotEqual(0UL, channelId);
-			Assert.IsTrue(channelId <= channelIdAllocator);
+			Assert.IsTrue(knownChannels.ContainsKey(channelId));
 
 			cancellationToken.ThrowIfCancellationRequested();
 
@@ -196,19 +196,43 @@ namespace Tgstation.Server.Tests.Live
 							channel,
 							new List<ChannelRepresentation>
 							{
-								new ChannelRepresentation
-								{
-									IsAdminChannel = channel.IsAdminChannel.Value,
-									ConnectionName = $"Connection_{channelIdAllocator + 1}",
-									EmbedsSupported = ChatBot.Provider.Value != Api.Models.ChatProvider.Irc,
-									FriendlyName = $"(Friendly) Channel_ID_{channelIdAllocator + 1}",
-									IsPrivateChannel = false,
-									RealId = ++channelIdAllocator,
-									Tag = channel.Tag,
-								}
+								CreateChannel(channel)
 							}))));
 		}
 
+		private ChannelRepresentation CreateChannel(ChatChannel channel)
+		{
+
+			ulong channelId;
+			if (channel.DiscordChannelId.HasValue)
+				channelId = channel.DiscordChannelId.Value;
+			else
+				channelId = (ulong)channel.IrcChannel.GetHashCode();
+
+			var entry = new ChannelRepresentation
+			{
+				IsAdminChannel = channel.IsAdminChannel.Value,
+				ConnectionName = $"Connection_{channelId}",
+				EmbedsSupported = ChatBot.Provider.Value != Api.Models.ChatProvider.Irc,
+				FriendlyName = $"(Friendly) Channel_ID_{channelId}",
+				IsPrivateChannel = false,
+				RealId = channelId,
+				Tag = channel.Tag,
+			};
+
+			knownChannels.Remove(channelId);
+			knownChannels.Add(channelId, entry);
+
+			return CloneChannel(entry);
+		}
+
+		ChannelRepresentation CloneChannel(ChannelRepresentation channel)
+		{
+			return JsonConvert.DeserializeObject<ChannelRepresentation>(
+				JsonConvert.SerializeObject(
+					channel));
+		}
+
 		async Task RandomMessageLoop(CancellationToken cancellationToken)
 		{
 			Logger.LogTrace("RandomMessageLoop");
@@ -220,35 +244,68 @@ namespace Tgstation.Server.Tests.Live
 					var delay = random.Next(0, 10000);
 					await Task.Delay(delay, cancellationToken);
 
-					if (!connected)
-						continue;
-
 					// %5 chance to disconnect randomly
 					if (enableRandomDisconnections != 0 && random.Next(0, 100) > 95)
 						connected = false;
 
-					if (channelIdAllocator >= Int32.MaxValue / 2)
-						Assert.Fail("Too many channels have been allocated!");
+					if (!connected)
+						continue;
 
-					var isPm = channelIdAllocator == 0 || random.Next(0, 100) > 20;
-					var realId = (ulong)random.Next(1, (int)channelIdAllocator);
-
-					if (isPm)
-						realId += Int32.MaxValue / 2;
+					var isPm = channelIdAllocator == 0 || random.Next(0, 100) > 80;
 
+					ChannelRepresentation channel;
 					var username = $"RandomUser{i}";
+					if (isPm)
+						// 50% chance to be a new user
+						if (random.Next(0, 100) > 50)
+						{
+							var enumerator = knownChannels
+								.Where(x => x.Value.IsPrivateChannel)
+								.ToList();
+							if (enumerator.Count == 0)
+								continue;
+
+							var index = random.Next(0, enumerator.Count);
+							channel = enumerator[index].Value;
+							username = channel.ConnectionName.Substring(0, channel.ConnectionName.Length - 11);
+						}
+						else
+						{
+							ulong channelId;
+							do
+							{
+								channelId = ++channelIdAllocator;
+							}
+							while (knownChannels.ContainsKey(channelId));
+
+							channel = new ChannelRepresentation
+							{
+								RealId = channelId,
+								IsPrivateChannel = true,
+								ConnectionName = $"{username}_Connection",
+								FriendlyName = $"{username}_Channel",
+								EmbedsSupported = ChatBot.Provider.Value != Api.Models.ChatProvider.Irc,
+
+								// isAdmin and Tag populated by manager
+							};
+
+							knownChannels.Add(channelId, channel);
+						}
+					else
+					{
+						var enumerator = knownChannels
+							.Where(x => !x.Value.IsPrivateChannel)
+							.ToList();
+						if (enumerator.Count == 0)
+							continue;
+
+						var index = random.Next(0, enumerator.Count);
+						channel = enumerator[index].Value;
+					}
+
 					var sender = new ChatUser
 					{
-						Channel = new ChannelRepresentation
-						{
-							RealId = realId,
-							IsPrivateChannel = isPm,
-							ConnectionName = isPm ? $"{username}_Connection" : $"Connection_{realId}",
-							FriendlyName = isPm ? $"{username}_Channel" : $"(Friendly) Channel_ID_{realId}",
-							EmbedsSupported = ChatBot.Provider.Value != Api.Models.ChatProvider.Irc,
-
-							// isAdmin and Tag populated by manager
-						},
+						Channel = CloneChannel(channel),
 						FriendlyName = username,
 						RealId = i + 50000,
 						Mention = $"@{username}",

tests/Tgstation.Server.Tests/Live/DummyChatProviderFactory.cs

@@ -47,7 +47,15 @@ namespace Tgstation.Server.Tests.Live
 			commandFactory.SetWatchdog(Mock.Of<IWatchdog>());
 			commands = commandFactory.GenerateCommands();
 
-			var baseRng = new Random(22475);
+			const bool UseUnseededProviderRng = false;
+
+			var randomSeed = UseUnseededProviderRng
+				? new Random().Next()
+				: 22475;
+
+			logger.LogInformation("Random seed: {0}", randomSeed);
+
+			var baseRng = new Random(randomSeed);
 			seededRng = new Dictionary<ChatProvider, Random>{
 				{ ChatProvider.Irc, new Random(baseRng.Next()) },
 				{ ChatProvider.Discord, new Random(baseRng.Next()) },

tests/Tgstation.Server.Tests/Live/HardFailLogger.cs

@@ -0,0 +1,36 @@
+using System;
+using System.Text.RegularExpressions;
+using System.Threading.Tasks;
+
+using Microsoft.Extensions.Logging;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace Tgstation.Server.Tests.Live
+{
+	sealed class HardFailLogger : ILogger
+	{
+		readonly Action<Exception> failureSink;
+
+		public HardFailLogger(Action<Exception> failureSink)
+		{
+			this.failureSink = failureSink ?? throw new ArgumentNullException(nameof(failureSink));
+		}
+
+		public IDisposable BeginScope<TState>(TState state) => Task.CompletedTask;
+
+		public bool IsEnabled(LogLevel logLevel) => true;
+
+		public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func<TState, Exception, string> formatter)
+		{
+			var logMessage = formatter(state, exception);
+			if ((logLevel == LogLevel.Error
+				&& !logMessage.StartsWith("Error disconnecting connection ")
+				&& !(logMessage.StartsWith("An exception occurred while iterating over the results of a query for context type") && (exception is TaskCanceledException || exception?.InnerException is TaskCanceledException)))
+				|| (logLevel == LogLevel.Critical && logMessage != "DropDatabase configuration option set! Dropping any existing database..."))
+			{
+				Console.WriteLine("UNEXPECTED ERROR OCCURS NEAR HERE");
+				failureSink(new AssertFailedException("TGS logged an unexpected error!"));
+			}
+		}
+	}
+}

tests/Tgstation.Server.Tests/Live/HardFailLoggerProvider.cs

@@ -0,0 +1,23 @@
+using System.Threading.Tasks;
+
+using Microsoft.Extensions.Logging;
+
+namespace Tgstation.Server.Tests.Live
+{
+	sealed class HardFailLoggerProvider : ILoggerProvider
+	{
+		public static bool BlockFails { get; set; }
+
+		public static Task FailureSource => failureSink.Task;
+
+		static readonly TaskCompletionSource failureSink = new (TaskCreationOptions.RunContinuationsAsynchronously);
+
+		public ILogger CreateLogger(string categoryName) => new HardFailLogger(ex =>
+		{
+			if (!BlockFails)
+				failureSink.TrySetException(ex);
+		});
+
+		public void Dispose() { }
+	}
+}

tests/Tgstation.Server.Tests/Live/Instance/ChatTest.cs

@@ -97,7 +97,7 @@ namespace Tgstation.Server.Tests.Live.Instance
 			var jobs = await JobsClient.List(null, cancellationToken);
 			var reconnectJob = jobs
 				.Where(x => x.StartedAt >= beforeChatBotEnabled && x.Description.Contains(updatedBot.Name))
-				.OrderByDescending(x => x.StartedAt)
+				.OrderBy(x => x.StartedAt)
 				.FirstOrDefault();
 
 			Assert.IsNotNull(reconnectJob);
@@ -198,7 +198,7 @@ namespace Tgstation.Server.Tests.Live.Instance
 			var jobs = await JobsClient.List(null, cancellationToken);
 			var reconnectJob = jobs
 				.Where(x => x.StartedAt >= beforeChatBotEnabled && x.Description.Contains(updatedBot.Name))
-				.OrderByDescending(x => x.StartedAt)
+				.OrderBy(x => x.StartedAt)
 				.FirstOrDefault();
 
 			Assert.IsNotNull(reconnectJob);

tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs

@@ -519,7 +519,8 @@ namespace Tgstation.Server.Tests.Live.Instance
 				TopicResponse topicRequestResult = null;
 				try
 				{
-					topicRequestResult = await TopicClient.SendTopic(
+					System.Console.WriteLine($"Topic limit test S:{payloadSize}...");
+					topicRequestResult = await TopicClientNoLogger.SendTopic(
 						IPAddress.Loopback,
 						$"tgs_integration_test_tactics3={TopicClient.SanitizeString(JsonConvert.SerializeObject(topic, DMApiConstants.SerializerSettings))}",
 						TestLiveServer.DDPort,
@@ -869,7 +870,21 @@ namespace Tgstation.Server.Tests.Live.Instance
 			return ddProc != null;
 		}
 
+		static readonly ILoggerFactory loggerFactory = LoggerFactory.Create(builder =>
+		{
+			builder.AddConsole();
+			builder.SetMinimumLevel(LogLevel.Trace);
+		});
+
 		public static readonly TopicClient TopicClient = new(new SocketParameters
+		{
+			SendTimeout = TimeSpan.FromSeconds(30),
+			ReceiveTimeout = TimeSpan.FromSeconds(30),
+			ConnectTimeout = TimeSpan.FromSeconds(30),
+			DisconnectTimeout = TimeSpan.FromSeconds(30)
+		}, new Logger<TopicClient>(DummyChatProvider.CreateLoggerFactoryForLogger(loggerFactory.CreateLogger("WatchdogTest.TopicClient"), out var mockLoggerFactory)));
+
+		public static readonly TopicClient TopicClientNoLogger = new(new SocketParameters
 		{
 			SendTimeout = TimeSpan.FromSeconds(30),
 			ReceiveTimeout = TimeSpan.FromSeconds(30),

tests/Tgstation.Server.Tests/Live/InstanceManagerTest.cs

@@ -38,9 +38,20 @@ namespace Tgstation.Server.Tests.Live
 			this.testRootPath = testRootPath ?? throw new ArgumentNullException(nameof(testRootPath));
 		}
 
-		public Task<InstanceResponse> CreateTestInstance(CancellationToken cancellationToken) => instanceManagerClient.CreateOrAttach(new InstanceCreateRequest
+		public async Task<InstanceResponse> CreateTestInstance(CancellationToken cancellationToken)
 		{
-			Name = TestInstanceName,
+			var instance = await CreateTestInstanceStub("LiveTestsInstance", cancellationToken);
+			return await instanceManagerClient.Update(new InstanceUpdateRequest
+			{
+				Id = instance.Id,
+				Online = true,
+				ConfigurationType = ConfigurationType.HostWrite
+			}, cancellationToken);
+		}
+
+		Task<InstanceResponse> CreateTestInstanceStub(string name, CancellationToken cancellationToken) => instanceManagerClient.CreateOrAttach(new InstanceCreateRequest
+		{
+			Name = name,
 			Path = Path.Combine(testRootPath, Guid.NewGuid().ToString()),
 			Online = true,
 			ChatBotLimit = 2
@@ -57,9 +68,9 @@ namespace Tgstation.Server.Tests.Live
 			Online = response.Online
 		};
 
-		public async Task<Api.Models.Instance> RunPreInstanceTest(CancellationToken cancellationToken)
+		public async Task RunPreTest(CancellationToken cancellationToken)
 		{
-			var firstTest = await CreateTestInstance(cancellationToken);
+			var firstTest = await CreateTestInstanceStub(TestInstanceName, cancellationToken);
 			//instances always start offline
 			Assert.AreEqual(false, firstTest.Online);
 			//check it exists
@@ -161,8 +172,14 @@ namespace Tgstation.Server.Tests.Live
 			Assert.IsTrue(Directory.Exists(firstTest.Path));
 
 			await RegressionTest1256(cancellationToken);
+			firstTest = await instanceManagerClient.Update(new InstanceUpdateRequest
+			{
+				Id = firstTest.Id,
+				Online = false,
+			}, cancellationToken);
 
-			return firstTest;
+			Assert.AreEqual(false, firstTest.Online);
+			await instanceManagerClient.Detach(firstTest, cancellationToken);
 		}
 
 		async Task RegressionTest1256(CancellationToken cancellationToken)
@@ -197,10 +214,10 @@ namespace Tgstation.Server.Tests.Live
 			Assert.AreEqual(3, paginated.TotalPages);
 		}
 
-		public async Task RunPostTest(CancellationToken cancellationToken)
+		public async Task RunPostTest(Api.Models.Instance instance, CancellationToken cancellationToken)
 		{
 			var instances = await instanceManagerClient.List(null, cancellationToken);
-			var firstTest = instances.Single(x => x.Name == TestInstanceName);
+			var firstTest = instances.Single(x => x.Name == instance.Name);
 			var instanceClient = instanceManagerClient.CreateClient(firstTest);
 
 			Assert.IsTrue(firstTest.Accessible);

tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs

@@ -46,7 +46,7 @@ namespace Tgstation.Server.Tests.Live
 			SerilogContextHelper.AddSwarmNodeIdentifierToTemplate();
 		}
 
-		public LiveTestingServer(SwarmConfiguration swarmConfiguration, bool enableOAuth, ushort port = 5010)
+		public LiveTestingServer(SwarmConfiguration swarmConfiguration, bool enableOAuth, ushort port = 5010, bool dumpOnMissingUpdate = true)
 		{
 			Directory = Environment.GetEnvironmentVariable("TGS_TEST_TEMP_DIRECTORY");
 			if (string.IsNullOrWhiteSpace(Directory))
@@ -86,6 +86,7 @@ namespace Tgstation.Server.Tests.Live
 			args = new List<string>()
 			{
 				string.Format(CultureInfo.InvariantCulture, "Database:DropDatabase={0}", true), // Replaced after first Run
+				string.Format(CultureInfo.InvariantCulture, "General:ConfigVersion={0}", GeneralConfiguration.CurrentConfigVersion),
 				string.Format(CultureInfo.InvariantCulture, "General:ApiPort={0}", port),
 				string.Format(CultureInfo.InvariantCulture, "Database:DatabaseType={0}", DatabaseType),
 				string.Format(CultureInfo.InvariantCulture, "Database:ConnectionString={0}", connectionString),
@@ -101,6 +102,9 @@ namespace Tgstation.Server.Tests.Live
 				"General:ByondTopicTimeout=3000"
 			};
 
+			if (dumpOnMissingUpdate)
+				args.Add("Updates:DumpReleasesOnNotFound=true");
+
 			swarmArgs = new List<string>();
 			if (swarmConfiguration != null)
 			{

tests/Tgstation.Server.Tests/Live/TestLiveServer.cs

@@ -215,6 +215,38 @@ namespace Tgstation.Server.Tests.Live
 			Assert.IsTrue(server.RestartRequested, "Server not requesting restart!");
 		}
 
+		[TestMethod]
+		public async Task TestUpdateBadVersion()
+		{
+			using var server = new LiveTestingServer(null, false, dumpOnMissingUpdate: false);
+			using var serverCts = new CancellationTokenSource();
+			var cancellationToken = serverCts.Token;
+			var serverTask = server.Run(cancellationToken);
+			try
+			{
+				var testUpdateVersion = new Version(5, 11, 20);
+				using var adminClient = await CreateAdminClient(server.Url, cancellationToken);
+				await ApiAssert.ThrowsException<ConflictException>(() => adminClient.Administration.Update(new ServerUpdateRequest
+				{
+					NewVersion = testUpdateVersion
+				}, cancellationToken), ErrorCode.ResourceNotPresent);
+			}
+			finally
+			{
+				serverCts.Cancel();
+				try
+				{
+					await serverTask;
+				}
+				catch (OperationCanceledException) { }
+				catch (AggregateException ex)
+				{
+					if (ex.InnerException is NotSupportedException notSupportedException)
+						Assert.Inconclusive(notSupportedException.Message);
+				}
+			}
+		}
+
 		[TestMethod]
 		public async Task TestOneServerSwarmUpdate()
 		{
@@ -697,6 +729,45 @@ namespace Tgstation.Server.Tests.Live
 
 		[TestMethod]
 		public async Task TestStandardTgsOperation()
+		{
+			const int MaximumTestMinutes = 30;
+			using var hardCancellationTokenSource = new CancellationTokenSource(TimeSpan.FromMinutes(MaximumTestMinutes));
+			var hardCancellationToken = hardCancellationTokenSource.Token;
+
+			ServiceCollectionExtensions.UseAdditionalLoggerProvider<HardFailLoggerProvider>();
+
+			var failureTask = HardFailLoggerProvider.FailureSource;
+			var internalTask = TestTgsInternal(hardCancellationToken);
+			await Task.WhenAny(
+				internalTask,
+				failureTask);
+
+			if (!internalTask.IsCompleted)
+			{
+				hardCancellationTokenSource.Cancel();
+				try
+				{
+					await failureTask;
+				}
+				finally
+				{
+					try
+					{
+						await internalTask;
+					}
+					catch (OperationCanceledException)
+					{
+						Console.WriteLine("TEST CANCELLED!");
+					}
+					catch (Exception ex)
+					{
+						Console.WriteLine($"ADDITIONAL TEST ERROR: {ex}");
+					}
+				}
+			}
+		}
+
+		async Task TestTgsInternal(CancellationToken hardCancellationToken)
 		{
 			var discordConnectionString = Environment.GetEnvironmentVariable("TGS_TEST_DISCORD_TOKEN");
 			var ircConnectionString = Environment.GetEnvironmentVariable("TGS_TEST_IRC_CONNECTION_STRING");
@@ -734,9 +805,6 @@ namespace Tgstation.Server.Tests.Live
 
 			using var server = new LiveTestingServer(null, true);
 
-			const int MaximumTestMinutes = 180;
-			using var hardTimeoutCancellationTokenSource = new CancellationTokenSource(TimeSpan.FromMinutes(MaximumTestMinutes));
-			var hardCancellationToken = hardTimeoutCancellationTokenSource.Token;
 			using var serverCts = CancellationTokenSource.CreateLinkedTokenSource(hardCancellationToken);
 			var cancellationToken = serverCts.Token;
 
@@ -786,7 +854,9 @@ namespace Tgstation.Server.Tests.Live
 					var rootTest = FailFast(RawRequestTests.Run(clientFactory, adminClient, cancellationToken));
 					var adminTest = FailFast(new AdministrationTest(adminClient.Administration).Run(cancellationToken));
 					var usersTest = FailFast(new UsersTest(adminClient).Run(cancellationToken));
-					instance = await new InstanceManagerTest(adminClient, server.Directory).RunPreInstanceTest(cancellationToken);
+					var instanceMangagerTest = new InstanceManagerTest(adminClient, server.Directory);
+					var instancesTest = FailFast(instanceMangagerTest.RunPreTest(cancellationToken));
+					instance = await instanceMangagerTest.CreateTestInstance(cancellationToken);
 					Assert.IsTrue(Directory.Exists(instance.Path));
 					var instanceClient = adminClient.Instances.CreateClient(instance);
 
@@ -794,7 +864,7 @@ namespace Tgstation.Server.Tests.Live
 
 					var instanceTests = FailFast(new InstanceTest(instanceClient, adminClient.Instances, GetInstanceManager(), (ushort)server.Url.Port).RunTests(cancellationToken));
 
-					await Task.WhenAll(rootTest, adminTest, instanceTests, usersTest);
+					await Task.WhenAll(rootTest, adminTest, instancesTest, instanceTests, usersTest);
 
 					await adminClient.Administration.Restart(cancellationToken);
 				}
@@ -816,12 +886,13 @@ namespace Tgstation.Server.Tests.Live
 				// http bind test https://github.com/tgstation/tgstation-server/issues/1065
 				if (new PlatformIdentifier().IsWindows)
 				{
-					using var blockingSocket = new Socket(SocketType.Stream, ProtocolType.Tcp);
-					blockingSocket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ExclusiveAddressUse, true);
-					blockingSocket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, false);
-					blockingSocket.Bind(new IPEndPoint(IPAddress.Any, server.Url.Port));
+					HardFailLoggerProvider.BlockFails = true;
 					try
 					{
+						using var blockingSocket = new Socket(SocketType.Stream, ProtocolType.Tcp);
+						blockingSocket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ExclusiveAddressUse, true);
+						blockingSocket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, false);
+						blockingSocket.Bind(new IPEndPoint(IPAddress.Any, server.Url.Port));
 						// bind test run
 						await server.Run(cancellationToken);
 						Assert.Fail("Expected server task to end with a SocketException");
@@ -830,6 +901,10 @@ namespace Tgstation.Server.Tests.Live
 					{
 						Assert.AreEqual(ex.SocketErrorCode, SocketError.AddressAlreadyInUse);
 					}
+					finally
+					{
+						HardFailLoggerProvider.BlockFails = false;
+					}
 				}
 
 				await Task.WhenAny(serverTask, Task.Delay(TimeSpan.FromMinutes(1), cancellationToken));
@@ -999,7 +1074,7 @@ namespace Tgstation.Server.Tests.Live
 					await new ChatTest(instanceClient.ChatBots, adminClient.Instances, instanceClient.Jobs, instance).RunPostTest(cancellationToken);
 					await repoTest;
 
-					await new InstanceManagerTest(adminClient, server.Directory).RunPostTest(cancellationToken);
+					await new InstanceManagerTest(adminClient, server.Directory).RunPostTest(instance, cancellationToken);
 				}
 			}
 			catch (ApiException ex)