Merge pull request #1820 from tgstation/1779-Auditing [TGSDeploy]

v6.6.1: Deployment Lock State Auditing

build/Version.props

@@ -3,7 +3,7 @@
   <!-- Integration tests will ensure they match across the board -->
   <Import Project="WebpanelVersion.props" />
   <PropertyGroup>
-    <TgsCoreVersion>6.6.0</TgsCoreVersion>
+    <TgsCoreVersion>6.6.1</TgsCoreVersion>
     <TgsConfigVersion>5.1.0</TgsConfigVersion>
     <TgsApiVersion>10.4.0</TgsApiVersion>
     <TgsCommonLibraryVersion>7.0.0</TgsCommonLibraryVersion>

src/DMAPI/tgs/v5/api.dm

@@ -50,7 +50,9 @@
 	version = null // we want this to be the TGS version, not the interop version
 
 	// sleep once to prevent an issue where world.Export on the first tick can hang indefinitely
+	TGS_DEBUG_LOG("Starting Export bug prevention sleep tick. time:[world.time] sleep_offline:[world.sleep_offline]")
 	sleep(world.tick_lag)
+	TGS_DEBUG_LOG("Export bug prevention sleep complete")
 
 	var/list/bridge_response = Bridge(DMAPI5_BRIDGE_COMMAND_STARTUP, list(DMAPI5_BRIDGE_PARAMETER_MINIMUM_SECURITY_LEVEL = minimum_required_security_level, DMAPI5_BRIDGE_PARAMETER_VERSION = api_version.raw_parameter, DMAPI5_PARAMETER_CUSTOM_COMMANDS = ListCustomCommands(), DMAPI5_PARAMETER_TOPIC_PORT = GetTopicPort()))
 	if(!istype(bridge_response))

src/Tgstation.Server.Api/Models/EngineVersion.cs

@@ -12,6 +12,11 @@ namespace Tgstation.Server.Api.Models
 	/// </summary>
 	public sealed class EngineVersion : IEquatable<EngineVersion>
 	{
+		/// <summary>
+		/// An array of a single '-' <see cref="char"/>.
+		/// </summary>
+		static readonly char[] DashChar = ['-'];
+
 		/// <summary>
 		/// The <see cref="EngineType"/>.
 		/// </summary>
@@ -48,7 +53,7 @@ namespace Tgstation.Server.Api.Models
 			if (input == null)
 				throw new ArgumentNullException(nameof(input));
 
-			var splits = input.Split(new char[] { '-' }, StringSplitOptions.RemoveEmptyEntries);
+			var splits = input.Split(DashChar, StringSplitOptions.RemoveEmptyEntries);
 			engineVersion = null;
 
 			if (splits.Length > 3)

src/Tgstation.Server.Host/Components/Chat/Commands/PullRequestsCommand.cs

@@ -137,7 +137,7 @@ namespace Tgstation.Server.Host.Components.Chat.Commands
 				var compileJobToUse = watchdog.ActiveCompileJob;
 				if (hasStaged)
 				{
-					var latestCompileJob = compileJobProvider.LatestCompileJob();
+					var latestCompileJob = await compileJobProvider.LatestCompileJob();
 					if (latestCompileJob?.Id != compileJobToUse?.Id)
 						compileJobToUse = latestCompileJob;
 					else

src/Tgstation.Server.Host/Components/Deployment/DeploymentLockManager.cs

@@ -0,0 +1,166 @@
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Runtime.CompilerServices;
+using System.Text;
+using System.Threading.Tasks;
+
+using Microsoft.Extensions.Logging;
+
+using Tgstation.Server.Host.Models;
+
+namespace Tgstation.Server.Host.Components.Deployment
+{
+	/// <summary>
+	/// Manages locks on a given <see cref="IDmbProvider"/>.
+	/// </summary>
+	sealed class DeploymentLockManager : IAsyncDisposable
+	{
+		/// <summary>
+		/// The <see cref="Models.CompileJob"/> represented by the <see cref="DeploymentLockManager"/>.
+		/// </summary>
+		public CompileJob CompileJob => dmbProvider.CompileJob;
+
+		/// <summary>
+		/// The <see cref="ILogger"/> for the <see cref="DeploymentLockManager"/>.
+		/// </summary>
+		readonly ILogger logger;
+
+		/// <summary>
+		/// The <see cref="IDmbProvider"/> the <see cref="DeploymentLockManager"/> is managing.
+		/// </summary>
+		readonly IDmbProvider dmbProvider;
+
+		/// <summary>
+		/// The <see cref="DmbLock"/>s on the <see cref="dmbProvider"/>.
+		/// </summary>
+		readonly HashSet<DmbLock> locks;
+
+		/// <summary>
+		/// The first lock acquired by the <see cref="DeploymentLockManager"/>.
+		/// </summary>
+		readonly DmbLock firstLock;
+
+		/// <summary>
+		/// Create a <see cref="DeploymentLockManager"/>.
+		/// </summary>
+		/// <param name="dmbProvider">The value of <see cref="dmbProvider"/>.</param>
+		/// <param name="logger">The value of <see cref="logger"/>.</param>
+		/// <param name="initialLockReason">The reason for the first lock.</param>
+		/// <param name="firstLock">The <see cref="IDmbProvider"/> that represents the first lock.</param>
+		/// <param name="callerFile">The file path of the calling function.</param>
+		/// <param name="callerLine">The line number of the call invocation.</param>
+		/// <returns>A new <see cref="DeploymentLockManager"/>.</returns>
+		public static DeploymentLockManager Create(IDmbProvider dmbProvider, ILogger logger, string initialLockReason, out IDmbProvider firstLock, [CallerFilePath] string? callerFile = null, [CallerLineNumber] int callerLine = default)
+		{
+			var manager = new DeploymentLockManager(dmbProvider, logger, initialLockReason, callerFile!, callerLine);
+			firstLock = manager.firstLock;
+			return manager;
+		}
+
+		/// <summary>
+		/// Generates a verbose description of a given <paramref name="dmbLock"/>.
+		/// </summary>
+		/// <param name="dmbLock">The <see cref="DmbLock"/> to get a description of.</param>
+		/// <returns>A verbose description of <paramref name="dmbLock"/>.</returns>
+		static string GetFullLockDescriptor(DmbLock dmbLock) => $"{dmbLock.LockID} {dmbLock.Descriptor} (Created at {dmbLock.LockTime}){(dmbLock.KeptAlive ? " (RELEASED)" : String.Empty)}";
+
+		/// <summary>
+		/// Initializes a new instance of the <see cref="DeploymentLockManager"/> class.
+		/// </summary>
+		/// <param name="dmbProvider">The value of <see cref="dmbProvider"/>.</param>
+		/// <param name="logger">The value of <see cref="logger"/>.</param>
+		/// <param name="initialLockReason">The reason for the first lock.</param>
+		/// <param name="callerFile">The file path of the calling function.</param>
+		/// <param name="callerLine">The line number of the call invocation.</param>
+		/// <returns>A new <see cref="DeploymentLockManager"/>.</returns>
+		DeploymentLockManager(IDmbProvider dmbProvider, ILogger logger, string initialLockReason, string callerFile, int callerLine)
+		{
+			this.dmbProvider = dmbProvider ?? throw new ArgumentNullException(nameof(dmbProvider));
+			this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
+
+			logger.LogTrace("Initializing lock manager for compile job {id}", dmbProvider.CompileJob.Id);
+			locks = new HashSet<DmbLock>();
+			firstLock = CreateLock(initialLockReason, callerFile, callerLine);
+		}
+
+		/// <inheritdoc />
+		public ValueTask DisposeAsync()
+			=> firstLock.DisposeAsync();
+
+		/// <summary>
+		/// Add a lock to the managed <see cref="IDmbProvider"/>.
+		/// </summary>
+		/// <param name="reason">The reason for the lock.</param>
+		/// <param name="callerFile">The file path of the calling function.</param>
+		/// <param name="callerLine">The line number of the call invocation.</param>
+		/// <returns>A <see cref="IDmbProvider"/> whose lifetime represents the lock.</returns>
+		public IDmbProvider AddLock(string reason, [CallerFilePath] string? callerFile = null, [CallerLineNumber]int callerLine = default)
+		{
+			ArgumentNullException.ThrowIfNull(reason);
+			lock (locks)
+			{
+				if (locks.Count == 0)
+					throw new InvalidOperationException($"No locks exist on the DmbProvider for CompileJob {dmbProvider.CompileJob.Id}!");
+
+				return CreateLock(reason, callerFile!, callerLine);
+			}
+		}
+
+		/// <summary>
+		/// Add lock stats to a given <paramref name="stringBuilder"/>.
+		/// </summary>
+		/// <param name="stringBuilder">The <see cref="StringBuilder"/> to append to.</param>
+		public void LogLockStats(StringBuilder stringBuilder)
+		{
+			ArgumentNullException.ThrowIfNull(stringBuilder);
+
+			stringBuilder.AppendLine(CultureInfo.InvariantCulture, $"Compile Job #{CompileJob.Id}: {CompileJob.DirectoryName}");
+			lock (locks)
+				foreach (var dmbLock in locks)
+					stringBuilder.AppendLine(CultureInfo.InvariantCulture, $"\t-{GetFullLockDescriptor(dmbLock)}");
+		}
+
+		/// <summary>
+		/// Creates a <see cref="DmbLock"/> and adds it to <see cref="locks"/>.
+		/// </summary>
+		/// <param name="reason">The reason for the lock.</param>
+		/// <param name="callerFile">The file path of the calling function.</param>
+		/// <param name="callerLine">The line number of the call invocation.</param>
+		/// <returns>A new <see cref="DmbLock"/>.</returns>
+		/// <remarks>Requires exclusive write access to <see cref="locks"/> be held by the caller.</remarks>
+		DmbLock CreateLock(string reason, string callerFile, int callerLine)
+		{
+			DmbLock? newLock = null;
+			string? descriptor = null;
+			ValueTask LockCleanupAction()
+			{
+				ValueTask disposeTask = ValueTask.CompletedTask;
+				lock (locks)
+				{
+					logger.LogTrace("Removing .dmb Lock: {descriptor}", descriptor);
+
+					if (locks.Remove(newLock!))
+						logger.LogTrace("Lock was removed from list successfully");
+					else
+						logger.LogError("A .dmb lock was disposed more than once: {descriptor}", descriptor);
+
+					if (locks.Count == 0)
+						disposeTask = dmbProvider.DisposeAsync();
+					else if (newLock == firstLock)
+						logger.LogDebug("First lock on CompileJob #{compileJobId} removed, it must cleanup {remaining} remaining locks to be cleaned", CompileJob.Id, locks.Count);
+				}
+
+				return disposeTask;
+			}
+
+			newLock = new DmbLock(LockCleanupAction, dmbProvider, $"{callerFile}#{callerLine}: {reason}");
+			locks.Add(newLock);
+
+			descriptor = GetFullLockDescriptor(newLock!);
+			logger.LogTrace("Created .dmb Lock: {descriptor}", descriptor);
+
+			return newLock;
+		}
+	}
+}

src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs

@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using System.Diagnostics.CodeAnalysis;
 using System.IO;
 using System.Linq;
+using System.Runtime.CompilerServices;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
@@ -31,14 +32,14 @@ namespace Tgstation.Server.Host.Components.Deployment
 		{
 			get
 			{
-				lock (jobLockCounts)
+				lock (jobLockManagers)
 					return newerDmbTcs.Task;
 			}
 		}
 
 		/// <inheritdoc />
-		[MemberNotNullWhen(true, nameof(nextDmbProvider))]
-		public bool DmbAvailable => nextDmbProvider != null;
+		[MemberNotNullWhen(true, nameof(nextLockManager))]
+		public bool DmbAvailable => nextLockManager != null;
 
 		/// <summary>
 		/// The <see cref="IDatabaseContextFactory"/> for the <see cref="DmbFactory"/>.
@@ -61,10 +62,15 @@ namespace Tgstation.Server.Host.Components.Deployment
 		readonly ILogger<DmbFactory> logger;
 
 		/// <summary>
-		/// The <see cref="IEventConsumer"/> for <see cref="DmbFactory"/>.
+		/// The <see cref="IEventConsumer"/> for the <see cref="DmbFactory"/>.
 		/// </summary>
 		readonly IEventConsumer eventConsumer;
 
+		/// <summary>
+		/// The <see cref="IAsyncDelayer"/> for the <see cref="DmbFactory"/>.
+		/// </summary>
+		readonly IAsyncDelayer asyncDelayer;
+
 		/// <summary>
 		/// The <see cref="Api.Models.Instance"/> for the <see cref="DmbFactory"/>.
 		/// </summary>
@@ -75,10 +81,15 @@ namespace Tgstation.Server.Host.Components.Deployment
 		/// </summary>
 		readonly CancellationTokenSource cleanupCts;
 
+		/// <summary>
+		/// The <see cref="CancellationTokenSource"/> for <see cref="LogLockStates"/>.
+		/// </summary>
+		readonly CancellationTokenSource lockLogCts;
+
 		/// <summary>
 		/// Map of <see cref="CompileJob.JobId"/>s to locks on them.
 		/// </summary>
-		readonly Dictionary<long, int> jobLockCounts;
+		readonly Dictionary<long, DeploymentLockManager> jobLockManagers;
 
 		/// <summary>
 		/// <see cref="TaskCompletionSource"/> resulting in the latest <see cref="DmbProvider"/> yet to exist.
@@ -91,9 +102,9 @@ namespace Tgstation.Server.Host.Components.Deployment
 		Task cleanupTask;
 
 		/// <summary>
-		/// The latest <see cref="DmbProvider"/>.
+		/// The <see cref="DeploymentLockManager"/> for the latest <see cref="DmbProvider"/>.
 		/// </summary>
-		IDmbProvider? nextDmbProvider;
+		DeploymentLockManager? nextLockManager;
 
 		/// <summary>
 		/// If the <see cref="DmbFactory"/> is "started" via <see cref="IComponentService"/>.
@@ -107,6 +118,7 @@ namespace Tgstation.Server.Host.Components.Deployment
 		/// <param name="ioManager">The value of <see cref="ioManager"/>.</param>
 		/// <param name="remoteDeploymentManagerFactory">The value of <see cref="remoteDeploymentManagerFactory"/>.</param>
 		/// <param name="eventConsumer">The value of <see cref="eventConsumer"/>.</param>
+		/// <param name="asyncDelayer">The value of <see cref="asyncDelayer"/>.</param>
 		/// <param name="logger">The value of <see cref="logger"/>.</param>
 		/// <param name="metadata">The value of <see cref="metadata"/>.</param>
 		public DmbFactory(
@@ -114,6 +126,7 @@ namespace Tgstation.Server.Host.Components.Deployment
 			IIOManager ioManager,
 			IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory,
 			IEventConsumer eventConsumer,
+			IAsyncDelayer asyncDelayer,
 			ILogger<DmbFactory> logger,
 			Api.Models.Instance metadata)
 		{
@@ -121,27 +134,37 @@ namespace Tgstation.Server.Host.Components.Deployment
 			this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager));
 			this.remoteDeploymentManagerFactory = remoteDeploymentManagerFactory ?? throw new ArgumentNullException(nameof(remoteDeploymentManagerFactory));
 			this.eventConsumer = eventConsumer ?? throw new ArgumentNullException(nameof(eventConsumer));
+			this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer));
 			this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
 			this.metadata = metadata ?? throw new ArgumentNullException(nameof(metadata));
 
 			cleanupTask = Task.CompletedTask;
 			newerDmbTcs = new TaskCompletionSource();
 			cleanupCts = new CancellationTokenSource();
-			jobLockCounts = new Dictionary<long, int>();
+			lockLogCts = new CancellationTokenSource();
+			jobLockManagers = new Dictionary<long, DeploymentLockManager>();
 		}
 
 		/// <inheritdoc />
-		public void Dispose() => cleanupCts.Dispose(); // we don't dispose nextDmbProvider here, since it might be the only thing we have
+		public void Dispose()
+		{
+			// we don't dispose nextDmbProvider here, since it might be the only thing we have
+			lockLogCts.Dispose();
+			cleanupCts.Dispose();
+		}
 
 		/// <inheritdoc />
 		public async ValueTask LoadCompileJob(CompileJob job, Action<bool>? activationAction, CancellationToken cancellationToken)
 		{
 			ArgumentNullException.ThrowIfNull(job);
 
-			var newProvider = await FromCompileJob(job, cancellationToken);
-			if (newProvider == null)
+			var (dmbProvider, lockManager) = await FromCompileJobInternal(job, "Compile job loading", cancellationToken);
+			if (dmbProvider == null)
 				return;
 
+			if (lockManager == null)
+				throw new InvalidOperationException($"We did not acquire the first lock for compile job {job.Id}!");
+
 			// Do this first, because it's entirely possible when we set the tcs it will immediately need to be applied
 			if (started)
 			{
@@ -149,16 +172,16 @@ namespace Tgstation.Server.Host.Components.Deployment
 					metadata,
 					job);
 				await remoteDeploymentManager.StageDeployment(
-					newProvider.CompileJob,
+					lockManager.CompileJob,
 					activationAction,
 					cancellationToken);
 			}
 
 			ValueTask dmbDisposeTask;
-			lock (jobLockCounts)
+			lock (jobLockManagers)
 			{
-				dmbDisposeTask = nextDmbProvider?.DisposeAsync() ?? ValueTask.CompletedTask;
-				nextDmbProvider = newProvider;
+				dmbDisposeTask = nextLockManager?.DisposeAsync() ?? ValueTask.CompletedTask;
+				nextLockManager = lockManager;
 
 				// Oh god dammit
 				var temp = Interlocked.Exchange(ref newerDmbTcs, new TaskCompletionSource());
@@ -169,20 +192,12 @@ namespace Tgstation.Server.Host.Components.Deployment
 		}
 
 		/// <inheritdoc />
-		public IDmbProvider LockNextDmb(int lockCount)
+		public IDmbProvider LockNextDmb(string reason, [CallerFilePath] string? callerFile = null, [CallerLineNumber] int callerLine = default)
 		{
 			if (!DmbAvailable)
 				throw new InvalidOperationException("No .dmb available!");
-			if (lockCount < 0)
-				throw new ArgumentOutOfRangeException(nameof(lockCount), lockCount, "lockCount must be greater than or equal to 0!");
-			lock (jobLockCounts)
-			{
-				var jobId = nextDmbProvider.CompileJob.Require(x => x.Id);
-				var incremented = jobLockCounts[jobId] += lockCount;
-				logger.LogTrace("Compile job {jobId} lock increased by: {increment}", jobId, lockCount);
-				LogLockCounts();
-				return nextDmbProvider;
-			}
+
+			return nextLockManager.AddLock(reason, callerFile, callerLine);
 		}
 
 		/// <inheritdoc />
@@ -210,6 +225,7 @@ namespace Tgstation.Server.Host.Components.Deployment
 			}
 
 			// we dont do CleanUnusedCompileJobs here because the watchdog may have plans for them yet
+			cleanupTask = Task.WhenAll(cleanupTask, LogLockStates());
 		}
 
 		/// <inheritdoc />
@@ -217,8 +233,10 @@ namespace Tgstation.Server.Host.Components.Deployment
 		{
 			try
 			{
-				lock (jobLockCounts)
-					remoteDeploymentManagerFactory.ForgetLocalStateForCompileJobs(jobLockCounts.Keys);
+				lockLogCts.Cancel();
+
+				lock (jobLockManagers)
+					remoteDeploymentManagerFactory.ForgetLocalStateForCompileJobs(jobLockManagers.Keys);
 
 				using (cancellationToken.Register(() => cleanupCts.Cancel()))
 					await cleanupTask;
@@ -231,124 +249,15 @@ namespace Tgstation.Server.Host.Components.Deployment
 
 		/// <inheritdoc />
 #pragma warning disable CA1506 // TODO: Decomplexify
-		public async ValueTask<IDmbProvider?> FromCompileJob(CompileJob compileJob, CancellationToken cancellationToken)
+		public async ValueTask<IDmbProvider?> FromCompileJob(CompileJob compileJob, string reason, CancellationToken cancellationToken, [CallerFilePath] string? callerFile = null, [CallerLineNumber] int callerLine = default)
 		{
 			ArgumentNullException.ThrowIfNull(compileJob);
+			ArgumentNullException.ThrowIfNull(reason);
 
-			// ensure we have the entire metadata tree
-			var compileJobId = compileJob.Require(x => x.Id);
-			logger.LogTrace("Loading compile job {id}...", compileJobId);
-			await databaseContextFactory.UseContext(
-				async db => compileJob = await db
-					.CompileJobs
-					.AsQueryable()
-					.Where(x => x!.Id == compileJobId)
-					.Include(x => x.Job!)
-						.ThenInclude(x => x.StartedBy)
-					.Include(x => x.Job!)
-						.ThenInclude(x => x.Instance)
-					.Include(x => x.RevisionInformation!)
-						.ThenInclude(x => x.PrimaryTestMerge!)
-							.ThenInclude(x => x.MergedBy)
-					.Include(x => x.RevisionInformation!)
-						.ThenInclude(x => x.ActiveTestMerges!)
-							.ThenInclude(x => x.TestMerge!)
-								.ThenInclude(x => x.MergedBy)
-					.FirstAsync(cancellationToken)); // can't wait to see that query
+			var (dmb, _) = await FromCompileJobInternal(compileJob, reason, cancellationToken, callerFile, callerLine);
 
-			EngineVersion engineVersion;
-			if (!EngineVersion.TryParse(compileJob.EngineVersion, out var engineVersionNullable))
-			{
-				logger.LogWarning("Error loading compile job, bad engine version: {engineVersion}", compileJob.EngineVersion);
-				return null; // omae wa mou shinderu
-			}
-			else
-				engineVersion = engineVersionNullable!;
-
-			if (!compileJob.Job.StoppedAt.HasValue)
-			{
-				// This happens when we're told to load the compile job that is currently finished up
-				// It constitutes an API violation if it's returned by the DreamDaemonController so just set it here
-				// Bit of a hack, but it works out to be nearly if not the same value that's put in the DB
-				logger.LogTrace("Setting missing StoppedAt for CompileJob.Job #{id}...", compileJob.Job.Id);
-				compileJob.Job.StoppedAt = DateTimeOffset.UtcNow;
-			}
-
-			var providerSubmitted = false;
-
-			void CleanupAction()
-			{
-				if (providerSubmitted)
-					CleanRegisteredCompileJob(compileJob);
-			}
-
-			var newProvider = new DmbProvider(compileJob, engineVersion, ioManager, new DisposeInvoker(CleanupAction));
-			try
-			{
-				const string LegacyADirectoryName = "A";
-				const string LegacyBDirectoryName = "B";
-
-				var dmbExistsAtRoot = await ioManager.FileExists(
-					ioManager.ConcatPath(
-						newProvider.Directory,
-						newProvider.DmbName),
-					cancellationToken);
-
-				if (!dmbExistsAtRoot)
-				{
-					logger.LogTrace("Didn't find .dmb at game directory root, checking A/B dirs...");
-					var primaryCheckTask = ioManager.FileExists(
-						ioManager.ConcatPath(
-							newProvider.Directory,
-							LegacyADirectoryName,
-							newProvider.DmbName),
-						cancellationToken);
-					var secondaryCheckTask = ioManager.FileExists(
-						ioManager.ConcatPath(
-							newProvider.Directory,
-							LegacyBDirectoryName,
-							newProvider.DmbName),
-						cancellationToken);
-
-					if (!(await primaryCheckTask && await secondaryCheckTask))
-					{
-						logger.LogWarning("Error loading compile job, .dmb missing!");
-						return null; // omae wa mou shinderu
-					}
-
-					// rebuild the provider because it's using the legacy style directories
-					// Don't dispose it
-					logger.LogDebug("Creating legacy two folder .dmb provider targeting {aDirName} directory...", LegacyADirectoryName);
-					newProvider = new DmbProvider(compileJob, engineVersion, ioManager, new DisposeInvoker(CleanupAction), Path.DirectorySeparatorChar + LegacyADirectoryName);
-				}
-
-				lock (jobLockCounts)
-				{
-					if (!jobLockCounts.TryGetValue(compileJobId, out int value))
-					{
-						value = 1;
-						logger.LogTrace("Initializing lock count for compile job {id}", compileJobId);
-						jobLockCounts.Add(compileJobId, 1);
-					}
-					else
-					{
-						logger.LogTrace("FromCompileJob already had a jobLockCounts entry for {id}. Incrementing lock count to {value}.", compileJobId, value);
-						jobLockCounts[compileJobId] = ++value;
-					}
-
-					providerSubmitted = true;
-
-					LogLockCounts();
-					return newProvider;
-				}
-			}
-			finally
-			{
-				if (!providerSubmitted)
-					await newProvider.DisposeAsync();
-			}
+			return dmb;
 		}
-#pragma warning restore CA1506
 
 		/// <inheritdoc />
 #pragma warning disable CA1506 // TODO: Decomplexify
@@ -357,8 +266,8 @@ namespace Tgstation.Server.Host.Components.Deployment
 			List<long> jobIdsToSkip;
 
 			// don't clean locked directories
-			lock (jobLockCounts)
-				jobIdsToSkip = jobLockCounts.Keys.ToList();
+			lock (jobLockManagers)
+				jobIdsToSkip = jobLockManagers.Keys.ToList();
 
 			List<string>? jobUidsToNotErase = null;
 
@@ -413,11 +322,143 @@ namespace Tgstation.Server.Host.Components.Deployment
 #pragma warning restore CA1506
 
 		/// <inheritdoc />
-		public CompileJob? LatestCompileJob()
+		public async ValueTask<CompileJob?> LatestCompileJob()
 		{
 			if (!DmbAvailable)
 				return null;
-			return LockNextDmb(0)?.CompileJob;
+
+			await using IDmbProvider provider = LockNextDmb("Checking latest CompileJob");
+
+			return provider.CompileJob;
+		}
+
+		/// <summary>
+		/// Gets a <see cref="IDmbProvider"/> and potentially the <see cref="DeploymentLockManager"/> for a given <see cref="CompileJob"/>.
+		/// </summary>
+		/// <param name="compileJob">The <see cref="CompileJob"/> to make the <see cref="IDmbProvider"/> for.</param>
+		/// <param name="reason">The reason the compile job needed to be loaded.</param>
+		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
+		/// <param name="callerFile">The file path of the calling function.</param>
+		/// <param name="callerLine">The line number of the call invocation.</param>
+		/// <returns>A <see cref="ValueTask{TResult}"/> resulting in, on success, a tuple containing new <see cref="IDmbProvider"/> representing the <see cref="CompileJob"/>. If the first lock on <paramref name="compileJob"/> was acquired, the <see cref="DeploymentLockManager"/> will also be returned. On failure, <see langword="null"/> Will be returned.</returns>
+		async ValueTask<(IDmbProvider? DmbProvider, DeploymentLockManager? LockManager)> FromCompileJobInternal(CompileJob compileJob, string reason, CancellationToken cancellationToken, [CallerFilePath] string? callerFile = null, [CallerLineNumber] int callerLine = default)
+		{
+			// ensure we have the entire metadata tree
+			var compileJobId = compileJob.Require(x => x.Id);
+			lock (jobLockManagers)
+				if (jobLockManagers.TryGetValue(compileJobId, out var lockManager))
+					return (DmbProvider: lockManager.AddLock(reason, callerFile, callerLine), LockManager: null); // fast path
+
+			logger.LogTrace("Loading compile job {id}...", compileJobId);
+			await databaseContextFactory.UseContext(
+				async db => compileJob = await db
+					.CompileJobs
+					.AsQueryable()
+					.Where(x => x!.Id == compileJobId)
+					.Include(x => x.Job!)
+						.ThenInclude(x => x.StartedBy)
+					.Include(x => x.Job!)
+						.ThenInclude(x => x.Instance)
+					.Include(x => x.RevisionInformation!)
+						.ThenInclude(x => x.PrimaryTestMerge!)
+							.ThenInclude(x => x.MergedBy)
+					.Include(x => x.RevisionInformation!)
+						.ThenInclude(x => x.ActiveTestMerges!)
+							.ThenInclude(x => x.TestMerge!)
+								.ThenInclude(x => x.MergedBy)
+					.FirstAsync(cancellationToken)); // can't wait to see that query
+
+			EngineVersion engineVersion;
+			if (!EngineVersion.TryParse(compileJob.EngineVersion, out var engineVersionNullable))
+			{
+				logger.LogError("Error loading compile job, bad engine version: {engineVersion}", compileJob.EngineVersion);
+				return (null, null); // omae wa mou shinderu
+			}
+			else
+				engineVersion = engineVersionNullable!;
+
+			if (!compileJob.Job.StoppedAt.HasValue)
+			{
+				// This happens when we're told to load the compile job that is currently finished up
+				// It constitutes an API violation if it's returned by the DreamDaemonController so just set it here
+				// Bit of a hack, but it works out to be nearly if not the same value that's put in the DB
+				logger.LogTrace("Setting missing StoppedAt for CompileJob.Job #{id}...", compileJob.Job.Id);
+				compileJob.Job.StoppedAt = DateTimeOffset.UtcNow;
+			}
+
+			var providerSubmitted = false;
+			void CleanupAction()
+			{
+				if (providerSubmitted)
+					CleanRegisteredCompileJob(compileJob);
+			}
+
+			var newProvider = new DmbProvider(compileJob, engineVersion, ioManager, new DisposeInvoker(CleanupAction));
+			try
+			{
+				const string LegacyADirectoryName = "A";
+				const string LegacyBDirectoryName = "B";
+
+				var dmbExistsAtRoot = await ioManager.FileExists(
+					ioManager.ConcatPath(
+						newProvider.Directory,
+						newProvider.DmbName),
+					cancellationToken);
+
+				if (!dmbExistsAtRoot)
+				{
+					logger.LogTrace("Didn't find .dmb at game directory root, checking A/B dirs...");
+					var primaryCheckTask = ioManager.FileExists(
+						ioManager.ConcatPath(
+							newProvider.Directory,
+							LegacyADirectoryName,
+							newProvider.DmbName),
+						cancellationToken);
+					var secondaryCheckTask = ioManager.FileExists(
+						ioManager.ConcatPath(
+							newProvider.Directory,
+							LegacyBDirectoryName,
+							newProvider.DmbName),
+						cancellationToken);
+
+					if (!(await primaryCheckTask && await secondaryCheckTask))
+					{
+						logger.LogWarning("Error loading compile job, .dmb missing!");
+						return (null, null); // omae wa mou shinderu
+					}
+
+					// rebuild the provider because it's using the legacy style directories
+					// Don't dispose it
+					logger.LogDebug("Creating legacy two folder .dmb provider targeting {aDirName} directory...", LegacyADirectoryName);
+#pragma warning disable CA2000 // Dispose objects before losing scope (false positive)
+					newProvider = new DmbProvider(compileJob, engineVersion, ioManager, new DisposeInvoker(CleanupAction), Path.DirectorySeparatorChar + LegacyADirectoryName);
+#pragma warning restore CA2000 // Dispose objects before losing scope
+				}
+
+				lock (jobLockManagers)
+				{
+					IDmbProvider lockedProvider;
+					if (!jobLockManagers.TryGetValue(compileJobId, out var lockManager))
+					{
+						lockManager = DeploymentLockManager.Create(newProvider, logger, reason, out lockedProvider);
+						jobLockManagers.Add(compileJobId, lockManager);
+
+						providerSubmitted = true;
+					}
+					else
+					{
+						lockedProvider = lockManager.AddLock(reason, callerFile, callerLine); // race condition
+						lockManager = null;
+					}
+
+					return (DmbProvider: lockedProvider, LockManager: lockManager);
+				}
+			}
+			finally
+			{
+				if (!providerSubmitted)
+					await newProvider.DisposeAsync();
+			}
 		}
 
 		/// <summary>
@@ -428,6 +469,9 @@ namespace Tgstation.Server.Host.Components.Deployment
 		{
 			Task HandleCleanup()
 			{
+				lock (jobLockManagers)
+					jobLockManagers.Remove(job.Require(x => x.Id));
+
 				var otherTask = cleanupTask;
 
 				async Task WrapThrowableTasks()
@@ -453,26 +497,8 @@ namespace Tgstation.Server.Host.Components.Deployment
 				return Task.WhenAll(otherTask, WrapThrowableTasks());
 			}
 
-			lock (jobLockCounts)
-			{
-				var jobId = job.Require(x => x.Id);
-				if (jobLockCounts.TryGetValue(jobId, out var currentVal))
-					if (currentVal == 1)
-					{
-						jobLockCounts.Remove(jobId);
-						logger.LogDebug("Cleaning lock-free compile job {id} => {dirName}", jobId, job.DirectoryName);
-						cleanupTask = HandleCleanup();
-					}
-					else
-					{
-						var decremented = --jobLockCounts[jobId];
-						logger.LogTrace("Compile job {id} lock count now: {lockCount}", jobId, decremented);
-					}
-				else
-					logger.LogError("Extra Dispose of DmbProvider for CompileJob {compileJobId}!", jobId);
-
-				LogLockCounts();
-			}
+			lock (cleanupCts)
+				cleanupTask = HandleCleanup();
 		}
 
 		/// <summary>
@@ -489,28 +515,32 @@ namespace Tgstation.Server.Host.Components.Deployment
 		}
 
 		/// <summary>
-		/// Log out the current lock counts to Trace.
+		/// Lock all <see cref="DeploymentLockManager"/>s states.
 		/// </summary>
-		/// <remarks><see cref="jobLockCounts"/> must be locked before calling this function.</remarks>
-		void LogLockCounts()
+		/// <returns>A <see cref="Task"/> representing the running operation.</returns>
+		async Task LogLockStates()
 		{
-			if (jobLockCounts.Count == 0)
-			{
-				logger.LogWarning("No compile jobs registered!");
-				return;
-			}
+			logger.LogTrace("Entering lock logging loop");
+			CancellationToken cancellationToken = lockLogCts.Token;
 
-			var builder = new StringBuilder();
-			foreach (var jobId in jobLockCounts.Keys)
-			{
-				builder.AppendLine();
-				builder.Append("\t- ");
-				builder.Append(jobId);
-				builder.Append(": ");
-				builder.Append(jobLockCounts[jobId]);
-			}
+			while (!cancellationToken.IsCancellationRequested)
+				try
+				{
+					var builder = new StringBuilder();
 
-			logger.LogTrace("Compile Job Lock Counts:{details}", builder.ToString());
+					lock (jobLockManagers)
+						foreach (var lockManager in jobLockManagers.Values)
+							lockManager.LogLockStats(builder);
+
+					logger.LogDebug("Periodic deployment log states report (R.e. Issue #1779):{newLine}{report}", Environment.NewLine, builder); // TODO: Reduce to trace once #1779 is fixed
+
+					await asyncDelayer.Delay(TimeSpan.FromMinutes(10), cancellationToken);
+				}
+				catch (OperationCanceledException ex)
+				{
+					logger.LogTrace(ex, "Exiting lock logging loop");
+					break;
+				}
 		}
 	}
 }

src/Tgstation.Server.Host/Components/Deployment/DmbLock.cs

@@ -0,0 +1,82 @@
+using System;
+using System.Threading.Tasks;
+
+using Tgstation.Server.Api.Models;
+using Tgstation.Server.Host.Models;
+
+namespace Tgstation.Server.Host.Components.Deployment
+{
+	/// <summary>
+	/// Represents a lock on a given <see cref="IDmbProvider"/>.
+	/// </summary>
+	sealed class DmbLock : IDmbProvider
+	{
+		/// <inheritdoc />
+		public string DmbName => baseProvider.DmbName;
+
+		/// <inheritdoc />
+		public string Directory => baseProvider.Directory;
+
+		/// <inheritdoc />
+		public CompileJob CompileJob => baseProvider.CompileJob;
+
+		/// <inheritdoc />
+		public EngineVersion EngineVersion => baseProvider.EngineVersion;
+
+		/// <summary>
+		/// Unique ID of the lock.
+		/// </summary>
+		public Guid LockID { get; }
+
+		/// <summary>
+		/// The <see cref="DateTimeOffset"/> of when the lock was acquired.
+		/// </summary>
+		public DateTimeOffset LockTime { get; }
+
+		/// <summary>
+		/// A description of the <see cref="DmbLock"/>'s purpose.
+		/// </summary>
+		public string Descriptor { get; }
+
+		/// <summary>
+		/// If <see cref="KeepAlive"/> was called on the <see cref="DmbLock"/>.
+		/// </summary>
+		public bool KeptAlive { get; private set; }
+
+		/// <summary>
+		/// The <see cref="IDmbProvider"/> being wrapped.
+		/// </summary>
+		readonly IDmbProvider baseProvider;
+
+		/// <summary>
+		/// A <see cref="Func{TResult}"/> to use as the implementation of <see cref="DisposeAsync"/>.
+		/// </summary>
+		readonly Func<ValueTask> disposeAction;
+
+		/// <summary>
+		/// Initializes a new instance of the <see cref="DmbLock"/> class.
+		/// </summary>
+		/// <param name="disposeAction">The value of <see cref="disposeAction"/>.</param>
+		/// <param name="baseProvider">The value of <see cref="baseProvider"/>.</param>
+		/// <param name="descriptor">The value of <see cref="Descriptor"/>.</param>
+		public DmbLock(Func<ValueTask> disposeAction, IDmbProvider baseProvider, string descriptor)
+		{
+			this.disposeAction = disposeAction ?? throw new ArgumentNullException(nameof(disposeAction));
+			this.baseProvider = baseProvider ?? throw new ArgumentNullException(nameof(baseProvider));
+			Descriptor = descriptor ?? throw new ArgumentNullException(nameof(descriptor));
+
+			LockID = Guid.NewGuid();
+			LockTime = DateTimeOffset.UtcNow;
+		}
+
+		/// <inheritdoc />
+		public ValueTask DisposeAsync() => disposeAction();
+
+		/// <inheritdoc />
+		public void KeepAlive()
+		{
+			KeptAlive = true;
+			baseProvider.KeepAlive();
+		}
+	}
+}

src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs

@@ -326,7 +326,7 @@ namespace Tgstation.Server.Host.Components.Deployment
 						likelyPushedTestMergeCommit,
 						cancellationToken);
 
-				var activeCompileJob = compileJobConsumer.LatestCompileJob();
+				var activeCompileJob = await compileJobConsumer.LatestCompileJob();
 				try
 				{
 					await databaseContextFactory.UseContext(

src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs

@@ -31,7 +31,7 @@ namespace Tgstation.Server.Host.Components.Deployment
 		/// <summary>
 		/// The <see cref="Task"/> representing the base provider mirroring operation.
 		/// </summary>
-		readonly Task<string> mirroringTask;
+		readonly Task<string?> mirroringTask;
 
 		/// <summary>
 		/// The <see cref="ILogger"/> for the <see cref="HardLinkDmbProvider"/>.
@@ -77,13 +77,27 @@ namespace Tgstation.Server.Host.Components.Deployment
 		{
 			cancellationTokenSource.Cancel();
 			cancellationTokenSource.Dispose();
-			try
+			var mirroredDir = await mirroringTask;
+			if (mirroredDir != null && !Swapped)
 			{
-				await mirroringTask;
-			}
-			catch (OperationCanceledException ex)
-			{
-				logger.LogDebug(ex, "Mirroring task cancelled!");
+				logger.LogDebug("Cancelled mirroring task, we must cleanup!");
+
+				// We shouldn't be doing long running I/O ops because this could be under an HTTP request (DELETE /api/DreamDaemon)
+				// dirty shit to follow:
+				async void AsyncCleanup()
+				{
+					try
+					{
+						await IOManager.DeleteDirectory(mirroredDir, CancellationToken.None); // DCT: None available
+						logger.LogTrace("Completed async cleanup of unused mirror directory: {mirroredDir}", mirroredDir);
+					}
+					catch (Exception ex)
+					{
+						logger.LogError(ex, "Error cleaning up mirrored directory {mirroredDir}!", mirroredDir);
+					}
+				}
+
+				AsyncCleanup();
 			}
 
 			await base.DisposeAsync();
@@ -103,6 +117,13 @@ namespace Tgstation.Server.Host.Components.Deployment
 		{
 			logger.LogTrace("Begin DoSwap, mirroring task complete: {complete}...", mirroringTask.IsCompleted);
 			var mirroredDir = await mirroringTask.WaitAsync(cancellationToken);
+			if (mirroredDir == null)
+			{
+				// huh, how?
+				cancellationToken.ThrowIfCancellationRequested();
+				throw new InvalidOperationException("mirroringTask was cancelled without us being cancelled?");
+			}
+
 			var goAheadTcs = new TaskCompletionSource();
 
 			// I feel dirty...
@@ -119,6 +140,7 @@ namespace Tgstation.Server.Host.Components.Deployment
 					goAheadTcs.SetResult();
 					logger.LogTrace("Deleting old Live directory {path}...", disposePath);
 					await IOManager.DeleteDirectory(disposePath, CancellationToken.None); // DCT: We're detached at this point
+					logger.LogTrace("Completed async cleanup of old Live directory: {disposePath}", disposePath);
 				}
 				catch (DirectoryNotFoundException ex)
 				{
@@ -148,31 +170,59 @@ namespace Tgstation.Server.Host.Components.Deployment
 		/// <param name="securityLevel">The launch <see cref="DreamDaemonSecurity"/> level.</param>
 		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
 		/// <returns>A <see cref="Task{TResult}"/> resulting in the full path to the mirrored directory.</returns>
-		async Task<string> MirrorSourceDirectory(int? taskThrottle, DreamDaemonSecurity securityLevel, CancellationToken cancellationToken)
+		async Task<string?> MirrorSourceDirectory(int? taskThrottle, DreamDaemonSecurity securityLevel, CancellationToken cancellationToken)
 		{
-			var stopwatch = Stopwatch.StartNew();
-			var mirrorGuid = Guid.NewGuid();
-			logger.LogDebug("Starting to mirror {sourceDir} as hard links to {mirrorGuid}...", CompileJob.DirectoryName, mirrorGuid);
 			if (taskThrottle.HasValue && taskThrottle < 1)
 				throw new ArgumentOutOfRangeException(nameof(taskThrottle), taskThrottle, "taskThrottle must be at least 1!");
 
-			var src = IOManager.ResolvePath(CompileJob.DirectoryName!.Value.ToString());
-			var dest = IOManager.ResolvePath(mirrorGuid.ToString());
+			string? dest = null;
+			try
+			{
+				var stopwatch = Stopwatch.StartNew();
+				var mirrorGuid = Guid.NewGuid();
 
-			using var semaphore = taskThrottle.HasValue ? new SemaphoreSlim(taskThrottle.Value) : null;
-			await Task.WhenAll(MirrorDirectoryImpl(
-				src,
-				dest,
-				semaphore,
-				securityLevel,
-				cancellationToken));
-			stopwatch.Stop();
+				logger.LogDebug("Starting to mirror {sourceDir} as hard links to {mirrorGuid}...", CompileJob.DirectoryName, mirrorGuid);
 
-			logger.LogDebug(
-				"Finished mirror of {sourceDir} to {mirrorGuid} in {seconds}s...",
-				CompileJob.DirectoryName,
-				mirrorGuid,
-				stopwatch.Elapsed.TotalSeconds.ToString("0.##", CultureInfo.InvariantCulture));
+				var src = IOManager.ResolvePath(CompileJob.DirectoryName!.Value.ToString());
+				dest = IOManager.ResolvePath(mirrorGuid.ToString());
+
+				using var semaphore = taskThrottle.HasValue ? new SemaphoreSlim(taskThrottle.Value) : null;
+				await Task.WhenAll(MirrorDirectoryImpl(
+					src,
+					dest,
+					semaphore,
+					securityLevel,
+					cancellationToken));
+				stopwatch.Stop();
+
+				logger.LogDebug(
+					"Finished mirror of {sourceDir} to {mirrorGuid} in {seconds}s...",
+					CompileJob.DirectoryName,
+					mirrorGuid,
+					stopwatch.Elapsed.TotalSeconds.ToString("0.##", CultureInfo.InvariantCulture));
+			}
+			catch (OperationCanceledException ex)
+			{
+				logger.LogDebug(ex, "Cancelled while mirroring");
+			}
+			catch (Exception ex)
+			{
+				logger.LogError(ex, "Could not mirror!");
+
+				if (dest != null)
+					try
+					{
+						logger.LogDebug("Cleaning up mirror attempt: {dest}", dest);
+						await IOManager.DeleteDirectory(dest, cancellationToken);
+					}
+					catch (OperationCanceledException ex2)
+					{
+						logger.LogDebug(ex2, "Errored cleanup cancellation edge case!");
+						return dest;
+					}
+
+				return null;
+			}
 
 			return dest;
 		}

src/Tgstation.Server.Host/Components/Deployment/IDmbFactory.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Runtime.CompilerServices;
 using System.Threading;
 using System.Threading.Tasks;
 
@@ -25,17 +26,22 @@ namespace Tgstation.Server.Host.Components.Deployment
 		/// <summary>
 		/// Gets the next <see cref="IDmbProvider"/>. <see cref="DmbAvailable"/> is a precondition.
 		/// </summary>
-		/// <param name="lockCount">The amount of locks to give the resulting <see cref="IDmbProvider"/>. It's <see cref="IDisposable.Dispose"/> must be called this many times to properly clean the job.</param>
+		/// <param name="reason">The reason the lock is being acquired.</param>
+		/// <param name="callerFile">The file path of the calling function.</param>
+		/// <param name="callerLine">The line number of the call invocation.</param>
 		/// <returns>A new <see cref="IDmbProvider"/>.</returns>
-		IDmbProvider LockNextDmb(int lockCount);
+		IDmbProvider LockNextDmb(string reason, [CallerFilePath] string? callerFile = null, [CallerLineNumber] int callerLine = default);
 
 		/// <summary>
 		/// Gets a <see cref="IDmbProvider"/> for a given <see cref="CompileJob"/>.
 		/// </summary>
 		/// <param name="compileJob">The <see cref="CompileJob"/> to make the <see cref="IDmbProvider"/> for.</param>
+		/// <param name="reason">The reason the compile job needed to be loaded.</param>
 		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
+		/// <param name="callerFile">The file path of the calling function.</param>
+		/// <param name="callerLine">The line number of the call invocation.</param>
 		/// <returns>A <see cref="ValueTask{TResult}"/> resulting in a new <see cref="IDmbProvider"/> representing the <see cref="CompileJob"/> on success, <see langword="null"/> on failure.</returns>
-		ValueTask<IDmbProvider?> FromCompileJob(CompileJob compileJob, CancellationToken cancellationToken);
+		ValueTask<IDmbProvider?> FromCompileJob(CompileJob compileJob, string reason, CancellationToken cancellationToken, [CallerFilePath] string? callerFile = null, [CallerLineNumber] int callerLine = default);
 
 		/// <summary>
 		/// Deletes all compile jobs that are inactive in the Game folder.

src/Tgstation.Server.Host/Components/Deployment/ILatestCompileJobProvider.cs

@@ -1,4 +1,6 @@
-using Tgstation.Server.Host.Models;
+using System.Threading.Tasks;
+
+using Tgstation.Server.Host.Models;
 
 namespace Tgstation.Server.Host.Components.Deployment
 {
@@ -10,7 +12,7 @@ namespace Tgstation.Server.Host.Components.Deployment
 		/// <summary>
 		/// Gets the latest <see cref="CompileJob"/>.
 		/// </summary>
-		/// <returns>The latest <see cref="CompileJob"/> or <see langword="null"/> if none are available.</returns>
-		CompileJob? LatestCompileJob();
+		/// <returns>A <see cref="ValueTask{TResult}"/> resulting in the latest <see cref="CompileJob"/> or <see langword="null"/> if none are available.</returns>
+		ValueTask<CompileJob?> LatestCompileJob();
 	}
 }

src/Tgstation.Server.Host/Components/Engine/EngineManager.cs

@@ -466,7 +466,7 @@ namespace Tgstation.Server.Host.Components.Engine
 						logger.LogWarning("The required engine version ({version}) is not readily available! We will have to install it.", version);
 					}
 					else
-						logger.LogDebug("Requested engine version {version} not currently installed. Doing so now...", version);
+						logger.LogInformation("Requested engine version {version} not currently installed. Doing so now...", version);
 
 					if (progressReporter != null)
 						progressReporter.StageName = "Running event";

src/Tgstation.Server.Host/Components/Engine/OpenDreamInstallation.cs

@@ -137,6 +137,8 @@ namespace Tgstation.Server.Host.Components.Engine
 			logger.LogTrace("Attempting Robust.Server graceful exit (Timeout: {seconds}s)...", MaximumTerminationSeconds);
 			var timeout = asyncDelayer.Delay(TimeSpan.FromSeconds(MaximumTerminationSeconds), cancellationToken);
 			var lifetime = process.Lifetime;
+			if (lifetime.IsCompleted)
+				logger.LogTrace("Robust.Server already exited");
 
 			var stopwatch = Stopwatch.StartNew();
 			try

src/Tgstation.Server.Host/Components/Instance.cs

@@ -258,7 +258,7 @@ namespace Tgstation.Server.Host.Components
 		}
 
 		/// <inheritdoc />
-		public CompileJob? LatestCompileJob() => dmbFactory.LatestCompileJob();
+		public ValueTask<CompileJob?> LatestCompileJob() => dmbFactory.LatestCompileJob();
 
 		/// <summary>
 		/// The <see cref="JobEntrypoint"/> for updating the repository.
@@ -576,7 +576,7 @@ namespace Tgstation.Server.Host.Components
 							continue;
 						}
 
-						if (deploySha == LatestCompileJob()?.RevisionInformation.CommitSha)
+						if (deploySha == (await LatestCompileJob())?.RevisionInformation.CommitSha)
 						{
 							logger.LogTrace("Aborting auto update, same revision as latest CompileJob");
 							continue;

src/Tgstation.Server.Host/Components/InstanceFactory.cs

@@ -290,6 +290,7 @@ namespace Tgstation.Server.Host.Components
 					gameIoManager,
 					remoteDeploymentManagerFactory,
 					eventConsumer,
+					asyncDelayer,
 					loggerFactory.CreateLogger<DmbFactory>(),
 					metadata);
 				try

src/Tgstation.Server.Host/Components/InstanceWrapper.cs

@@ -58,6 +58,6 @@ namespace Tgstation.Server.Host.Components
 		public ValueTask ScheduleAutoUpdate(uint newInterval, string? newCron) => Instance.ScheduleAutoUpdate(newInterval, newCron);
 
 		/// <inheritdoc />
-		public CompileJob? LatestCompileJob() => Instance.LatestCompileJob();
+		public ValueTask<CompileJob?> LatestCompileJob() => Instance.LatestCompileJob();
 	}
 }

src/Tgstation.Server.Host/Components/Session/SessionController.cs

@@ -7,6 +7,7 @@ using System.Threading;
 using System.Threading.Tasks;
 
 using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging.Abstractions;
 
 using Newtonsoft.Json;
 
@@ -942,7 +943,9 @@ namespace Tgstation.Server.Host.Components.Session
 			using (await TopicSendSemaphore.Lock(cancellationToken))
 				byondResponse = await byondTopicSender.SendWithOptionalPriority(
 					asyncDelayer,
-					Logger,
+					LogTopicRequests
+						? Logger
+						: NullLogger.Instance,
 					queryString,
 					targetPort,
 					priority,

src/Tgstation.Server.Host/Components/Session/SessionPersistor.cs

@@ -208,7 +208,7 @@ namespace Tgstation.Server.Host.Components.Session
 				return null;
 			}
 
-			var dmb = await dmbFactory.FromCompileJob(result!.CompileJob!, cancellationToken);
+			var dmb = await dmbFactory.FromCompileJob(result!.CompileJob!, "Session Loading Main Deployment", cancellationToken);
 			if (dmb == null)
 			{
 				logger.LogError("Unable to reattach! Could not load .dmb!");
@@ -230,7 +230,7 @@ namespace Tgstation.Server.Host.Components.Session
 			if (result.InitialCompileJob != null)
 			{
 				logger.LogTrace("Loading initial compile job...");
-				initialDmb = await dmbFactory.FromCompileJob(result.InitialCompileJob, cancellationToken);
+				initialDmb = await dmbFactory.FromCompileJob(result.InitialCompileJob, "Session Loading Initial Deployment", cancellationToken);
 			}
 
 			logger.LogTrace("Retrieved ReattachInformation");

src/Tgstation.Server.Host/Components/Watchdog/AdvancedWatchdog.cs

@@ -232,7 +232,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// <inheritdoc />
 		protected sealed override async ValueTask HandleNewDmbAvailable(CancellationToken cancellationToken)
 		{
-			IDmbProvider compileJobProvider = DmbFactory.LockNextDmb(1);
+			IDmbProvider compileJobProvider = DmbFactory.LockNextDmb("AdvancedWatchdog next compile job preload");
 			bool canSeamlesslySwap = CanUseSwappableDmbProvider(compileJobProvider);
 			if (canSeamlesslySwap)
 				if (compileJobProvider.CompileJob.EngineVersion != ActiveCompileJob!.EngineVersion)

src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs

@@ -181,7 +181,8 @@ namespace Tgstation.Server.Host.Components.Watchdog
 					await HandleEventImpl(EventType.WorldPrime, Enumerable.Empty<string>(), false, cancellationToken);
 					break;
 				case MonitorActivationReason.ActiveServerStartup:
-					break; // unused in BasicWatchdog
+					Status = Api.Models.WatchdogStatus.Online;
+					break;
 				case MonitorActivationReason.HealthCheck:
 				default:
 					throw new InvalidOperationException($"Invalid activation reason: {reason}");
@@ -213,7 +214,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		{
 			// don't need a new dmb if reattaching
 			var reattachInProgress = reattachInfo != null;
-			var dmbToUse = reattachInProgress ? null : DmbFactory.LockNextDmb(1);
+			var dmbToUse = reattachInProgress ? null : DmbFactory.LockNextDmb("Watchdog initialization");
 
 			// if this try catches something, both servers are killed
 			try
@@ -296,7 +297,12 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
 		/// <returns>A <see cref="ValueTask{TResult}"/> resulting in the <see cref="MonitorAction"/> to take.</returns>
 		protected virtual ValueTask<MonitorAction> HandleNormalReboot(CancellationToken cancellationToken)
-			=> ValueTask.FromResult(MonitorAction.Continue);
+		{
+			if (Server!.CompileJob.DMApiVersion != null)
+				Status = Api.Models.WatchdogStatus.Restoring;
+
+			return ValueTask.FromResult(MonitorAction.Continue);
+		}
 
 		/// <summary>
 		/// Handler for <see cref="MonitorActivationReason.NewDmbAvailable"/>.

src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs

@@ -42,10 +42,11 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		public WatchdogStatus Status
 		{
 			get => status;
-			set
+			protected set
 			{
+				var oldStatus = status;
 				status = value;
-				Logger.LogTrace("Status set to {status}", status);
+				Logger.LogTrace("Status set from {oldStatus} to {status}", oldStatus, status);
 			}
 		}
 
@@ -814,21 +815,18 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// </summary>
 		/// <param name="currentCompileJob">The session's current <see cref="CompileJob"/>.</param>
 		/// <returns>A <see cref="Task"/> that completes if and when a newer <see cref="CompileJob"/> is available.</returns>
-		Task InitialCheckDmbUpdated(CompileJob currentCompileJob)
+		async Task InitialCheckDmbUpdated(CompileJob currentCompileJob)
 		{
 			var factoryTask = DmbFactory.OnNewerDmb;
 
-			var latestCompileJob = DmbFactory.LatestCompileJob();
-			if (latestCompileJob == null)
-				return factoryTask;
-
-			if (latestCompileJob.Id != currentCompileJob.Id)
+			var latestCompileJob = await DmbFactory.LatestCompileJob();
+			if (latestCompileJob != null && latestCompileJob.Id != currentCompileJob.Id)
 			{
 				Logger.LogDebug("Found new CompileJob without waiting");
-				return Task.CompletedTask;
+				return;
 			}
 
-			return factoryTask;
+			await factoryTask;
 		}
 
 		/// <summary>

src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs

@@ -87,7 +87,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 				return;
 			}
 
-			Server.ReattachInformation.InitialDmb = await DmbFactory.FromCompileJob(Server.CompileJob, cancellationToken);
+			Server.ReattachInformation.InitialDmb = await DmbFactory.FromCompileJob(Server.CompileJob, "WindowsWatchdog Initial Deployment", cancellationToken);
 		}
 
 		/// <inheritdoc />

src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs

@@ -386,7 +386,7 @@ namespace Tgstation.Server.Host.Controllers
 
 				if (revision)
 				{
-					var latestCompileJob = instance.LatestCompileJob();
+					var latestCompileJob = await instance.LatestCompileJob();
 					result.ActiveCompileJob = ((instance.Watchdog.Status != WatchdogStatus.Offline
 						? dd.ActiveCompileJob
 						: latestCompileJob) ?? latestCompileJob)

src/Tgstation.Server.Host/Extensions/TopicClientExtensions.cs

@@ -16,6 +16,11 @@ namespace Tgstation.Server.Host.Extensions
 	/// </summary>
 	static class TopicClientExtensions
 	{
+		/// <summary>
+		/// Counter for topic request logging.
+		/// </summary>
+		static ulong topicRequestId;
+
 		/// <summary>
 		/// Send a <paramref name="queryString"/> with optional repeated priority.
 		/// </summary>
@@ -45,13 +50,14 @@ namespace Tgstation.Server.Host.Extensions
 				{
 					firstSend = false;
 
-					logger.LogTrace("Begin topic request");
+					var localRequestId = Interlocked.Increment(ref topicRequestId);
+					logger.LogTrace("Begin topic request #{requestId}: {query}", localRequestId, queryString);
 					var byondResponse = await topicClient.SendTopic(
 						endpoint,
 						queryString,
 						cancellationToken);
 
-					logger.LogTrace("End topic request");
+					logger.LogTrace("End topic request #{requestId}", localRequestId);
 					return byondResponse;
 				}
 				catch (Exception ex) when (ex is not OperationCanceledException)

tests/DMAPI/LongRunning/Test.dm

@@ -3,7 +3,7 @@
 	loop_checks = FALSE
 
 /world/proc/RunTest()
-	log << "Initial value of sleep_offline: [sleep_offline]"
+	log << "Initial value of sleep_offline: [sleep_offline], setting to FALSE"
 	sleep_offline = FALSE
 
 	if(params["slow_start"])
@@ -215,6 +215,7 @@ var/run_bridge_test
 var/kajigger_test = FALSE
 
 /world/Reboot(reason)
+	log << "Reboot Start"
 	TgsChatBroadcast("World Rebooting")
 
 	if(kajigger_test && !fexists("kajigger.txt"))
@@ -222,6 +223,7 @@ var/kajigger_test = FALSE
 
 	TgsReboot()
 
+	log << "Calling base reboot"
 	..()
 
 var/received_health_check = FALSE

tests/Tgstation.Server.Tests/Live/Instance/DeploymentTest.cs

@@ -24,7 +24,7 @@ namespace Tgstation.Server.Tests.Live.Instance
 		readonly ushort dmPort;
 		readonly ushort ddPort;
 		readonly bool lowPriorityDeployments;
-		readonly EngineType testEngine;
+		readonly EngineVersion testEngine;
 
 		Task vpTest;
 
@@ -34,7 +34,7 @@ namespace Tgstation.Server.Tests.Live.Instance
 			ushort dmPort,
 			ushort ddPort,
 			bool lowPriorityDeployments,
-			EngineType testEngine) : base(jobsClient)
+			EngineVersion testEngine) : base(jobsClient)
 		{
 			this.instanceClient = instanceClient ?? throw new ArgumentNullException(nameof(instanceClient));
 			dreamMakerClient = instanceClient.DreamMaker;
@@ -65,7 +65,7 @@ namespace Tgstation.Server.Tests.Live.Instance
 			// this doesn't check dm's priority, but it really should
 			while (!deploymentJobWaitTask.IsCompleted)
 			{
-				var allProcesses = TestLiveServer.GetEngineServerProcessesOnPort(testEngine, dmPort);
+				var allProcesses = TestLiveServer.GetEngineServerProcessesOnPort(testEngine.Engine.Value, dmPort);
 				if (allProcesses.Count == 0)
 					continue;
 
@@ -133,13 +133,14 @@ namespace Tgstation.Server.Tests.Live.Instance
 			}
 			else
 			{
+				var canUseDashD = testEngine.Engine == EngineType.Byond && testEngine.Version >= new Version(515, 1597);
 				var updatedDM = await dreamMakerClient.Update(new DreamMakerRequest
 				{
 					ApiValidationPort = dmPort,
-					CompilerAdditionalArguments = testEngine == EngineType.Byond ? " -DBABABOOEY" : "     ",
+					CompilerAdditionalArguments = canUseDashD ? " -DBABABOOEY" : "     ",
 				}, cancellationToken);
 				Assert.AreEqual(dmPort, updatedDM.ApiValidationPort);
-				if (testEngine == EngineType.Byond)
+				if (canUseDashD)
 					Assert.AreEqual("-DBABABOOEY", updatedDM.CompilerAdditionalArguments);
 				else
 					Assert.IsNull(updatedDM.CompilerAdditionalArguments);

tests/Tgstation.Server.Tests/Live/Instance/EngineTest.cs

@@ -93,7 +93,7 @@ namespace Tgstation.Server.Tests.Live.Instance
 				}
 				else
 				{
-					var masterBranch = await TestingGitHubService.RealTestClient.Repository.Branch.Get("OpenDreamProject", "OpenDream", "master");
+					var masterBranch = await TestingGitHubService.RealClient.Repository.Branch.Get("OpenDreamProject", "OpenDream", "master");
 
 					engineVersion = new EngineVersion
 					{

tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs

@@ -1,8 +1,6 @@
 using System;
 using System.Collections.Generic;
-using System.IO;
 using System.Linq;
-using System.Reflection;
 using System.Threading;
 using System.Threading.Tasks;
 
@@ -24,7 +22,6 @@ using Tgstation.Server.Host.Components.Events;
 using Tgstation.Server.Host.Components.Repository;
 using Tgstation.Server.Host.Configuration;
 using Tgstation.Server.Host.IO;
-using Tgstation.Server.Host.Jobs;
 using Tgstation.Server.Host.System;
 using Tgstation.Server.Host.Utils;
 
@@ -51,7 +48,7 @@ namespace Tgstation.Server.Tests.Live.Instance
 			await using var chatTest = new ChatTest(instanceClient.ChatBots, instanceManagerClient, instanceClient.Jobs, instanceClient.Metadata);
 			var configTest = new ConfigurationTest(instanceClient.Configuration, instanceClient.Metadata);
 			await using var repoTest = new RepositoryTest(instanceClient.Repository, instanceClient.Jobs);
-			await using var dmTest = new DeploymentTest(instanceClient, instanceClient.Jobs, dmPort, ddPort, lowPrioDeployment, testVersion.Engine.Value);
+			await using var dmTest = new DeploymentTest(instanceClient, instanceClient.Jobs, dmPort, ddPort, lowPrioDeployment, testVersion);
 
 			var byondTask = engineTest.Run(cancellationToken, out var firstInstall);
 			var chatTask = chatTest.RunPreWatchdog(cancellationToken);
@@ -77,6 +74,12 @@ namespace Tgstation.Server.Tests.Live.Instance
 				ddPort,
 				usingBasicWatchdog);
 			await wdt.Run(cancellationToken);
+
+			await wdt.ExpectGameDirectoryCount(
+				usingBasicWatchdog || new PlatformIdentifier().IsWindows
+					? 2 // old + new deployment
+					: 3, // + new mirrored deployment waiting to take over Live
+				cancellationToken);
 		}
 
 		public static async ValueTask<IEngineInstallationData> DownloadEngineVersion(
@@ -285,6 +288,12 @@ namespace Tgstation.Server.Tests.Live.Instance
 			await using var wdt = new WatchdogTest(compatVersion, instanceClient, instanceManager, serverPort, highPrioDD, ddPort, usingBasicWatchdog);
 			await wdt.Run(cancellationToken);
 
+			await instanceClient.DreamDaemon.Shutdown(cancellationToken);
+
+			await wdt.ExpectGameDirectoryCount(
+				1, // current deployment
+				cancellationToken);
+
 			await instanceManagerClient.Update(new InstanceUpdateRequest
 			{
 				Id = instanceClient.Metadata.Id,

tests/Tgstation.Server.Tests/Live/Instance/JobsRequiredTest.cs

@@ -162,7 +162,8 @@ namespace Tgstation.Server.Tests.Live.Instance
 
 			await JobsClient.Cancel(job, cancellationToken);
 
-			timeout -= (int)Math.Ceiling((DateTimeOffset.UtcNow - start).TotalSeconds);
+			timeout -= Math.Max(0, (int)Math.Ceiling((DateTimeOffset.UtcNow - start).TotalSeconds));
+			timeout = Math.Max(0, timeout);
 			return await WaitForJob(job, timeout, false, null, cancellationToken);
 		}
 	}

tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs

@@ -595,6 +595,8 @@ namespace Tgstation.Server.Tests.Live.Instance
 			Assert.IsNull(daemonStatus.ActiveCompileJob.DMApiVersion);
 			Assert.AreEqual(DreamDaemonSecurity.Ultrasafe, daemonStatus.ActiveCompileJob.MinimumSecurityLevel);
 
+			await ExpectGameDirectoryCount(1, cancellationToken);
+
 			var startJob = await StartDD(cancellationToken);
 
 			await WaitForJob(startJob, 40, false, null, cancellationToken);
@@ -618,11 +620,39 @@ namespace Tgstation.Server.Tests.Live.Instance
 			Assert.AreEqual(initialCompileJob.Id, daemonStatus.ActiveCompileJob.Id);
 			var newerCompileJob = daemonStatus.StagedCompileJob;
 
+			await ExpectGameDirectoryCount(2, cancellationToken);
+
 			Assert.IsNotNull(newerCompileJob);
 			Assert.AreNotEqual(initialCompileJob.Id, newerCompileJob.Id);
 			Assert.AreEqual(DreamDaemonSecurity.Trusted, newerCompileJob.MinimumSecurityLevel);
 			Assert.AreEqual(DMApiConstants.InteropVersion, daemonStatus.StagedCompileJob.DMApiVersion);
 			await instanceClient.DreamDaemon.Shutdown(cancellationToken);
+
+			await ExpectGameDirectoryCount(1, cancellationToken);
+		}
+
+		public async ValueTask ExpectGameDirectoryCount(int expected, CancellationToken cancellationToken)
+		{
+			string[] lastDirectories;
+			int CountNonLiveDirs()
+			{
+				lastDirectories = Directory.GetDirectories(Path.Combine(instanceClient.Metadata.Path, "Game"));
+				return lastDirectories.Where(directory => Path.GetFileName(directory) != "Live").Count();
+			}
+
+			int nonLiveDirs = 0;
+			// cleanup task is async
+			for(var i = 0; i < 20; ++i)
+			{
+				nonLiveDirs = CountNonLiveDirs();
+				if (expected == nonLiveDirs)
+					return;
+
+				await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken);
+			}
+
+			nonLiveDirs = CountNonLiveDirs();
+			Assert.AreEqual(expected, nonLiveDirs, $"Directories present: {String.Join(", ", lastDirectories.Select(Path.GetFileName))}");
 		}
 
 		async Task RunBasicTest(CancellationToken cancellationToken)
@@ -643,6 +673,8 @@ namespace Tgstation.Server.Tests.Live.Instance
 			Assert.AreEqual(DMApiConstants.InteropVersion, daemonStatus.ActiveCompileJob.DMApiVersion);
 			Assert.AreEqual(DreamDaemonSecurity.Trusted, daemonStatus.ActiveCompileJob.MinimumSecurityLevel);
 
+			await ExpectGameDirectoryCount(1, cancellationToken);
+
 			JobResponse startJob;
 			if (new PlatformIdentifier().IsWindows) // Can't get address reuse to trigger on linux for some reason
 				using (var blockSocket = new Socket(
@@ -683,6 +715,7 @@ namespace Tgstation.Server.Tests.Live.Instance
 			daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
 			Assert.AreEqual(WatchdogStatus.Offline, daemonStatus.Status.Value);
 			Assert.IsFalse(daemonStatus.SessionId.HasValue);
+			await ExpectGameDirectoryCount(1, cancellationToken);
 
 			await CheckDMApiFail(daemonStatus.ActiveCompileJob, cancellationToken, false);
 
@@ -1069,6 +1102,15 @@ namespace Tgstation.Server.Tests.Live.Instance
 		// - Injects a custom bridge handler into the bridge registrar and makes the test hack into the DMAPI and change its access_identifier
 		async Task WhiteBoxChatCommandTest(CancellationToken cancellationToken)
 		{
+			var ddInfo = await instanceClient.DreamDaemon.Read(cancellationToken);
+			for (int i = 0; ddInfo.Status != WatchdogStatus.Online && i < 15; ++i)
+			{
+				await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken);
+				ddInfo = await instanceClient.DreamDaemon.Read(cancellationToken);
+			}
+
+			Assert.AreEqual(WatchdogStatus.Online, ddInfo.Status);
+
 			MessageContent embedsResponse, overloadResponse, overloadResponse2, embedsResponse2;
 			var startTime = DateTimeOffset.UtcNow - TimeSpan.FromSeconds(5);
 			using (var instanceReference = instanceManager.GetInstanceReference(instanceClient.Metadata))
@@ -1115,7 +1157,7 @@ namespace Tgstation.Server.Tests.Live.Instance
 
 			var endTime = DateTimeOffset.UtcNow + TimeSpan.FromSeconds(5);
 
-			var ddInfo = await instanceClient.DreamDaemon.Read(cancellationToken);
+			ddInfo = await instanceClient.DreamDaemon.Read(cancellationToken);
 			await CheckDMApiFail(ddInfo.ActiveCompileJob, cancellationToken);
 
 			CheckEmbedsTest(embedsResponse, startTime, endTime);

tests/Tgstation.Server.Tests/Live/TestLiveServer.cs

@@ -1724,7 +1724,7 @@ namespace Tgstation.Server.Tests.Live
 					Assert.AreEqual(dd.StagedCompileJob.Job.Id, compileJob.Id);
 
 					expectedCompileJobId = compileJob.Id.Value;
-					dd = await wdt.TellWorldToReboot(server.UsingBasicWatchdog, cancellationToken);
+					dd = await wdt.TellWorldToReboot(true, cancellationToken);
 
 					Assert.AreEqual(dd.ActiveCompileJob.Job.Id, expectedCompileJobId);
 					Assert.AreEqual(WatchdogStatus.Online, dd.Status.Value);

tests/Tgstation.Server.Tests/Live/TestingGitHubService.cs

@@ -28,7 +28,7 @@ namespace Tgstation.Server.Tests.Live
 		readonly ICryptographySuite cryptographySuite;
 		readonly ILogger<TestingGitHubService> logger;
 
-		public static readonly IGitHubClient RealTestClient;
+		public static readonly IGitHubClient RealClient;
 
 		static TestingGitHubService()
 		{
@@ -39,7 +39,7 @@ namespace Tgstation.Server.Tests.Live
 			});
 
 			var gitHubClientFactory = new GitHubClientFactory(new AssemblyInformationProvider(), Mock.Of<ILogger<GitHubClientFactory>>(), mockOptions.Object);
-			RealTestClient = gitHubClientFactory.CreateClient();
+			RealClient = gitHubClientFactory.CreateClient();
 		}
 
 		public static async Task InitializeAndInject(CancellationToken cancellationToken)
@@ -47,7 +47,7 @@ namespace Tgstation.Server.Tests.Live
 			Release targetRelease;
 			do
 			{
-				var releases = await RealTestClient
+				var releases = await RealClient
 					.Repository
 					.Release
 					.GetAll("tgstation", "tgstation-server")
@@ -62,12 +62,12 @@ namespace Tgstation.Server.Tests.Live
 				{ TestLiveServer.TestUpdateVersion, targetRelease }
 			};
 
-			var testCommitTask = RealTestClient
+			var testCommitTask = RealClient
 				.Repository
 				.Commit
 				.Get("Cyberboss", "common_core", "4b4926dfaf6295f19f8ae7abf03cb357dbb05b29")
 				.WaitAsync(cancellationToken);
-			testPr = await RealTestClient
+			testPr = await RealClient
 				.PullRequest
 				.Get("Cyberboss", "common_core", 2)
 				.WaitAsync(cancellationToken);