Rewrite IDmbProvider locking so that locks are tracked and reasons are given

src/Tgstation.Server.Host/Components/Chat/Commands/PullRequestsCommand.cs

@@ -137,7 +137,7 @@ namespace Tgstation.Server.Host.Components.Chat.Commands
 				var compileJobToUse = watchdog.ActiveCompileJob;
 				if (hasStaged)
 				{
-					var latestCompileJob = compileJobProvider.LatestCompileJob();
+					var latestCompileJob = await compileJobProvider.LatestCompileJob();
 					if (latestCompileJob?.Id != compileJobToUse?.Id)
 						compileJobToUse = latestCompileJob;
 					else

src/Tgstation.Server.Host/Components/Deployment/DeploymentLockManager.cs

@@ -0,0 +1,149 @@
+using System;
+using System.Collections.Generic;
+using System.Runtime.CompilerServices;
+using System.Threading.Tasks;
+
+using Microsoft.Extensions.Logging;
+
+using Tgstation.Server.Host.Models;
+
+namespace Tgstation.Server.Host.Components.Deployment
+{
+	/// <summary>
+	/// Manages locks on a given <see cref="IDmbProvider"/>.
+	/// </summary>
+	sealed class DeploymentLockManager : IAsyncDisposable
+	{
+		/// <summary>
+		/// The <see cref="Models.CompileJob"/> represented by the <see cref="DeploymentLockManager"/>.
+		/// </summary>
+		public CompileJob CompileJob => dmbProvider.CompileJob;
+
+		/// <summary>
+		/// The <see cref="ILogger"/> for the <see cref="DeploymentLockManager"/>.
+		/// </summary>
+		readonly ILogger logger;
+
+		/// <summary>
+		/// The <see cref="IDmbProvider"/> the <see cref="DeploymentLockManager"/> is managing.
+		/// </summary>
+		readonly IDmbProvider dmbProvider;
+
+		/// <summary>
+		/// The <see cref="DmbLock"/>s on the <see cref="dmbProvider"/>.
+		/// </summary>
+		readonly HashSet<DmbLock> locks;
+
+		/// <summary>
+		/// The first lock acquired by the <see cref="DeploymentLockManager"/>.
+		/// </summary>
+		readonly DmbLock firstLock;
+
+		/// <summary>
+		/// Create a <see cref="DeploymentLockManager"/>.
+		/// </summary>
+		/// <param name="dmbProvider">The value of <see cref="dmbProvider"/>.</param>
+		/// <param name="logger">The value of <see cref="logger"/>.</param>
+		/// <param name="initialLockReason">The reason for the first lock.</param>
+		/// <param name="firstLock">The <see cref="IDmbProvider"/> that represents the first lock.</param>
+		/// <param name="callerFile">The file path of the calling function.</param>
+		/// <param name="callerLine">The line number of the call invocation.</param>
+		/// <returns>A new <see cref="DeploymentLockManager"/>.</returns>
+		public static DeploymentLockManager Create(IDmbProvider dmbProvider, ILogger logger, string initialLockReason, out IDmbProvider firstLock, [CallerFilePath] string? callerFile = null, [CallerLineNumber] int callerLine = default)
+		{
+			var manager = new DeploymentLockManager(dmbProvider, logger, initialLockReason, callerFile!, callerLine);
+			firstLock = manager.firstLock;
+			return manager;
+		}
+
+		/// <summary>
+		/// Generates a verbose description of a given <paramref name="dmbLock"/>.
+		/// </summary>
+		/// <param name="dmbLock">The <see cref="DmbLock"/> to get a description of.</param>
+		/// <returns>A verbose description of <paramref name="dmbLock"/>.</returns>
+		static string GetFullLockDescriptor(DmbLock dmbLock) => $"{dmbLock.LockID} {dmbLock.Descriptor} (Created at {dmbLock.LockTime}){(dmbLock.KeptAlive ? " (RELEASED)" : String.Empty)}";
+
+		/// <summary>
+		/// Initializes a new instance of the <see cref="DeploymentLockManager"/> class.
+		/// </summary>
+		/// <param name="dmbProvider">The value of <see cref="dmbProvider"/>.</param>
+		/// <param name="logger">The value of <see cref="logger"/>.</param>
+		/// <param name="initialLockReason">The reason for the first lock.</param>
+		/// <param name="callerFile">The file path of the calling function.</param>
+		/// <param name="callerLine">The line number of the call invocation.</param>
+		/// <returns>A new <see cref="DeploymentLockManager"/>.</returns>
+		DeploymentLockManager(IDmbProvider dmbProvider, ILogger logger, string initialLockReason, string callerFile, int callerLine)
+		{
+			this.dmbProvider = dmbProvider ?? throw new ArgumentNullException(nameof(dmbProvider));
+			this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
+
+			logger.LogTrace("Initializing lock manager for compile job {id}", dmbProvider.CompileJob.Id);
+			locks = new HashSet<DmbLock>();
+			firstLock = CreateLock(initialLockReason, callerFile, callerLine);
+		}
+
+		/// <inheritdoc />
+		public ValueTask DisposeAsync()
+			=> firstLock.DisposeAsync();
+
+		/// <summary>
+		/// Add a lock to the managed <see cref="IDmbProvider"/>.
+		/// </summary>
+		/// <param name="reason">The reason for the lock.</param>
+		/// <param name="callerFile">The file path of the calling function.</param>
+		/// <param name="callerLine">The line number of the call invocation.</param>
+		/// <returns>A <see cref="IDmbProvider"/> whose lifetime represents the lock.</returns>
+		public IDmbProvider AddLock(string reason, [CallerFilePath] string? callerFile = null, [CallerLineNumber]int callerLine = default)
+		{
+			lock (locks)
+			{
+				if (locks.Count == 0)
+					throw new InvalidOperationException($"No locks exist on the DmbProvider for CompileJob {dmbProvider.CompileJob.Id}!");
+
+				return CreateLock(reason, callerFile!, callerLine);
+			}
+		}
+
+		/// <summary>
+		/// Creates a <see cref="DmbLock"/> and adds it to <see cref="locks"/>.
+		/// </summary>
+		/// <param name="reason">The reason for the lock.</param>
+		/// <param name="callerFile">The file path of the calling function.</param>
+		/// <param name="callerLine">The line number of the call invocation.</param>
+		/// <returns>A new <see cref="DmbLock"/>.</returns>
+		/// <remarks>Requires exclusive write access to <see cref="locks"/> be held by the caller.</remarks>
+		DmbLock CreateLock(string reason, string callerFile, int callerLine)
+		{
+			DmbLock? newLock = null;
+			string? descriptor = null;
+			ValueTask LockCleanupAction()
+			{
+				ValueTask disposeTask = ValueTask.CompletedTask;
+				lock (locks)
+				{
+					logger.LogTrace("Removing .dmb Lock: {descriptor}", descriptor);
+
+					if (locks.Remove(newLock!))
+						logger.LogTrace("Lock was removed from list successfully");
+					else
+						logger.LogError("A .dmb lock was disposed more than once: {descriptor}", descriptor);
+
+					if (locks.Count == 0)
+						disposeTask = dmbProvider.DisposeAsync();
+					else if (newLock == firstLock)
+						logger.LogDebug("First lock on CompileJob #{compileJobId} removed, it must cleanup {remaining} remaining locks to be cleaned", CompileJob.Id, locks.Count);
+				}
+
+				return disposeTask;
+			}
+
+			newLock = new DmbLock(LockCleanupAction, dmbProvider, $"{callerFile}#{callerLine}: {reason}");
+			locks.Add(newLock);
+
+			descriptor = GetFullLockDescriptor(newLock!);
+			logger.LogTrace("Created .dmb Lock: {descriptor}", descriptor);
+
+			return newLock;
+		}
+	}
+}

src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs

@@ -3,7 +3,7 @@ using System.Collections.Generic;
 using System.Diagnostics.CodeAnalysis;
 using System.IO;
 using System.Linq;
-using System.Text;
+using System.Runtime.CompilerServices;
 using System.Threading;
 using System.Threading.Tasks;
 
@@ -31,14 +31,14 @@ namespace Tgstation.Server.Host.Components.Deployment
 		{
 			get
 			{
-				lock (jobLockCounts)
+				lock (jobLockManagers)
 					return newerDmbTcs.Task;
 			}
 		}
 
 		/// <inheritdoc />
-		[MemberNotNullWhen(true, nameof(nextDmbProvider))]
-		public bool DmbAvailable => nextDmbProvider != null;
+		[MemberNotNullWhen(true, nameof(nextLockManager))]
+		public bool DmbAvailable => nextLockManager != null;
 
 		/// <summary>
 		/// The <see cref="IDatabaseContextFactory"/> for the <see cref="DmbFactory"/>.
@@ -78,7 +78,7 @@ namespace Tgstation.Server.Host.Components.Deployment
 		/// <summary>
 		/// Map of <see cref="CompileJob.JobId"/>s to locks on them.
 		/// </summary>
-		readonly Dictionary<long, int> jobLockCounts;
+		readonly Dictionary<long, DeploymentLockManager> jobLockManagers;
 
 		/// <summary>
 		/// <see cref="TaskCompletionSource"/> resulting in the latest <see cref="DmbProvider"/> yet to exist.
@@ -91,9 +91,9 @@ namespace Tgstation.Server.Host.Components.Deployment
 		Task cleanupTask;
 
 		/// <summary>
-		/// The latest <see cref="DmbProvider"/>.
+		/// The <see cref="DeploymentLockManager"/> for the latest <see cref="DmbProvider"/>.
 		/// </summary>
-		IDmbProvider? nextDmbProvider;
+		DeploymentLockManager? nextLockManager;
 
 		/// <summary>
 		/// If the <see cref="DmbFactory"/> is "started" via <see cref="IComponentService"/>.
@@ -127,7 +127,7 @@ namespace Tgstation.Server.Host.Components.Deployment
 			cleanupTask = Task.CompletedTask;
 			newerDmbTcs = new TaskCompletionSource();
 			cleanupCts = new CancellationTokenSource();
-			jobLockCounts = new Dictionary<long, int>();
+			jobLockManagers = new Dictionary<long, DeploymentLockManager>();
 		}
 
 		/// <inheritdoc />
@@ -138,10 +138,13 @@ namespace Tgstation.Server.Host.Components.Deployment
 		{
 			ArgumentNullException.ThrowIfNull(job);
 
-			var newProvider = await FromCompileJob(job, cancellationToken);
-			if (newProvider == null)
+			var (dmbProvider, lockManager) = await FromCompileJobInternal(job, "Compile job loading", cancellationToken);
+			if (dmbProvider == null)
 				return;
 
+			if (lockManager == null)
+				throw new InvalidOperationException($"We did not acquire the first lock for compile job {job.Id}!");
+
 			// Do this first, because it's entirely possible when we set the tcs it will immediately need to be applied
 			if (started)
 			{
@@ -149,16 +152,16 @@ namespace Tgstation.Server.Host.Components.Deployment
 					metadata,
 					job);
 				await remoteDeploymentManager.StageDeployment(
-					newProvider.CompileJob,
+					lockManager.CompileJob,
 					activationAction,
 					cancellationToken);
 			}
 
 			ValueTask dmbDisposeTask;
-			lock (jobLockCounts)
+			lock (jobLockManagers)
 			{
-				dmbDisposeTask = nextDmbProvider?.DisposeAsync() ?? ValueTask.CompletedTask;
-				nextDmbProvider = newProvider;
+				dmbDisposeTask = nextLockManager?.DisposeAsync() ?? ValueTask.CompletedTask;
+				nextLockManager = lockManager;
 
 				// Oh god dammit
 				var temp = Interlocked.Exchange(ref newerDmbTcs, new TaskCompletionSource());
@@ -169,20 +172,12 @@ namespace Tgstation.Server.Host.Components.Deployment
 		}
 
 		/// <inheritdoc />
-		public IDmbProvider LockNextDmb(int lockCount)
+		public IDmbProvider LockNextDmb(string reason, [CallerFilePath] string? callerFile = null, [CallerLineNumber] int callerLine = default)
 		{
 			if (!DmbAvailable)
 				throw new InvalidOperationException("No .dmb available!");
-			if (lockCount < 0)
-				throw new ArgumentOutOfRangeException(nameof(lockCount), lockCount, "lockCount must be greater than or equal to 0!");
-			lock (jobLockCounts)
-			{
-				var jobId = nextDmbProvider.CompileJob.Require(x => x.Id);
-				var incremented = jobLockCounts[jobId] += lockCount;
-				logger.LogTrace("Compile job {jobId} lock increased by: {increment}", jobId, lockCount);
-				LogLockCounts();
-				return nextDmbProvider;
-			}
+
+			return nextLockManager.AddLock(reason, callerFile, callerLine);
 		}
 
 		/// <inheritdoc />
@@ -217,8 +212,8 @@ namespace Tgstation.Server.Host.Components.Deployment
 		{
 			try
 			{
-				lock (jobLockCounts)
-					remoteDeploymentManagerFactory.ForgetLocalStateForCompileJobs(jobLockCounts.Keys);
+				lock (jobLockManagers)
+					remoteDeploymentManagerFactory.ForgetLocalStateForCompileJobs(jobLockManagers.Keys);
 
 				using (cancellationToken.Register(() => cleanupCts.Cancel()))
 					await cleanupTask;
@@ -231,124 +226,15 @@ namespace Tgstation.Server.Host.Components.Deployment
 
 		/// <inheritdoc />
 #pragma warning disable CA1506 // TODO: Decomplexify
-		public async ValueTask<IDmbProvider?> FromCompileJob(CompileJob compileJob, CancellationToken cancellationToken)
+		public async ValueTask<IDmbProvider?> FromCompileJob(CompileJob compileJob, string reason, CancellationToken cancellationToken, [CallerFilePath] string? callerFile = null, [CallerLineNumber] int callerLine = default)
 		{
 			ArgumentNullException.ThrowIfNull(compileJob);
+			ArgumentNullException.ThrowIfNull(reason);
 
-			// ensure we have the entire metadata tree
-			var compileJobId = compileJob.Require(x => x.Id);
-			logger.LogTrace("Loading compile job {id}...", compileJobId);
-			await databaseContextFactory.UseContext(
-				async db => compileJob = await db
-					.CompileJobs
-					.AsQueryable()
-					.Where(x => x!.Id == compileJobId)
-					.Include(x => x.Job!)
-						.ThenInclude(x => x.StartedBy)
-					.Include(x => x.Job!)
-						.ThenInclude(x => x.Instance)
-					.Include(x => x.RevisionInformation!)
-						.ThenInclude(x => x.PrimaryTestMerge!)
-							.ThenInclude(x => x.MergedBy)
-					.Include(x => x.RevisionInformation!)
-						.ThenInclude(x => x.ActiveTestMerges!)
-							.ThenInclude(x => x.TestMerge!)
-								.ThenInclude(x => x.MergedBy)
-					.FirstAsync(cancellationToken)); // can't wait to see that query
+			var (dmb, _) = await FromCompileJobInternal(compileJob, reason, cancellationToken, callerFile, callerLine);
 
-			EngineVersion engineVersion;
-			if (!EngineVersion.TryParse(compileJob.EngineVersion, out var engineVersionNullable))
-			{
-				logger.LogWarning("Error loading compile job, bad engine version: {engineVersion}", compileJob.EngineVersion);
-				return null; // omae wa mou shinderu
-			}
-			else
-				engineVersion = engineVersionNullable!;
-
-			if (!compileJob.Job.StoppedAt.HasValue)
-			{
-				// This happens when we're told to load the compile job that is currently finished up
-				// It constitutes an API violation if it's returned by the DreamDaemonController so just set it here
-				// Bit of a hack, but it works out to be nearly if not the same value that's put in the DB
-				logger.LogTrace("Setting missing StoppedAt for CompileJob.Job #{id}...", compileJob.Job.Id);
-				compileJob.Job.StoppedAt = DateTimeOffset.UtcNow;
-			}
-
-			var providerSubmitted = false;
-
-			void CleanupAction()
-			{
-				if (providerSubmitted)
-					CleanRegisteredCompileJob(compileJob);
-			}
-
-			var newProvider = new DmbProvider(compileJob, engineVersion, ioManager, new DisposeInvoker(CleanupAction));
-			try
-			{
-				const string LegacyADirectoryName = "A";
-				const string LegacyBDirectoryName = "B";
-
-				var dmbExistsAtRoot = await ioManager.FileExists(
-					ioManager.ConcatPath(
-						newProvider.Directory,
-						newProvider.DmbName),
-					cancellationToken);
-
-				if (!dmbExistsAtRoot)
-				{
-					logger.LogTrace("Didn't find .dmb at game directory root, checking A/B dirs...");
-					var primaryCheckTask = ioManager.FileExists(
-						ioManager.ConcatPath(
-							newProvider.Directory,
-							LegacyADirectoryName,
-							newProvider.DmbName),
-						cancellationToken);
-					var secondaryCheckTask = ioManager.FileExists(
-						ioManager.ConcatPath(
-							newProvider.Directory,
-							LegacyBDirectoryName,
-							newProvider.DmbName),
-						cancellationToken);
-
-					if (!(await primaryCheckTask && await secondaryCheckTask))
-					{
-						logger.LogWarning("Error loading compile job, .dmb missing!");
-						return null; // omae wa mou shinderu
-					}
-
-					// rebuild the provider because it's using the legacy style directories
-					// Don't dispose it
-					logger.LogDebug("Creating legacy two folder .dmb provider targeting {aDirName} directory...", LegacyADirectoryName);
-					newProvider = new DmbProvider(compileJob, engineVersion, ioManager, new DisposeInvoker(CleanupAction), Path.DirectorySeparatorChar + LegacyADirectoryName);
-				}
-
-				lock (jobLockCounts)
-				{
-					if (!jobLockCounts.TryGetValue(compileJobId, out int value))
-					{
-						value = 1;
-						logger.LogTrace("Initializing lock count for compile job {id}", compileJobId);
-						jobLockCounts.Add(compileJobId, 1);
-					}
-					else
-					{
-						logger.LogTrace("FromCompileJob already had a jobLockCounts entry for {id}. Incrementing lock count to {value}.", compileJobId, value);
-						jobLockCounts[compileJobId] = ++value;
-					}
-
-					providerSubmitted = true;
-
-					LogLockCounts();
-					return newProvider;
-				}
-			}
-			finally
-			{
-				if (!providerSubmitted)
-					await newProvider.DisposeAsync();
-			}
+			return dmb;
 		}
-#pragma warning restore CA1506
 
 		/// <inheritdoc />
 #pragma warning disable CA1506 // TODO: Decomplexify
@@ -357,8 +243,8 @@ namespace Tgstation.Server.Host.Components.Deployment
 			List<long> jobIdsToSkip;
 
 			// don't clean locked directories
-			lock (jobLockCounts)
-				jobIdsToSkip = jobLockCounts.Keys.ToList();
+			lock (jobLockManagers)
+				jobIdsToSkip = jobLockManagers.Keys.ToList();
 
 			List<string>? jobUidsToNotErase = null;
 
@@ -413,11 +299,143 @@ namespace Tgstation.Server.Host.Components.Deployment
 #pragma warning restore CA1506
 
 		/// <inheritdoc />
-		public CompileJob? LatestCompileJob()
+		public async ValueTask<CompileJob?> LatestCompileJob()
 		{
 			if (!DmbAvailable)
 				return null;
-			return LockNextDmb(0)?.CompileJob;
+
+			await using IDmbProvider provider = LockNextDmb("Checking latest CompileJob");
+
+			return provider.CompileJob;
+		}
+
+		/// <summary>
+		/// Gets a <see cref="IDmbProvider"/> and potentially the <see cref="DeploymentLockManager"/> for a given <see cref="CompileJob"/>.
+		/// </summary>
+		/// <param name="compileJob">The <see cref="CompileJob"/> to make the <see cref="IDmbProvider"/> for.</param>
+		/// <param name="reason">The reason the compile job needed to be loaded.</param>
+		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
+		/// <param name="callerFile">The file path of the calling function.</param>
+		/// <param name="callerLine">The line number of the call invocation.</param>
+		/// <returns>A <see cref="ValueTask{TResult}"/> resulting in, on success, a tuple containing new <see cref="IDmbProvider"/> representing the <see cref="CompileJob"/>. If the first lock on <paramref name="compileJob"/> was acquired, the <see cref="DeploymentLockManager"/> will also be returned. On failure, <see langword="null"/> Will be returned.</returns>
+		async ValueTask<(IDmbProvider? DmbProvider, DeploymentLockManager? LockManager)> FromCompileJobInternal(CompileJob compileJob, string reason, CancellationToken cancellationToken, [CallerFilePath] string? callerFile = null, [CallerLineNumber] int callerLine = default)
+		{
+			// ensure we have the entire metadata tree
+			var compileJobId = compileJob.Require(x => x.Id);
+			lock (jobLockManagers)
+				if (jobLockManagers.TryGetValue(compileJobId, out var lockManager))
+					return (DmbProvider: lockManager.AddLock(reason, callerFile, callerLine), LockManager: null); // fast path
+
+			logger.LogTrace("Loading compile job {id}...", compileJobId);
+			await databaseContextFactory.UseContext(
+				async db => compileJob = await db
+					.CompileJobs
+					.AsQueryable()
+					.Where(x => x!.Id == compileJobId)
+					.Include(x => x.Job!)
+						.ThenInclude(x => x.StartedBy)
+					.Include(x => x.Job!)
+						.ThenInclude(x => x.Instance)
+					.Include(x => x.RevisionInformation!)
+						.ThenInclude(x => x.PrimaryTestMerge!)
+							.ThenInclude(x => x.MergedBy)
+					.Include(x => x.RevisionInformation!)
+						.ThenInclude(x => x.ActiveTestMerges!)
+							.ThenInclude(x => x.TestMerge!)
+								.ThenInclude(x => x.MergedBy)
+					.FirstAsync(cancellationToken)); // can't wait to see that query
+
+			EngineVersion engineVersion;
+			if (!EngineVersion.TryParse(compileJob.EngineVersion, out var engineVersionNullable))
+			{
+				logger.LogWarning("Error loading compile job, bad engine version: {engineVersion}", compileJob.EngineVersion);
+				return (null, null); // omae wa mou shinderu
+			}
+			else
+				engineVersion = engineVersionNullable!;
+
+			if (!compileJob.Job.StoppedAt.HasValue)
+			{
+				// This happens when we're told to load the compile job that is currently finished up
+				// It constitutes an API violation if it's returned by the DreamDaemonController so just set it here
+				// Bit of a hack, but it works out to be nearly if not the same value that's put in the DB
+				logger.LogTrace("Setting missing StoppedAt for CompileJob.Job #{id}...", compileJob.Job.Id);
+				compileJob.Job.StoppedAt = DateTimeOffset.UtcNow;
+			}
+
+			var providerSubmitted = false;
+			void CleanupAction()
+			{
+				if (providerSubmitted)
+					CleanRegisteredCompileJob(compileJob);
+			}
+
+			var newProvider = new DmbProvider(compileJob, engineVersion, ioManager, new DisposeInvoker(CleanupAction));
+			try
+			{
+				const string LegacyADirectoryName = "A";
+				const string LegacyBDirectoryName = "B";
+
+				var dmbExistsAtRoot = await ioManager.FileExists(
+					ioManager.ConcatPath(
+						newProvider.Directory,
+						newProvider.DmbName),
+					cancellationToken);
+
+				if (!dmbExistsAtRoot)
+				{
+					logger.LogTrace("Didn't find .dmb at game directory root, checking A/B dirs...");
+					var primaryCheckTask = ioManager.FileExists(
+						ioManager.ConcatPath(
+							newProvider.Directory,
+							LegacyADirectoryName,
+							newProvider.DmbName),
+						cancellationToken);
+					var secondaryCheckTask = ioManager.FileExists(
+						ioManager.ConcatPath(
+							newProvider.Directory,
+							LegacyBDirectoryName,
+							newProvider.DmbName),
+						cancellationToken);
+
+					if (!(await primaryCheckTask && await secondaryCheckTask))
+					{
+						logger.LogWarning("Error loading compile job, .dmb missing!");
+						return (null, null); // omae wa mou shinderu
+					}
+
+					// rebuild the provider because it's using the legacy style directories
+					// Don't dispose it
+					logger.LogDebug("Creating legacy two folder .dmb provider targeting {aDirName} directory...", LegacyADirectoryName);
+#pragma warning disable CA2000 // Dispose objects before losing scope (false positive)
+					newProvider = new DmbProvider(compileJob, engineVersion, ioManager, new DisposeInvoker(CleanupAction), Path.DirectorySeparatorChar + LegacyADirectoryName);
+#pragma warning restore CA2000 // Dispose objects before losing scope
+				}
+
+				lock (jobLockManagers)
+				{
+					IDmbProvider lockedProvider;
+					if (!jobLockManagers.TryGetValue(compileJobId, out var lockManager))
+					{
+						lockManager = DeploymentLockManager.Create(newProvider, logger, reason, out lockedProvider);
+						jobLockManagers.Add(compileJobId, lockManager);
+
+						providerSubmitted = true;
+					}
+					else
+					{
+						lockedProvider = lockManager.AddLock(reason, callerFile, callerLine); // race condition
+						lockManager = null;
+					}
+
+					return (DmbProvider: lockedProvider, LockManager: lockManager);
+				}
+			}
+			finally
+			{
+				if (!providerSubmitted)
+					await newProvider.DisposeAsync();
+			}
 		}
 
 		/// <summary>
@@ -428,6 +446,9 @@ namespace Tgstation.Server.Host.Components.Deployment
 		{
 			Task HandleCleanup()
 			{
+				lock (jobLockManagers)
+					jobLockManagers.Remove(job.Require(x => x.Id));
+
 				var otherTask = cleanupTask;
 
 				async Task WrapThrowableTasks()
@@ -453,26 +474,8 @@ namespace Tgstation.Server.Host.Components.Deployment
 				return Task.WhenAll(otherTask, WrapThrowableTasks());
 			}
 
-			lock (jobLockCounts)
-			{
-				var jobId = job.Require(x => x.Id);
-				if (jobLockCounts.TryGetValue(jobId, out var currentVal))
-					if (currentVal == 1)
-					{
-						jobLockCounts.Remove(jobId);
-						logger.LogDebug("Cleaning lock-free compile job {id} => {dirName}", jobId, job.DirectoryName);
-						cleanupTask = HandleCleanup();
-					}
-					else
-					{
-						var decremented = --jobLockCounts[jobId];
-						logger.LogTrace("Compile job {id} lock count now: {lockCount}", jobId, decremented);
-					}
-				else
-					logger.LogError("Extra Dispose of DmbProvider for CompileJob {compileJobId}!", jobId);
-
-				LogLockCounts();
-			}
+			lock (cleanupCts)
+				cleanupTask = HandleCleanup();
 		}
 
 		/// <summary>
@@ -487,30 +490,5 @@ namespace Tgstation.Server.Host.Components.Deployment
 			await eventConsumer.HandleEvent(EventType.DeploymentCleanup, new List<string> { ioManager.ResolvePath(directory) }, true, cancellationToken);
 			await ioManager.DeleteDirectory(directory, cancellationToken);
 		}
-
-		/// <summary>
-		/// Log out the current lock counts to Trace.
-		/// </summary>
-		/// <remarks><see cref="jobLockCounts"/> must be locked before calling this function.</remarks>
-		void LogLockCounts()
-		{
-			if (jobLockCounts.Count == 0)
-			{
-				logger.LogWarning("No compile jobs registered!");
-				return;
-			}
-
-			var builder = new StringBuilder();
-			foreach (var jobId in jobLockCounts.Keys)
-			{
-				builder.AppendLine();
-				builder.Append("\t- ");
-				builder.Append(jobId);
-				builder.Append(": ");
-				builder.Append(jobLockCounts[jobId]);
-			}
-
-			logger.LogTrace("Compile Job Lock Counts:{details}", builder.ToString());
-		}
 	}
 }

src/Tgstation.Server.Host/Components/Deployment/DmbLock.cs

@@ -0,0 +1,82 @@
+using System;
+using System.Threading.Tasks;
+
+using Tgstation.Server.Api.Models;
+using Tgstation.Server.Host.Models;
+
+namespace Tgstation.Server.Host.Components.Deployment
+{
+	/// <summary>
+	/// Represents a lock on a given <see cref="IDmbProvider"/>.
+	/// </summary>
+	sealed class DmbLock : IDmbProvider
+	{
+		/// <inheritdoc />
+		public string DmbName => baseProvider.DmbName;
+
+		/// <inheritdoc />
+		public string Directory => baseProvider.Directory;
+
+		/// <inheritdoc />
+		public CompileJob CompileJob => baseProvider.CompileJob;
+
+		/// <inheritdoc />
+		public EngineVersion EngineVersion => baseProvider.EngineVersion;
+
+		/// <summary>
+		/// Unique ID of the lock.
+		/// </summary>
+		public Guid LockID { get; }
+
+		/// <summary>
+		/// The <see cref="DateTimeOffset"/> of when the lock was acquired.
+		/// </summary>
+		public DateTimeOffset LockTime { get; }
+
+		/// <summary>
+		/// A description of the <see cref="DmbLock"/>'s purpose.
+		/// </summary>
+		public string Descriptor { get; }
+
+		/// <summary>
+		/// If <see cref="KeepAlive"/> was called on the <see cref="DmbLock"/>.
+		/// </summary>
+		public bool KeptAlive { get; private set; }
+
+		/// <summary>
+		/// The <see cref="IDmbProvider"/> being wrapped.
+		/// </summary>
+		readonly IDmbProvider baseProvider;
+
+		/// <summary>
+		/// A <see cref="Func{TResult}"/> to use as the implementation of <see cref="DisposeAsync"/>.
+		/// </summary>
+		readonly Func<ValueTask> disposeAction;
+
+		/// <summary>
+		/// Initializes a new instance of the <see cref="DmbLock"/> class.
+		/// </summary>
+		/// <param name="disposeAction">The value of <see cref="disposeAction"/>.</param>
+		/// <param name="baseProvider">The value of <see cref="baseProvider"/>.</param>
+		/// <param name="descriptor">The value of <see cref="Descriptor"/>.</param>
+		public DmbLock(Func<ValueTask> disposeAction, IDmbProvider baseProvider, string descriptor)
+		{
+			this.disposeAction = disposeAction ?? throw new ArgumentNullException(nameof(disposeAction));
+			this.baseProvider = baseProvider ?? throw new ArgumentNullException(nameof(baseProvider));
+			Descriptor = descriptor ?? throw new ArgumentNullException(nameof(descriptor));
+
+			LockID = Guid.NewGuid();
+			LockTime = DateTimeOffset.UtcNow;
+		}
+
+		/// <inheritdoc />
+		public ValueTask DisposeAsync() => disposeAction();
+
+		/// <inheritdoc />
+		public void KeepAlive()
+		{
+			KeptAlive = true;
+			baseProvider.KeepAlive();
+		}
+	}
+}

src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs

@@ -326,7 +326,7 @@ namespace Tgstation.Server.Host.Components.Deployment
 						likelyPushedTestMergeCommit,
 						cancellationToken);
 
-				var activeCompileJob = compileJobConsumer.LatestCompileJob();
+				var activeCompileJob = await compileJobConsumer.LatestCompileJob();
 				try
 				{
 					await databaseContextFactory.UseContext(

src/Tgstation.Server.Host/Components/Deployment/IDmbFactory.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Runtime.CompilerServices;
 using System.Threading;
 using System.Threading.Tasks;
 
@@ -25,17 +26,22 @@ namespace Tgstation.Server.Host.Components.Deployment
 		/// <summary>
 		/// Gets the next <see cref="IDmbProvider"/>. <see cref="DmbAvailable"/> is a precondition.
 		/// </summary>
-		/// <param name="lockCount">The amount of locks to give the resulting <see cref="IDmbProvider"/>. It's <see cref="IDisposable.Dispose"/> must be called this many times to properly clean the job.</param>
+		/// <param name="reason">The reason the lock is being acquired.</param>
+		/// <param name="callerFile">The file path of the calling function.</param>
+		/// <param name="callerLine">The line number of the call invocation.</param>
 		/// <returns>A new <see cref="IDmbProvider"/>.</returns>
-		IDmbProvider LockNextDmb(int lockCount);
+		IDmbProvider LockNextDmb(string reason, [CallerFilePath] string? callerFile = null, [CallerLineNumber] int callerLine = default);
 
 		/// <summary>
 		/// Gets a <see cref="IDmbProvider"/> for a given <see cref="CompileJob"/>.
 		/// </summary>
 		/// <param name="compileJob">The <see cref="CompileJob"/> to make the <see cref="IDmbProvider"/> for.</param>
+		/// <param name="reason">The reason the compile job needed to be loaded.</param>
 		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
+		/// <param name="callerFile">The file path of the calling function.</param>
+		/// <param name="callerLine">The line number of the call invocation.</param>
 		/// <returns>A <see cref="ValueTask{TResult}"/> resulting in a new <see cref="IDmbProvider"/> representing the <see cref="CompileJob"/> on success, <see langword="null"/> on failure.</returns>
-		ValueTask<IDmbProvider?> FromCompileJob(CompileJob compileJob, CancellationToken cancellationToken);
+		ValueTask<IDmbProvider?> FromCompileJob(CompileJob compileJob, string reason, CancellationToken cancellationToken, [CallerFilePath] string? callerFile = null, [CallerLineNumber] int callerLine = default);
 
 		/// <summary>
 		/// Deletes all compile jobs that are inactive in the Game folder.

src/Tgstation.Server.Host/Components/Deployment/ILatestCompileJobProvider.cs

@@ -1,4 +1,6 @@
-using Tgstation.Server.Host.Models;
+using System.Threading.Tasks;
+
+using Tgstation.Server.Host.Models;
 
 namespace Tgstation.Server.Host.Components.Deployment
 {
@@ -10,7 +12,7 @@ namespace Tgstation.Server.Host.Components.Deployment
 		/// <summary>
 		/// Gets the latest <see cref="CompileJob"/>.
 		/// </summary>
-		/// <returns>The latest <see cref="CompileJob"/> or <see langword="null"/> if none are available.</returns>
-		CompileJob? LatestCompileJob();
+		/// <returns>A <see cref="ValueTask{TResult}"/> resulting in the latest <see cref="CompileJob"/> or <see langword="null"/> if none are available.</returns>
+		ValueTask<CompileJob?> LatestCompileJob();
 	}
 }

src/Tgstation.Server.Host/Components/Instance.cs

@@ -258,7 +258,7 @@ namespace Tgstation.Server.Host.Components
 		}
 
 		/// <inheritdoc />
-		public CompileJob? LatestCompileJob() => dmbFactory.LatestCompileJob();
+		public ValueTask<CompileJob?> LatestCompileJob() => dmbFactory.LatestCompileJob();
 
 		/// <summary>
 		/// The <see cref="JobEntrypoint"/> for updating the repository.
@@ -576,7 +576,7 @@ namespace Tgstation.Server.Host.Components
 							continue;
 						}
 
-						if (deploySha == LatestCompileJob()?.RevisionInformation.CommitSha)
+						if (deploySha == (await LatestCompileJob())?.RevisionInformation.CommitSha)
 						{
 							logger.LogTrace("Aborting auto update, same revision as latest CompileJob");
 							continue;

src/Tgstation.Server.Host/Components/InstanceWrapper.cs

@@ -58,6 +58,6 @@ namespace Tgstation.Server.Host.Components
 		public ValueTask ScheduleAutoUpdate(uint newInterval, string? newCron) => Instance.ScheduleAutoUpdate(newInterval, newCron);
 
 		/// <inheritdoc />
-		public CompileJob? LatestCompileJob() => Instance.LatestCompileJob();
+		public ValueTask<CompileJob?> LatestCompileJob() => Instance.LatestCompileJob();
 	}
 }

src/Tgstation.Server.Host/Components/Session/SessionPersistor.cs

@@ -208,7 +208,7 @@ namespace Tgstation.Server.Host.Components.Session
 				return null;
 			}
 
-			var dmb = await dmbFactory.FromCompileJob(result!.CompileJob!, cancellationToken);
+			var dmb = await dmbFactory.FromCompileJob(result!.CompileJob!, "Session Loading Main Deployment", cancellationToken);
 			if (dmb == null)
 			{
 				logger.LogError("Unable to reattach! Could not load .dmb!");
@@ -230,7 +230,7 @@ namespace Tgstation.Server.Host.Components.Session
 			if (result.InitialCompileJob != null)
 			{
 				logger.LogTrace("Loading initial compile job...");
-				initialDmb = await dmbFactory.FromCompileJob(result.InitialCompileJob, cancellationToken);
+				initialDmb = await dmbFactory.FromCompileJob(result.InitialCompileJob, "Session Loading Initial Deployment", cancellationToken);
 			}
 
 			logger.LogTrace("Retrieved ReattachInformation");

src/Tgstation.Server.Host/Components/Watchdog/AdvancedWatchdog.cs

@@ -232,7 +232,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// <inheritdoc />
 		protected sealed override async ValueTask HandleNewDmbAvailable(CancellationToken cancellationToken)
 		{
-			IDmbProvider compileJobProvider = DmbFactory.LockNextDmb(1);
+			IDmbProvider compileJobProvider = DmbFactory.LockNextDmb("AdvancedWatchdog next compile job preload");
 			bool canSeamlesslySwap = CanUseSwappableDmbProvider(compileJobProvider);
 			if (canSeamlesslySwap)
 				if (compileJobProvider.CompileJob.EngineVersion != ActiveCompileJob!.EngineVersion)

src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs

@@ -214,7 +214,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		{
 			// don't need a new dmb if reattaching
 			var reattachInProgress = reattachInfo != null;
-			var dmbToUse = reattachInProgress ? null : DmbFactory.LockNextDmb(1);
+			var dmbToUse = reattachInProgress ? null : DmbFactory.LockNextDmb("Watchdog initialization");
 
 			// if this try catches something, both servers are killed
 			try

src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs

@@ -815,21 +815,18 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// </summary>
 		/// <param name="currentCompileJob">The session's current <see cref="CompileJob"/>.</param>
 		/// <returns>A <see cref="Task"/> that completes if and when a newer <see cref="CompileJob"/> is available.</returns>
-		Task InitialCheckDmbUpdated(CompileJob currentCompileJob)
+		async Task InitialCheckDmbUpdated(CompileJob currentCompileJob)
 		{
 			var factoryTask = DmbFactory.OnNewerDmb;
 
-			var latestCompileJob = DmbFactory.LatestCompileJob();
-			if (latestCompileJob == null)
-				return factoryTask;
-
-			if (latestCompileJob.Id != currentCompileJob.Id)
+			var latestCompileJob = await DmbFactory.LatestCompileJob();
+			if (latestCompileJob != null && latestCompileJob.Id != currentCompileJob.Id)
 			{
 				Logger.LogDebug("Found new CompileJob without waiting");
-				return Task.CompletedTask;
+				return;
 			}
 
-			return factoryTask;
+			await factoryTask;
 		}
 
 		/// <summary>

src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs

@@ -87,7 +87,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 				return;
 			}
 
-			Server.ReattachInformation.InitialDmb = await DmbFactory.FromCompileJob(Server.CompileJob, cancellationToken);
+			Server.ReattachInformation.InitialDmb = await DmbFactory.FromCompileJob(Server.CompileJob, "WindowsWatchdog Initial Deployment", cancellationToken);
 		}
 
 		/// <inheritdoc />

src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs

@@ -386,7 +386,7 @@ namespace Tgstation.Server.Host.Controllers
 
 				if (revision)
 				{
-					var latestCompileJob = instance.LatestCompileJob();
+					var latestCompileJob = await instance.LatestCompileJob();
 					result.ActiveCompileJob = ((instance.Watchdog.Status != WatchdogStatus.Offline
 						? dd.ActiveCompileJob
 						: latestCompileJob) ?? latestCompileJob)