Add periodic deployment lock state logging

src/Tgstation.Server.Host/Components/Deployment/DeploymentLockManager.cs

@@ -1,6 +1,8 @@
 using System;
 using System.Collections.Generic;
+using System.Globalization;
 using System.Runtime.CompilerServices;
+using System.Text;
 using System.Threading.Tasks;
 
 using Microsoft.Extensions.Logging;
@@ -95,6 +97,7 @@ namespace Tgstation.Server.Host.Components.Deployment
 		/// <returns>A <see cref="IDmbProvider"/> whose lifetime represents the lock.</returns>
 		public IDmbProvider AddLock(string reason, [CallerFilePath] string? callerFile = null, [CallerLineNumber]int callerLine = default)
 		{
+			ArgumentNullException.ThrowIfNull(reason);
 			lock (locks)
 			{
 				if (locks.Count == 0)
@@ -104,6 +107,20 @@ namespace Tgstation.Server.Host.Components.Deployment
 			}
 		}
 
+		/// <summary>
+		/// Add lock stats to a given <paramref name="stringBuilder"/>.
+		/// </summary>
+		/// <param name="stringBuilder">The <see cref="StringBuilder"/> to append to.</param>
+		public void LogLockStats(StringBuilder stringBuilder)
+		{
+			ArgumentNullException.ThrowIfNull(stringBuilder);
+
+			stringBuilder.AppendLine(CultureInfo.InvariantCulture, $"Compile Job #{CompileJob.Id}: {CompileJob.DirectoryName}");
+			lock (locks)
+				foreach (var dmbLock in locks)
+					stringBuilder.AppendLine(CultureInfo.InvariantCulture, $"\t-{GetFullLockDescriptor(dmbLock)}");
+		}
+
 		/// <summary>
 		/// Creates a <see cref="DmbLock"/> and adds it to <see cref="locks"/>.
 		/// </summary>

src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs

@@ -4,6 +4,7 @@ using System.Diagnostics.CodeAnalysis;
 using System.IO;
 using System.Linq;
 using System.Runtime.CompilerServices;
+using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 
@@ -61,10 +62,15 @@ namespace Tgstation.Server.Host.Components.Deployment
 		readonly ILogger<DmbFactory> logger;
 
 		/// <summary>
-		/// The <see cref="IEventConsumer"/> for <see cref="DmbFactory"/>.
+		/// The <see cref="IEventConsumer"/> for the <see cref="DmbFactory"/>.
 		/// </summary>
 		readonly IEventConsumer eventConsumer;
 
+		/// <summary>
+		/// The <see cref="IAsyncDelayer"/> for the <see cref="DmbFactory"/>.
+		/// </summary>
+		readonly IAsyncDelayer asyncDelayer;
+
 		/// <summary>
 		/// The <see cref="Api.Models.Instance"/> for the <see cref="DmbFactory"/>.
 		/// </summary>
@@ -75,6 +81,11 @@ namespace Tgstation.Server.Host.Components.Deployment
 		/// </summary>
 		readonly CancellationTokenSource cleanupCts;
 
+		/// <summary>
+		/// The <see cref="CancellationTokenSource"/> for <see cref="LogLockStates"/>.
+		/// </summary>
+		readonly CancellationTokenSource lockLogCts;
+
 		/// <summary>
 		/// Map of <see cref="CompileJob.JobId"/>s to locks on them.
 		/// </summary>
@@ -107,6 +118,7 @@ namespace Tgstation.Server.Host.Components.Deployment
 		/// <param name="ioManager">The value of <see cref="ioManager"/>.</param>
 		/// <param name="remoteDeploymentManagerFactory">The value of <see cref="remoteDeploymentManagerFactory"/>.</param>
 		/// <param name="eventConsumer">The value of <see cref="eventConsumer"/>.</param>
+		/// <param name="asyncDelayer">The value of <see cref="asyncDelayer"/>.</param>
 		/// <param name="logger">The value of <see cref="logger"/>.</param>
 		/// <param name="metadata">The value of <see cref="metadata"/>.</param>
 		public DmbFactory(
@@ -114,6 +126,7 @@ namespace Tgstation.Server.Host.Components.Deployment
 			IIOManager ioManager,
 			IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory,
 			IEventConsumer eventConsumer,
+			IAsyncDelayer asyncDelayer,
 			ILogger<DmbFactory> logger,
 			Api.Models.Instance metadata)
 		{
@@ -121,17 +134,24 @@ namespace Tgstation.Server.Host.Components.Deployment
 			this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager));
 			this.remoteDeploymentManagerFactory = remoteDeploymentManagerFactory ?? throw new ArgumentNullException(nameof(remoteDeploymentManagerFactory));
 			this.eventConsumer = eventConsumer ?? throw new ArgumentNullException(nameof(eventConsumer));
+			this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer));
 			this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
 			this.metadata = metadata ?? throw new ArgumentNullException(nameof(metadata));
 
 			cleanupTask = Task.CompletedTask;
 			newerDmbTcs = new TaskCompletionSource();
 			cleanupCts = new CancellationTokenSource();
+			lockLogCts = new CancellationTokenSource();
 			jobLockManagers = new Dictionary<long, DeploymentLockManager>();
 		}
 
 		/// <inheritdoc />
-		public void Dispose() => cleanupCts.Dispose(); // we don't dispose nextDmbProvider here, since it might be the only thing we have
+		public void Dispose()
+		{
+			// we don't dispose nextDmbProvider here, since it might be the only thing we have
+			lockLogCts.Dispose();
+			cleanupCts.Dispose();
+		}
 
 		/// <inheritdoc />
 		public async ValueTask LoadCompileJob(CompileJob job, Action<bool>? activationAction, CancellationToken cancellationToken)
@@ -205,6 +225,7 @@ namespace Tgstation.Server.Host.Components.Deployment
 			}
 
 			// we dont do CleanUnusedCompileJobs here because the watchdog may have plans for them yet
+			cleanupTask = Task.WhenAll(cleanupTask, LogLockStates());
 		}
 
 		/// <inheritdoc />
@@ -212,6 +233,8 @@ namespace Tgstation.Server.Host.Components.Deployment
 		{
 			try
 			{
+				lockLogCts.Cancel();
+
 				lock (jobLockManagers)
 					remoteDeploymentManagerFactory.ForgetLocalStateForCompileJobs(jobLockManagers.Keys);
 
@@ -490,5 +513,34 @@ namespace Tgstation.Server.Host.Components.Deployment
 			await eventConsumer.HandleEvent(EventType.DeploymentCleanup, new List<string> { ioManager.ResolvePath(directory) }, true, cancellationToken);
 			await ioManager.DeleteDirectory(directory, cancellationToken);
 		}
+
+		/// <summary>
+		/// Lock all <see cref="DeploymentLockManager"/>s states.
+		/// </summary>
+		/// <returns>A <see cref="Task"/> representing the running operation.</returns>
+		async Task LogLockStates()
+		{
+			logger.LogTrace("Entering lock logging loop");
+			CancellationToken cancellationToken = lockLogCts.Token;
+
+			while (!cancellationToken.IsCancellationRequested)
+				try
+				{
+					var builder = new StringBuilder();
+
+					lock (jobLockManagers)
+						foreach (var lockManager in jobLockManagers.Values)
+							lockManager.LogLockStats(builder);
+
+					logger.LogDebug("Periodic deployment log states report (R.e. Issue #1779):{newLine}{report}", Environment.NewLine, builder); // TODO: Reduce to trace once #1779 is fixed
+
+					await asyncDelayer.Delay(TimeSpan.FromMinutes(10), cancellationToken);
+				}
+				catch (OperationCanceledException ex)
+				{
+					logger.LogTrace(ex, "Exiting lock logging loop");
+					break;
+				}
+		}
 	}
 }

src/Tgstation.Server.Host/Components/InstanceFactory.cs

@@ -290,6 +290,7 @@ namespace Tgstation.Server.Host.Components
 					gameIoManager,
 					remoteDeploymentManagerFactory,
 					eventConsumer,
+					asyncDelayer,
 					loggerFactory.CreateLogger<DmbFactory>(),
 					metadata);
 				try