diff --git a/src/Tgstation.Server.Host/Components/Deployment/DeploymentLockManager.cs b/src/Tgstation.Server.Host/Components/Deployment/DeploymentLockManager.cs
index 552fda6c8d..46efbfdbc9 100644
--- a/src/Tgstation.Server.Host/Components/Deployment/DeploymentLockManager.cs
+++ b/src/Tgstation.Server.Host/Components/Deployment/DeploymentLockManager.cs
@@ -1,6 +1,8 @@
using System;
using System.Collections.Generic;
+using System.Globalization;
using System.Runtime.CompilerServices;
+using System.Text;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
@@ -95,6 +97,7 @@ namespace Tgstation.Server.Host.Components.Deployment
/// A whose lifetime represents the lock.
public IDmbProvider AddLock(string reason, [CallerFilePath] string? callerFile = null, [CallerLineNumber]int callerLine = default)
{
+ ArgumentNullException.ThrowIfNull(reason);
lock (locks)
{
if (locks.Count == 0)
@@ -104,6 +107,20 @@ namespace Tgstation.Server.Host.Components.Deployment
}
}
+ ///
+ /// Add lock stats to a given .
+ ///
+ /// The to append to.
+ public void LogLockStats(StringBuilder stringBuilder)
+ {
+ ArgumentNullException.ThrowIfNull(stringBuilder);
+
+ stringBuilder.AppendLine(CultureInfo.InvariantCulture, $"Compile Job #{CompileJob.Id}: {CompileJob.DirectoryName}");
+ lock (locks)
+ foreach (var dmbLock in locks)
+ stringBuilder.AppendLine(CultureInfo.InvariantCulture, $"\t-{GetFullLockDescriptor(dmbLock)}");
+ }
+
///
/// Creates a and adds it to .
///
diff --git a/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs b/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs
index de885df172..ab2349d3a2 100644
--- a/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs
+++ b/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs
@@ -4,6 +4,7 @@ using System.Diagnostics.CodeAnalysis;
using System.IO;
using System.Linq;
using System.Runtime.CompilerServices;
+using System.Text;
using System.Threading;
using System.Threading.Tasks;
@@ -61,10 +62,15 @@ namespace Tgstation.Server.Host.Components.Deployment
readonly ILogger logger;
///
- /// The for .
+ /// The for the .
///
readonly IEventConsumer eventConsumer;
+ ///
+ /// The for the .
+ ///
+ readonly IAsyncDelayer asyncDelayer;
+
///
/// The for the .
///
@@ -75,6 +81,11 @@ namespace Tgstation.Server.Host.Components.Deployment
///
readonly CancellationTokenSource cleanupCts;
+ ///
+ /// The for .
+ ///
+ readonly CancellationTokenSource lockLogCts;
+
///
/// Map of s to locks on them.
///
@@ -107,6 +118,7 @@ namespace Tgstation.Server.Host.Components.Deployment
/// The value of .
/// The value of .
/// The value of .
+ /// The value of .
/// The value of .
/// The value of .
public DmbFactory(
@@ -114,6 +126,7 @@ namespace Tgstation.Server.Host.Components.Deployment
IIOManager ioManager,
IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory,
IEventConsumer eventConsumer,
+ IAsyncDelayer asyncDelayer,
ILogger logger,
Api.Models.Instance metadata)
{
@@ -121,17 +134,24 @@ namespace Tgstation.Server.Host.Components.Deployment
this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager));
this.remoteDeploymentManagerFactory = remoteDeploymentManagerFactory ?? throw new ArgumentNullException(nameof(remoteDeploymentManagerFactory));
this.eventConsumer = eventConsumer ?? throw new ArgumentNullException(nameof(eventConsumer));
+ this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer));
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
this.metadata = metadata ?? throw new ArgumentNullException(nameof(metadata));
cleanupTask = Task.CompletedTask;
newerDmbTcs = new TaskCompletionSource();
cleanupCts = new CancellationTokenSource();
+ lockLogCts = new CancellationTokenSource();
jobLockManagers = new Dictionary();
}
///
- public void Dispose() => cleanupCts.Dispose(); // we don't dispose nextDmbProvider here, since it might be the only thing we have
+ public void Dispose()
+ {
+ // we don't dispose nextDmbProvider here, since it might be the only thing we have
+ lockLogCts.Dispose();
+ cleanupCts.Dispose();
+ }
///
public async ValueTask LoadCompileJob(CompileJob job, Action? activationAction, CancellationToken cancellationToken)
@@ -205,6 +225,7 @@ namespace Tgstation.Server.Host.Components.Deployment
}
// we dont do CleanUnusedCompileJobs here because the watchdog may have plans for them yet
+ cleanupTask = Task.WhenAll(cleanupTask, LogLockStates());
}
///
@@ -212,6 +233,8 @@ namespace Tgstation.Server.Host.Components.Deployment
{
try
{
+ lockLogCts.Cancel();
+
lock (jobLockManagers)
remoteDeploymentManagerFactory.ForgetLocalStateForCompileJobs(jobLockManagers.Keys);
@@ -490,5 +513,34 @@ namespace Tgstation.Server.Host.Components.Deployment
await eventConsumer.HandleEvent(EventType.DeploymentCleanup, new List { ioManager.ResolvePath(directory) }, true, cancellationToken);
await ioManager.DeleteDirectory(directory, cancellationToken);
}
+
+ ///
+ /// Lock all s states.
+ ///
+ /// A representing the running operation.
+ async Task LogLockStates()
+ {
+ logger.LogTrace("Entering lock logging loop");
+ CancellationToken cancellationToken = lockLogCts.Token;
+
+ while (!cancellationToken.IsCancellationRequested)
+ try
+ {
+ var builder = new StringBuilder();
+
+ lock (jobLockManagers)
+ foreach (var lockManager in jobLockManagers.Values)
+ lockManager.LogLockStats(builder);
+
+ logger.LogDebug("Periodic deployment log states report (R.e. Issue #1779):{newLine}{report}", Environment.NewLine, builder); // TODO: Reduce to trace once #1779 is fixed
+
+ await asyncDelayer.Delay(TimeSpan.FromMinutes(10), cancellationToken);
+ }
+ catch (OperationCanceledException ex)
+ {
+ logger.LogTrace(ex, "Exiting lock logging loop");
+ break;
+ }
+ }
}
}
diff --git a/src/Tgstation.Server.Host/Components/InstanceFactory.cs b/src/Tgstation.Server.Host/Components/InstanceFactory.cs
index f12eb28fa5..70ef1dd028 100644
--- a/src/Tgstation.Server.Host/Components/InstanceFactory.cs
+++ b/src/Tgstation.Server.Host/Components/InstanceFactory.cs
@@ -290,6 +290,7 @@ namespace Tgstation.Server.Host.Components
gameIoManager,
remoteDeploymentManagerFactory,
eventConsumer,
+ asyncDelayer,
loggerFactory.CreateLogger(),
metadata);
try