From cbb1c27b19fe5d3ee811c89b2d082bda2330e609 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 16 Jun 2024 12:13:05 -0400 Subject: [PATCH] Add periodic deployment lock state logging --- .../Deployment/DeploymentLockManager.cs | 17 ++++++ .../Components/Deployment/DmbFactory.cs | 56 ++++++++++++++++++- .../Components/InstanceFactory.cs | 1 + 3 files changed, 72 insertions(+), 2 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Deployment/DeploymentLockManager.cs b/src/Tgstation.Server.Host/Components/Deployment/DeploymentLockManager.cs index 552fda6c8d..46efbfdbc9 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/DeploymentLockManager.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/DeploymentLockManager.cs @@ -1,6 +1,8 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.Runtime.CompilerServices; +using System.Text; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -95,6 +97,7 @@ namespace Tgstation.Server.Host.Components.Deployment /// A whose lifetime represents the lock. public IDmbProvider AddLock(string reason, [CallerFilePath] string? callerFile = null, [CallerLineNumber]int callerLine = default) { + ArgumentNullException.ThrowIfNull(reason); lock (locks) { if (locks.Count == 0) @@ -104,6 +107,20 @@ namespace Tgstation.Server.Host.Components.Deployment } } + /// + /// Add lock stats to a given . + /// + /// The to append to. + public void LogLockStats(StringBuilder stringBuilder) + { + ArgumentNullException.ThrowIfNull(stringBuilder); + + stringBuilder.AppendLine(CultureInfo.InvariantCulture, $"Compile Job #{CompileJob.Id}: {CompileJob.DirectoryName}"); + lock (locks) + foreach (var dmbLock in locks) + stringBuilder.AppendLine(CultureInfo.InvariantCulture, $"\t-{GetFullLockDescriptor(dmbLock)}"); + } + /// /// Creates a and adds it to . /// diff --git a/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs b/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs index de885df172..ab2349d3a2 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs @@ -4,6 +4,7 @@ using System.Diagnostics.CodeAnalysis; using System.IO; using System.Linq; using System.Runtime.CompilerServices; +using System.Text; using System.Threading; using System.Threading.Tasks; @@ -61,10 +62,15 @@ namespace Tgstation.Server.Host.Components.Deployment readonly ILogger logger; /// - /// The for . + /// The for the . /// readonly IEventConsumer eventConsumer; + /// + /// The for the . + /// + readonly IAsyncDelayer asyncDelayer; + /// /// The for the . /// @@ -75,6 +81,11 @@ namespace Tgstation.Server.Host.Components.Deployment /// readonly CancellationTokenSource cleanupCts; + /// + /// The for . + /// + readonly CancellationTokenSource lockLogCts; + /// /// Map of s to locks on them. /// @@ -107,6 +118,7 @@ namespace Tgstation.Server.Host.Components.Deployment /// The value of . /// The value of . /// The value of . + /// The value of . /// The value of . /// The value of . public DmbFactory( @@ -114,6 +126,7 @@ namespace Tgstation.Server.Host.Components.Deployment IIOManager ioManager, IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory, IEventConsumer eventConsumer, + IAsyncDelayer asyncDelayer, ILogger logger, Api.Models.Instance metadata) { @@ -121,17 +134,24 @@ namespace Tgstation.Server.Host.Components.Deployment this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); this.remoteDeploymentManagerFactory = remoteDeploymentManagerFactory ?? throw new ArgumentNullException(nameof(remoteDeploymentManagerFactory)); this.eventConsumer = eventConsumer ?? throw new ArgumentNullException(nameof(eventConsumer)); + this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer)); this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); this.metadata = metadata ?? throw new ArgumentNullException(nameof(metadata)); cleanupTask = Task.CompletedTask; newerDmbTcs = new TaskCompletionSource(); cleanupCts = new CancellationTokenSource(); + lockLogCts = new CancellationTokenSource(); jobLockManagers = new Dictionary(); } /// - public void Dispose() => cleanupCts.Dispose(); // we don't dispose nextDmbProvider here, since it might be the only thing we have + public void Dispose() + { + // we don't dispose nextDmbProvider here, since it might be the only thing we have + lockLogCts.Dispose(); + cleanupCts.Dispose(); + } /// public async ValueTask LoadCompileJob(CompileJob job, Action? activationAction, CancellationToken cancellationToken) @@ -205,6 +225,7 @@ namespace Tgstation.Server.Host.Components.Deployment } // we dont do CleanUnusedCompileJobs here because the watchdog may have plans for them yet + cleanupTask = Task.WhenAll(cleanupTask, LogLockStates()); } /// @@ -212,6 +233,8 @@ namespace Tgstation.Server.Host.Components.Deployment { try { + lockLogCts.Cancel(); + lock (jobLockManagers) remoteDeploymentManagerFactory.ForgetLocalStateForCompileJobs(jobLockManagers.Keys); @@ -490,5 +513,34 @@ namespace Tgstation.Server.Host.Components.Deployment await eventConsumer.HandleEvent(EventType.DeploymentCleanup, new List { ioManager.ResolvePath(directory) }, true, cancellationToken); await ioManager.DeleteDirectory(directory, cancellationToken); } + + /// + /// Lock all s states. + /// + /// A representing the running operation. + async Task LogLockStates() + { + logger.LogTrace("Entering lock logging loop"); + CancellationToken cancellationToken = lockLogCts.Token; + + while (!cancellationToken.IsCancellationRequested) + try + { + var builder = new StringBuilder(); + + lock (jobLockManagers) + foreach (var lockManager in jobLockManagers.Values) + lockManager.LogLockStats(builder); + + logger.LogDebug("Periodic deployment log states report (R.e. Issue #1779):{newLine}{report}", Environment.NewLine, builder); // TODO: Reduce to trace once #1779 is fixed + + await asyncDelayer.Delay(TimeSpan.FromMinutes(10), cancellationToken); + } + catch (OperationCanceledException ex) + { + logger.LogTrace(ex, "Exiting lock logging loop"); + break; + } + } } } diff --git a/src/Tgstation.Server.Host/Components/InstanceFactory.cs b/src/Tgstation.Server.Host/Components/InstanceFactory.cs index f12eb28fa5..70ef1dd028 100644 --- a/src/Tgstation.Server.Host/Components/InstanceFactory.cs +++ b/src/Tgstation.Server.Host/Components/InstanceFactory.cs @@ -290,6 +290,7 @@ namespace Tgstation.Server.Host.Components gameIoManager, remoteDeploymentManagerFactory, eventConsumer, + asyncDelayer, loggerFactory.CreateLogger(), metadata); try