diff --git a/src/Tgstation.Server.Host/Components/Chat/Commands/PullRequestsCommand.cs b/src/Tgstation.Server.Host/Components/Chat/Commands/PullRequestsCommand.cs index 3fd8c9e64b..5cc52d2b83 100644 --- a/src/Tgstation.Server.Host/Components/Chat/Commands/PullRequestsCommand.cs +++ b/src/Tgstation.Server.Host/Components/Chat/Commands/PullRequestsCommand.cs @@ -137,7 +137,7 @@ namespace Tgstation.Server.Host.Components.Chat.Commands var compileJobToUse = watchdog.ActiveCompileJob; if (hasStaged) { - var latestCompileJob = compileJobProvider.LatestCompileJob(); + var latestCompileJob = await compileJobProvider.LatestCompileJob(); if (latestCompileJob?.Id != compileJobToUse?.Id) compileJobToUse = latestCompileJob; else diff --git a/src/Tgstation.Server.Host/Components/Deployment/DeploymentLockManager.cs b/src/Tgstation.Server.Host/Components/Deployment/DeploymentLockManager.cs new file mode 100644 index 0000000000..552fda6c8d --- /dev/null +++ b/src/Tgstation.Server.Host/Components/Deployment/DeploymentLockManager.cs @@ -0,0 +1,149 @@ +using System; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Threading.Tasks; + +using Microsoft.Extensions.Logging; + +using Tgstation.Server.Host.Models; + +namespace Tgstation.Server.Host.Components.Deployment +{ + /// + /// Manages locks on a given . + /// + sealed class DeploymentLockManager : IAsyncDisposable + { + /// + /// The represented by the . + /// + public CompileJob CompileJob => dmbProvider.CompileJob; + + /// + /// The for the . + /// + readonly ILogger logger; + + /// + /// The the is managing. + /// + readonly IDmbProvider dmbProvider; + + /// + /// The s on the . + /// + readonly HashSet locks; + + /// + /// The first lock acquired by the . + /// + readonly DmbLock firstLock; + + /// + /// Create a . + /// + /// The value of . + /// The value of . + /// The reason for the first lock. + /// The that represents the first lock. + /// The file path of the calling function. + /// The line number of the call invocation. + /// A new . + public static DeploymentLockManager Create(IDmbProvider dmbProvider, ILogger logger, string initialLockReason, out IDmbProvider firstLock, [CallerFilePath] string? callerFile = null, [CallerLineNumber] int callerLine = default) + { + var manager = new DeploymentLockManager(dmbProvider, logger, initialLockReason, callerFile!, callerLine); + firstLock = manager.firstLock; + return manager; + } + + /// + /// Generates a verbose description of a given . + /// + /// The to get a description of. + /// A verbose description of . + static string GetFullLockDescriptor(DmbLock dmbLock) => $"{dmbLock.LockID} {dmbLock.Descriptor} (Created at {dmbLock.LockTime}){(dmbLock.KeptAlive ? " (RELEASED)" : String.Empty)}"; + + /// + /// Initializes a new instance of the class. + /// + /// The value of . + /// The value of . + /// The reason for the first lock. + /// The file path of the calling function. + /// The line number of the call invocation. + /// A new . + DeploymentLockManager(IDmbProvider dmbProvider, ILogger logger, string initialLockReason, string callerFile, int callerLine) + { + this.dmbProvider = dmbProvider ?? throw new ArgumentNullException(nameof(dmbProvider)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + logger.LogTrace("Initializing lock manager for compile job {id}", dmbProvider.CompileJob.Id); + locks = new HashSet(); + firstLock = CreateLock(initialLockReason, callerFile, callerLine); + } + + /// + public ValueTask DisposeAsync() + => firstLock.DisposeAsync(); + + /// + /// Add a lock to the managed . + /// + /// The reason for the lock. + /// The file path of the calling function. + /// The line number of the call invocation. + /// A whose lifetime represents the lock. + public IDmbProvider AddLock(string reason, [CallerFilePath] string? callerFile = null, [CallerLineNumber]int callerLine = default) + { + lock (locks) + { + if (locks.Count == 0) + throw new InvalidOperationException($"No locks exist on the DmbProvider for CompileJob {dmbProvider.CompileJob.Id}!"); + + return CreateLock(reason, callerFile!, callerLine); + } + } + + /// + /// Creates a and adds it to . + /// + /// The reason for the lock. + /// The file path of the calling function. + /// The line number of the call invocation. + /// A new . + /// Requires exclusive write access to be held by the caller. + DmbLock CreateLock(string reason, string callerFile, int callerLine) + { + DmbLock? newLock = null; + string? descriptor = null; + ValueTask LockCleanupAction() + { + ValueTask disposeTask = ValueTask.CompletedTask; + lock (locks) + { + logger.LogTrace("Removing .dmb Lock: {descriptor}", descriptor); + + if (locks.Remove(newLock!)) + logger.LogTrace("Lock was removed from list successfully"); + else + logger.LogError("A .dmb lock was disposed more than once: {descriptor}", descriptor); + + if (locks.Count == 0) + disposeTask = dmbProvider.DisposeAsync(); + else if (newLock == firstLock) + logger.LogDebug("First lock on CompileJob #{compileJobId} removed, it must cleanup {remaining} remaining locks to be cleaned", CompileJob.Id, locks.Count); + } + + return disposeTask; + } + + newLock = new DmbLock(LockCleanupAction, dmbProvider, $"{callerFile}#{callerLine}: {reason}"); + locks.Add(newLock); + + descriptor = GetFullLockDescriptor(newLock!); + logger.LogTrace("Created .dmb Lock: {descriptor}", descriptor); + + return newLock; + } + } +} diff --git a/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs b/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs index fbeb7d3f92..de885df172 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs @@ -3,7 +3,7 @@ using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.IO; using System.Linq; -using System.Text; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; @@ -31,14 +31,14 @@ namespace Tgstation.Server.Host.Components.Deployment { get { - lock (jobLockCounts) + lock (jobLockManagers) return newerDmbTcs.Task; } } /// - [MemberNotNullWhen(true, nameof(nextDmbProvider))] - public bool DmbAvailable => nextDmbProvider != null; + [MemberNotNullWhen(true, nameof(nextLockManager))] + public bool DmbAvailable => nextLockManager != null; /// /// The for the . @@ -78,7 +78,7 @@ namespace Tgstation.Server.Host.Components.Deployment /// /// Map of s to locks on them. /// - readonly Dictionary jobLockCounts; + readonly Dictionary jobLockManagers; /// /// resulting in the latest yet to exist. @@ -91,9 +91,9 @@ namespace Tgstation.Server.Host.Components.Deployment Task cleanupTask; /// - /// The latest . + /// The for the latest . /// - IDmbProvider? nextDmbProvider; + DeploymentLockManager? nextLockManager; /// /// If the is "started" via . @@ -127,7 +127,7 @@ namespace Tgstation.Server.Host.Components.Deployment cleanupTask = Task.CompletedTask; newerDmbTcs = new TaskCompletionSource(); cleanupCts = new CancellationTokenSource(); - jobLockCounts = new Dictionary(); + jobLockManagers = new Dictionary(); } /// @@ -138,10 +138,13 @@ namespace Tgstation.Server.Host.Components.Deployment { ArgumentNullException.ThrowIfNull(job); - var newProvider = await FromCompileJob(job, cancellationToken); - if (newProvider == null) + var (dmbProvider, lockManager) = await FromCompileJobInternal(job, "Compile job loading", cancellationToken); + if (dmbProvider == null) return; + if (lockManager == null) + throw new InvalidOperationException($"We did not acquire the first lock for compile job {job.Id}!"); + // Do this first, because it's entirely possible when we set the tcs it will immediately need to be applied if (started) { @@ -149,16 +152,16 @@ namespace Tgstation.Server.Host.Components.Deployment metadata, job); await remoteDeploymentManager.StageDeployment( - newProvider.CompileJob, + lockManager.CompileJob, activationAction, cancellationToken); } ValueTask dmbDisposeTask; - lock (jobLockCounts) + lock (jobLockManagers) { - dmbDisposeTask = nextDmbProvider?.DisposeAsync() ?? ValueTask.CompletedTask; - nextDmbProvider = newProvider; + dmbDisposeTask = nextLockManager?.DisposeAsync() ?? ValueTask.CompletedTask; + nextLockManager = lockManager; // Oh god dammit var temp = Interlocked.Exchange(ref newerDmbTcs, new TaskCompletionSource()); @@ -169,20 +172,12 @@ namespace Tgstation.Server.Host.Components.Deployment } /// - public IDmbProvider LockNextDmb(int lockCount) + public IDmbProvider LockNextDmb(string reason, [CallerFilePath] string? callerFile = null, [CallerLineNumber] int callerLine = default) { if (!DmbAvailable) throw new InvalidOperationException("No .dmb available!"); - if (lockCount < 0) - throw new ArgumentOutOfRangeException(nameof(lockCount), lockCount, "lockCount must be greater than or equal to 0!"); - lock (jobLockCounts) - { - var jobId = nextDmbProvider.CompileJob.Require(x => x.Id); - var incremented = jobLockCounts[jobId] += lockCount; - logger.LogTrace("Compile job {jobId} lock increased by: {increment}", jobId, lockCount); - LogLockCounts(); - return nextDmbProvider; - } + + return nextLockManager.AddLock(reason, callerFile, callerLine); } /// @@ -217,8 +212,8 @@ namespace Tgstation.Server.Host.Components.Deployment { try { - lock (jobLockCounts) - remoteDeploymentManagerFactory.ForgetLocalStateForCompileJobs(jobLockCounts.Keys); + lock (jobLockManagers) + remoteDeploymentManagerFactory.ForgetLocalStateForCompileJobs(jobLockManagers.Keys); using (cancellationToken.Register(() => cleanupCts.Cancel())) await cleanupTask; @@ -231,124 +226,15 @@ namespace Tgstation.Server.Host.Components.Deployment /// #pragma warning disable CA1506 // TODO: Decomplexify - public async ValueTask FromCompileJob(CompileJob compileJob, CancellationToken cancellationToken) + public async ValueTask FromCompileJob(CompileJob compileJob, string reason, CancellationToken cancellationToken, [CallerFilePath] string? callerFile = null, [CallerLineNumber] int callerLine = default) { ArgumentNullException.ThrowIfNull(compileJob); + ArgumentNullException.ThrowIfNull(reason); - // ensure we have the entire metadata tree - var compileJobId = compileJob.Require(x => x.Id); - logger.LogTrace("Loading compile job {id}...", compileJobId); - await databaseContextFactory.UseContext( - async db => compileJob = await db - .CompileJobs - .AsQueryable() - .Where(x => x!.Id == compileJobId) - .Include(x => x.Job!) - .ThenInclude(x => x.StartedBy) - .Include(x => x.Job!) - .ThenInclude(x => x.Instance) - .Include(x => x.RevisionInformation!) - .ThenInclude(x => x.PrimaryTestMerge!) - .ThenInclude(x => x.MergedBy) - .Include(x => x.RevisionInformation!) - .ThenInclude(x => x.ActiveTestMerges!) - .ThenInclude(x => x.TestMerge!) - .ThenInclude(x => x.MergedBy) - .FirstAsync(cancellationToken)); // can't wait to see that query + var (dmb, _) = await FromCompileJobInternal(compileJob, reason, cancellationToken, callerFile, callerLine); - EngineVersion engineVersion; - if (!EngineVersion.TryParse(compileJob.EngineVersion, out var engineVersionNullable)) - { - logger.LogWarning("Error loading compile job, bad engine version: {engineVersion}", compileJob.EngineVersion); - return null; // omae wa mou shinderu - } - else - engineVersion = engineVersionNullable!; - - if (!compileJob.Job.StoppedAt.HasValue) - { - // This happens when we're told to load the compile job that is currently finished up - // It constitutes an API violation if it's returned by the DreamDaemonController so just set it here - // Bit of a hack, but it works out to be nearly if not the same value that's put in the DB - logger.LogTrace("Setting missing StoppedAt for CompileJob.Job #{id}...", compileJob.Job.Id); - compileJob.Job.StoppedAt = DateTimeOffset.UtcNow; - } - - var providerSubmitted = false; - - void CleanupAction() - { - if (providerSubmitted) - CleanRegisteredCompileJob(compileJob); - } - - var newProvider = new DmbProvider(compileJob, engineVersion, ioManager, new DisposeInvoker(CleanupAction)); - try - { - const string LegacyADirectoryName = "A"; - const string LegacyBDirectoryName = "B"; - - var dmbExistsAtRoot = await ioManager.FileExists( - ioManager.ConcatPath( - newProvider.Directory, - newProvider.DmbName), - cancellationToken); - - if (!dmbExistsAtRoot) - { - logger.LogTrace("Didn't find .dmb at game directory root, checking A/B dirs..."); - var primaryCheckTask = ioManager.FileExists( - ioManager.ConcatPath( - newProvider.Directory, - LegacyADirectoryName, - newProvider.DmbName), - cancellationToken); - var secondaryCheckTask = ioManager.FileExists( - ioManager.ConcatPath( - newProvider.Directory, - LegacyBDirectoryName, - newProvider.DmbName), - cancellationToken); - - if (!(await primaryCheckTask && await secondaryCheckTask)) - { - logger.LogWarning("Error loading compile job, .dmb missing!"); - return null; // omae wa mou shinderu - } - - // rebuild the provider because it's using the legacy style directories - // Don't dispose it - logger.LogDebug("Creating legacy two folder .dmb provider targeting {aDirName} directory...", LegacyADirectoryName); - newProvider = new DmbProvider(compileJob, engineVersion, ioManager, new DisposeInvoker(CleanupAction), Path.DirectorySeparatorChar + LegacyADirectoryName); - } - - lock (jobLockCounts) - { - if (!jobLockCounts.TryGetValue(compileJobId, out int value)) - { - value = 1; - logger.LogTrace("Initializing lock count for compile job {id}", compileJobId); - jobLockCounts.Add(compileJobId, 1); - } - else - { - logger.LogTrace("FromCompileJob already had a jobLockCounts entry for {id}. Incrementing lock count to {value}.", compileJobId, value); - jobLockCounts[compileJobId] = ++value; - } - - providerSubmitted = true; - - LogLockCounts(); - return newProvider; - } - } - finally - { - if (!providerSubmitted) - await newProvider.DisposeAsync(); - } + return dmb; } -#pragma warning restore CA1506 /// #pragma warning disable CA1506 // TODO: Decomplexify @@ -357,8 +243,8 @@ namespace Tgstation.Server.Host.Components.Deployment List jobIdsToSkip; // don't clean locked directories - lock (jobLockCounts) - jobIdsToSkip = jobLockCounts.Keys.ToList(); + lock (jobLockManagers) + jobIdsToSkip = jobLockManagers.Keys.ToList(); List? jobUidsToNotErase = null; @@ -413,11 +299,143 @@ namespace Tgstation.Server.Host.Components.Deployment #pragma warning restore CA1506 /// - public CompileJob? LatestCompileJob() + public async ValueTask LatestCompileJob() { if (!DmbAvailable) return null; - return LockNextDmb(0)?.CompileJob; + + await using IDmbProvider provider = LockNextDmb("Checking latest CompileJob"); + + return provider.CompileJob; + } + + /// + /// Gets a and potentially the for a given . + /// + /// The to make the for. + /// The reason the compile job needed to be loaded. + /// The for the operation. + /// The file path of the calling function. + /// The line number of the call invocation. + /// A resulting in, on success, a tuple containing new representing the . If the first lock on was acquired, the will also be returned. On failure, Will be returned. + async ValueTask<(IDmbProvider? DmbProvider, DeploymentLockManager? LockManager)> FromCompileJobInternal(CompileJob compileJob, string reason, CancellationToken cancellationToken, [CallerFilePath] string? callerFile = null, [CallerLineNumber] int callerLine = default) + { + // ensure we have the entire metadata tree + var compileJobId = compileJob.Require(x => x.Id); + lock (jobLockManagers) + if (jobLockManagers.TryGetValue(compileJobId, out var lockManager)) + return (DmbProvider: lockManager.AddLock(reason, callerFile, callerLine), LockManager: null); // fast path + + logger.LogTrace("Loading compile job {id}...", compileJobId); + await databaseContextFactory.UseContext( + async db => compileJob = await db + .CompileJobs + .AsQueryable() + .Where(x => x!.Id == compileJobId) + .Include(x => x.Job!) + .ThenInclude(x => x.StartedBy) + .Include(x => x.Job!) + .ThenInclude(x => x.Instance) + .Include(x => x.RevisionInformation!) + .ThenInclude(x => x.PrimaryTestMerge!) + .ThenInclude(x => x.MergedBy) + .Include(x => x.RevisionInformation!) + .ThenInclude(x => x.ActiveTestMerges!) + .ThenInclude(x => x.TestMerge!) + .ThenInclude(x => x.MergedBy) + .FirstAsync(cancellationToken)); // can't wait to see that query + + EngineVersion engineVersion; + if (!EngineVersion.TryParse(compileJob.EngineVersion, out var engineVersionNullable)) + { + logger.LogWarning("Error loading compile job, bad engine version: {engineVersion}", compileJob.EngineVersion); + return (null, null); // omae wa mou shinderu + } + else + engineVersion = engineVersionNullable!; + + if (!compileJob.Job.StoppedAt.HasValue) + { + // This happens when we're told to load the compile job that is currently finished up + // It constitutes an API violation if it's returned by the DreamDaemonController so just set it here + // Bit of a hack, but it works out to be nearly if not the same value that's put in the DB + logger.LogTrace("Setting missing StoppedAt for CompileJob.Job #{id}...", compileJob.Job.Id); + compileJob.Job.StoppedAt = DateTimeOffset.UtcNow; + } + + var providerSubmitted = false; + void CleanupAction() + { + if (providerSubmitted) + CleanRegisteredCompileJob(compileJob); + } + + var newProvider = new DmbProvider(compileJob, engineVersion, ioManager, new DisposeInvoker(CleanupAction)); + try + { + const string LegacyADirectoryName = "A"; + const string LegacyBDirectoryName = "B"; + + var dmbExistsAtRoot = await ioManager.FileExists( + ioManager.ConcatPath( + newProvider.Directory, + newProvider.DmbName), + cancellationToken); + + if (!dmbExistsAtRoot) + { + logger.LogTrace("Didn't find .dmb at game directory root, checking A/B dirs..."); + var primaryCheckTask = ioManager.FileExists( + ioManager.ConcatPath( + newProvider.Directory, + LegacyADirectoryName, + newProvider.DmbName), + cancellationToken); + var secondaryCheckTask = ioManager.FileExists( + ioManager.ConcatPath( + newProvider.Directory, + LegacyBDirectoryName, + newProvider.DmbName), + cancellationToken); + + if (!(await primaryCheckTask && await secondaryCheckTask)) + { + logger.LogWarning("Error loading compile job, .dmb missing!"); + return (null, null); // omae wa mou shinderu + } + + // rebuild the provider because it's using the legacy style directories + // Don't dispose it + logger.LogDebug("Creating legacy two folder .dmb provider targeting {aDirName} directory...", LegacyADirectoryName); +#pragma warning disable CA2000 // Dispose objects before losing scope (false positive) + newProvider = new DmbProvider(compileJob, engineVersion, ioManager, new DisposeInvoker(CleanupAction), Path.DirectorySeparatorChar + LegacyADirectoryName); +#pragma warning restore CA2000 // Dispose objects before losing scope + } + + lock (jobLockManagers) + { + IDmbProvider lockedProvider; + if (!jobLockManagers.TryGetValue(compileJobId, out var lockManager)) + { + lockManager = DeploymentLockManager.Create(newProvider, logger, reason, out lockedProvider); + jobLockManagers.Add(compileJobId, lockManager); + + providerSubmitted = true; + } + else + { + lockedProvider = lockManager.AddLock(reason, callerFile, callerLine); // race condition + lockManager = null; + } + + return (DmbProvider: lockedProvider, LockManager: lockManager); + } + } + finally + { + if (!providerSubmitted) + await newProvider.DisposeAsync(); + } } /// @@ -428,6 +446,9 @@ namespace Tgstation.Server.Host.Components.Deployment { Task HandleCleanup() { + lock (jobLockManagers) + jobLockManagers.Remove(job.Require(x => x.Id)); + var otherTask = cleanupTask; async Task WrapThrowableTasks() @@ -453,26 +474,8 @@ namespace Tgstation.Server.Host.Components.Deployment return Task.WhenAll(otherTask, WrapThrowableTasks()); } - lock (jobLockCounts) - { - var jobId = job.Require(x => x.Id); - if (jobLockCounts.TryGetValue(jobId, out var currentVal)) - if (currentVal == 1) - { - jobLockCounts.Remove(jobId); - logger.LogDebug("Cleaning lock-free compile job {id} => {dirName}", jobId, job.DirectoryName); - cleanupTask = HandleCleanup(); - } - else - { - var decremented = --jobLockCounts[jobId]; - logger.LogTrace("Compile job {id} lock count now: {lockCount}", jobId, decremented); - } - else - logger.LogError("Extra Dispose of DmbProvider for CompileJob {compileJobId}!", jobId); - - LogLockCounts(); - } + lock (cleanupCts) + cleanupTask = HandleCleanup(); } /// @@ -487,30 +490,5 @@ namespace Tgstation.Server.Host.Components.Deployment await eventConsumer.HandleEvent(EventType.DeploymentCleanup, new List { ioManager.ResolvePath(directory) }, true, cancellationToken); await ioManager.DeleteDirectory(directory, cancellationToken); } - - /// - /// Log out the current lock counts to Trace. - /// - /// must be locked before calling this function. - void LogLockCounts() - { - if (jobLockCounts.Count == 0) - { - logger.LogWarning("No compile jobs registered!"); - return; - } - - var builder = new StringBuilder(); - foreach (var jobId in jobLockCounts.Keys) - { - builder.AppendLine(); - builder.Append("\t- "); - builder.Append(jobId); - builder.Append(": "); - builder.Append(jobLockCounts[jobId]); - } - - logger.LogTrace("Compile Job Lock Counts:{details}", builder.ToString()); - } } } diff --git a/src/Tgstation.Server.Host/Components/Deployment/DmbLock.cs b/src/Tgstation.Server.Host/Components/Deployment/DmbLock.cs new file mode 100644 index 0000000000..677da1d915 --- /dev/null +++ b/src/Tgstation.Server.Host/Components/Deployment/DmbLock.cs @@ -0,0 +1,82 @@ +using System; +using System.Threading.Tasks; + +using Tgstation.Server.Api.Models; +using Tgstation.Server.Host.Models; + +namespace Tgstation.Server.Host.Components.Deployment +{ + /// + /// Represents a lock on a given . + /// + sealed class DmbLock : IDmbProvider + { + /// + public string DmbName => baseProvider.DmbName; + + /// + public string Directory => baseProvider.Directory; + + /// + public CompileJob CompileJob => baseProvider.CompileJob; + + /// + public EngineVersion EngineVersion => baseProvider.EngineVersion; + + /// + /// Unique ID of the lock. + /// + public Guid LockID { get; } + + /// + /// The of when the lock was acquired. + /// + public DateTimeOffset LockTime { get; } + + /// + /// A description of the 's purpose. + /// + public string Descriptor { get; } + + /// + /// If was called on the . + /// + public bool KeptAlive { get; private set; } + + /// + /// The being wrapped. + /// + readonly IDmbProvider baseProvider; + + /// + /// A to use as the implementation of . + /// + readonly Func disposeAction; + + /// + /// Initializes a new instance of the class. + /// + /// The value of . + /// The value of . + /// The value of . + public DmbLock(Func disposeAction, IDmbProvider baseProvider, string descriptor) + { + this.disposeAction = disposeAction ?? throw new ArgumentNullException(nameof(disposeAction)); + this.baseProvider = baseProvider ?? throw new ArgumentNullException(nameof(baseProvider)); + Descriptor = descriptor ?? throw new ArgumentNullException(nameof(descriptor)); + + LockID = Guid.NewGuid(); + LockTime = DateTimeOffset.UtcNow; + } + + /// + public ValueTask DisposeAsync() => disposeAction(); + + /// + public void KeepAlive() + { + KeptAlive = true; + baseProvider.KeepAlive(); + } + } +} diff --git a/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs b/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs index 76c415bac4..1b8eed488d 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs @@ -326,7 +326,7 @@ namespace Tgstation.Server.Host.Components.Deployment likelyPushedTestMergeCommit, cancellationToken); - var activeCompileJob = compileJobConsumer.LatestCompileJob(); + var activeCompileJob = await compileJobConsumer.LatestCompileJob(); try { await databaseContextFactory.UseContext( diff --git a/src/Tgstation.Server.Host/Components/Deployment/IDmbFactory.cs b/src/Tgstation.Server.Host/Components/Deployment/IDmbFactory.cs index 08d1a782b5..c737b8b358 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/IDmbFactory.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/IDmbFactory.cs @@ -1,4 +1,5 @@ using System; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; @@ -25,17 +26,22 @@ namespace Tgstation.Server.Host.Components.Deployment /// /// Gets the next . is a precondition. /// - /// The amount of locks to give the resulting . It's must be called this many times to properly clean the job. + /// The reason the lock is being acquired. + /// The file path of the calling function. + /// The line number of the call invocation. /// A new . - IDmbProvider LockNextDmb(int lockCount); + IDmbProvider LockNextDmb(string reason, [CallerFilePath] string? callerFile = null, [CallerLineNumber] int callerLine = default); /// /// Gets a for a given . /// /// The to make the for. + /// The reason the compile job needed to be loaded. /// The for the operation. + /// The file path of the calling function. + /// The line number of the call invocation. /// A resulting in a new representing the on success, on failure. - ValueTask FromCompileJob(CompileJob compileJob, CancellationToken cancellationToken); + ValueTask FromCompileJob(CompileJob compileJob, string reason, CancellationToken cancellationToken, [CallerFilePath] string? callerFile = null, [CallerLineNumber] int callerLine = default); /// /// Deletes all compile jobs that are inactive in the Game folder. diff --git a/src/Tgstation.Server.Host/Components/Deployment/ILatestCompileJobProvider.cs b/src/Tgstation.Server.Host/Components/Deployment/ILatestCompileJobProvider.cs index 021f62602d..253fe939fa 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/ILatestCompileJobProvider.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/ILatestCompileJobProvider.cs @@ -1,4 +1,6 @@ -using Tgstation.Server.Host.Models; +using System.Threading.Tasks; + +using Tgstation.Server.Host.Models; namespace Tgstation.Server.Host.Components.Deployment { @@ -10,7 +12,7 @@ namespace Tgstation.Server.Host.Components.Deployment /// /// Gets the latest . /// - /// The latest or if none are available. - CompileJob? LatestCompileJob(); + /// A resulting in the latest or if none are available. + ValueTask LatestCompileJob(); } } diff --git a/src/Tgstation.Server.Host/Components/Instance.cs b/src/Tgstation.Server.Host/Components/Instance.cs index f6d417eb7a..d178eb94e0 100644 --- a/src/Tgstation.Server.Host/Components/Instance.cs +++ b/src/Tgstation.Server.Host/Components/Instance.cs @@ -258,7 +258,7 @@ namespace Tgstation.Server.Host.Components } /// - public CompileJob? LatestCompileJob() => dmbFactory.LatestCompileJob(); + public ValueTask LatestCompileJob() => dmbFactory.LatestCompileJob(); /// /// The for updating the repository. @@ -576,7 +576,7 @@ namespace Tgstation.Server.Host.Components continue; } - if (deploySha == LatestCompileJob()?.RevisionInformation.CommitSha) + if (deploySha == (await LatestCompileJob())?.RevisionInformation.CommitSha) { logger.LogTrace("Aborting auto update, same revision as latest CompileJob"); continue; diff --git a/src/Tgstation.Server.Host/Components/InstanceWrapper.cs b/src/Tgstation.Server.Host/Components/InstanceWrapper.cs index 249342fc37..b5a835420d 100644 --- a/src/Tgstation.Server.Host/Components/InstanceWrapper.cs +++ b/src/Tgstation.Server.Host/Components/InstanceWrapper.cs @@ -58,6 +58,6 @@ namespace Tgstation.Server.Host.Components public ValueTask ScheduleAutoUpdate(uint newInterval, string? newCron) => Instance.ScheduleAutoUpdate(newInterval, newCron); /// - public CompileJob? LatestCompileJob() => Instance.LatestCompileJob(); + public ValueTask LatestCompileJob() => Instance.LatestCompileJob(); } } diff --git a/src/Tgstation.Server.Host/Components/Session/SessionPersistor.cs b/src/Tgstation.Server.Host/Components/Session/SessionPersistor.cs index d661564723..5c87bfda5f 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionPersistor.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionPersistor.cs @@ -208,7 +208,7 @@ namespace Tgstation.Server.Host.Components.Session return null; } - var dmb = await dmbFactory.FromCompileJob(result!.CompileJob!, cancellationToken); + var dmb = await dmbFactory.FromCompileJob(result!.CompileJob!, "Session Loading Main Deployment", cancellationToken); if (dmb == null) { logger.LogError("Unable to reattach! Could not load .dmb!"); @@ -230,7 +230,7 @@ namespace Tgstation.Server.Host.Components.Session if (result.InitialCompileJob != null) { logger.LogTrace("Loading initial compile job..."); - initialDmb = await dmbFactory.FromCompileJob(result.InitialCompileJob, cancellationToken); + initialDmb = await dmbFactory.FromCompileJob(result.InitialCompileJob, "Session Loading Initial Deployment", cancellationToken); } logger.LogTrace("Retrieved ReattachInformation"); diff --git a/src/Tgstation.Server.Host/Components/Watchdog/AdvancedWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/AdvancedWatchdog.cs index 610fe25bba..b58db53601 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/AdvancedWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/AdvancedWatchdog.cs @@ -232,7 +232,7 @@ namespace Tgstation.Server.Host.Components.Watchdog /// protected sealed override async ValueTask HandleNewDmbAvailable(CancellationToken cancellationToken) { - IDmbProvider compileJobProvider = DmbFactory.LockNextDmb(1); + IDmbProvider compileJobProvider = DmbFactory.LockNextDmb("AdvancedWatchdog next compile job preload"); bool canSeamlesslySwap = CanUseSwappableDmbProvider(compileJobProvider); if (canSeamlesslySwap) if (compileJobProvider.CompileJob.EngineVersion != ActiveCompileJob!.EngineVersion) diff --git a/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs index edc50518d4..5bc8089824 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs @@ -214,7 +214,7 @@ namespace Tgstation.Server.Host.Components.Watchdog { // don't need a new dmb if reattaching var reattachInProgress = reattachInfo != null; - var dmbToUse = reattachInProgress ? null : DmbFactory.LockNextDmb(1); + var dmbToUse = reattachInProgress ? null : DmbFactory.LockNextDmb("Watchdog initialization"); // if this try catches something, both servers are killed try diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs index ec3ae85346..e8d8290aa0 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs @@ -815,21 +815,18 @@ namespace Tgstation.Server.Host.Components.Watchdog /// /// The session's current . /// A that completes if and when a newer is available. - Task InitialCheckDmbUpdated(CompileJob currentCompileJob) + async Task InitialCheckDmbUpdated(CompileJob currentCompileJob) { var factoryTask = DmbFactory.OnNewerDmb; - var latestCompileJob = DmbFactory.LatestCompileJob(); - if (latestCompileJob == null) - return factoryTask; - - if (latestCompileJob.Id != currentCompileJob.Id) + var latestCompileJob = await DmbFactory.LatestCompileJob(); + if (latestCompileJob != null && latestCompileJob.Id != currentCompileJob.Id) { Logger.LogDebug("Found new CompileJob without waiting"); - return Task.CompletedTask; + return; } - return factoryTask; + await factoryTask; } /// diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs index e75ba74511..b03cad9bae 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs @@ -87,7 +87,7 @@ namespace Tgstation.Server.Host.Components.Watchdog return; } - Server.ReattachInformation.InitialDmb = await DmbFactory.FromCompileJob(Server.CompileJob, cancellationToken); + Server.ReattachInformation.InitialDmb = await DmbFactory.FromCompileJob(Server.CompileJob, "WindowsWatchdog Initial Deployment", cancellationToken); } /// diff --git a/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs b/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs index 17006bfa2e..11e8fc1d16 100644 --- a/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs +++ b/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs @@ -386,7 +386,7 @@ namespace Tgstation.Server.Host.Controllers if (revision) { - var latestCompileJob = instance.LatestCompileJob(); + var latestCompileJob = await instance.LatestCompileJob(); result.ActiveCompileJob = ((instance.Watchdog.Status != WatchdogStatus.Offline ? dd.ActiveCompileJob : latestCompileJob) ?? latestCompileJob)