diff --git a/src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs b/src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs index 648a7a095a..b2b319473b 100644 --- a/src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs +++ b/src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs @@ -55,7 +55,7 @@ namespace Tgstation.Server.Host.Components.Compiler /// /// resulting in the latest yet to exist /// - TaskCompletionSource newerDmbTcs; + TaskCompletionSource newerDmbTcs; /// /// The latest /// @@ -78,12 +78,13 @@ namespace Tgstation.Server.Host.Components.Compiler this.instance = instance ?? throw new ArgumentNullException(nameof(instance)); cleanupTask = Task.CompletedTask; + newerDmbTcs = new TaskCompletionSource(); cleanupCts = new CancellationTokenSource(); jobLockCounts = new Dictionary(); } /// - public void Dispose() => cleanupCts.Dispose(); + public void Dispose() => cleanupCts.Dispose(); //we don't dispose nextDmbProvider here, since it might be the only thing we have /// /// Delete the of @@ -91,75 +92,71 @@ namespace Tgstation.Server.Host.Components.Compiler /// The to clean void CleanJob(CompileJob job) { + logger.LogTrace("Cleaning compile job {0} => {1}", job.Id, job.DirectoryName); async Task HandleCleanup() { var deleteJob = ioManager.DeleteDirectory(job.DirectoryName.ToString(), cleanupCts.Token); Task otherTask; - lock (this) - otherTask = cleanupTask; + //lock (this) //already locked below + otherTask = cleanupTask; await Task.WhenAll(otherTask, deleteJob).ConfigureAwait(false); } lock (this) { - var currentVal = jobLockCounts[job.Id]; - if (--jobLockCounts[job.Id] == 0) + if (!jobLockCounts.TryGetValue(job.Id, out var currentVal) || --jobLockCounts[job.Id] == 0) + { + jobLockCounts.Remove(job.Id); cleanupTask = HandleCleanup(); + } } } /// - public Task LoadCompileJob(CompileJob job, CancellationToken cancellationToken) => LoadCompileJob(job, true, cancellationToken); - - async Task LoadCompileJob(CompileJob job, bool setAsStagedInDb, CancellationToken cancellationToken) + public async Task LoadCompileJob(CompileJob job, CancellationToken cancellationToken) { if (job == null) throw new ArgumentNullException(nameof(job)); - if (job.DMApiValidated != true || job.Job.Cancelled.Value || job.Job.ExceptionDetails != null || job.Job.StoppedAt == null) + if (job.DMApiValidated != true || job.Job.Cancelled == true || job.Job.ExceptionDetails != null) throw new InvalidOperationException("Cannot load incomplete compile job!"); - if (setAsStagedInDb) - await databaseContextFactory.UseContext(async db => - { - var ddsettings = new DreamDaemonSettings - { - InstanceId = instance.Id - }; - db.DreamDaemonSettings.Attach(ddsettings); - ddsettings.StagedCompileJob = job; - await db.Save(cancellationToken).ConfigureAwait(false); - }).ConfigureAwait(false); + var newProvider = await FromCompileJob(job, cancellationToken).ConfigureAwait(false); + if (newProvider == null) + return; lock (this) { - var oldDmbProvider = nextDmbProvider; - if (oldDmbProvider != null && oldDmbProvider.CompileJob.Job.StoppedAt < oldDmbProvider.CompileJob.Job.StoppedAt) - throw new InvalidOperationException("Loaded compile job older than current job!"); - nextDmbProvider = FromCompileJob(job); + nextDmbProvider?.Dispose(); + nextDmbProvider = newProvider; newerDmbTcs.SetResult(nextDmbProvider); - newerDmbTcs = new TaskCompletionSource(); + newerDmbTcs = new TaskCompletionSource(); } } /// public async Task LockNextDmb(CancellationToken cancellationToken) { - Task task; - lock (this) - if (nextDmbProvider != null) - return nextDmbProvider; - else + if (nextDmbProvider == null) + { + Task task; + lock (this) task = newerDmbTcs.Task; - - return await task.ConfigureAwait(false); + await task.ConfigureAwait(false); + } + lock (this) + { + ++jobLockCounts[nextDmbProvider.CompileJob.Id]; + return nextDmbProvider; + } } /// public Task StartAsync(CancellationToken cancellationToken) => databaseContextFactory.UseContext(async (db) => { //where complete clause not necessary, only successful COMPILEjobs get in the db - var cj = await db.CompileJobs.Where(x => x.Job.Instance.Id == instance.Id).OrderByDescending(x => x.Job.StoppedAt).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + var cj = await db.CompileJobs.Where(x => x.Job.Instance.Id == instance.Id && !x.Job.Cancelled.Value && x.Job.ExceptionDetails == null && x.Job.StoppedAt != null) + .Include(x => x.Job) + .OrderByDescending(x => x.Job.StoppedAt).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); if (cj == default(CompileJob)) return; - await LoadCompileJob(cj, false, cancellationToken).ConfigureAwait(false); - + await LoadCompileJob(cj, cancellationToken).ConfigureAwait(false); //we dont do CleanUnusedCompileJobs here because the watchdog may have plans for them yet }); @@ -171,15 +168,41 @@ namespace Tgstation.Server.Host.Components.Compiler } /// - public IDmbProvider FromCompileJob(CompileJob compileJob) + public async Task FromCompileJob(CompileJob compileJob, CancellationToken cancellationToken) { - lock (this) + logger.LogTrace("Loading compile job {0}...", compileJob.Id); + var providerSubmitted = false; + var newProvider = new DmbProvider(compileJob, ioManager, () => { - if (!jobLockCounts.TryGetValue(compileJob.Id, out int value)) - jobLockCounts.Add(compileJob.Id, 1); - else - jobLockCounts[compileJob.Id] = ++value; - return new DmbProvider(compileJob, ioManager, () => CleanJob(compileJob)); + if (providerSubmitted) + CleanJob(compileJob); + }); + + try + { + var primaryCheckTask = ioManager.FileExists(ioManager.ConcatPath(newProvider.PrimaryDirectory, newProvider.DmbName), cancellationToken); + var secondaryCheckTask = ioManager.FileExists(ioManager.ConcatPath(newProvider.PrimaryDirectory, newProvider.DmbName), cancellationToken); + + if (!(await primaryCheckTask.ConfigureAwait(false) && await secondaryCheckTask.ConfigureAwait(false))) + { + logger.LogWarning("Error loading compile job, .dmb missing!"); + return null; //omae wa mou shinderu + } + + lock (this) + { + if (!jobLockCounts.TryGetValue(compileJob.Id, out int value)) + jobLockCounts.Add(compileJob.Id, 1); + else + jobLockCounts[compileJob.Id] = ++value; + providerSubmitted = true; + return newProvider; + } + } + finally + { + if (!providerSubmitted) + newProvider.Dispose(); } } diff --git a/src/Tgstation.Server.Host/Components/Compiler/IDmbFactory.cs b/src/Tgstation.Server.Host/Components/Compiler/IDmbFactory.cs index 6efc0f69b9..a76611577c 100644 --- a/src/Tgstation.Server.Host/Components/Compiler/IDmbFactory.cs +++ b/src/Tgstation.Server.Host/Components/Compiler/IDmbFactory.cs @@ -27,8 +27,9 @@ namespace Tgstation.Server.Host.Components.Compiler /// Gets a for a given /// /// The to make the for - /// A new - IDmbProvider FromCompileJob(CompileJob compileJob); + /// The for the operation + /// A resulting in a new representing the on success, on failure + Task FromCompileJob(CompileJob compileJob, CancellationToken cancellationToken); /// /// Deletes all compile jobs that are inactive in the Game folder diff --git a/src/Tgstation.Server.Host/Components/ReattachInfoHandler.cs b/src/Tgstation.Server.Host/Components/ReattachInfoHandler.cs index a4135a29fa..5e831d0651 100644 --- a/src/Tgstation.Server.Host/Components/ReattachInfoHandler.cs +++ b/src/Tgstation.Server.Host/Components/ReattachInfoHandler.cs @@ -80,7 +80,8 @@ namespace Tgstation.Server.Host.Components await databaseContextFactory.UseContext(async (db) => result = await db.Instances.Where(x => x.Id == metadata.Id).Select(x => x.WatchdogReattachInformation).FirstAsync(cancellationToken).ConfigureAwait(false) ).ConfigureAwait(false); - return new WatchdogReattachInformation(result, dmbFactory); + var bravoDmbTask = dmbFactory.FromCompileJob(result.Bravo.CompileJob, cancellationToken); + return new WatchdogReattachInformation(result, await dmbFactory.FromCompileJob(result.Alpha.CompileJob, cancellationToken).ConfigureAwait(false), await bravoDmbTask.ConfigureAwait(false)); } } } diff --git a/src/Tgstation.Server.Host/Components/Watchdog/ReattachInformation.cs b/src/Tgstation.Server.Host/Components/Watchdog/ReattachInformation.cs index 271d840692..c403d7cae6 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/ReattachInformation.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/ReattachInformation.cs @@ -1,4 +1,4 @@ -using Tgstation.Server.Host.Components.Compiler; +using System; using Tgstation.Server.Host.Models; namespace Tgstation.Server.Host.Components.Watchdog @@ -22,10 +22,10 @@ namespace Tgstation.Server.Host.Components.Watchdog /// Construct a from a given and /// /// The to copy values from - /// The used to assign - public ReattachInformation(Models.ReattachInformation copy, IDmbFactory dmbFactory) : base(copy) + /// The value of + public ReattachInformation(Models.ReattachInformation copy, IDmbProvider dmb) : base(copy) { - Dmb = dmbFactory.FromCompileJob(copy.CompileJob); + Dmb = dmb ?? throw new ArgumentNullException(nameof(dmb)); } } } \ No newline at end of file diff --git a/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs index 75cea931bb..3248a31598 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs @@ -314,7 +314,12 @@ namespace Tgstation.Server.Host.Components.Watchdog //either way try to start it using the active server's dmb as a backup try { - var dmbBackup = dmbFactory.FromCompileJob(monitorState.ActiveServer.Dmb.CompileJob); + var dmbBackup = await dmbFactory.FromCompileJob(monitorState.ActiveServer.Dmb.CompileJob, cancellationToken).ConfigureAwait(false); + + if (dmbBackup == null) //NANI!? + //just give up, if THAT compile job is failing then the ActiveServer is gonna crash soon too or already has + throw new Exception("Creating backup DMB provider failed!"); + monitorState.InactiveServer = await sessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbBackup, null, false, !monitorState.ActiveServer.IsPrimary, false, cancellationToken).ConfigureAwait(false); usedMostRecentDmb = false; await chat.SendWatchdogMessage("Staging newest DMB on inactive server failed: {0} Falling back to previous dmb...", cancellationToken).ConfigureAwait(false); @@ -326,7 +331,7 @@ namespace Tgstation.Server.Host.Components.Watchdog catch (Exception e2) { //fuuuuucckkk - logger.LogError("Backup strategy failed! Monitor will restart when active server reboots! This Exception: {0}", e2.ToString()); + logger.LogError("Backup strategy failed! Monitor will restart when active server reboots! Exception: {0}", e2.ToString()); monitorState.InactiveServerCritFail = true; await chat.SendWatchdogMessage("Attempted reboot of inactive server failed. Watchdog will reset when active server fails or exits", cancellationToken).ConfigureAwait(false); return true; //we didn't use the old dmb diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogReattachInformation.cs b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogReattachInformation.cs index efe645c044..0ed75fed6f 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogReattachInformation.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogReattachInformation.cs @@ -1,5 +1,4 @@ -using Tgstation.Server.Host.Components.Compiler; -using Tgstation.Server.Host.Models; +using Tgstation.Server.Host.Models; namespace Tgstation.Server.Host.Components.Watchdog { @@ -27,13 +26,14 @@ namespace Tgstation.Server.Host.Components.Watchdog /// Construct a from a given with a given /// /// The to copy information from - /// The used to build the s - public WatchdogReattachInformation(Models.WatchdogReattachInformation copy, IDmbFactory dmbFactory): base(copy) + /// The used to build + /// The used to build + public WatchdogReattachInformation(Models.WatchdogReattachInformation copy, IDmbProvider dmbAlpha, IDmbProvider dmbBravo): base(copy) { if (copy.Alpha != null) - Alpha = new ReattachInformation(copy.Alpha, dmbFactory); + Alpha = new ReattachInformation(copy.Alpha, dmbAlpha); if (copy.Bravo != null) - Bravo = new ReattachInformation(copy.Bravo, dmbFactory); + Bravo = new ReattachInformation(copy.Bravo, dmbBravo); } } }