cyberpresidentvanellope/tgstation-server
1
0
Fork 0
mirror of https://github.com/tgstation/tgstation-server.git synced 2026-08-27 15:07:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

Remove Job requirement from DmbProviders

Browse Source
This commit is contained in:
Jordan Brown
2018-08-07 21:33:34 -04:00
parent 568d169731
commit 1ddce44310
6 changed files with 88 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs
+66 -43
View File
@@ -55,7 +55,7 @@ namespace Tgstation.Server.Host.Components.Compiler
/// <summary>
/// <see cref="TaskCompletionSource{TResult}"/> resulting in the latest <see cref="DmbProvider"/> yet to exist
/// </summary>
TaskCompletionSource<IDmbProvider> newerDmbTcs;
TaskCompletionSource<object> newerDmbTcs;
/// <summary>
/// The latest <see cref="DmbProvider"/>
/// </summary>
@@ -78,12 +78,13 @@ namespace Tgstation.Server.Host.Components.Compiler
this.instance = instance ?? throw new ArgumentNullException(nameof(instance));
cleanupTask = Task.CompletedTask;
newerDmbTcs = new TaskCompletionSource<object>();
cleanupCts = new CancellationTokenSource();
jobLockCounts = new Dictionary<long, int>();
}
/// <inheritdoc />
public void Dispose() => cleanupCts.Dispose();
public void Dispose() => cleanupCts.Dispose(); //we don't dispose nextDmbProvider here, since it might be the only thing we have
/// <summary>
/// Delete the <see cref="Api.Models.Internal.CompileJob.DirectoryName"/> of <paramref name="job"/>
@@ -91,75 +92,71 @@ namespace Tgstation.Server.Host.Components.Compiler
/// <param name="job">The <see cref="CompileJob"/> to clean</param>
void CleanJob(CompileJob job)
{
logger.LogTrace("Cleaning compile job {0} => {1}", job.Id, job.DirectoryName);
async Task HandleCleanup()
{
var deleteJob = ioManager.DeleteDirectory(job.DirectoryName.ToString(), cleanupCts.Token);
Task otherTask;
lock (this)
otherTask = cleanupTask;
//lock (this) //already locked below
otherTask = cleanupTask;
await Task.WhenAll(otherTask, deleteJob).ConfigureAwait(false);
}
lock (this)
{
var currentVal = jobLockCounts[job.Id];
if (--jobLockCounts[job.Id] == 0)
if (!jobLockCounts.TryGetValue(job.Id, out var currentVal) || --jobLockCounts[job.Id] == 0)
{
jobLockCounts.Remove(job.Id);
cleanupTask = HandleCleanup();
}
}
}
/// <inheritdoc />
public Task LoadCompileJob(CompileJob job, CancellationToken cancellationToken) => LoadCompileJob(job, true, cancellationToken);
async Task LoadCompileJob(CompileJob job, bool setAsStagedInDb, CancellationToken cancellationToken)
public async Task LoadCompileJob(CompileJob job, CancellationToken cancellationToken)
{
if (job == null)
throw new ArgumentNullException(nameof(job));
if (job.DMApiValidated != true || job.Job.Cancelled.Value || job.Job.ExceptionDetails != null || job.Job.StoppedAt == null)
if (job.DMApiValidated != true || job.Job.Cancelled == true || job.Job.ExceptionDetails != null)
throw new InvalidOperationException("Cannot load incomplete compile job!");
if (setAsStagedInDb)
await databaseContextFactory.UseContext(async db =>
{
var ddsettings = new DreamDaemonSettings
{
InstanceId = instance.Id
};
db.DreamDaemonSettings.Attach(ddsettings);
ddsettings.StagedCompileJob = job;
await db.Save(cancellationToken).ConfigureAwait(false);
}).ConfigureAwait(false);
var newProvider = await FromCompileJob(job, cancellationToken).ConfigureAwait(false);
if (newProvider == null)
return;
lock (this)
{
var oldDmbProvider = nextDmbProvider;
if (oldDmbProvider != null && oldDmbProvider.CompileJob.Job.StoppedAt < oldDmbProvider.CompileJob.Job.StoppedAt)
throw new InvalidOperationException("Loaded compile job older than current job!");
nextDmbProvider = FromCompileJob(job);
nextDmbProvider?.Dispose();
nextDmbProvider = newProvider;
newerDmbTcs.SetResult(nextDmbProvider);
newerDmbTcs = new TaskCompletionSource<IDmbProvider>();
newerDmbTcs = new TaskCompletionSource<object>();
}
}
/// <inheritdoc />
public async Task<IDmbProvider> LockNextDmb(CancellationToken cancellationToken)
{
Task<IDmbProvider> task;
lock (this)
if (nextDmbProvider != null)
return nextDmbProvider;
else
if (nextDmbProvider == null)
{
Task task;
lock (this)
task = newerDmbTcs.Task;
return await task.ConfigureAwait(false);
await task.ConfigureAwait(false);
}
lock (this)
{
++jobLockCounts[nextDmbProvider.CompileJob.Id];
return nextDmbProvider;
}
}
/// <inheritdoc />
public Task StartAsync(CancellationToken cancellationToken) => databaseContextFactory.UseContext(async (db) =>
{
//where complete clause not necessary, only successful COMPILEjobs get in the db
var cj = await db.CompileJobs.Where(x => x.Job.Instance.Id == instance.Id).OrderByDescending(x => x.Job.StoppedAt).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false);
var cj = await db.CompileJobs.Where(x => x.Job.Instance.Id == instance.Id && !x.Job.Cancelled.Value && x.Job.ExceptionDetails == null && x.Job.StoppedAt != null)
.Include(x => x.Job)
.OrderByDescending(x => x.Job.StoppedAt).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false);
if (cj == default(CompileJob))
return;
await LoadCompileJob(cj, false, cancellationToken).ConfigureAwait(false);
await LoadCompileJob(cj, cancellationToken).ConfigureAwait(false);
//we dont do CleanUnusedCompileJobs here because the watchdog may have plans for them yet
});
@@ -171,15 +168,41 @@ namespace Tgstation.Server.Host.Components.Compiler
}
/// <inheritdoc />
public IDmbProvider FromCompileJob(CompileJob compileJob)
public async Task<IDmbProvider> FromCompileJob(CompileJob compileJob, CancellationToken cancellationToken)
{
lock (this)
logger.LogTrace("Loading compile job {0}...", compileJob.Id);
var providerSubmitted = false;
var newProvider = new DmbProvider(compileJob, ioManager, () =>
{
if (!jobLockCounts.TryGetValue(compileJob.Id, out int value))
jobLockCounts.Add(compileJob.Id, 1);
else
jobLockCounts[compileJob.Id] = ++value;
return new DmbProvider(compileJob, ioManager, () => CleanJob(compileJob));
if (providerSubmitted)
CleanJob(compileJob);
});
try
{
var primaryCheckTask = ioManager.FileExists(ioManager.ConcatPath(newProvider.PrimaryDirectory, newProvider.DmbName), cancellationToken);
var secondaryCheckTask = ioManager.FileExists(ioManager.ConcatPath(newProvider.PrimaryDirectory, newProvider.DmbName), cancellationToken);
if (!(await primaryCheckTask.ConfigureAwait(false) && await secondaryCheckTask.ConfigureAwait(false)))
{
logger.LogWarning("Error loading compile job, .dmb missing!");
return null; //omae wa mou shinderu
}
lock (this)
{
if (!jobLockCounts.TryGetValue(compileJob.Id, out int value))
jobLockCounts.Add(compileJob.Id, 1);
else
jobLockCounts[compileJob.Id] = ++value;
providerSubmitted = true;
return newProvider;
}
}
finally
{
if (!providerSubmitted)
newProvider.Dispose();
}
}
src/Tgstation.Server.Host/Components/Compiler/IDmbFactory.cs
+3 -2
View File
@@ -27,8 +27,9 @@ namespace Tgstation.Server.Host.Components.Compiler
/// Gets a <see cref="IDmbProvider"/> for a given <see cref="CompileJob"/>
/// </summary>
/// <param name="compileJob">The <see cref="CompileJob"/> to make the <see cref="IDmbProvider"/> for</param>
/// <returns>A new <see cref="IDmbProvider"/></returns>
IDmbProvider FromCompileJob(CompileJob compileJob);
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
/// <returns>A <see cref="Task{TResult}"/> resulting in a new <see cref="IDmbProvider"/> representing the <see cref="CompileJob"/> on success, <see langword="null"/> on failure</returns>
Task<IDmbProvider> FromCompileJob(CompileJob compileJob, CancellationToken cancellationToken);
/// <summary>
/// Deletes all compile jobs that are inactive in the Game folder <paramref name="exceptThisOne"/>
src/Tgstation.Server.Host/Components/ReattachInfoHandler.cs
+2 -1
View File
@@ -80,7 +80,8 @@ namespace Tgstation.Server.Host.Components
await databaseContextFactory.UseContext(async (db) =>
result = await db.Instances.Where(x => x.Id == metadata.Id).Select(x => x.WatchdogReattachInformation).FirstAsync(cancellationToken).ConfigureAwait(false)
).ConfigureAwait(false);
return new WatchdogReattachInformation(result, dmbFactory);
var bravoDmbTask = dmbFactory.FromCompileJob(result.Bravo.CompileJob, cancellationToken);
return new WatchdogReattachInformation(result, await dmbFactory.FromCompileJob(result.Alpha.CompileJob, cancellationToken).ConfigureAwait(false), await bravoDmbTask.ConfigureAwait(false));
}
}
}
src/Tgstation.Server.Host/Components/Watchdog/ReattachInformation.cs
+4 -4
View File
@@ -1,4 +1,4 @@
using Tgstation.Server.Host.Components.Compiler;
using System;
using Tgstation.Server.Host.Models;
namespace Tgstation.Server.Host.Components.Watchdog
@@ -22,10 +22,10 @@ namespace Tgstation.Server.Host.Components.Watchdog
/// Construct a <see cref="ReattachInformation"/> from a given <paramref name="copy"/> and <paramref name="dmbFactory"/>
/// </summary>
/// <param name="copy">The <see cref="Models.ReattachInformation"/> to copy values from</param>
/// <param name="dmbFactory">The <see cref="IDmbFactory"/> used to assign <see cref="Dmb"/></param>
public ReattachInformation(Models.ReattachInformation copy, IDmbFactory dmbFactory) : base(copy)
/// <param name="dmb">The value of <see cref="Dmb"/></param>
public ReattachInformation(Models.ReattachInformation copy, IDmbProvider dmb) : base(copy)
{
Dmb = dmbFactory.FromCompileJob(copy.CompileJob);
Dmb = dmb ?? throw new ArgumentNullException(nameof(dmb));
}
}
}
src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs
+7 -2
View File
@@ -314,7 +314,12 @@ namespace Tgstation.Server.Host.Components.Watchdog
//either way try to start it using the active server's dmb as a backup
try
{
var dmbBackup = dmbFactory.FromCompileJob(monitorState.ActiveServer.Dmb.CompileJob);
var dmbBackup = await dmbFactory.FromCompileJob(monitorState.ActiveServer.Dmb.CompileJob, cancellationToken).ConfigureAwait(false);
if (dmbBackup == null) //NANI!?
//just give up, if THAT compile job is failing then the ActiveServer is gonna crash soon too or already has
throw new Exception("Creating backup DMB provider failed!");
monitorState.InactiveServer = await sessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbBackup, null, false, !monitorState.ActiveServer.IsPrimary, false, cancellationToken).ConfigureAwait(false);
usedMostRecentDmb = false;
await chat.SendWatchdogMessage("Staging newest DMB on inactive server failed: {0} Falling back to previous dmb...", cancellationToken).ConfigureAwait(false);
@@ -326,7 +331,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
catch (Exception e2)
{
//fuuuuucckkk
logger.LogError("Backup strategy failed! Monitor will restart when active server reboots! This Exception: {0}", e2.ToString());
logger.LogError("Backup strategy failed! Monitor will restart when active server reboots! Exception: {0}", e2.ToString());
monitorState.InactiveServerCritFail = true;
await chat.SendWatchdogMessage("Attempted reboot of inactive server failed. Watchdog will reset when active server fails or exits", cancellationToken).ConfigureAwait(false);
return true; //we didn't use the old dmb
src/Tgstation.Server.Host/Components/Watchdog/WatchdogReattachInformation.cs
+6 -6
View File
@@ -1,5 +1,4 @@
using Tgstation.Server.Host.Components.Compiler;
using Tgstation.Server.Host.Models;
using Tgstation.Server.Host.Models;
namespace Tgstation.Server.Host.Components.Watchdog
{
@@ -27,13 +26,14 @@ namespace Tgstation.Server.Host.Components.Watchdog
/// Construct a <see cref="WatchdogReattachInformation"/> from a given <paramref name="copy"/> with a given <paramref name="dmbFactory"/>
/// </summary>
/// <param name="copy">The <see cref="WatchdogReattachInformationBase"/> to copy information from</param>
/// <param name="dmbFactory">The <see cref="IDmbFactory"/> used to build the <see cref="ReattachInformation.Dmb"/>s</param>
public WatchdogReattachInformation(Models.WatchdogReattachInformation copy, IDmbFactory dmbFactory): base(copy)
/// <param name="dmbAlpha">The <see cref="IDmbProvider"/> used to build <see cref="Alpha"/></param>
/// <param name="dmbBravo">The <see cref="IDmbProvider"/> used to build <see cref="Bravo"/></param>
public WatchdogReattachInformation(Models.WatchdogReattachInformation copy, IDmbProvider dmbAlpha, IDmbProvider dmbBravo): base(copy)
{
if (copy.Alpha != null)
Alpha = new ReattachInformation(copy.Alpha, dmbFactory);
Alpha = new ReattachInformation(copy.Alpha, dmbAlpha);
if (copy.Bravo != null)
Bravo = new ReattachInformation(copy.Bravo, dmbFactory);
Bravo = new ReattachInformation(copy.Bravo, dmbBravo);
}
}
}