Add initial CompileJob to reattach info in WindowsWatchdog

This commit is contained in:
Dominion
2023-04-03 01:50:40 -04:00
parent 30d2e48200
commit 5d2bbc907c
21 changed files with 4619 additions and 45 deletions
@@ -416,17 +416,20 @@ namespace Tgstation.Server.Host.Components.Deployment
}
lock (jobLockCounts)
if (!jobLockCounts.TryGetValue(job.Id.Value, out var currentVal) || currentVal == 1)
{
jobLockCounts.Remove(job.Id.Value);
logger.LogDebug("Cleaning lock-free compile job {0} => {1}", job.Id, job.DirectoryName);
cleanupTask = HandleCleanup();
}
if (jobLockCounts.TryGetValue(job.Id.Value, out var currentVal))
if (currentVal == 1)
{
jobLockCounts.Remove(job.Id.Value);
logger.LogDebug("Cleaning lock-free compile job {0} => {1}", job.Id, job.DirectoryName);
cleanupTask = HandleCleanup();
}
else
{
var decremented = --jobLockCounts[job.Id.Value];
logger.LogTrace("Compile job {0} lock count now: {1}", job.Id, decremented);
}
else
{
var decremented = --jobLockCounts[job.Id.Value];
logger.LogTrace("Compile job {0} lock count now: {1}", job.Id, decremented);
}
logger.LogError("Extra Dispose of DmbProvider for CompileJob {compileJobId}!", job.Id);
}
/// <summary>
@@ -17,6 +17,11 @@ namespace Tgstation.Server.Host.Components.Session
/// </summary>
public IDmbProvider Dmb { get; set; }
/// <summary>
/// The <see cref="IDmbProvider"/> initially used to launch DreamDaemon. Should be a different <see cref="IDmbProvider"/> than <see cref="Dmb"/>. Should not be set if persisting the initial <see cref="CompileJob"/> isn't necessary.
/// </summary>
public IDmbProvider InitialDmb { get; set; }
/// <summary>
/// The <see cref="Interop.Bridge.RuntimeInformation"/> for the DMAPI.
/// </summary>
@@ -37,13 +42,16 @@ namespace Tgstation.Server.Host.Components.Session
/// </summary>
/// <param name="copy">The <see cref="Models.ReattachInformation"/> to copy values from.</param>
/// <param name="dmb">The value of <see cref="Dmb"/>.</param>
/// <param name="initialDmb">The value of <see cref="InitialDmb"/>.</param>
/// <param name="topicRequestTimeout">The value of <see cref="TopicRequestTimeout"/>.</param>
public ReattachInformation(
Models.ReattachInformation copy,
IDmbProvider dmb,
IDmbProvider initialDmb,
TimeSpan topicRequestTimeout) : base(copy)
{
Dmb = dmb ?? throw new ArgumentNullException(nameof(dmb));
InitialDmb = initialDmb;
TopicRequestTimeout = topicRequestTimeout;
runtimeInformationLock = new object();
@@ -270,22 +270,18 @@ namespace Tgstation.Server.Host.Components.Session
logger.LogTrace("Disposing...");
if (!released)
{
process.Terminate();
byondLock.Dispose();
}
await process.DisposeAsync();
byondLock.Dispose();
bridgeRegistration?.Dispose();
ReattachInformation.Dmb?.Dispose(); // will be null when released
ReattachInformation.Dmb.Dispose();
ReattachInformation.InitialDmb?.Dispose();
chatTrackingContext.Dispose();
reattachTopicCts.Dispose();
if (!released)
{
// finish the async callback
await Lifetime;
}
await Lifetime; // finish the async callback
}
/// <inheritdoc />
@@ -441,14 +437,11 @@ namespace Tgstation.Server.Host.Components.Session
{
CheckDisposed();
// we still don't want to dispose the dmb yet, even though we're keeping it alive
var tmpProvider = ReattachInformation.Dmb;
ReattachInformation.Dmb = null;
ReattachInformation.Dmb.KeepAlive();
ReattachInformation.InitialDmb?.KeepAlive();
byondLock.DoNotDeleteThisSession();
released = true;
await DisposeAsync();
byondLock.DoNotDeleteThisSession();
tmpProvider.KeepAlive();
ReattachInformation.Dmb = tmpProvider;
}
/// <inheritdoc />
@@ -78,6 +78,7 @@ namespace Tgstation.Server.Host.Components.Session
{
AccessIdentifier = reattachInformation.AccessIdentifier,
CompileJobId = reattachInformation.Dmb.CompileJob.Id.Value,
InitialCompileJobId = reattachInformation.InitialDmb?.CompileJob.Id.Value,
Port = reattachInformation.Port,
ProcessId = reattachInformation.ProcessId,
RebootState = reattachInformation.RebootState,
@@ -128,6 +129,7 @@ namespace Tgstation.Server.Host.Components.Session
.AsQueryable()
.Where(x => x.CompileJob.Job.Instance.Id == metadata.Id)
.Include(x => x.CompileJob)
.Include(x => x.InitialCompileJob)
.ToListAsync(cancellationToken);
result = dbReattachInfos.FirstOrDefault();
if (result == default)
@@ -191,9 +193,19 @@ namespace Tgstation.Server.Host.Components.Session
return null;
}
IDmbProvider initialDmb = null;
if (result.InitialCompileJob != null)
{
logger.LogTrace("Loading initial compile job...");
initialDmb = await dmbFactory.FromCompileJob(result.InitialCompileJob, cancellationToken);
}
logger.LogTrace("Retrieved ReattachInformation");
var info = new ReattachInformation(
result,
dmb,
initialDmb,
topicTimeout.Value);
logger.LogDebug("Reattach information loaded: {info}", info);
@@ -249,7 +249,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
// Server.AdjustPriority(true);
if (!reattachInProgress)
await SessionPersistor.Save(Server.ReattachInformation, cancellationToken);
await SessionStartupPersist(cancellationToken);
await CheckLaunchResult(Server, "Server", cancellationToken);
@@ -273,6 +273,16 @@ namespace Tgstation.Server.Host.Components.Watchdog
}
}
/// <summary>
/// Called to save the current <see cref="Server"/> into the <see cref="WatchdogBase.SessionPersistor"/> when initially launched.
/// </summary>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
/// <returns>A <see cref="Task"/> representing the running operation.</returns>
protected virtual Task SessionStartupPersist(CancellationToken cancellationToken)
{
return SessionPersistor.Save(Server.ReattachInformation, cancellationToken);
}
/// <summary>
/// Handler for <see cref="MonitorActivationReason.ActiveServerRebooted"/> when the <see cref="RebootState"/> is <see cref="RebootState.Normal"/>.
/// </summary>
@@ -82,6 +82,9 @@ namespace Tgstation.Server.Host.Components.Watchdog
{
}
/// <inheritdoc />
protected override Task ApplyInitialDmb(CancellationToken cancellationToken) => Task.CompletedTask;
/// <inheritdoc />
protected override async Task InitialLink(CancellationToken cancellationToken)
{
@@ -107,6 +110,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
/// <inheritdoc />
protected override async Task InitController(Task chatTask, ReattachInformation reattachInfo, CancellationToken cancellationToken)
{
var suspended = false;
try
{
await base.InitController(chatTask, reattachInfo, cancellationToken);
@@ -120,6 +124,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
{
Logger.LogTrace("Unhardlinking compile job...");
Server?.Suspend();
suspended = true;
var hardLink = hardLinkedDmb.Directory;
var originalPosition = hardLinkedDmb.CompileJob.DirectoryName.ToString();
await GameIOManager.MoveDirectory(
@@ -149,7 +154,8 @@ namespace Tgstation.Server.Host.Components.Watchdog
Logger.LogTrace("Symlinking compile job...");
await ActiveSwappable.MakeActive(cancellationToken);
Server.Resume();
if (suspended)
Server.Resume();
}
}
}
@@ -727,7 +727,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
/// </summary>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
/// <returns>A <see cref="Task"/> representing the running operation.</returns>
private async Task MonitorRestart(CancellationToken cancellationToken)
async Task MonitorRestart(CancellationToken cancellationToken)
{
Logger.LogTrace("Monitor restart!");
@@ -41,11 +41,6 @@ namespace Tgstation.Server.Host.Components.Watchdog
/// </summary>
SwappableDmbProvider pendingSwappable;
/// <summary>
/// The <see cref="IDmbProvider"/> the <see cref="WindowsWatchdog"/> was started with.
/// </summary>
IDmbProvider startupDmbProvider;
/// <summary>
/// Initializes a new instance of the <see cref="WindowsWatchdog"/> class.
/// </summary>
@@ -120,9 +115,6 @@ namespace Tgstation.Server.Host.Components.Watchdog
ActiveSwappable = null;
pendingSwappable?.Dispose();
pendingSwappable = null;
startupDmbProvider?.Dispose();
startupDmbProvider = null;
}
/// <inheritdoc />
@@ -136,6 +128,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
ActiveSwappable = pendingSwappable;
pendingSwappable = null;
await SessionPersistor.Save(Server.ReattachInformation, cancellationToken);
await updateTask;
}
else
@@ -218,18 +211,12 @@ namespace Tgstation.Server.Host.Components.Watchdog
{
if (ActiveSwappable != null)
throw new InvalidOperationException("Expected activeSwappable to be null!");
if (startupDmbProvider != null)
throw new InvalidOperationException("Expected startupDmbProvider to be null!");
if (pendingSwappable != null)
throw new InvalidOperationException("Expected pendingSwappable to be null!");
Logger.LogTrace("Prep for server launch. pendingSwappable is {0}available", pendingSwappable == null ? "not " : String.Empty);
// Add another lock to the startup DMB because it'll be used throughout the lifetime of the watchdog
startupDmbProvider = await DmbFactory.FromCompileJob(dmbToUse.CompileJob, cancellationToken);
pendingSwappable ??= new SwappableDmbProvider(dmbToUse, GameIOManager, symlinkFactory);
ActiveSwappable = pendingSwappable;
pendingSwappable = null;
Logger.LogTrace("Prep for server launch");
ActiveSwappable = new SwappableDmbProvider(dmbToUse, GameIOManager, symlinkFactory);
try
{
await InitialLink(cancellationToken);
@@ -245,6 +232,23 @@ namespace Tgstation.Server.Host.Components.Watchdog
return ActiveSwappable;
}
/// <summary>
/// Set the <see cref="ReattachInformation.InitialDmb"/> for the <see cref="BasicWatchdog.Server"/>.
/// </summary>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
/// <returns>A <see cref="Task"/> representing the running operation.</returns>
protected virtual async Task ApplyInitialDmb(CancellationToken cancellationToken)
{
Server.ReattachInformation.InitialDmb = await DmbFactory.FromCompileJob(Server.CompileJob, cancellationToken);
}
/// <inheritdoc />
protected override async Task SessionStartupPersist(CancellationToken cancellationToken)
{
await ApplyInitialDmb(cancellationToken);
await base.SessionStartupPersist(cancellationToken);
}
/// <summary>
/// Create the initial link to the live game directory using <see cref="ActiveSwappable"/>.
/// </summary>
@@ -0,0 +1,58 @@
using System;
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace Tgstation.Server.Host.Database.Migrations
{
/// <summary>
/// Adds the InitialCompileJobId to the ReattachInformations table for MSSQL.
/// </summary>
public partial class MSAddReattachInfoInitialCompileJob : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
if (migrationBuilder == null)
throw new ArgumentNullException(nameof(migrationBuilder));
migrationBuilder.AddColumn<long>(
name: "InitialCompileJobId",
table: "ReattachInformations",
type: "bigint",
nullable: true);
migrationBuilder.CreateIndex(
name: "IX_ReattachInformations_InitialCompileJobId",
table: "ReattachInformations",
column: "InitialCompileJobId");
migrationBuilder.AddForeignKey(
name: "FK_ReattachInformations_CompileJobs_InitialCompileJobId",
table: "ReattachInformations",
column: "InitialCompileJobId",
principalTable: "CompileJobs",
principalColumn: "Id");
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
if (migrationBuilder == null)
throw new ArgumentNullException(nameof(migrationBuilder));
migrationBuilder.DropForeignKey(
name: "FK_ReattachInformations_CompileJobs_InitialCompileJobId",
table: "ReattachInformations");
migrationBuilder.DropIndex(
name: "IX_ReattachInformations_InitialCompileJobId",
table: "ReattachInformations");
migrationBuilder.DropColumn(
name: "InitialCompileJobId",
table: "ReattachInformations");
}
}
}
@@ -0,0 +1,58 @@
using System;
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace Tgstation.Server.Host.Database.Migrations
{
/// <summary>
/// Adds the InitialCompileJobId to the ReattachInformations table for MYSQL.
/// </summary>
public partial class MYAddReattachInfoInitialCompileJob : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
if (migrationBuilder == null)
throw new ArgumentNullException(nameof(migrationBuilder));
migrationBuilder.AddColumn<long>(
name: "InitialCompileJobId",
table: "ReattachInformations",
type: "bigint",
nullable: true);
migrationBuilder.CreateIndex(
name: "IX_ReattachInformations_InitialCompileJobId",
table: "ReattachInformations",
column: "InitialCompileJobId");
migrationBuilder.AddForeignKey(
name: "FK_ReattachInformations_CompileJobs_InitialCompileJobId",
table: "ReattachInformations",
column: "InitialCompileJobId",
principalTable: "CompileJobs",
principalColumn: "Id");
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
if (migrationBuilder == null)
throw new ArgumentNullException(nameof(migrationBuilder));
migrationBuilder.DropForeignKey(
name: "FK_ReattachInformations_CompileJobs_InitialCompileJobId",
table: "ReattachInformations");
migrationBuilder.DropIndex(
name: "IX_ReattachInformations_InitialCompileJobId",
table: "ReattachInformations");
migrationBuilder.DropColumn(
name: "InitialCompileJobId",
table: "ReattachInformations");
}
}
}
@@ -0,0 +1,58 @@
using System;
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace Tgstation.Server.Host.Database.Migrations
{
/// <summary>
/// Adds the InitialCompileJobId to the ReattachInformations table for PostgresSQL.
/// </summary>
public partial class PGAddReattachInfoInitialCompileJob : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
if (migrationBuilder == null)
throw new ArgumentNullException(nameof(migrationBuilder));
migrationBuilder.AddColumn<long>(
name: "InitialCompileJobId",
table: "ReattachInformations",
type: "bigint",
nullable: true);
migrationBuilder.CreateIndex(
name: "IX_ReattachInformations_InitialCompileJobId",
table: "ReattachInformations",
column: "InitialCompileJobId");
migrationBuilder.AddForeignKey(
name: "FK_ReattachInformations_CompileJobs_InitialCompileJobId",
table: "ReattachInformations",
column: "InitialCompileJobId",
principalTable: "CompileJobs",
principalColumn: "Id");
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
if (migrationBuilder == null)
throw new ArgumentNullException(nameof(migrationBuilder));
migrationBuilder.DropForeignKey(
name: "FK_ReattachInformations_CompileJobs_InitialCompileJobId",
table: "ReattachInformations");
migrationBuilder.DropIndex(
name: "IX_ReattachInformations_InitialCompileJobId",
table: "ReattachInformations");
migrationBuilder.DropColumn(
name: "InitialCompileJobId",
table: "ReattachInformations");
}
}
}
@@ -0,0 +1,58 @@
using System;
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace Tgstation.Server.Host.Database.Migrations
{
/// <summary>
/// Adds the InitialCompileJobId to the ReattachInformations table for SQLite.
/// </summary>
public partial class SLAddReattachInfoInitialCompileJob : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
if (migrationBuilder == null)
throw new ArgumentNullException(nameof(migrationBuilder));
migrationBuilder.AddColumn<long>(
name: "InitialCompileJobId",
table: "ReattachInformations",
type: "INTEGER",
nullable: true);
migrationBuilder.CreateIndex(
name: "IX_ReattachInformations_InitialCompileJobId",
table: "ReattachInformations",
column: "InitialCompileJobId");
migrationBuilder.AddForeignKey(
name: "FK_ReattachInformations_CompileJobs_InitialCompileJobId",
table: "ReattachInformations",
column: "InitialCompileJobId",
principalTable: "CompileJobs",
principalColumn: "Id");
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
if (migrationBuilder == null)
throw new ArgumentNullException(nameof(migrationBuilder));
migrationBuilder.DropForeignKey(
name: "FK_ReattachInformations_CompileJobs_InitialCompileJobId",
table: "ReattachInformations");
migrationBuilder.DropIndex(
name: "IX_ReattachInformations_InitialCompileJobId",
table: "ReattachInformations");
migrationBuilder.DropColumn(
name: "InitialCompileJobId",
table: "ReattachInformations");
}
}
}
@@ -502,6 +502,9 @@ namespace Tgstation.Server.Host.Database.Migrations
b.Property<long>("CompileJobId")
.HasColumnType("bigint");
b.Property<long?>("InitialCompileJobId")
.HasColumnType("bigint");
b.Property<int>("LaunchSecurityLevel")
.HasColumnType("int");
@@ -521,6 +524,8 @@ namespace Tgstation.Server.Host.Database.Migrations
b.HasIndex("CompileJobId");
b.HasIndex("InitialCompileJobId");
b.ToTable("ReattachInformations");
});
@@ -942,7 +947,13 @@ namespace Tgstation.Server.Host.Database.Migrations
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("Tgstation.Server.Host.Models.CompileJob", "InitialCompileJob")
.WithMany()
.HasForeignKey("InitialCompileJobId");
b.Navigation("CompileJob");
b.Navigation("InitialCompileJob");
});
modelBuilder.Entity("Tgstation.Server.Host.Models.RepositorySettings", b =>
@@ -485,6 +485,9 @@ namespace Tgstation.Server.Host.Database.Migrations
b.Property<long>("CompileJobId")
.HasColumnType("bigint");
b.Property<long?>("InitialCompileJobId")
.HasColumnType("bigint");
b.Property<int>("LaunchSecurityLevel")
.HasColumnType("integer");
@@ -504,6 +507,8 @@ namespace Tgstation.Server.Host.Database.Migrations
b.HasIndex("CompileJobId");
b.HasIndex("InitialCompileJobId");
b.ToTable("ReattachInformations");
});
@@ -903,7 +908,13 @@ namespace Tgstation.Server.Host.Database.Migrations
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("Tgstation.Server.Host.Models.CompileJob", "InitialCompileJob")
.WithMany()
.HasForeignKey("InitialCompileJobId");
b.Navigation("CompileJob");
b.Navigation("InitialCompileJob");
});
modelBuilder.Entity("Tgstation.Server.Host.Models.RepositorySettings", b =>
@@ -490,6 +490,9 @@ namespace Tgstation.Server.Host.Database.Migrations
b.Property<long>("CompileJobId")
.HasColumnType("bigint");
b.Property<long?>("InitialCompileJobId")
.HasColumnType("bigint");
b.Property<int>("LaunchSecurityLevel")
.HasColumnType("int");
@@ -509,6 +512,8 @@ namespace Tgstation.Server.Host.Database.Migrations
b.HasIndex("CompileJobId");
b.HasIndex("InitialCompileJobId");
b.ToTable("ReattachInformations");
});
@@ -909,7 +914,13 @@ namespace Tgstation.Server.Host.Database.Migrations
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("Tgstation.Server.Host.Models.CompileJob", "InitialCompileJob")
.WithMany()
.HasForeignKey("InitialCompileJobId");
b.Navigation("CompileJob");
b.Navigation("InitialCompileJob");
});
modelBuilder.Entity("Tgstation.Server.Host.Models.RepositorySettings", b =>
@@ -468,6 +468,9 @@ namespace Tgstation.Server.Host.Database.Migrations
b.Property<long>("CompileJobId")
.HasColumnType("INTEGER");
b.Property<long?>("InitialCompileJobId")
.HasColumnType("INTEGER");
b.Property<int>("LaunchSecurityLevel")
.HasColumnType("INTEGER");
@@ -487,6 +490,8 @@ namespace Tgstation.Server.Host.Database.Migrations
b.HasIndex("CompileJobId");
b.HasIndex("InitialCompileJobId");
b.ToTable("ReattachInformations");
});
@@ -874,7 +879,13 @@ namespace Tgstation.Server.Host.Database.Migrations
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("Tgstation.Server.Host.Models.CompileJob", "InitialCompileJob")
.WithMany()
.HasForeignKey("InitialCompileJobId");
b.Navigation("CompileJob");
b.Navigation("InitialCompileJob");
});
modelBuilder.Entity("Tgstation.Server.Host.Models.RepositorySettings", b =>
@@ -22,5 +22,15 @@ namespace Tgstation.Server.Host.Models
/// The <see cref="Api.Models.EntityId.Id"/> of <see cref="CompileJob"/>.
/// </summary>
public long CompileJobId { get; set; }
/// <summary>
/// The <see cref="Models.CompileJob"/> the server was initially launched with in the case of Windows.
/// </summary>
public CompileJob InitialCompileJob { get; set; }
/// <summary>
/// The <see cref="Api.Models.EntityId.Id"/> of <see cref="InitialCompileJob"/>.
/// </summary>
public long? InitialCompileJobId { get; set; }
}
}