diff --git a/src/Tgstation.Server.Host/Components/Session/ISessionController.cs b/src/Tgstation.Server.Host/Components/Session/ISessionController.cs
index 21b45dc434..5e3e9de16b 100644
--- a/src/Tgstation.Server.Host/Components/Session/ISessionController.cs
+++ b/src/Tgstation.Server.Host/Components/Session/ISessionController.cs
@@ -54,6 +54,11 @@ namespace Tgstation.Server.Host.Components.Session
///
RebootState RebootState { get; }
+ ///
+ /// A that completes when the server calls /world/TgsNew().
+ ///
+ Task OnStartup { get; }
+
///
/// A that completes when the server calls /world/TgsReboot().
///
@@ -113,6 +118,7 @@ namespace Tgstation.Server.Host.Components.Session
/// Replace the in use with a given , disposing the old one.
///
/// The new .
- void ReplaceDmbProvider(IDmbProvider newProvider);
+ /// An to be disposed once certain that the original is no longer in use.
+ IDisposable ReplaceDmbProvider(IDmbProvider newProvider);
}
}
diff --git a/src/Tgstation.Server.Host/Components/Session/SessionController.cs b/src/Tgstation.Server.Host/Components/Session/SessionController.cs
index b7978ff051..a1d7e8db9b 100644
--- a/src/Tgstation.Server.Host/Components/Session/SessionController.cs
+++ b/src/Tgstation.Server.Host/Components/Session/SessionController.cs
@@ -69,6 +69,9 @@ namespace Tgstation.Server.Host.Components.Session
///
public Task Lifetime { get; }
+ ///
+ public Task OnStartup => startupTcs.Task;
+
///
public Task OnReboot => rebootTcs.Task;
@@ -143,20 +146,30 @@ namespace Tgstation.Server.Host.Components.Session
///
TaskCompletionSource portAssignmentTcs;
+ ///
+ /// The that completes when DD sends a valid startup bridge request.
+ ///
+ volatile TaskCompletionSource startupTcs;
+
+ ///
+ /// The that completes when DD tells us about a reboot.
+ ///
+ volatile TaskCompletionSource rebootTcs;
+
+ ///
+ /// The that completes when DD tells us it's primed.
+ ///
+ volatile TaskCompletionSource primeTcs;
+
///
/// The port to assign DreamDaemon when it queries for it.
///
ushort? nextPort;
///
- /// The that completes when DD tells us about a reboot.
+ /// The for the .
///
- TaskCompletionSource rebootTcs;
-
- ///
- /// The that completes when DD tells us it's primed.
- ///
- TaskCompletionSource primeTcs;
+ ApiValidationStatus apiValidationStatus;
///
/// If we know DreamDaemon currently has it's port closed.
@@ -168,11 +181,6 @@ namespace Tgstation.Server.Host.Components.Session
///
bool disposed;
- ///
- /// The for the .
- ///
- ApiValidationStatus apiValidationStatus;
-
///
/// If should be kept alive instead.
///
@@ -234,6 +242,7 @@ namespace Tgstation.Server.Host.Components.Session
apiValidationStatus = ApiValidationStatus.NeverValidated;
released = false;
+ startupTcs = new TaskCompletionSource();
rebootTcs = new TaskCompletionSource();
primeTcs = new TaskCompletionSource();
@@ -500,11 +509,11 @@ namespace Tgstation.Server.Host.Components.Session
public void Resume() => process.Resume();
///
- public void ReplaceDmbProvider(IDmbProvider dmbProvider)
+ public IDisposable ReplaceDmbProvider(IDmbProvider dmbProvider)
{
var oldDmb = ReattachInformation.Dmb;
ReattachInformation.Dmb = dmbProvider ?? throw new ArgumentNullException(nameof(dmbProvider));
- oldDmb.Dispose();
+ return oldDmb;
}
///
@@ -639,9 +648,7 @@ namespace Tgstation.Server.Host.Components.Session
parsedChannels);
break;
case BridgeCommandType.Prime:
- var oldPrimeTcs = primeTcs;
- primeTcs = new TaskCompletionSource();
- oldPrimeTcs.SetResult();
+ Interlocked.Exchange(ref primeTcs, new TaskCompletionSource()).SetResult();
break;
case BridgeCommandType.Kill:
Logger.LogInformation("Bridge requested process termination!");
@@ -722,6 +729,7 @@ namespace Tgstation.Server.Host.Components.Session
// Load custom commands
chatTrackingContext.CustomCommands = parameters.CustomCommands;
+ Interlocked.Exchange(ref startupTcs, new TaskCompletionSource()).SetResult();
break;
case BridgeCommandType.Reboot:
if (ClosePortOnReboot)
@@ -731,9 +739,7 @@ namespace Tgstation.Server.Host.Components.Session
portClosedForReboot = true;
}
- var oldRebootTcs = rebootTcs;
- rebootTcs = new TaskCompletionSource();
- oldRebootTcs.SetResult();
+ Interlocked.Exchange(ref rebootTcs, new TaskCompletionSource()).SetResult();
break;
case BridgeCommandType.Chunk:
return await ProcessChunk(ProcessBridgeCommand, BridgeError, parameters.Chunk, cancellationToken);
diff --git a/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs
index f7bc152132..f42b708489 100644
--- a/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs
+++ b/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs
@@ -170,6 +170,8 @@ namespace Tgstation.Server.Host.Components.Watchdog
case MonitorActivationReason.ActiveServerPrimed:
await HandleEvent(EventType.WorldPrime, Enumerable.Empty(), false, cancellationToken);
break;
+ case MonitorActivationReason.ActiveServerStartup:
+ break; // unused in BasicWatchdog
case MonitorActivationReason.Heartbeat:
default:
throw new InvalidOperationException($"Invalid activation reason: {reason}");
diff --git a/src/Tgstation.Server.Host/Components/Watchdog/MonitorActivationReason.cs b/src/Tgstation.Server.Host/Components/Watchdog/MonitorActivationReason.cs
index eaa4b4e441..e3f2a5d48a 100644
--- a/src/Tgstation.Server.Host/Components/Watchdog/MonitorActivationReason.cs
+++ b/src/Tgstation.Server.Host/Components/Watchdog/MonitorActivationReason.cs
@@ -34,5 +34,11 @@
/// Server primed.
///
ActiveServerPrimed,
+
+ ///
+ /// Server started.
+ ///
+ /// The monitor misses the first startup of a session.
+ ActiveServerStartup,
}
}
diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs
index 08ba1c8e2d..126f36628e 100644
--- a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs
+++ b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs
@@ -796,6 +796,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
MonitorAction nextAction = MonitorAction.Continue;
Task activeServerLifetime = null,
activeServerReboot = null,
+ activeServerStartup = null,
serverPrimed = null,
activeLaunchParametersChanged = null,
newDmbAvailable = null;
@@ -825,12 +826,14 @@ namespace Tgstation.Server.Host.Components.Watchdog
TryUpdateTask(ref activeServerLifetime, () => controller.Lifetime);
TryUpdateTask(ref activeServerReboot, () => controller.OnReboot);
TryUpdateTask(ref serverPrimed, () => controller.OnPrime);
+ TryUpdateTask(ref activeServerStartup, () => controller.OnStartup);
}
else
{
activeServerLifetime = controller.Lifetime;
activeServerReboot = controller.OnReboot;
serverPrimed = controller.OnPrime;
+ activeServerStartup = controller.OnStartup;
lastController = controller;
}
@@ -862,6 +865,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
var toWaitOn = Task.WhenAny(
activeServerLifetime,
activeServerReboot,
+ activeServerStartup,
heartbeat,
newDmbAvailable,
cancelTcs.Task,
@@ -908,7 +912,8 @@ namespace Tgstation.Server.Host.Components.Watchdog
|| CheckActivationReason(ref newDmbAvailable, MonitorActivationReason.NewDmbAvailable)
|| CheckActivationReason(ref activeLaunchParametersChanged, MonitorActivationReason.ActiveLaunchParametersUpdated)
|| CheckActivationReason(ref heartbeat, MonitorActivationReason.Heartbeat)
- || CheckActivationReason(ref serverPrimed, MonitorActivationReason.ActiveServerPrimed);
+ || CheckActivationReason(ref serverPrimed, MonitorActivationReason.ActiveServerPrimed)
+ || CheckActivationReason(ref activeServerStartup, MonitorActivationReason.ActiveServerStartup);
UpdateMonitoredTasks();
diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs
index 2f80de7f9c..04175fde30 100644
--- a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs
+++ b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs
@@ -1,4 +1,6 @@
using System;
+using System.Collections.Generic;
+using System.Diagnostics;
using System.Threading;
using System.Threading.Tasks;
@@ -37,11 +39,21 @@ namespace Tgstation.Server.Host.Components.Watchdog
///
readonly ISymlinkFactory symlinkFactory;
+ ///
+ /// of s that are waiting to clean up old deployments.
+ ///
+ readonly List deploymentCleanupTasks;
+
///
/// The active for .
///
SwappableDmbProvider pendingSwappable;
+ ///
+ /// The representing the cleanup of an unused .
+ ///
+ volatile TaskCompletionSource deploymentCleanupGate;
+
///
/// Initializes a new instance of the class.
///
@@ -98,11 +110,16 @@ namespace Tgstation.Server.Host.Components.Watchdog
{
GameIOManager = gameIOManager ?? throw new ArgumentNullException(nameof(gameIOManager));
this.symlinkFactory = symlinkFactory ?? throw new ArgumentNullException(nameof(symlinkFactory));
+
+ deploymentCleanupTasks = new List();
}
catch
{
// Async dispose is for if we have controllers running, not the case here
- DisposeAsync().AsTask().GetAwaiter().GetResult();
+ var disposeTask = DisposeAsync();
+ Debug.Assert(disposeTask.IsCompleted, "This should always be true during construction!");
+ disposeTask.GetAwaiter().GetResult();
+
throw;
}
}
@@ -116,6 +133,8 @@ namespace Tgstation.Server.Host.Components.Watchdog
ActiveSwappable = null;
pendingSwappable?.Dispose();
pendingSwappable = null;
+
+ await DrainDeploymentCleanupTasks(true);
}
///
@@ -124,11 +143,55 @@ namespace Tgstation.Server.Host.Components.Watchdog
if (pendingSwappable != null)
{
var updateTask = BeforeApplyDmb(pendingSwappable.CompileJob, cancellationToken);
- Logger.LogTrace("Replacing activeSwappable with pendingSwappable...");
+
if (!pendingSwappable.Swapped)
await PerformDmbSwap(pendingSwappable, cancellationToken);
- Server.ReplaceDmbProvider(pendingSwappable);
+ var currentCompileJobId = Server.ReattachInformation.Dmb.CompileJob.Id;
+
+ await DrainDeploymentCleanupTasks(false);
+
+ IDisposable lingeringDeployment;
+ var localDeploymentCleanupGate = new TaskCompletionSource();
+ async Task CleanupLingeringDeployment()
+ {
+ var lingeringDeploymentExpirySeconds = ActiveLaunchParameters.StartupTimeout.Value;
+ Logger.LogDebug(
+ "Holding old deployment {compileJobId} for up to {expiry} seconds...",
+ currentCompileJobId,
+ lingeringDeploymentExpirySeconds);
+
+ var timeout = AsyncDelayer.Delay(TimeSpan.FromSeconds(lingeringDeploymentExpirySeconds), cancellationToken);
+
+ var completedTask = await Task.WhenAny(
+ localDeploymentCleanupGate.Task,
+ timeout);
+
+ var timedOut = completedTask == timeout;
+ Logger.Log(
+ timedOut
+ ? LogLevel.Warning
+ : LogLevel.Trace,
+ "Releasing old deployment {compileJobId}{afterTimeout}",
+ timedOut
+ ? " due to timeout!"
+ : "...");
+
+ lingeringDeployment.Dispose();
+ }
+
+ var oldDeploymentCleanupGate = Interlocked.Exchange(ref deploymentCleanupGate, localDeploymentCleanupGate);
+ oldDeploymentCleanupGate?.TrySetResult();
+
+ Logger.LogTrace("Replacing activeSwappable with pendingSwappable...");
+
+ lock (deploymentCleanupTasks)
+ {
+ lingeringDeployment = Server.ReplaceDmbProvider(pendingSwappable);
+ deploymentCleanupTasks.Add(
+ CleanupLingeringDeployment());
+ }
+
ActiveSwappable = pendingSwappable;
pendingSwappable = null;
@@ -240,6 +303,16 @@ namespace Tgstation.Server.Host.Components.Watchdog
await base.SessionStartupPersist(cancellationToken);
}
+ ///
+ protected override async Task HandleMonitorWakeup(MonitorActivationReason reason, CancellationToken cancellationToken)
+ {
+ var result = await base.HandleMonitorWakeup(reason, cancellationToken);
+ if (reason == MonitorActivationReason.ActiveServerStartup)
+ await DrainDeploymentCleanupTasks(false);
+
+ return result;
+ }
+
///
/// Create the initial link to the live game directory using .
///
@@ -284,5 +357,35 @@ namespace Tgstation.Server.Host.Components.Watchdog
server.Resume();
}
}
+
+ ///
+ /// Asynchronously drain .
+ ///
+ /// If , all s will be ed. Otherwise, only s with set will be ed.
+ /// A representing the running operation.
+ Task DrainDeploymentCleanupTasks(bool blocking)
+ {
+ Logger.LogTrace("DrainDeploymentCleanupTasks...");
+ var localDeploymentCleanupGate = Interlocked.Exchange(ref deploymentCleanupGate, null);
+ localDeploymentCleanupGate?.TrySetResult();
+
+ List localDeploymentCleanupTasks;
+ lock (deploymentCleanupTasks)
+ {
+ var totalActiveTasks = deploymentCleanupTasks.Count;
+ localDeploymentCleanupTasks = new List(totalActiveTasks);
+ for (var i = totalActiveTasks - 1; i >= 0; --i)
+ {
+ var currentTask = deploymentCleanupTasks[i];
+ if (!blocking && !currentTask.IsCompleted)
+ continue;
+
+ localDeploymentCleanupTasks.Add(currentTask);
+ deploymentCleanupTasks.RemoveAt(i);
+ }
+ }
+
+ return Task.WhenAll(localDeploymentCleanupTasks);
+ }
}
}