From 975fb984186f807c2ae511c7cfb132e1b52fe6b7 Mon Sep 17 00:00:00 2001 From: Jordan Date: Fri, 16 Jun 2023 14:14:48 -0400 Subject: [PATCH] Fixes #1548 --- .../Components/Session/ISessionController.cs | 8 +- .../Components/Session/SessionController.cs | 46 ++++---- .../Components/Watchdog/BasicWatchdog.cs | 2 + .../Watchdog/MonitorActivationReason.cs | 6 + .../Components/Watchdog/WatchdogBase.cs | 7 +- .../Components/Watchdog/WindowsWatchdog.cs | 109 +++++++++++++++++- 6 files changed, 153 insertions(+), 25 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Session/ISessionController.cs b/src/Tgstation.Server.Host/Components/Session/ISessionController.cs index 21b45dc434..5e3e9de16b 100644 --- a/src/Tgstation.Server.Host/Components/Session/ISessionController.cs +++ b/src/Tgstation.Server.Host/Components/Session/ISessionController.cs @@ -54,6 +54,11 @@ namespace Tgstation.Server.Host.Components.Session /// RebootState RebootState { get; } + /// + /// A that completes when the server calls /world/TgsNew(). + /// + Task OnStartup { get; } + /// /// A that completes when the server calls /world/TgsReboot(). /// @@ -113,6 +118,7 @@ namespace Tgstation.Server.Host.Components.Session /// Replace the in use with a given , disposing the old one. /// /// The new . - void ReplaceDmbProvider(IDmbProvider newProvider); + /// An to be disposed once certain that the original is no longer in use. + IDisposable ReplaceDmbProvider(IDmbProvider newProvider); } } diff --git a/src/Tgstation.Server.Host/Components/Session/SessionController.cs b/src/Tgstation.Server.Host/Components/Session/SessionController.cs index b7978ff051..a1d7e8db9b 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionController.cs @@ -69,6 +69,9 @@ namespace Tgstation.Server.Host.Components.Session /// public Task Lifetime { get; } + /// + public Task OnStartup => startupTcs.Task; + /// public Task OnReboot => rebootTcs.Task; @@ -143,20 +146,30 @@ namespace Tgstation.Server.Host.Components.Session /// TaskCompletionSource portAssignmentTcs; + /// + /// The that completes when DD sends a valid startup bridge request. + /// + volatile TaskCompletionSource startupTcs; + + /// + /// The that completes when DD tells us about a reboot. + /// + volatile TaskCompletionSource rebootTcs; + + /// + /// The that completes when DD tells us it's primed. + /// + volatile TaskCompletionSource primeTcs; + /// /// The port to assign DreamDaemon when it queries for it. /// ushort? nextPort; /// - /// The that completes when DD tells us about a reboot. + /// The for the . /// - TaskCompletionSource rebootTcs; - - /// - /// The that completes when DD tells us it's primed. - /// - TaskCompletionSource primeTcs; + ApiValidationStatus apiValidationStatus; /// /// If we know DreamDaemon currently has it's port closed. @@ -168,11 +181,6 @@ namespace Tgstation.Server.Host.Components.Session /// bool disposed; - /// - /// The for the . - /// - ApiValidationStatus apiValidationStatus; - /// /// If should be kept alive instead. /// @@ -234,6 +242,7 @@ namespace Tgstation.Server.Host.Components.Session apiValidationStatus = ApiValidationStatus.NeverValidated; released = false; + startupTcs = new TaskCompletionSource(); rebootTcs = new TaskCompletionSource(); primeTcs = new TaskCompletionSource(); @@ -500,11 +509,11 @@ namespace Tgstation.Server.Host.Components.Session public void Resume() => process.Resume(); /// - public void ReplaceDmbProvider(IDmbProvider dmbProvider) + public IDisposable ReplaceDmbProvider(IDmbProvider dmbProvider) { var oldDmb = ReattachInformation.Dmb; ReattachInformation.Dmb = dmbProvider ?? throw new ArgumentNullException(nameof(dmbProvider)); - oldDmb.Dispose(); + return oldDmb; } /// @@ -639,9 +648,7 @@ namespace Tgstation.Server.Host.Components.Session parsedChannels); break; case BridgeCommandType.Prime: - var oldPrimeTcs = primeTcs; - primeTcs = new TaskCompletionSource(); - oldPrimeTcs.SetResult(); + Interlocked.Exchange(ref primeTcs, new TaskCompletionSource()).SetResult(); break; case BridgeCommandType.Kill: Logger.LogInformation("Bridge requested process termination!"); @@ -722,6 +729,7 @@ namespace Tgstation.Server.Host.Components.Session // Load custom commands chatTrackingContext.CustomCommands = parameters.CustomCommands; + Interlocked.Exchange(ref startupTcs, new TaskCompletionSource()).SetResult(); break; case BridgeCommandType.Reboot: if (ClosePortOnReboot) @@ -731,9 +739,7 @@ namespace Tgstation.Server.Host.Components.Session portClosedForReboot = true; } - var oldRebootTcs = rebootTcs; - rebootTcs = new TaskCompletionSource(); - oldRebootTcs.SetResult(); + Interlocked.Exchange(ref rebootTcs, new TaskCompletionSource()).SetResult(); break; case BridgeCommandType.Chunk: return await ProcessChunk(ProcessBridgeCommand, BridgeError, parameters.Chunk, cancellationToken); diff --git a/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs index f7bc152132..f42b708489 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs @@ -170,6 +170,8 @@ namespace Tgstation.Server.Host.Components.Watchdog case MonitorActivationReason.ActiveServerPrimed: await HandleEvent(EventType.WorldPrime, Enumerable.Empty(), false, cancellationToken); break; + case MonitorActivationReason.ActiveServerStartup: + break; // unused in BasicWatchdog case MonitorActivationReason.Heartbeat: default: throw new InvalidOperationException($"Invalid activation reason: {reason}"); diff --git a/src/Tgstation.Server.Host/Components/Watchdog/MonitorActivationReason.cs b/src/Tgstation.Server.Host/Components/Watchdog/MonitorActivationReason.cs index eaa4b4e441..e3f2a5d48a 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/MonitorActivationReason.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/MonitorActivationReason.cs @@ -34,5 +34,11 @@ /// Server primed. /// ActiveServerPrimed, + + /// + /// Server started. + /// + /// The monitor misses the first startup of a session. + ActiveServerStartup, } } diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs index 08ba1c8e2d..126f36628e 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs @@ -796,6 +796,7 @@ namespace Tgstation.Server.Host.Components.Watchdog MonitorAction nextAction = MonitorAction.Continue; Task activeServerLifetime = null, activeServerReboot = null, + activeServerStartup = null, serverPrimed = null, activeLaunchParametersChanged = null, newDmbAvailable = null; @@ -825,12 +826,14 @@ namespace Tgstation.Server.Host.Components.Watchdog TryUpdateTask(ref activeServerLifetime, () => controller.Lifetime); TryUpdateTask(ref activeServerReboot, () => controller.OnReboot); TryUpdateTask(ref serverPrimed, () => controller.OnPrime); + TryUpdateTask(ref activeServerStartup, () => controller.OnStartup); } else { activeServerLifetime = controller.Lifetime; activeServerReboot = controller.OnReboot; serverPrimed = controller.OnPrime; + activeServerStartup = controller.OnStartup; lastController = controller; } @@ -862,6 +865,7 @@ namespace Tgstation.Server.Host.Components.Watchdog var toWaitOn = Task.WhenAny( activeServerLifetime, activeServerReboot, + activeServerStartup, heartbeat, newDmbAvailable, cancelTcs.Task, @@ -908,7 +912,8 @@ namespace Tgstation.Server.Host.Components.Watchdog || CheckActivationReason(ref newDmbAvailable, MonitorActivationReason.NewDmbAvailable) || CheckActivationReason(ref activeLaunchParametersChanged, MonitorActivationReason.ActiveLaunchParametersUpdated) || CheckActivationReason(ref heartbeat, MonitorActivationReason.Heartbeat) - || CheckActivationReason(ref serverPrimed, MonitorActivationReason.ActiveServerPrimed); + || CheckActivationReason(ref serverPrimed, MonitorActivationReason.ActiveServerPrimed) + || CheckActivationReason(ref activeServerStartup, MonitorActivationReason.ActiveServerStartup); UpdateMonitoredTasks(); diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs index 2f80de7f9c..04175fde30 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs @@ -1,4 +1,6 @@ using System; +using System.Collections.Generic; +using System.Diagnostics; using System.Threading; using System.Threading.Tasks; @@ -37,11 +39,21 @@ namespace Tgstation.Server.Host.Components.Watchdog /// readonly ISymlinkFactory symlinkFactory; + /// + /// of s that are waiting to clean up old deployments. + /// + readonly List deploymentCleanupTasks; + /// /// The active for . /// SwappableDmbProvider pendingSwappable; + /// + /// The representing the cleanup of an unused . + /// + volatile TaskCompletionSource deploymentCleanupGate; + /// /// Initializes a new instance of the class. /// @@ -98,11 +110,16 @@ namespace Tgstation.Server.Host.Components.Watchdog { GameIOManager = gameIOManager ?? throw new ArgumentNullException(nameof(gameIOManager)); this.symlinkFactory = symlinkFactory ?? throw new ArgumentNullException(nameof(symlinkFactory)); + + deploymentCleanupTasks = new List(); } catch { // Async dispose is for if we have controllers running, not the case here - DisposeAsync().AsTask().GetAwaiter().GetResult(); + var disposeTask = DisposeAsync(); + Debug.Assert(disposeTask.IsCompleted, "This should always be true during construction!"); + disposeTask.GetAwaiter().GetResult(); + throw; } } @@ -116,6 +133,8 @@ namespace Tgstation.Server.Host.Components.Watchdog ActiveSwappable = null; pendingSwappable?.Dispose(); pendingSwappable = null; + + await DrainDeploymentCleanupTasks(true); } /// @@ -124,11 +143,55 @@ namespace Tgstation.Server.Host.Components.Watchdog if (pendingSwappable != null) { var updateTask = BeforeApplyDmb(pendingSwappable.CompileJob, cancellationToken); - Logger.LogTrace("Replacing activeSwappable with pendingSwappable..."); + if (!pendingSwappable.Swapped) await PerformDmbSwap(pendingSwappable, cancellationToken); - Server.ReplaceDmbProvider(pendingSwappable); + var currentCompileJobId = Server.ReattachInformation.Dmb.CompileJob.Id; + + await DrainDeploymentCleanupTasks(false); + + IDisposable lingeringDeployment; + var localDeploymentCleanupGate = new TaskCompletionSource(); + async Task CleanupLingeringDeployment() + { + var lingeringDeploymentExpirySeconds = ActiveLaunchParameters.StartupTimeout.Value; + Logger.LogDebug( + "Holding old deployment {compileJobId} for up to {expiry} seconds...", + currentCompileJobId, + lingeringDeploymentExpirySeconds); + + var timeout = AsyncDelayer.Delay(TimeSpan.FromSeconds(lingeringDeploymentExpirySeconds), cancellationToken); + + var completedTask = await Task.WhenAny( + localDeploymentCleanupGate.Task, + timeout); + + var timedOut = completedTask == timeout; + Logger.Log( + timedOut + ? LogLevel.Warning + : LogLevel.Trace, + "Releasing old deployment {compileJobId}{afterTimeout}", + timedOut + ? " due to timeout!" + : "..."); + + lingeringDeployment.Dispose(); + } + + var oldDeploymentCleanupGate = Interlocked.Exchange(ref deploymentCleanupGate, localDeploymentCleanupGate); + oldDeploymentCleanupGate?.TrySetResult(); + + Logger.LogTrace("Replacing activeSwappable with pendingSwappable..."); + + lock (deploymentCleanupTasks) + { + lingeringDeployment = Server.ReplaceDmbProvider(pendingSwappable); + deploymentCleanupTasks.Add( + CleanupLingeringDeployment()); + } + ActiveSwappable = pendingSwappable; pendingSwappable = null; @@ -240,6 +303,16 @@ namespace Tgstation.Server.Host.Components.Watchdog await base.SessionStartupPersist(cancellationToken); } + /// + protected override async Task HandleMonitorWakeup(MonitorActivationReason reason, CancellationToken cancellationToken) + { + var result = await base.HandleMonitorWakeup(reason, cancellationToken); + if (reason == MonitorActivationReason.ActiveServerStartup) + await DrainDeploymentCleanupTasks(false); + + return result; + } + /// /// Create the initial link to the live game directory using . /// @@ -284,5 +357,35 @@ namespace Tgstation.Server.Host.Components.Watchdog server.Resume(); } } + + /// + /// Asynchronously drain . + /// + /// If , all s will be ed. Otherwise, only s with set will be ed. + /// A representing the running operation. + Task DrainDeploymentCleanupTasks(bool blocking) + { + Logger.LogTrace("DrainDeploymentCleanupTasks..."); + var localDeploymentCleanupGate = Interlocked.Exchange(ref deploymentCleanupGate, null); + localDeploymentCleanupGate?.TrySetResult(); + + List localDeploymentCleanupTasks; + lock (deploymentCleanupTasks) + { + var totalActiveTasks = deploymentCleanupTasks.Count; + localDeploymentCleanupTasks = new List(totalActiveTasks); + for (var i = totalActiveTasks - 1; i >= 0; --i) + { + var currentTask = deploymentCleanupTasks[i]; + if (!blocking && !currentTask.IsCompleted) + continue; + + localDeploymentCleanupTasks.Add(currentTask); + deploymentCleanupTasks.RemoveAt(i); + } + } + + return Task.WhenAll(localDeploymentCleanupTasks); + } } }