From 337e653ebda03d88d27d3f810f9e4c4c10931565 Mon Sep 17 00:00:00 2001 From: Cyberboss Date: Mon, 28 Dec 2020 13:31:29 -0500 Subject: [PATCH] Further swarm update commit fixes Also adds a 10 minute timeout on the controller. --- .../Swarm/SwarmService.cs | 170 ++++++++++-------- 1 file changed, 100 insertions(+), 70 deletions(-) diff --git a/src/Tgstation.Server.Host/Swarm/SwarmService.cs b/src/Tgstation.Server.Host/Swarm/SwarmService.cs index 20a178c852..9843b8657a 100644 --- a/src/Tgstation.Server.Host/Swarm/SwarmService.cs +++ b/src/Tgstation.Server.Host/Swarm/SwarmService.cs @@ -36,6 +36,11 @@ namespace Tgstation.Server.Host.Swarm /// const int NodeHealthCheckIntervalMinutes = 5; + /// + /// Number of minutes the controller waits to receive a ready-commit from all nodes before aborting an update. + /// + const int UpdateCommitTimeoutMinutes = 10; + /// /// See for the swarm system. /// @@ -334,10 +339,26 @@ namespace Tgstation.Server.Host.Swarm return false; } - var commitGoAhead = await commitTcsTask.ConfigureAwait(false) && updateCommitTcs?.Task == commitTcsTask; + var timeoutTask = swarmController + ? asyncDelayer.Delay( + TimeSpan.FromMinutes(UpdateCommitTimeoutMinutes), + cancellationToken) + : Extensions.TaskExtensions.InfiniteTask(); + + var commitTask = Task.WhenAny(commitTcsTask, timeoutTask); + + await commitTask.ConfigureAwait(false); + + var commitGoAhead = commitTcsTask.IsCompleted + && commitTcsTask.Result + && updateCommitTcs?.Task == commitTcsTask; if (!commitGoAhead) { - logger.LogDebug("Update commit failed!"); + logger.LogDebug( + "Update commit failed!{0}", + timeoutTask.IsCompleted + ? " Timed out!" + : String.Empty); await AbortUpdate(cancellationToken).ConfigureAwait(false); return false; } @@ -395,15 +416,36 @@ namespace Tgstation.Server.Host.Swarm } /// - public async Task PrepareUpdate(Version version, CancellationToken cancellationToken) + public Task PrepareUpdate(Version version, CancellationToken cancellationToken) + { + logger.LogTrace("Begin PrepareUpdate..."); + return PrepareUpdateImpl(version, true, cancellationToken); + } + + /// + public Task PrepareUpdateFromController(Version version, CancellationToken cancellationToken) + { + logger.LogTrace("Received remote update request from {0}", !swarmController ? "controller" : "node"); + return PrepareUpdateImpl(version, false, cancellationToken); + } + + /// + /// Implementation of , + /// + /// The being updated to. + /// Whether or not the update request originated on this server. + /// The for the operation. + /// A resulting in whether or not the update should proceed. + async Task PrepareUpdateImpl(Version version, bool initiator, CancellationToken cancellationToken) { if (version == null) throw new ArgumentNullException(nameof(version)); + logger.LogTrace("PrepareUpdateImpl {0}...", version); + if (!SwarmMode) return true; - logger.LogTrace("Begin PrepareUpdate..."); if (version == targetUpdateVersion) { logger.LogDebug("Prepare update short circuit!"); @@ -426,60 +468,6 @@ namespace Tgstation.Server.Host.Swarm return response.IsSuccessStatusCode; } - if (!swarmController) - { - logger.LogDebug("Forwarding update request to swarm controller..."); - - return await RemotePrepareUpdate(null).ConfigureAwait(false); - } - - var selfPrepare = await PrepareUpdateImpl(version, true, cancellationToken).ConfigureAwait(false); - if (!selfPrepare) - return false; - - try - { - logger.LogTrace("Sending remote prepare nodes..."); - List> tasks; - lock (swarmServers) - tasks = swarmServers - .Where(x => !x.Controller) - .Select(x => RemotePrepareUpdate(x)) - .ToList(); - - await Task.WhenAll(tasks); - - // if all succeeds... - if (tasks.All(x => x.Result)) - { - logger.LogDebug("Distributed prepare for update to version {0} complete.", version); - return true; - } - } - catch (Exception ex) - { - logger.LogWarning(ex, "Error remotely preparing updates!"); - } - - logger.LogDebug("Distrubuted prepare failed!"); - await AbortUpdate(cancellationToken).ConfigureAwait(false); - return false; - } - - /// - public Task PrepareUpdateFromController(Version version, CancellationToken cancellationToken) - => PrepareUpdateImpl(version, false, cancellationToken); - - /// - /// Implementation of , - /// - /// The being updated to. - /// Whether or not the update request originated on this server. - /// The for the operation. - /// A resulting in whether or not the update should proceed. - async Task PrepareUpdateImpl(Version version, bool initiator, CancellationToken cancellationToken) - { - logger.LogTrace("PrepareUpdateImpl {0}...", version); var shouldAbort = false; try { @@ -501,9 +489,20 @@ namespace Tgstation.Server.Host.Swarm targetUpdateVersion = version; } - updateCommitTcs = new TaskCompletionSource(); + if (!swarmController && initiator) + { + logger.LogDebug("Forwarding update request to swarm controller..."); + var result = await RemotePrepareUpdate(null).ConfigureAwait(false); + if (result) + updateCommitTcs = new TaskCompletionSource(); + + return result; + } + if (!initiator) - { + { + logger.LogTrace("Beginning local update process..."); + updateCommitTcs = new TaskCompletionSource(); var updateApplyResult = await serverUpdater.BeginUpdate( version, cancellationToken) @@ -514,16 +513,9 @@ namespace Tgstation.Server.Host.Swarm shouldAbort = true; return false; } - - lock (swarmServers) - nodesThatNeedToBeReadyToCommit = new List( - swarmServers - .Where(x => !x.Controller) - .Select(x => x.Identifier)); } logger.LogDebug("Prepared for update to version {0}", version); - return true; } catch (Exception ex) { @@ -536,6 +528,43 @@ namespace Tgstation.Server.Host.Swarm if (shouldAbort) await AbortUpdate(cancellationToken).ConfigureAwait(false); } + + if (!swarmController) + return true; + + try + { + logger.LogTrace("Sending remote prepare to nodes..."); + List> tasks; + lock (swarmServers) + { + nodesThatNeedToBeReadyToCommit = new List( + swarmServers + .Where(x => !x.Controller) + .Select(x => x.Identifier)); + tasks = swarmServers + .Where(x => !x.Controller) + .Select(x => RemotePrepareUpdate(x)) + .ToList(); + } + + await Task.WhenAll(tasks); + + // if all succeeds... + if (tasks.All(x => x.Result)) + { + logger.LogDebug("Distributed prepare for update to version {0} complete.", version); + return true; + } + } + catch (Exception ex) + { + logger.LogWarning(ex, "Error remotely preparing updates!"); + } + + logger.LogDebug("Distrubuted prepare failed!"); + await AbortUpdate(cancellationToken).ConfigureAwait(false); + return false; } /// @@ -543,10 +572,11 @@ namespace Tgstation.Server.Host.Swarm { if (SwarmMode) logger.LogInformation( - "Swarm mode enabled ({0})", + "Swarm mode enabled: {0} {1}", swarmController - ? "controller" - : "node"); + ? "Controller" + : "Node", + swarmConfiguration.Identifier); else logger.LogTrace("Swarm mode disabled");