diff --git a/src/Tgstation.Server.Host/Swarm/SwarmService.cs b/src/Tgstation.Server.Host/Swarm/SwarmService.cs
index 20a178c852..9843b8657a 100644
--- a/src/Tgstation.Server.Host/Swarm/SwarmService.cs
+++ b/src/Tgstation.Server.Host/Swarm/SwarmService.cs
@@ -36,6 +36,11 @@ namespace Tgstation.Server.Host.Swarm
///
const int NodeHealthCheckIntervalMinutes = 5;
+ ///
+ /// Number of minutes the controller waits to receive a ready-commit from all nodes before aborting an update.
+ ///
+ const int UpdateCommitTimeoutMinutes = 10;
+
///
/// See for the swarm system.
///
@@ -334,10 +339,26 @@ namespace Tgstation.Server.Host.Swarm
return false;
}
- var commitGoAhead = await commitTcsTask.ConfigureAwait(false) && updateCommitTcs?.Task == commitTcsTask;
+ var timeoutTask = swarmController
+ ? asyncDelayer.Delay(
+ TimeSpan.FromMinutes(UpdateCommitTimeoutMinutes),
+ cancellationToken)
+ : Extensions.TaskExtensions.InfiniteTask();
+
+ var commitTask = Task.WhenAny(commitTcsTask, timeoutTask);
+
+ await commitTask.ConfigureAwait(false);
+
+ var commitGoAhead = commitTcsTask.IsCompleted
+ && commitTcsTask.Result
+ && updateCommitTcs?.Task == commitTcsTask;
if (!commitGoAhead)
{
- logger.LogDebug("Update commit failed!");
+ logger.LogDebug(
+ "Update commit failed!{0}",
+ timeoutTask.IsCompleted
+ ? " Timed out!"
+ : String.Empty);
await AbortUpdate(cancellationToken).ConfigureAwait(false);
return false;
}
@@ -395,15 +416,36 @@ namespace Tgstation.Server.Host.Swarm
}
///
- public async Task PrepareUpdate(Version version, CancellationToken cancellationToken)
+ public Task PrepareUpdate(Version version, CancellationToken cancellationToken)
+ {
+ logger.LogTrace("Begin PrepareUpdate...");
+ return PrepareUpdateImpl(version, true, cancellationToken);
+ }
+
+ ///
+ public Task PrepareUpdateFromController(Version version, CancellationToken cancellationToken)
+ {
+ logger.LogTrace("Received remote update request from {0}", !swarmController ? "controller" : "node");
+ return PrepareUpdateImpl(version, false, cancellationToken);
+ }
+
+ ///
+ /// Implementation of ,
+ ///
+ /// The being updated to.
+ /// Whether or not the update request originated on this server.
+ /// The for the operation.
+ /// A resulting in whether or not the update should proceed.
+ async Task PrepareUpdateImpl(Version version, bool initiator, CancellationToken cancellationToken)
{
if (version == null)
throw new ArgumentNullException(nameof(version));
+ logger.LogTrace("PrepareUpdateImpl {0}...", version);
+
if (!SwarmMode)
return true;
- logger.LogTrace("Begin PrepareUpdate...");
if (version == targetUpdateVersion)
{
logger.LogDebug("Prepare update short circuit!");
@@ -426,60 +468,6 @@ namespace Tgstation.Server.Host.Swarm
return response.IsSuccessStatusCode;
}
- if (!swarmController)
- {
- logger.LogDebug("Forwarding update request to swarm controller...");
-
- return await RemotePrepareUpdate(null).ConfigureAwait(false);
- }
-
- var selfPrepare = await PrepareUpdateImpl(version, true, cancellationToken).ConfigureAwait(false);
- if (!selfPrepare)
- return false;
-
- try
- {
- logger.LogTrace("Sending remote prepare nodes...");
- List> tasks;
- lock (swarmServers)
- tasks = swarmServers
- .Where(x => !x.Controller)
- .Select(x => RemotePrepareUpdate(x))
- .ToList();
-
- await Task.WhenAll(tasks);
-
- // if all succeeds...
- if (tasks.All(x => x.Result))
- {
- logger.LogDebug("Distributed prepare for update to version {0} complete.", version);
- return true;
- }
- }
- catch (Exception ex)
- {
- logger.LogWarning(ex, "Error remotely preparing updates!");
- }
-
- logger.LogDebug("Distrubuted prepare failed!");
- await AbortUpdate(cancellationToken).ConfigureAwait(false);
- return false;
- }
-
- ///
- public Task PrepareUpdateFromController(Version version, CancellationToken cancellationToken)
- => PrepareUpdateImpl(version, false, cancellationToken);
-
- ///
- /// Implementation of ,
- ///
- /// The being updated to.
- /// Whether or not the update request originated on this server.
- /// The for the operation.
- /// A resulting in whether or not the update should proceed.
- async Task PrepareUpdateImpl(Version version, bool initiator, CancellationToken cancellationToken)
- {
- logger.LogTrace("PrepareUpdateImpl {0}...", version);
var shouldAbort = false;
try
{
@@ -501,9 +489,20 @@ namespace Tgstation.Server.Host.Swarm
targetUpdateVersion = version;
}
- updateCommitTcs = new TaskCompletionSource();
+ if (!swarmController && initiator)
+ {
+ logger.LogDebug("Forwarding update request to swarm controller...");
+ var result = await RemotePrepareUpdate(null).ConfigureAwait(false);
+ if (result)
+ updateCommitTcs = new TaskCompletionSource();
+
+ return result;
+ }
+
if (!initiator)
- {
+ {
+ logger.LogTrace("Beginning local update process...");
+ updateCommitTcs = new TaskCompletionSource();
var updateApplyResult = await serverUpdater.BeginUpdate(
version,
cancellationToken)
@@ -514,16 +513,9 @@ namespace Tgstation.Server.Host.Swarm
shouldAbort = true;
return false;
}
-
- lock (swarmServers)
- nodesThatNeedToBeReadyToCommit = new List(
- swarmServers
- .Where(x => !x.Controller)
- .Select(x => x.Identifier));
}
logger.LogDebug("Prepared for update to version {0}", version);
- return true;
}
catch (Exception ex)
{
@@ -536,6 +528,43 @@ namespace Tgstation.Server.Host.Swarm
if (shouldAbort)
await AbortUpdate(cancellationToken).ConfigureAwait(false);
}
+
+ if (!swarmController)
+ return true;
+
+ try
+ {
+ logger.LogTrace("Sending remote prepare to nodes...");
+ List> tasks;
+ lock (swarmServers)
+ {
+ nodesThatNeedToBeReadyToCommit = new List(
+ swarmServers
+ .Where(x => !x.Controller)
+ .Select(x => x.Identifier));
+ tasks = swarmServers
+ .Where(x => !x.Controller)
+ .Select(x => RemotePrepareUpdate(x))
+ .ToList();
+ }
+
+ await Task.WhenAll(tasks);
+
+ // if all succeeds...
+ if (tasks.All(x => x.Result))
+ {
+ logger.LogDebug("Distributed prepare for update to version {0} complete.", version);
+ return true;
+ }
+ }
+ catch (Exception ex)
+ {
+ logger.LogWarning(ex, "Error remotely preparing updates!");
+ }
+
+ logger.LogDebug("Distrubuted prepare failed!");
+ await AbortUpdate(cancellationToken).ConfigureAwait(false);
+ return false;
}
///
@@ -543,10 +572,11 @@ namespace Tgstation.Server.Host.Swarm
{
if (SwarmMode)
logger.LogInformation(
- "Swarm mode enabled ({0})",
+ "Swarm mode enabled: {0} {1}",
swarmController
- ? "controller"
- : "node");
+ ? "Controller"
+ : "Node",
+ swarmConfiguration.Identifier);
else
logger.LogTrace("Swarm mode disabled");