From c1f68d9243c70f58b9de1ade0edb9d8a2dd458a8 Mon Sep 17 00:00:00 2001 From: Dominion Date: Sat, 22 Apr 2023 11:40:01 -0400 Subject: [PATCH] Added error code for swarm integrity check failures --- build/Version.props | 6 +++--- docs/API.dox | 2 +- src/Tgstation.Server.Api/Models/ErrorCode.cs | 6 ++++++ .../Tgstation.Server.Api.csproj | 2 +- .../Tgstation.Server.Client.csproj | 2 +- .../Configuration/SwarmConfiguration.cs | 2 +- .../Controllers/AdministrationController.cs | 19 +++++++++--------- .../Core/ServerUpdateResult.cs | 5 +++++ .../Core/ServerUpdater.cs | 3 +++ .../Swarm/ISwarmService.cs | 5 +++++ .../Swarm/SwarmService.cs | 20 +++++++++++++++---- 11 files changed, 52 insertions(+), 20 deletions(-) diff --git a/build/Version.props b/build/Version.props index e1395d2bb2..04e20b8f15 100644 --- a/build/Version.props +++ b/build/Version.props @@ -5,9 +5,9 @@ 5.11.0 4.6.0 - 9.9.0 - 10.3.0 - 11.3.1 + 9.10.0 + 10.4.0 + 11.4.0 6.4.2 5.6.0 1.2.2 diff --git a/docs/API.dox b/docs/API.dox index 49779e1f2d..90963deecf 100644 --- a/docs/API.dox +++ b/docs/API.dox @@ -70,7 +70,7 @@ TGS will only every return the response codes listed here - 409: Conflict. Documented in the requests that use them - 410: Gone. Attempted to access/modify a resource that ideally should have been ready, but isn't or no longer is - 422: Unprocessable Entity: Used specifically when an operation that requires a server restart is unable to be performed due to the @ref Tgstation.Server.Host.Watchdog not being present in the deployment. Should not happen with a proper server configuration. Response body contains an @ref Tgstation.Server.Api.Models.ErrorMessage -- 424: Failed Dependency: When a request that depends on the GitHub API fails for a reason other than rate limiting. Check server logs, usually this indicates a bad access token. +- 424: Failed Dependency: When a request that depends on an external API fails for a reason other than rate limiting. The response body will contain an @ref Tgstation.Server.Api.Models.ErrorMessage model detailing the error. - 426: Upgrade required: Used when the client's API version is not compatible with the server's. Response body contains an @ref Tgstation.Server.Api.Models.ErrorMessage - 429: Rate limited. Used with operations that rely on GitHub.com. If a rate limit is hit for an operation this will be returned. Response will contain a Retry-After header - 500: Server error. Please report the request and response body to the code repository diff --git a/src/Tgstation.Server.Api/Models/ErrorCode.cs b/src/Tgstation.Server.Api/Models/ErrorCode.cs index 2f42896133..585bd0b137 100644 --- a/src/Tgstation.Server.Api/Models/ErrorCode.cs +++ b/src/Tgstation.Server.Api/Models/ErrorCode.cs @@ -629,5 +629,11 @@ namespace Tgstation.Server.Api.Models /// [Description("The deployment took longer than the configured timeout!")] DeploymentTimeout, + + /// + /// The server swarm has less than the expected amount of nodes. + /// + [Description("The server swarm has less than the expected amount of nodes!")] + SwarmIntegrityCheckFailed, } } diff --git a/src/Tgstation.Server.Api/Tgstation.Server.Api.csproj b/src/Tgstation.Server.Api/Tgstation.Server.Api.csproj index 56f94b8618..77b0e05672 100644 --- a/src/Tgstation.Server.Api/Tgstation.Server.Api.csproj +++ b/src/Tgstation.Server.Api/Tgstation.Server.Api.csproj @@ -16,7 +16,7 @@ https://github.com/tgstation/tgstation-server 2018-2023 json web api tgstation-server tgstation ss13 byond - Added ChannelData field to ChatChannels model. + Added ErrorCode.SwarmIntegrityCheckFailed. true snupkg ../../build/analyzers.ruleset diff --git a/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj b/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj index e2272b2855..c5af5ef053 100644 --- a/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj +++ b/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj @@ -16,7 +16,7 @@ https://github.com/tgstation/tgstation-server 2018-2023 json web api tgstation-server tgstation ss13 byond client - Fix login refreshing not working. + Updated definitions for API version 9.10.0. true snupkg ../../build/analyzers.ruleset diff --git a/src/Tgstation.Server.Host/Configuration/SwarmConfiguration.cs b/src/Tgstation.Server.Host/Configuration/SwarmConfiguration.cs index 0e7c99f88d..56410f708c 100644 --- a/src/Tgstation.Server.Host/Configuration/SwarmConfiguration.cs +++ b/src/Tgstation.Server.Host/Configuration/SwarmConfiguration.cs @@ -35,7 +35,7 @@ namespace Tgstation.Server.Host.Configuration public string PrivateKey { get; set; } /// - /// The number of nodes required to be connected before performing an update. Must be set on controller. Does not include controller. + /// The number of nodes in addition to the controller required to be connected a server swarm before performing an update. /// public uint UpdateRequiredNodeCount { get; set; } } diff --git a/src/Tgstation.Server.Host/Controllers/AdministrationController.cs b/src/Tgstation.Server.Host/Controllers/AdministrationController.cs index f5c06338a4..deab3c1288 100644 --- a/src/Tgstation.Server.Host/Controllers/AdministrationController.cs +++ b/src/Tgstation.Server.Host/Controllers/AdministrationController.cs @@ -233,16 +233,17 @@ namespace Tgstation.Server.Host.Controllers try { var updateResult = await serverUpdateInitiator.InitiateUpdate(model.NewVersion, cancellationToken); - if (updateResult == ServerUpdateResult.ReleaseMissing) - return Gone(); - - if (updateResult == ServerUpdateResult.UpdateInProgress) - return BadRequest(new ErrorMessageResponse(ErrorCode.ServerUpdateInProgress)); - - return Accepted(new ServerUpdateResponse + return updateResult switch { - NewVersion = model.NewVersion, - }); + ServerUpdateResult.Started => Accepted(new ServerUpdateResponse + { + NewVersion = model.NewVersion, + }), + ServerUpdateResult.ReleaseMissing => Gone(), + ServerUpdateResult.UpdateInProgress => BadRequest(new ErrorMessageResponse(ErrorCode.ServerUpdateInProgress)), + ServerUpdateResult.SwarmIntegrityCheckFailed => StatusCode(HttpStatusCode.FailedDependency, new ErrorMessageResponse(ErrorCode.SwarmIntegrityCheckFailed)), + _ => throw new InvalidOperationException($"Unexpected ServerUpdateResult: {updateResult}"), + }; } catch (RateLimitExceededException e) { diff --git a/src/Tgstation.Server.Host/Core/ServerUpdateResult.cs b/src/Tgstation.Server.Host/Core/ServerUpdateResult.cs index 8146c03d4a..c5abb6aefb 100644 --- a/src/Tgstation.Server.Host/Core/ServerUpdateResult.cs +++ b/src/Tgstation.Server.Host/Core/ServerUpdateResult.cs @@ -19,5 +19,10 @@ /// Another update is already in progress. /// UpdateInProgress, + + /// + /// The server swarm does not contain the expected amount of nodes. + /// + SwarmIntegrityCheckFailed, } } diff --git a/src/Tgstation.Server.Host/Core/ServerUpdater.cs b/src/Tgstation.Server.Host/Core/ServerUpdater.cs index 29c075ce0f..d33ce063b0 100644 --- a/src/Tgstation.Server.Host/Core/ServerUpdater.cs +++ b/src/Tgstation.Server.Host/Core/ServerUpdater.cs @@ -89,6 +89,9 @@ namespace Tgstation.Server.Host.Core if (newVersion == null) throw new ArgumentNullException(nameof(newVersion)); + if (!swarmService.ExpectedNumberOfNodesConnected) + return ServerUpdateResult.SwarmIntegrityCheckFailed; + logger.LogDebug("Looking for GitHub releases version {version}...", newVersion); IEnumerable releases; var gitHubClient = gitHubClientFactory.CreateClient(); diff --git a/src/Tgstation.Server.Host/Swarm/ISwarmService.cs b/src/Tgstation.Server.Host/Swarm/ISwarmService.cs index a498374195..5f0010297d 100644 --- a/src/Tgstation.Server.Host/Swarm/ISwarmService.cs +++ b/src/Tgstation.Server.Host/Swarm/ISwarmService.cs @@ -12,6 +12,11 @@ namespace Tgstation.Server.Host.Swarm /// public interface ISwarmService { + /// + /// Gets a value indicating if the expected amount of nodes are connected to the swarm. + /// + bool ExpectedNumberOfNodesConnected { get; } + /// /// Signal to the swarm that an update is requested. /// diff --git a/src/Tgstation.Server.Host/Swarm/SwarmService.cs b/src/Tgstation.Server.Host/Swarm/SwarmService.cs index d67c6b854e..48cf0a43b1 100644 --- a/src/Tgstation.Server.Host/Swarm/SwarmService.cs +++ b/src/Tgstation.Server.Host/Swarm/SwarmService.cs @@ -67,6 +67,16 @@ namespace Tgstation.Server.Host.Swarm ReferenceLoopHandling = ReferenceLoopHandling.Ignore, }; + /// + public bool ExpectedNumberOfNodesConnected + { + get + { + lock (swarmServers) + return swarmServers.Count - 1 > swarmConfiguration.UpdateRequiredNodeCount; + } + } + /// /// If the swarm system is enabled. /// @@ -1087,10 +1097,14 @@ namespace Tgstation.Server.Host.Swarm /// A representing the running operation. async Task SendUpdatedServerListToNodes(CancellationToken cancellationToken) { - logger.LogDebug("Sending updated server list to all nodes..."); List currentSwarmServers; lock (swarmServers) + { + serversDirty = false; currentSwarmServers = swarmServers.ToList(); + } + + logger.LogDebug("Sending updated server list to all {nodeCount} nodes...", currentSwarmServers.Count); using var httpClient = httpClientFactory.CreateClient(); async Task UpdateRequestForServer(SwarmServerResponse swarmServer) @@ -1124,9 +1138,7 @@ namespace Tgstation.Server.Host.Swarm await Task.WhenAll( currentSwarmServers .Where(x => !x.Controller) - .Select(UpdateRequestForServer)) - ; - serversDirty = false; + .Select(UpdateRequestForServer)); } ///