From 630335ca5ec64dc0d6214dbfdf7599accf1b11fb Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Tue, 15 Jun 2021 13:55:24 -0400 Subject: [PATCH] Fix single server swarms update deadlock Credit @ZeWaka for finding this bug --- .../Swarm/SwarmService.cs | 9 ++ .../Tgstation.Server.Tests/IntegrationTest.cs | 82 ++++++++++++++++++- 2 files changed, 90 insertions(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Swarm/SwarmService.cs b/src/Tgstation.Server.Host/Swarm/SwarmService.cs index 642431e666..ad786320aa 100644 --- a/src/Tgstation.Server.Host/Swarm/SwarmService.cs +++ b/src/Tgstation.Server.Host/Swarm/SwarmService.cs @@ -856,6 +856,15 @@ namespace Tgstation.Server.Host.Swarm swarmServers .Where(x => !x.Controller) .Select(x => x.Identifier)); + + if (nodesThatNeedToBeReadyToCommit.Count == 0) + { + logger.LogTrace("Controller has no nodes, setting commit-ready."); + var commitTcs = updateCommitTcs; + commitTcs?.TrySetResult(true); + return commitTcs != null; + } + tasks = swarmServers .Where(x => !x.Controller) .Select(RemotePrepareUpdate) diff --git a/tests/Tgstation.Server.Tests/IntegrationTest.cs b/tests/Tgstation.Server.Tests/IntegrationTest.cs index 2c1f446d9f..24e3ebeee1 100644 --- a/tests/Tgstation.Server.Tests/IntegrationTest.cs +++ b/tests/Tgstation.Server.Tests/IntegrationTest.cs @@ -1,4 +1,4 @@ -using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.Extensions.DependencyInjection; @@ -118,6 +118,86 @@ namespace Tgstation.Server.Tests Assert.IsTrue(server.RestartRequested, "Server not requesting restart!"); } + [TestMethod] + public async Task TestOneServerSwarmUpdate() + { + // cleanup existing directories + new TestingServer(null, false).Dispose(); + + const string PrivateKey = "adlfj73ywifhks7iwrgfegjs"; + + var controllerAddress = new Uri("http://localhost:5011"); + using (var controller = new TestingServer(new SwarmConfiguration + { + Address = controllerAddress, + Identifier = "controller", + PrivateKey = PrivateKey + }, false, 5011)) + { + using var serverCts = new CancellationTokenSource(); + var cancellationToken = serverCts.Token; + var serverTask = controller.Run(cancellationToken); + + try + { + using var controllerClient = await CreateAdminClient(controller.Url, cancellationToken); + + var controllerInfo = await controllerClient.ServerInformation(cancellationToken); + + static void CheckInfo(ServerInformationResponse serverInformation) + { + Assert.IsNotNull(serverInformation.SwarmServers); + Assert.AreEqual(1, serverInformation.SwarmServers.Count); + var controller = serverInformation.SwarmServers.SingleOrDefault(x => x.Identifier == "controller"); + Assert.IsNotNull(controller); + Assert.AreEqual(controller.Address, "http://localhost:5011"); + Assert.IsTrue(controller.Controller); + } + + CheckInfo(controllerInfo); + + // test update + var testUpdateVersion = new Version(4, 8, 1); + await controllerClient.Administration.Update( + new ServerUpdateRequest + { + NewVersion = testUpdateVersion + }, + cancellationToken); + await Task.WhenAny(Task.Delay(TimeSpan.FromMinutes(2)), serverTask); + Assert.IsTrue(serverTask.IsCompleted); + + void CheckServerUpdated(TestingServer server) + { + Assert.IsTrue(Directory.Exists(server.UpdatePath), "Update directory not present!"); + + var updatedAssemblyPath = Path.Combine(server.UpdatePath, "Tgstation.Server.Host.dll"); + Assert.IsTrue(File.Exists(updatedAssemblyPath), "Updated assembly missing!"); + + var updatedAssemblyVersion = FileVersionInfo.GetVersionInfo(updatedAssemblyPath); + Assert.AreEqual(testUpdateVersion, Version.Parse(updatedAssemblyVersion.FileVersion).Semver()); + Directory.Delete(server.UpdatePath, true); + } + + CheckServerUpdated(controller); + } + catch (RateLimitException ex) + { + if (String.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("TGS4_TEST_GITHUB_TOKEN"))) + throw; + + Assert.Inconclusive("GitHub rate limit hit: {0}", ex); + } + finally + { + serverCts.Cancel(); + await serverTask; + } + } + + new TestingServer(null, false).Dispose(); + } + [TestMethod] public async Task TestSwarmSynchronizationAndUpdates() {