diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs index fcf1dac080..c8d6c7eb44 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs @@ -186,6 +186,7 @@ namespace Tgstation.Server.Host.Components.Watchdog ? LogLevel.Warning : LogLevel.Trace, "Releasing old deployment {compileJobId}{afterTimeout}", + currentCompileJobId, timedOut ? " due to timeout!" : "..."); diff --git a/src/Tgstation.Server.Host/Swarm/SwarmService.cs b/src/Tgstation.Server.Host/Swarm/SwarmService.cs index e947ff4ffa..317f97463e 100644 --- a/src/Tgstation.Server.Host/Swarm/SwarmService.cs +++ b/src/Tgstation.Server.Host/Swarm/SwarmService.cs @@ -106,9 +106,9 @@ namespace Tgstation.Server.Host.Swarm readonly List swarmServers; /// - /// of s to registration s. + /// of s to registration s and when they were created. /// - readonly Dictionary registrationIds; + readonly Dictionary registrationIdsAndTimes; /// /// If the current server is the swarm controller. @@ -192,7 +192,7 @@ namespace Tgstation.Server.Host.Swarm serverHealthCheckCancellationTokenSource = new CancellationTokenSource(); forceHealthCheckTcs = new TaskCompletionSource(); if (swarmController) - registrationIds = new Dictionary(); + registrationIdsAndTimes = new (); swarmServers = new List { @@ -486,7 +486,7 @@ namespace Tgstation.Server.Host.Swarm .Where(x => !x.Controller) .Select(SendUnregistrationRequest)); swarmServers.RemoveRange(1, swarmServers.Count - 1); - registrationIds.Clear(); + registrationIdsAndTimes.Clear(); } await task; @@ -517,7 +517,7 @@ namespace Tgstation.Server.Host.Swarm { if (swarmController) lock (swarmServers) - return registrationIds.Values.Any(x => x == registrationId); + return registrationIdsAndTimes.Values.Any(x => x.Item1 == registrationId); if (registrationId != controllerRegistration) return false; @@ -546,9 +546,9 @@ namespace Tgstation.Server.Host.Swarm lock (swarmServers) { - if (registrationIds.Any(x => x.Value == registrationId)) + if (registrationIdsAndTimes.Any(x => x.Value.Item1 == registrationId)) { - var preExistingRegistrationKvp = registrationIds.FirstOrDefault(x => x.Value == registrationId); + var preExistingRegistrationKvp = registrationIdsAndTimes.FirstOrDefault(x => x.Value.Item1 == registrationId); if (preExistingRegistrationKvp.Key == node.Identifier) { logger.LogWarning("Node {nodeId} has already registered!", node.Identifier); @@ -563,11 +563,11 @@ namespace Tgstation.Server.Host.Swarm return false; } - if (registrationIds.TryGetValue(node.Identifier, out var oldRegistration)) + if (registrationIdsAndTimes.TryGetValue(node.Identifier, out var oldRegistration)) { logger.LogInformation("Node {nodeId} is re-registering without first unregistering. Indicative of restart.", node.Identifier); swarmServers.RemoveAll(x => x.Identifier == node.Identifier); - registrationIds.Remove(node.Identifier); + registrationIdsAndTimes.Remove(node.Identifier); } swarmServers.Add(new SwarmServerResponse @@ -577,7 +577,7 @@ namespace Tgstation.Server.Host.Swarm Identifier = node.Identifier, Controller = false, }); - registrationIds.Add(node.Identifier, registrationId); + registrationIdsAndTimes.Add(node.Identifier, (registrationId, DateTimeOffset.UtcNow)); } logger.LogInformation("Registered node {nodeId} ({nodeIP}) with ID {registrationId}", node.Identifier, node.Address, registrationId); @@ -658,7 +658,7 @@ namespace Tgstation.Server.Host.Swarm lock (swarmServers) { swarmServers.RemoveAll(x => x.Identifier == nodeIdentifier); - registrationIds.Remove(nodeIdentifier); + registrationIdsAndTimes.Remove(nodeIdentifier); } MarkServersDirty(); @@ -1127,13 +1127,15 @@ namespace Tgstation.Server.Host.Swarm lock (swarmServers) { swarmServers.Remove(swarmServer); - registrationIds.Remove(swarmServer.Identifier); + registrationIdsAndTimes.Remove(swarmServer.Identifier); } } await Task.WhenAll( currentSwarmServers - .Where(x => !x.Controller) + .Where(node => !node.Controller + && registrationIdsAndTimes.TryGetValue(node.Identifier, out var registrationAndTime) + && registrationAndTime.Item2.AddMinutes(SwarmConstants.ControllerHealthCheckIntervalMinutes) < DateTimeOffset.UtcNow) .Select(HealthRequestForServer)); lock (swarmServers) @@ -1329,7 +1331,7 @@ namespace Tgstation.Server.Host.Swarm lock (swarmServers) { swarmServers.Remove(swarmServer); - registrationIds.Remove(swarmServer.Identifier); + registrationIdsAndTimes.Remove(swarmServer.Identifier); } } } @@ -1382,8 +1384,8 @@ namespace Tgstation.Server.Host.Swarm else if (swarmController) { lock (swarmServers) - if (registrationIds.TryGetValue(swarmServer.Identifier, out var registrationId)) - request.Headers.Add(SwarmConstants.RegistrationIdHeader, registrationId.ToString()); + if (registrationIdsAndTimes.TryGetValue(swarmServer.Identifier, out var registrationIdAndTime)) + request.Headers.Add(SwarmConstants.RegistrationIdHeader, registrationIdAndTime.Item1.ToString()); } else if (controllerRegistration.HasValue) request.Headers.Add(SwarmConstants.RegistrationIdHeader, controllerRegistration.Value.ToString()); @@ -1498,14 +1500,14 @@ namespace Tgstation.Server.Host.Swarm lock (swarmServers) { - var exists = registrationIds.Any(x => x.Value == registrationId); + var exists = registrationIdsAndTimes.Any(x => x.Value.Item1 == registrationId); if (!exists) { logger.LogWarning("A node that was to be looked up ({registrationId}) disappeared from our records!", registrationId); return null; } - return registrationIds.First(x => x.Value == registrationId).Key; + return registrationIdsAndTimes.First(x => x.Value.Item1 == registrationId).Key; } } } diff --git a/tests/Tgstation.Server.Host.Tests/Swarm/TestSwarmProtocol.cs b/tests/Tgstation.Server.Host.Tests/Swarm/TestSwarmProtocol.cs index 80ef9bd8f0..5845fe177e 100644 --- a/tests/Tgstation.Server.Host.Tests/Swarm/TestSwarmProtocol.cs +++ b/tests/Tgstation.Server.Host.Tests/Swarm/TestSwarmProtocol.cs @@ -135,7 +135,7 @@ namespace Tgstation.Server.Host.Swarm.Tests Assert.AreEqual(2, controller.Service.GetSwarmServers().Count); Assert.AreEqual(1, node1.Service.GetSwarmServers().Count); - await DelayMax(() => Assert.AreEqual(2, node1.Service.GetSwarmServers().Count), 15); // node checks every 5 minutes, delays are 100ms + await DelayMax(() => Assert.AreEqual(2, node1.Service.GetSwarmServers().Count)); await controller.SimulateReboot(default); Assert.AreEqual(SwarmRegistrationResult.Success, await controller.TryInit());