From 8cfba1d79de23329460794de050cc52457c81b87 Mon Sep 17 00:00:00 2001 From: Dominion Date: Tue, 23 May 2023 22:02:11 -0400 Subject: [PATCH 01/45] Set General:ConfigVersion in LiveTestingServer --- tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs b/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs index 4508ee9fc1..a03ff12abe 100644 --- a/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs +++ b/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs @@ -86,6 +86,7 @@ namespace Tgstation.Server.Tests.Live args = new List() { string.Format(CultureInfo.InvariantCulture, "Database:DropDatabase={0}", true), // Replaced after first Run + string.Format(CultureInfo.InvariantCulture, "General:ConfigVersion={0}", GeneralConfiguration.CurrentConfigVersion), string.Format(CultureInfo.InvariantCulture, "General:ApiPort={0}", port), string.Format(CultureInfo.InvariantCulture, "Database:DatabaseType={0}", DatabaseType), string.Format(CultureInfo.InvariantCulture, "Database:ConnectionString={0}", connectionString), From b4634da0d27b398d3ca450ce70c3125f8fccfad6 Mon Sep 17 00:00:00 2001 From: Dominion Date: Tue, 23 May 2023 22:40:53 -0400 Subject: [PATCH 02/45] Add priority topic retries. --- .../Interop/Topic/TopicParameters.cs | 21 ++++++++ .../Components/Session/SessionController.cs | 52 ++++++++++++++----- 2 files changed, 59 insertions(+), 14 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Interop/Topic/TopicParameters.cs b/src/Tgstation.Server.Host/Components/Interop/Topic/TopicParameters.cs index 6e3e604d7c..d4373d244d 100644 --- a/src/Tgstation.Server.Host/Components/Interop/Topic/TopicParameters.cs +++ b/src/Tgstation.Server.Host/Components/Interop/Topic/TopicParameters.cs @@ -1,5 +1,7 @@ using System; +using Newtonsoft.Json; + using Tgstation.Server.Host.Components.Session; namespace Tgstation.Server.Host.Components.Interop.Topic @@ -55,6 +57,25 @@ namespace Tgstation.Server.Host.Components.Interop.Topic /// public ChunkData Chunk { get; } + /// + /// Whether or not the constitute a priority request. + /// + [JsonIgnore] + public bool IsPriority => CommandType switch + { + TopicCommandType.EventNotification + or TopicCommandType.ChangePort + or TopicCommandType.ChangeRebootState + or TopicCommandType.InstanceRenamed + or TopicCommandType.ChatChannelsUpdate + or TopicCommandType.ServerRestarted => true, + TopicCommandType.ChatCommand + or TopicCommandType.Heartbeat + or TopicCommandType.ReceiveChunk => false, + TopicCommandType.SendChunk => throw new InvalidOperationException("SendChunk topic priority should be based on the original TopicParameters!"), + _ => throw new InvalidOperationException($"Invalid value for {nameof(CommandType)}: {CommandType}"), + }; + /// /// Initializes a new instance of the class. /// diff --git a/src/Tgstation.Server.Host/Components/Session/SessionController.cs b/src/Tgstation.Server.Host/Components/Session/SessionController.cs index a831a37679..a8e033ac4f 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionController.cs @@ -760,8 +760,9 @@ namespace Tgstation.Server.Host.Components.Session var fullCommandString = GenerateQueryString(parameters, out var json); Logger.LogTrace("Topic request: {json}", json); var fullCommandByteCount = Encoding.UTF8.GetByteCount(fullCommandString); + var topicPriority = parameters.IsPriority; if (fullCommandByteCount <= DMApiConstants.MaximumTopicRequestLength) - return await SendRawTopic(fullCommandString, cancellationToken); + return await SendRawTopic(fullCommandString, topicPriority, cancellationToken); var interopChunkingVersion = new Version(5, 6, 0); if (ReattachInformation.Dmb.CompileJob.DMApiVersion < interopChunkingVersion) @@ -849,7 +850,7 @@ namespace Tgstation.Server.Host.Components.Session foreach (var chunkCommandString in chunkQueryStrings) { - combinedResponse = await SendRawTopic(chunkCommandString, cancellationToken); + combinedResponse = await SendRawTopic(chunkCommandString, topicPriority, cancellationToken); if (LogRequestIssue(chunkCommandString == chunkQueryStrings.Last())) return null; } @@ -861,7 +862,7 @@ namespace Tgstation.Server.Host.Components.Session foreach (var missingChunkIndex in combinedResponse.InteropResponse.MissingChunks) { var chunkCommandString = chunkQueryStrings[(int)missingChunkIndex]; - combinedResponse = await SendRawTopic(chunkCommandString, cancellationToken); + combinedResponse = await SendRawTopic(chunkCommandString, topicPriority, cancellationToken); if (LogRequestIssue(missingChunkIndex == lastIndex)) return null; } @@ -891,22 +892,45 @@ namespace Tgstation.Server.Host.Components.Session /// Send a given to DreamDaemon's /world/Topic. /// /// The sanitized topic query string to send. + /// If this is a priority message. If so, the topic will make 5 attempts to send unless BYOND reboots or exits. /// The for the operation. /// A resulting in the of the topic request. - async Task SendRawTopic(string queryString, CancellationToken cancellationToken) + async Task SendRawTopic(string queryString, bool priority, CancellationToken cancellationToken) { var targetPort = ReattachInformation.Port; - global::Byond.TopicSender.TopicResponse byondResponse; - try + var killedOrRebootedTask = Task.WhenAll(Lifetime, OnReboot); + global::Byond.TopicSender.TopicResponse byondResponse = null; + var firstSend = true; + for (var i = 4; i >= 0 && (priority || firstSend); --i) + try + { + firstSend = false; + if (!killedOrRebootedTask.IsCompleted) + byondResponse = await byondTopicSender.SendTopic( + new IPEndPoint(IPAddress.Loopback, targetPort), + queryString, + cancellationToken); + + break; + } + catch (Exception ex) + { + Logger.LogWarning(ex, "SendTopic exception!{retryDetails}", priority ? $"{i} attempts remaining." : String.Empty); + } + + if (byondResponse == null) { - byondResponse = await byondTopicSender.SendTopic( - new IPEndPoint(IPAddress.Loopback, targetPort), - queryString, - cancellationToken); - } - catch (Exception ex) - { - Logger.LogWarning(ex, "SendTopic exception!"); + if (priority) + if (killedOrRebootedTask.IsCompleted) + Logger.LogWarning( + "Unable to send priority topic \"{queryString}\" DreamDaemon {stateClearAction}!", + queryString, + Lifetime.IsCompleted ? "process ended" : "rebooted"); + else + Logger.LogError( + "Unable to send priority topic \"{queryString}\"!", + queryString); + return null; } From e12697fcc957499d4709e8dfb4d9e929fb0c8a07 Mon Sep 17 00:00:00 2001 From: Dominion Date: Tue, 23 May 2023 22:42:12 -0400 Subject: [PATCH 03/45] Error Live test if an unallowed error is logged --- .../Extensions/ServiceCollectionExtensions.cs | 20 +++++++++++ .../Live/HardFailLogger.cs | 29 +++++++++++++++ .../Live/HardFailLoggerProvider.cs | 23 ++++++++++++ .../Live/TestLiveServer.cs | 36 +++++++++++++++---- 4 files changed, 101 insertions(+), 7 deletions(-) create mode 100644 tests/Tgstation.Server.Tests/Live/HardFailLogger.cs create mode 100644 tests/Tgstation.Server.Tests/Live/HardFailLoggerProvider.cs diff --git a/src/Tgstation.Server.Host/Extensions/ServiceCollectionExtensions.cs b/src/Tgstation.Server.Host/Extensions/ServiceCollectionExtensions.cs index 4fcbc5d32e..4980d732d8 100644 --- a/src/Tgstation.Server.Host/Extensions/ServiceCollectionExtensions.cs +++ b/src/Tgstation.Server.Host/Extensions/ServiceCollectionExtensions.cs @@ -5,6 +5,7 @@ using System.Globalization; using Elastic.CommonSchema.Serilog; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Logging; using Serilog; @@ -27,6 +28,11 @@ namespace Tgstation.Server.Host.Extensions /// static Type chatProviderFactoryType = typeof(ProviderFactory); + /// + /// A for an additional to use. + /// + static ServiceDescriptor additionalLoggerProvider; + /// /// Change the used as an implementation for calls to . /// @@ -36,6 +42,17 @@ namespace Tgstation.Server.Host.Extensions chatProviderFactoryType = typeof(TProviderFactory); } + /// + /// Add an additional to s that call . + /// + /// The of to add. + public static void UseAdditionalLoggerProvider() where TLoggerProvider : class, ILoggerProvider + { + if (additionalLoggerProvider != null) + throw new InvalidOperationException("Cannot have multiple additionalLoggerProviders!"); + additionalLoggerProvider = ServiceDescriptor.Singleton(); + } + /// /// Adds a implementation to the given . /// @@ -133,6 +150,9 @@ namespace Tgstation.Server.Host.Extensions if (Debugger.IsAttached) builder.AddDebug(); + + if (additionalLoggerProvider != null) + builder.Services.TryAddEnumerable(additionalLoggerProvider); }); } } diff --git a/tests/Tgstation.Server.Tests/Live/HardFailLogger.cs b/tests/Tgstation.Server.Tests/Live/HardFailLogger.cs new file mode 100644 index 0000000000..22b46a5532 --- /dev/null +++ b/tests/Tgstation.Server.Tests/Live/HardFailLogger.cs @@ -0,0 +1,29 @@ +using System; +using System.Threading.Tasks; + +using Microsoft.Extensions.Logging; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace Tgstation.Server.Tests.Live +{ + sealed class HardFailLogger : ILogger + { + readonly Action failureSink; + + public HardFailLogger(Action failureSink) + { + this.failureSink = failureSink ?? throw new ArgumentNullException(nameof(failureSink)); + } + + public IDisposable BeginScope(TState state) => Task.CompletedTask; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func formatter) + { + var logMessage = formatter(state, exception); + if (logLevel == LogLevel.Error || (logLevel == LogLevel.Critical && logMessage != "DropDatabase configuration option set! Dropping any existing database...")) + failureSink(new AssertFailedException(logMessage)); + } + } +} diff --git a/tests/Tgstation.Server.Tests/Live/HardFailLoggerProvider.cs b/tests/Tgstation.Server.Tests/Live/HardFailLoggerProvider.cs new file mode 100644 index 0000000000..e72ea557db --- /dev/null +++ b/tests/Tgstation.Server.Tests/Live/HardFailLoggerProvider.cs @@ -0,0 +1,23 @@ +using System.Threading.Tasks; + +using Microsoft.Extensions.Logging; + +namespace Tgstation.Server.Tests.Live +{ + sealed class HardFailLoggerProvider : ILoggerProvider + { + public static bool BlockFails { get; set; } + + public static Task FailureSource => failureSink.Task; + + static readonly TaskCompletionSource failureSink = new TaskCompletionSource(); + + public ILogger CreateLogger(string categoryName) => new HardFailLogger(ex => + { + if (!BlockFails) + failureSink.TrySetException(ex); + }); + + public void Dispose() { } + } +} diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index e39d36650e..535d4d124c 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -698,6 +698,26 @@ namespace Tgstation.Server.Tests.Live [TestMethod] public async Task TestStandardTgsOperation() { + const int MaximumTestMinutes = 30; + using var hardCancellationTokenSource = new CancellationTokenSource(TimeSpan.FromMinutes(MaximumTestMinutes)); + var hardCancellationToken = hardCancellationTokenSource.Token; + + ServiceCollectionExtensions.UseAdditionalLoggerProvider(); + + var internalTask = TestTgsInternal(hardCancellationToken); + await Task.WhenAny( + internalTask, + HardFailLoggerProvider.FailureSource); + + if (!internalTask.IsCompleted) + { + hardCancellationTokenSource.Cancel(); + await Task.WhenAll(internalTask, HardFailLoggerProvider.FailureSource); + } + } + + async Task TestTgsInternal(CancellationToken hardCancellationToken) + { var discordConnectionString = Environment.GetEnvironmentVariable("TGS_TEST_DISCORD_TOKEN"); var ircConnectionString = Environment.GetEnvironmentVariable("TGS_TEST_IRC_CONNECTION_STRING"); var missingChatVarsCount = Convert.ToInt32(String.IsNullOrWhiteSpace(discordConnectionString)) @@ -734,9 +754,6 @@ namespace Tgstation.Server.Tests.Live using var server = new LiveTestingServer(null, true); - const int MaximumTestMinutes = 180; - using var hardTimeoutCancellationTokenSource = new CancellationTokenSource(TimeSpan.FromMinutes(MaximumTestMinutes)); - var hardCancellationToken = hardTimeoutCancellationTokenSource.Token; using var serverCts = CancellationTokenSource.CreateLinkedTokenSource(hardCancellationToken); var cancellationToken = serverCts.Token; @@ -816,12 +833,13 @@ namespace Tgstation.Server.Tests.Live // http bind test https://github.com/tgstation/tgstation-server/issues/1065 if (new PlatformIdentifier().IsWindows) { - using var blockingSocket = new Socket(SocketType.Stream, ProtocolType.Tcp); - blockingSocket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ExclusiveAddressUse, true); - blockingSocket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, false); - blockingSocket.Bind(new IPEndPoint(IPAddress.Any, server.Url.Port)); + HardFailLoggerProvider.BlockFails = true; try { + using var blockingSocket = new Socket(SocketType.Stream, ProtocolType.Tcp); + blockingSocket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ExclusiveAddressUse, true); + blockingSocket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, false); + blockingSocket.Bind(new IPEndPoint(IPAddress.Any, server.Url.Port)); // bind test run await server.Run(cancellationToken); Assert.Fail("Expected server task to end with a SocketException"); @@ -830,6 +848,10 @@ namespace Tgstation.Server.Tests.Live { Assert.AreEqual(ex.SocketErrorCode, SocketError.AddressAlreadyInUse); } + finally + { + HardFailLoggerProvider.BlockFails = false; + } } await Task.WhenAny(serverTask, Task.Delay(TimeSpan.FromMinutes(1), cancellationToken)); From 71593573dd4e86fc515905e2c43ce22767459bdd Mon Sep 17 00:00:00 2001 From: Dominion Date: Tue, 23 May 2023 22:45:47 -0400 Subject: [PATCH 04/45] Interop version bump to 5.6.1 --- build/Version.props | 2 +- src/DMAPI/tgs/v5/__interop_version.dm | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build/Version.props b/build/Version.props index bb6a17d416..5844d85fea 100644 --- a/build/Version.props +++ b/build/Version.props @@ -9,7 +9,7 @@ 10.4.1 11.4.2 6.4.4 - 5.6.0 + 5.6.1 1.2.2 1.2.1 1.0.1 diff --git a/src/DMAPI/tgs/v5/__interop_version.dm b/src/DMAPI/tgs/v5/__interop_version.dm index 6ef7c86ef7..5d3d491a73 100644 --- a/src/DMAPI/tgs/v5/__interop_version.dm +++ b/src/DMAPI/tgs/v5/__interop_version.dm @@ -1 +1 @@ -"5.6.0" +"5.6.1" From 6fc3584b30bb63c725c1c5adf24b6e163f6c7dc1 Mon Sep 17 00:00:00 2001 From: Dominion Date: Tue, 23 May 2023 23:01:31 -0400 Subject: [PATCH 05/45] Fix a bad assert --- tests/Tgstation.Server.Tests/Live/DummyChatProvider.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Tgstation.Server.Tests/Live/DummyChatProvider.cs b/tests/Tgstation.Server.Tests/Live/DummyChatProvider.cs index 331931cef6..74a5b9dae4 100644 --- a/tests/Tgstation.Server.Tests/Live/DummyChatProvider.cs +++ b/tests/Tgstation.Server.Tests/Live/DummyChatProvider.cs @@ -108,7 +108,7 @@ namespace Tgstation.Server.Tests.Live Logger.LogTrace("SendMessage"); Assert.AreNotEqual(0UL, channelId); - Assert.IsTrue(channelId <= channelIdAllocator); + Assert.IsTrue(channelId <= channelIdAllocator || channelId > (Int32.MaxValue / 2)); cancellationToken.ThrowIfCancellationRequested(); From de8d1b9c1c4ac7a5506bc09a2394b176f686aeae Mon Sep 17 00:00:00 2001 From: Dominion Date: Thu, 25 May 2023 17:40:07 -0400 Subject: [PATCH 06/45] Fix semicolon --- src/Tgstation.Server.Host/Components/Chat/ChatManager.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs b/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs index 5ae211107d..5c9208300b 100644 --- a/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs +++ b/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs @@ -729,8 +729,7 @@ namespace Tgstation.Server.Host.Components.Chat { Text = "TGS: Processing error, check logs!", }, - cancellationToken) - ; + cancellationToken); return; } From d3be54be15caaf826b4e48062895618570a6011e Mon Sep 17 00:00:00 2001 From: Dominion Date: Fri, 26 May 2023 12:59:06 -0400 Subject: [PATCH 07/45] Several fixes for the dummy chat provider --- .../Live/DummyChatProvider.cs | 23 ++++++++++--------- .../Live/HardFailLogger.cs | 5 ++-- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/tests/Tgstation.Server.Tests/Live/DummyChatProvider.cs b/tests/Tgstation.Server.Tests/Live/DummyChatProvider.cs index 74a5b9dae4..7020863bdf 100644 --- a/tests/Tgstation.Server.Tests/Live/DummyChatProvider.cs +++ b/tests/Tgstation.Server.Tests/Live/DummyChatProvider.cs @@ -83,8 +83,6 @@ namespace Tgstation.Server.Tests.Live this.commands = commands ?? throw new ArgumentNullException(nameof(commands)); this.random = random ?? throw new ArgumentNullException(nameof(random)); - // this could be random but there's no point - channelIdAllocator = 100000; logger.LogTrace("Base channel ID {baseChannelId}", channelIdAllocator); this.randomMessageCts = new CancellationTokenSource(); @@ -108,7 +106,8 @@ namespace Tgstation.Server.Tests.Live Logger.LogTrace("SendMessage"); Assert.AreNotEqual(0UL, channelId); - Assert.IsTrue(channelId <= channelIdAllocator || channelId > (Int32.MaxValue / 2)); + var condition = channelId <= channelIdAllocator || channelId >= (Int32.MaxValue / 2); + Assert.IsTrue(condition); cancellationToken.ThrowIfCancellationRequested(); @@ -172,6 +171,7 @@ namespace Tgstation.Server.Tests.Live Logger.LogTrace("DisconnectImpl"); cancellationToken.ThrowIfCancellationRequested(); connected = false; + channelIdAllocator = 0; if (random.Next(0, 100) > 70) throw new Exception("Random disconnection failure!"); @@ -220,21 +220,22 @@ namespace Tgstation.Server.Tests.Live var delay = random.Next(0, 10000); await Task.Delay(delay, cancellationToken); - if (!connected) - continue; - // %5 chance to disconnect randomly if (enableRandomDisconnections != 0 && random.Next(0, 100) > 95) + { connected = false; + channelIdAllocator = 0; + } + + if (!connected) + continue; if (channelIdAllocator >= Int32.MaxValue / 2) Assert.Fail("Too many channels have been allocated!"); - var isPm = channelIdAllocator == 0 || random.Next(0, 100) > 20; - var realId = (ulong)random.Next(1, (int)channelIdAllocator); - - if (isPm) - realId += Int32.MaxValue / 2; + var isPm = channelIdAllocator == 0 || random.Next(0, 100) > 80; + var idRandomizer = isPm ? channelIdAllocator + (Int32.MaxValue / 2) : channelIdAllocator; + var realId = (ulong)random.Next(isPm ? Int32.MaxValue / 2 : 1, (int)idRandomizer); var username = $"RandomUser{i}"; var sender = new ChatUser diff --git a/tests/Tgstation.Server.Tests/Live/HardFailLogger.cs b/tests/Tgstation.Server.Tests/Live/HardFailLogger.cs index 22b46a5532..7471e9c0c3 100644 --- a/tests/Tgstation.Server.Tests/Live/HardFailLogger.cs +++ b/tests/Tgstation.Server.Tests/Live/HardFailLogger.cs @@ -22,8 +22,9 @@ namespace Tgstation.Server.Tests.Live public void Log(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func formatter) { var logMessage = formatter(state, exception); - if (logLevel == LogLevel.Error || (logLevel == LogLevel.Critical && logMessage != "DropDatabase configuration option set! Dropping any existing database...")) - failureSink(new AssertFailedException(logMessage)); + if ((logLevel == LogLevel.Error && !logMessage.StartsWith("Error disconnecting connection ")) + || (logLevel == LogLevel.Critical && logMessage != "DropDatabase configuration option set! Dropping any existing database...")) + failureSink(new Exception(logMessage, exception)); } } } From 2a1bcdb6464fee6b890d15059cfd96d409534a4e Mon Sep 17 00:00:00 2001 From: Dominion Date: Fri, 26 May 2023 12:59:45 -0400 Subject: [PATCH 08/45] Fix unrequired channel remapping occurring upon DM --- src/Tgstation.Server.Host/Components/Chat/ChatManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs b/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs index 5c9208300b..a139bd6557 100644 --- a/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs +++ b/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs @@ -673,7 +673,7 @@ namespace Tgstation.Server.Host.Components.Chat .Any(); } - if (!recursed && !mappedChannel.HasValue && hasChannelZero) + if (!recursed && !mappedChannel.HasValue && !message.User.Channel.IsPrivateChannel && hasChannelZero) { logger.LogInformation("Receieved message from unmapped channel whose provider contains ID 0. Remapping..."); await RemapProvider(provider, cancellationToken); From f842c99e925f00da27c6234357c5a503eeea9b9a Mon Sep 17 00:00:00 2001 From: Dominion Date: Fri, 26 May 2023 13:00:34 -0400 Subject: [PATCH 09/45] Test instance manager in parallel with instance --- .../Live/InstanceManagerTest.cs | 31 ++++++++++++++----- .../Live/TestLiveServer.cs | 10 +++--- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/tests/Tgstation.Server.Tests/Live/InstanceManagerTest.cs b/tests/Tgstation.Server.Tests/Live/InstanceManagerTest.cs index 99125b5606..4dcefae0bc 100644 --- a/tests/Tgstation.Server.Tests/Live/InstanceManagerTest.cs +++ b/tests/Tgstation.Server.Tests/Live/InstanceManagerTest.cs @@ -38,9 +38,20 @@ namespace Tgstation.Server.Tests.Live this.testRootPath = testRootPath ?? throw new ArgumentNullException(nameof(testRootPath)); } - public Task CreateTestInstance(CancellationToken cancellationToken) => instanceManagerClient.CreateOrAttach(new InstanceCreateRequest + public async Task CreateTestInstance(CancellationToken cancellationToken) { - Name = TestInstanceName, + var instance = await CreateTestInstanceStub("LiveTestsInstance", cancellationToken); + return await instanceManagerClient.Update(new InstanceUpdateRequest + { + Id = instance.Id, + Online = true, + ConfigurationType = ConfigurationType.HostWrite + }, cancellationToken); + } + + Task CreateTestInstanceStub(string name, CancellationToken cancellationToken) => instanceManagerClient.CreateOrAttach(new InstanceCreateRequest + { + Name = name, Path = Path.Combine(testRootPath, Guid.NewGuid().ToString()), Online = true, ChatBotLimit = 2 @@ -57,9 +68,9 @@ namespace Tgstation.Server.Tests.Live Online = response.Online }; - public async Task RunPreInstanceTest(CancellationToken cancellationToken) + public async Task RunPreTest(CancellationToken cancellationToken) { - var firstTest = await CreateTestInstance(cancellationToken); + var firstTest = await CreateTestInstanceStub(TestInstanceName, cancellationToken); //instances always start offline Assert.AreEqual(false, firstTest.Online); //check it exists @@ -161,8 +172,14 @@ namespace Tgstation.Server.Tests.Live Assert.IsTrue(Directory.Exists(firstTest.Path)); await RegressionTest1256(cancellationToken); + firstTest = await instanceManagerClient.Update(new InstanceUpdateRequest + { + Id = firstTest.Id, + Online = false, + }, cancellationToken); - return firstTest; + Assert.AreEqual(false, firstTest.Online); + await instanceManagerClient.Detach(firstTest, cancellationToken); } async Task RegressionTest1256(CancellationToken cancellationToken) @@ -197,10 +214,10 @@ namespace Tgstation.Server.Tests.Live Assert.AreEqual(3, paginated.TotalPages); } - public async Task RunPostTest(CancellationToken cancellationToken) + public async Task RunPostTest(Api.Models.Instance instance, CancellationToken cancellationToken) { var instances = await instanceManagerClient.List(null, cancellationToken); - var firstTest = instances.Single(x => x.Name == TestInstanceName); + var firstTest = instances.Single(x => x.Name == instance.Name); var instanceClient = instanceManagerClient.CreateClient(firstTest); Assert.IsTrue(firstTest.Accessible); diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index 535d4d124c..fad45213ee 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -717,7 +717,7 @@ namespace Tgstation.Server.Tests.Live } async Task TestTgsInternal(CancellationToken hardCancellationToken) - { + { var discordConnectionString = Environment.GetEnvironmentVariable("TGS_TEST_DISCORD_TOKEN"); var ircConnectionString = Environment.GetEnvironmentVariable("TGS_TEST_IRC_CONNECTION_STRING"); var missingChatVarsCount = Convert.ToInt32(String.IsNullOrWhiteSpace(discordConnectionString)) @@ -803,7 +803,9 @@ namespace Tgstation.Server.Tests.Live var rootTest = FailFast(RawRequestTests.Run(clientFactory, adminClient, cancellationToken)); var adminTest = FailFast(new AdministrationTest(adminClient.Administration).Run(cancellationToken)); var usersTest = FailFast(new UsersTest(adminClient).Run(cancellationToken)); - instance = await new InstanceManagerTest(adminClient, server.Directory).RunPreInstanceTest(cancellationToken); + var instanceMangagerTest = new InstanceManagerTest(adminClient, server.Directory); + var instancesTest = FailFast(instanceMangagerTest.RunPreTest(cancellationToken)); + instance = await instanceMangagerTest.CreateTestInstance(cancellationToken); Assert.IsTrue(Directory.Exists(instance.Path)); var instanceClient = adminClient.Instances.CreateClient(instance); @@ -811,7 +813,7 @@ namespace Tgstation.Server.Tests.Live var instanceTests = FailFast(new InstanceTest(instanceClient, adminClient.Instances, GetInstanceManager(), (ushort)server.Url.Port).RunTests(cancellationToken)); - await Task.WhenAll(rootTest, adminTest, instanceTests, usersTest); + await Task.WhenAll(rootTest, adminTest, instancesTest, instanceTests, usersTest); await adminClient.Administration.Restart(cancellationToken); } @@ -1021,7 +1023,7 @@ namespace Tgstation.Server.Tests.Live await new ChatTest(instanceClient.ChatBots, adminClient.Instances, instanceClient.Jobs, instance).RunPostTest(cancellationToken); await repoTest; - await new InstanceManagerTest(adminClient, server.Directory).RunPostTest(cancellationToken); + await new InstanceManagerTest(adminClient, server.Directory).RunPostTest(instance, cancellationToken); } } catch (ApiException ex) From 23f73ec2ed3ca5ef41609ef1df926f39c5de9b3c Mon Sep 17 00:00:00 2001 From: Dominion Date: Fri, 26 May 2023 13:40:59 -0400 Subject: [PATCH 10/45] Additional trace logging on error --- src/Tgstation.Server.Host/Components/Chat/ChatManager.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs b/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs index a139bd6557..7ad410ea8a 100644 --- a/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs +++ b/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs @@ -719,6 +719,9 @@ namespace Tgstation.Server.Host.Components.Chat "Error mapping message: Provider ID: {providerId}, Channel Real ID: {realId}", providerId, message.User.Channel.RealId); + logger.LogTrace("message: {messageJson}", JsonConvert.SerializeObject(message)); + lock (mappedChannels) + logger.LogTrace("mappedChannels: {mappedChannelsJson}", JsonConvert.SerializeObject(mappedChannels)); await SendMessage( new List { From 397728738d69eaeecfd4d5532e0e31cf8c624e1f Mon Sep 17 00:00:00 2001 From: Dominion Date: Fri, 26 May 2023 13:41:21 -0400 Subject: [PATCH 11/45] Hard fail test logging doesn't parrot --- tests/Tgstation.Server.Tests/Live/HardFailLogger.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Tgstation.Server.Tests/Live/HardFailLogger.cs b/tests/Tgstation.Server.Tests/Live/HardFailLogger.cs index 7471e9c0c3..1b5c6de609 100644 --- a/tests/Tgstation.Server.Tests/Live/HardFailLogger.cs +++ b/tests/Tgstation.Server.Tests/Live/HardFailLogger.cs @@ -24,7 +24,7 @@ namespace Tgstation.Server.Tests.Live var logMessage = formatter(state, exception); if ((logLevel == LogLevel.Error && !logMessage.StartsWith("Error disconnecting connection ")) || (logLevel == LogLevel.Critical && logMessage != "DropDatabase configuration option set! Dropping any existing database...")) - failureSink(new Exception(logMessage, exception)); + failureSink(new AssertFailedException("TGS logged an error!")); } } } From 8244c470fc55de6b14a62e004fe24b2450fa8885 Mon Sep 17 00:00:00 2001 From: Dominion Date: Fri, 26 May 2023 14:53:21 -0400 Subject: [PATCH 12/45] Minor formatting change + comment --- tests/Tgstation.Server.Tests/Live/DummyChatProviderFactory.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/Tgstation.Server.Tests/Live/DummyChatProviderFactory.cs b/tests/Tgstation.Server.Tests/Live/DummyChatProviderFactory.cs index 55162d85c2..4610fefe4a 100644 --- a/tests/Tgstation.Server.Tests/Live/DummyChatProviderFactory.cs +++ b/tests/Tgstation.Server.Tests/Live/DummyChatProviderFactory.cs @@ -47,7 +47,9 @@ namespace Tgstation.Server.Tests.Live commandFactory.SetWatchdog(Mock.Of()); commands = commandFactory.GenerateCommands(); - var baseRng = new Random(22475); + var baseRng = new Random( + 22475 // comment this out to enable true randomness, not that seeding is imperfect because Task execution order is non-deterministic + ); seededRng = new Dictionary{ { ChatProvider.Irc, new Random(baseRng.Next()) }, { ChatProvider.Discord, new Random(baseRng.Next()) }, From 41702d90b6acb0e588091fe5910caeb4ab038dc3 Mon Sep 17 00:00:00 2001 From: Dominion Date: Fri, 26 May 2023 15:25:59 -0400 Subject: [PATCH 13/45] Fix Discord connect jobs erroring when cancelled --- .../Components/Chat/Providers/DiscordProvider.cs | 7 +++++-- .../Components/Chat/Providers/IrcProvider.cs | 6 +----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Chat/Providers/DiscordProvider.cs b/src/Tgstation.Server.Host/Components/Chat/Providers/DiscordProvider.cs index b15ccc02ef..be488af4bb 100644 --- a/src/Tgstation.Server.Host/Components/Chat/Providers/DiscordProvider.cs +++ b/src/Tgstation.Server.Host/Components/Chat/Providers/DiscordProvider.cs @@ -611,15 +611,18 @@ namespace Tgstation.Server.Host.Components.Chat.Providers { await Task.WhenAny(gatewayReadyTcs.Task, localGatewayTask); - if (localGatewayTask.IsCompleted || cancellationToken.IsCancellationRequested) + cancellationToken.ThrowIfCancellationRequested(); + if (localGatewayTask.IsCompleted) throw new JobException(ErrorCode.ChatCannotConnectProvider); var userClient = serviceProvider.GetRequiredService(); using var localCombinedCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, gatewayCancellationToken); - var currentUserResult = await userClient.GetCurrentUserAsync(localCombinedCts.Token); + var localCombinedCancellationToken = localCombinedCts.Token; + var currentUserResult = await userClient.GetCurrentUserAsync(localCombinedCancellationToken); if (!currentUserResult.IsSuccess) { + localCombinedCancellationToken.ThrowIfCancellationRequested(); Logger.LogWarning("Unable to retrieve current user: {result}", currentUserResult.LogFormat()); throw new JobException(ErrorCode.ChatCannotConnectProvider); } diff --git a/src/Tgstation.Server.Host/Components/Chat/Providers/IrcProvider.cs b/src/Tgstation.Server.Host/Components/Chat/Providers/IrcProvider.cs index 179105d248..08d4b7a1ce 100644 --- a/src/Tgstation.Server.Host/Components/Chat/Providers/IrcProvider.cs +++ b/src/Tgstation.Server.Host/Components/Chat/Providers/IrcProvider.cs @@ -455,11 +455,7 @@ namespace Tgstation.Server.Host.Components.Chat.Providers Logger.LogTrace("Connection established!"); } - catch (OperationCanceledException) - { - throw; - } - catch (Exception e) + catch (Exception e) when (e is not OperationCanceledException) { throw new JobException(ErrorCode.ChatCannotConnectProvider, e); } From a84abee4aa74efcd783e6cd33cc2b26740255248 Mon Sep 17 00:00:00 2001 From: Dominion Date: Fri, 26 May 2023 15:33:12 -0400 Subject: [PATCH 14/45] Fix issue cancelling Provider reconnection loop while a connection job was running --- .../Components/Chat/Providers/Provider.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Chat/Providers/Provider.cs b/src/Tgstation.Server.Host/Components/Chat/Providers/Provider.cs index e13f46ade8..f9ef3396c6 100644 --- a/src/Tgstation.Server.Host/Components/Chat/Providers/Provider.cs +++ b/src/Tgstation.Server.Host/Components/Chat/Providers/Provider.cs @@ -312,16 +312,18 @@ namespace Tgstation.Server.Host.Components.Chat.Providers await jobManager.WaitForJobCompletion(job, null, cancellationToken, default); } } - catch (OperationCanceledException) + catch (OperationCanceledException e) { - break; + Logger.LogTrace(e, "ReconnectionLoop cancelled"); } catch (Exception e) { Logger.LogError(e, "Error reconnecting!"); } } - while (true); + while (!cancellationToken.IsCancellationRequested); + + Logger.LogTrace("ReconnectionLoop exiting..."); } } } From 23664203bd5c62325a1f4754db45d6524c5f9679 Mon Sep 17 00:00:00 2001 From: Dominion Date: Fri, 26 May 2023 15:38:19 -0400 Subject: [PATCH 15/45] Removes an unnecessary throw/catch pair --- src/Tgstation.Server.Host/Jobs/JobManager.cs | 27 +++----------------- 1 file changed, 3 insertions(+), 24 deletions(-) diff --git a/src/Tgstation.Server.Host/Jobs/JobManager.cs b/src/Tgstation.Server.Host/Jobs/JobManager.cs index dbf9352ea7..51a3b69343 100644 --- a/src/Tgstation.Server.Host/Jobs/JobManager.cs +++ b/src/Tgstation.Server.Host/Jobs/JobManager.cs @@ -185,15 +185,9 @@ namespace Tgstation.Server.Host.Jobs JobHandler handler; lock (addCancelLock) { - try - { - handler = CheckGetJob(job); - } - catch (InvalidOperationException) - { - // this is fine - return null; - } + lock (synchronizationLock) + if (!jobs.TryGetValue(job.Id.Value, out handler)) + return null; handler.Cancel(); // this will ensure the db update is only done once } @@ -267,21 +261,6 @@ namespace Tgstation.Server.Host.Jobs activationTcs.SetResult(instanceCoreProvider); } - /// - /// Gets the for a given if it exists. - /// - /// The to get the for. - /// The . - JobHandler CheckGetJob(Job job) - { - lock (synchronizationLock) - { - if (!jobs.TryGetValue(job.Id.Value, out JobHandler jobHandler)) - throw new InvalidOperationException("Job not running!"); - return jobHandler; - } - } - /// /// Runner for s. /// From a3558f1c1e13ca1a2f2c2ebe3821f171f89e2f18 Mon Sep 17 00:00:00 2001 From: Dominion Date: Fri, 26 May 2023 15:39:56 -0400 Subject: [PATCH 16/45] Add debug logging when a job is cancelled --- src/Tgstation.Server.Host/Jobs/JobManager.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Tgstation.Server.Host/Jobs/JobManager.cs b/src/Tgstation.Server.Host/Jobs/JobManager.cs index 51a3b69343..88828baca2 100644 --- a/src/Tgstation.Server.Host/Jobs/JobManager.cs +++ b/src/Tgstation.Server.Host/Jobs/JobManager.cs @@ -189,6 +189,7 @@ namespace Tgstation.Server.Host.Jobs if (!jobs.TryGetValue(job.Id.Value, out handler)) return null; + logger.LogDebug("Cancelling job ID {jobId}...", job.Id.Value); handler.Cancel(); // this will ensure the db update is only done once } From da11e3d31311ac91dc77f6aa8a8ad15ba951db5b Mon Sep 17 00:00:00 2001 From: Dominion Date: Fri, 26 May 2023 15:44:40 -0400 Subject: [PATCH 17/45] Fix a typo --- tests/Tgstation.Server.Tests/Live/DummyChatProviderFactory.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Tgstation.Server.Tests/Live/DummyChatProviderFactory.cs b/tests/Tgstation.Server.Tests/Live/DummyChatProviderFactory.cs index 4610fefe4a..d8098054e3 100644 --- a/tests/Tgstation.Server.Tests/Live/DummyChatProviderFactory.cs +++ b/tests/Tgstation.Server.Tests/Live/DummyChatProviderFactory.cs @@ -48,7 +48,7 @@ namespace Tgstation.Server.Tests.Live commands = commandFactory.GenerateCommands(); var baseRng = new Random( - 22475 // comment this out to enable true randomness, not that seeding is imperfect because Task execution order is non-deterministic + 22475 // comment this out to enable true randomness, note that seeding is imperfect because Task execution order is non-deterministic ); seededRng = new Dictionary{ { ChatProvider.Irc, new Random(baseRng.Next()) }, From d23d75cab183c264b37a2819e368a31236a3cf65 Mon Sep 17 00:00:00 2001 From: Dominion Date: Fri, 26 May 2023 15:54:46 -0400 Subject: [PATCH 18/45] Fix spacing in log message --- .../Components/Session/SessionController.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Components/Session/SessionController.cs b/src/Tgstation.Server.Host/Components/Session/SessionController.cs index a8e033ac4f..7103763c9e 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionController.cs @@ -915,7 +915,7 @@ namespace Tgstation.Server.Host.Components.Session } catch (Exception ex) { - Logger.LogWarning(ex, "SendTopic exception!{retryDetails}", priority ? $"{i} attempts remaining." : String.Empty); + Logger.LogWarning(ex, "SendTopic exception!{retryDetails}", priority ? $" {i} attempts remaining." : String.Empty); } if (byondResponse == null) From 4d1e1898ceaa2d6eab36599dbe3144d6128d47f3 Mon Sep 17 00:00:00 2001 From: Dominion Date: Fri, 26 May 2023 15:57:48 -0400 Subject: [PATCH 19/45] Fixed topic retries not aborting unless a reboot AND process termination occurred --- .../Components/Session/SessionController.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Components/Session/SessionController.cs b/src/Tgstation.Server.Host/Components/Session/SessionController.cs index 7103763c9e..ec67f72e55 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionController.cs @@ -898,7 +898,7 @@ namespace Tgstation.Server.Host.Components.Session async Task SendRawTopic(string queryString, bool priority, CancellationToken cancellationToken) { var targetPort = ReattachInformation.Port; - var killedOrRebootedTask = Task.WhenAll(Lifetime, OnReboot); + var killedOrRebootedTask = Task.WhenAny(Lifetime, OnReboot); global::Byond.TopicSender.TopicResponse byondResponse = null; var firstSend = true; for (var i = 4; i >= 0 && (priority || firstSend); --i) From d610c05efba5a4ec758043215c397472dc016bb8 Mon Sep 17 00:00:00 2001 From: Dominion Date: Fri, 26 May 2023 16:39:59 -0400 Subject: [PATCH 20/45] Ignore query cancellation exceptions in hard fail logger --- tests/Tgstation.Server.Tests/Live/HardFailLogger.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/Tgstation.Server.Tests/Live/HardFailLogger.cs b/tests/Tgstation.Server.Tests/Live/HardFailLogger.cs index 1b5c6de609..1d9d234935 100644 --- a/tests/Tgstation.Server.Tests/Live/HardFailLogger.cs +++ b/tests/Tgstation.Server.Tests/Live/HardFailLogger.cs @@ -22,7 +22,9 @@ namespace Tgstation.Server.Tests.Live public void Log(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func formatter) { var logMessage = formatter(state, exception); - if ((logLevel == LogLevel.Error && !logMessage.StartsWith("Error disconnecting connection ")) + if ((logLevel == LogLevel.Error + && !logMessage.StartsWith("Error disconnecting connection ") + && !(logMessage.StartsWith("An exception occurred while iterating over the results of a query for context type") && exception is TaskCanceledException)) || (logLevel == LogLevel.Critical && logMessage != "DropDatabase configuration option set! Dropping any existing database...")) failureSink(new AssertFailedException("TGS logged an error!")); } From c85f4593330f419552834ff2940e7e280d7e8358 Mon Sep 17 00:00:00 2001 From: Dominion Date: Fri, 26 May 2023 16:41:59 -0400 Subject: [PATCH 21/45] Regex instead of StartsWith for better accuracy --- tests/Tgstation.Server.Tests/Live/HardFailLogger.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/Tgstation.Server.Tests/Live/HardFailLogger.cs b/tests/Tgstation.Server.Tests/Live/HardFailLogger.cs index 1d9d234935..ea9ce8f300 100644 --- a/tests/Tgstation.Server.Tests/Live/HardFailLogger.cs +++ b/tests/Tgstation.Server.Tests/Live/HardFailLogger.cs @@ -1,4 +1,5 @@ using System; +using System.Text.RegularExpressions; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -23,7 +24,7 @@ namespace Tgstation.Server.Tests.Live { var logMessage = formatter(state, exception); if ((logLevel == LogLevel.Error - && !logMessage.StartsWith("Error disconnecting connection ") + && !Regex.IsMatch(logMessage, "Error disconnecting connection [1-9][0-9]*!") && !(logMessage.StartsWith("An exception occurred while iterating over the results of a query for context type") && exception is TaskCanceledException)) || (logLevel == LogLevel.Critical && logMessage != "DropDatabase configuration option set! Dropping any existing database...")) failureSink(new AssertFailedException("TGS logged an error!")); From aca475dda22a7363a249a923ee44fd775653d1c9 Mon Sep 17 00:00:00 2001 From: Dominion Date: Sat, 27 May 2023 11:13:46 -0400 Subject: [PATCH 22/45] Ignore more cancellations --- tests/Tgstation.Server.Tests/Live/HardFailLogger.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Tgstation.Server.Tests/Live/HardFailLogger.cs b/tests/Tgstation.Server.Tests/Live/HardFailLogger.cs index ea9ce8f300..5eb968fb40 100644 --- a/tests/Tgstation.Server.Tests/Live/HardFailLogger.cs +++ b/tests/Tgstation.Server.Tests/Live/HardFailLogger.cs @@ -25,7 +25,7 @@ namespace Tgstation.Server.Tests.Live var logMessage = formatter(state, exception); if ((logLevel == LogLevel.Error && !Regex.IsMatch(logMessage, "Error disconnecting connection [1-9][0-9]*!") - && !(logMessage.StartsWith("An exception occurred while iterating over the results of a query for context type") && exception is TaskCanceledException)) + && !(logMessage.StartsWith("An exception occurred while iterating over the results of a query for context type") && (exception is TaskCanceledException || exception?.InnerException is TaskCanceledException))) || (logLevel == LogLevel.Critical && logMessage != "DropDatabase configuration option set! Dropping any existing database...")) failureSink(new AssertFailedException("TGS logged an error!")); } From 2e90a0bab6e63b60b6ac82e5e484743ba96e754c Mon Sep 17 00:00:00 2001 From: Dominion Date: Sat, 27 May 2023 11:30:09 -0400 Subject: [PATCH 23/45] Only set TGS_TEST_DUMP_API_SPEC in the job that needs it --- .github/workflows/ci-suite.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-suite.yml b/.github/workflows/ci-suite.yml index 5b934f08da..bd4f690ed2 100644 --- a/.github/workflows/ci-suite.yml +++ b/.github/workflows/ci-suite.yml @@ -309,8 +309,6 @@ jobs: name: Windows Live Tests needs: dmapi-build if: "!(cancelled() || failure()) && needs.dmapi-build.result == 'success'" - env: - TGS_TEST_DUMP_API_SPEC: yes strategy: fail-fast: false matrix: @@ -327,6 +325,10 @@ jobs: - name: Upgrade NPM run: npm install -g npm + - name: Set TGS_TEST_DUMP_API_SPEC + if: ${{ matrix.configuration == 'Release' && matrix.watchdog-type == 'System' && matrix.database-type == 'SqlServer' }} + run: echo "TGS_TEST_DUMP_API_SPEC=yes" >> $Env:GITHUB_ENV + - name: Set General__UseBasicWatchdog if: ${{ matrix.watchdog-type == 'Basic' }} run: echo "General__UseBasicWatchdog=true" >> $Env:GITHUB_ENV From a29331cb00df0060aea5764e720d97d4d2dc961c Mon Sep 17 00:00:00 2001 From: Dominion Date: Sat, 27 May 2023 11:33:16 -0400 Subject: [PATCH 24/45] Test error log failure fixes --- .../Live/TestLiveServer.cs | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index fad45213ee..9fe1dd22ed 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -704,15 +704,34 @@ namespace Tgstation.Server.Tests.Live ServiceCollectionExtensions.UseAdditionalLoggerProvider(); + var failureTask = HardFailLoggerProvider.FailureSource; var internalTask = TestTgsInternal(hardCancellationToken); await Task.WhenAny( internalTask, - HardFailLoggerProvider.FailureSource); + failureTask); if (!internalTask.IsCompleted) { hardCancellationTokenSource.Cancel(); - await Task.WhenAll(internalTask, HardFailLoggerProvider.FailureSource); + try + { + await failureTask; + } + finally + { + try + { + await internalTask; + } + catch (OperationCanceledException) + { + Console.WriteLine("TEST CANCELLED!"); + } + catch (Exception ex) + { + Console.WriteLine($"ADDITIONAL TEST ERROR: {ex}"); + } + } } } From 8dddb0d5e92a161ce7648dbdfa00e47a59c47922 Mon Sep 17 00:00:00 2001 From: Dominion Date: Sat, 27 May 2023 11:57:18 -0400 Subject: [PATCH 25/45] Use .StartsWith, logMessage formatter doesn't match exactly --- tests/Tgstation.Server.Tests/Live/HardFailLogger.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Tgstation.Server.Tests/Live/HardFailLogger.cs b/tests/Tgstation.Server.Tests/Live/HardFailLogger.cs index 5eb968fb40..5f5e1e1713 100644 --- a/tests/Tgstation.Server.Tests/Live/HardFailLogger.cs +++ b/tests/Tgstation.Server.Tests/Live/HardFailLogger.cs @@ -24,7 +24,7 @@ namespace Tgstation.Server.Tests.Live { var logMessage = formatter(state, exception); if ((logLevel == LogLevel.Error - && !Regex.IsMatch(logMessage, "Error disconnecting connection [1-9][0-9]*!") + && !logMessage.StartsWith("Error disconnecting connection ") && !(logMessage.StartsWith("An exception occurred while iterating over the results of a query for context type") && (exception is TaskCanceledException || exception?.InnerException is TaskCanceledException))) || (logLevel == LogLevel.Critical && logMessage != "DropDatabase configuration option set! Dropping any existing database...")) failureSink(new AssertFailedException("TGS logged an error!")); From c8c0be22fbc831c92a9459786355ca5ba3b5d694 Mon Sep 17 00:00:00 2001 From: Dominion Date: Sat, 27 May 2023 11:59:28 -0400 Subject: [PATCH 26/45] Log chat provider RNG --- .../Live/DummyChatProviderFactory.cs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/Tgstation.Server.Tests/Live/DummyChatProviderFactory.cs b/tests/Tgstation.Server.Tests/Live/DummyChatProviderFactory.cs index d8098054e3..3f6fd02275 100644 --- a/tests/Tgstation.Server.Tests/Live/DummyChatProviderFactory.cs +++ b/tests/Tgstation.Server.Tests/Live/DummyChatProviderFactory.cs @@ -47,9 +47,15 @@ namespace Tgstation.Server.Tests.Live commandFactory.SetWatchdog(Mock.Of()); commands = commandFactory.GenerateCommands(); - var baseRng = new Random( - 22475 // comment this out to enable true randomness, note that seeding is imperfect because Task execution order is non-deterministic - ); + const bool UseUnseededProviderRng = false; + + var randomSeed = UseUnseededProviderRng + ? new Random().Next() + : 22475; + + logger.LogInformation("Random seed: {0}", randomSeed); + + var baseRng = new Random(randomSeed); seededRng = new Dictionary{ { ChatProvider.Irc, new Random(baseRng.Next()) }, { ChatProvider.Discord, new Random(baseRng.Next()) }, From 8d4537c4e9d9bd3ac649e56586441ac579c206d9 Mon Sep 17 00:00:00 2001 From: Dominion Date: Sat, 27 May 2023 14:51:42 -0400 Subject: [PATCH 27/45] Log topics sent from watchdog test --- tests/Tgstation.Server.Tests/Live/DummyChatProvider.cs | 2 +- .../Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/Tgstation.Server.Tests/Live/DummyChatProvider.cs b/tests/Tgstation.Server.Tests/Live/DummyChatProvider.cs index 7020863bdf..8ecce8d2fa 100644 --- a/tests/Tgstation.Server.Tests/Live/DummyChatProvider.cs +++ b/tests/Tgstation.Server.Tests/Live/DummyChatProvider.cs @@ -40,7 +40,7 @@ namespace Tgstation.Server.Tests.Live ulong channelIdAllocator; - static ILoggerFactory CreateLoggerFactoryForLogger(ILogger logger, out Mock mockLoggerFactory) + public static ILoggerFactory CreateLoggerFactoryForLogger(ILogger logger, out Mock mockLoggerFactory) { mockLoggerFactory = new Mock(); mockLoggerFactory.Setup(x => x.CreateLogger(It.IsAny())).Returns(() => diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index 78b287a641..8ed4bf6850 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -869,13 +869,19 @@ namespace Tgstation.Server.Tests.Live.Instance return ddProc != null; } + static readonly ILoggerFactory loggerFactory = LoggerFactory.Create(builder => + { + builder.AddConsole(); + builder.SetMinimumLevel(LogLevel.Trace); + }); + public static readonly TopicClient TopicClient = new(new SocketParameters { SendTimeout = TimeSpan.FromSeconds(30), ReceiveTimeout = TimeSpan.FromSeconds(30), ConnectTimeout = TimeSpan.FromSeconds(30), DisconnectTimeout = TimeSpan.FromSeconds(30) - }); + }, new Logger(DummyChatProvider.CreateLoggerFactoryForLogger(loggerFactory.CreateLogger("WatchdogTest.TopicClient"), out var mockLoggerFactory))); public async Task TellWorldToReboot(CancellationToken cancellationToken) { From 9d919f101f6eca5c94eecc555292442b22cd4565 Mon Sep 17 00:00:00 2001 From: Dominion Date: Sat, 27 May 2023 18:10:01 -0400 Subject: [PATCH 28/45] Delay 2s between each priority topic send attempt --- .../Components/Session/SessionController.cs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Components/Session/SessionController.cs b/src/Tgstation.Server.Host/Components/Session/SessionController.cs index ec67f72e55..9c35318f29 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionController.cs @@ -123,6 +123,11 @@ namespace Tgstation.Server.Host.Components.Session /// readonly IChatManager chat; + /// + /// The for the . + /// + readonly IAsyncDelayer asyncDelayer; + /// /// for port updates and . /// @@ -212,7 +217,12 @@ namespace Tgstation.Server.Host.Components.Session this.chatTrackingContext = chatTrackingContext ?? throw new ArgumentNullException(nameof(chatTrackingContext)); if (bridgeRegistrar == null) throw new ArgumentNullException(nameof(bridgeRegistrar)); + this.chat = chat ?? throw new ArgumentNullException(nameof(chat)); + if (assemblyInformationProvider == null) + throw new ArgumentNullException(nameof(assemblyInformationProvider)); + + this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer)); portClosedForReboot = false; disposed = false; @@ -901,7 +911,9 @@ namespace Tgstation.Server.Host.Components.Session var killedOrRebootedTask = Task.WhenAny(Lifetime, OnReboot); global::Byond.TopicSender.TopicResponse byondResponse = null; var firstSend = true; - for (var i = 4; i >= 0 && (priority || firstSend); --i) + + const int PrioritySendAttempts = 5; + for (var i = PrioritySendAttempts - 1; i >= 0 && (priority || firstSend); --i) try { firstSend = false; @@ -916,6 +928,9 @@ namespace Tgstation.Server.Host.Components.Session catch (Exception ex) { Logger.LogWarning(ex, "SendTopic exception!{retryDetails}", priority ? $" {i} attempts remaining." : String.Empty); + + if (i > 0) + await asyncDelayer.Delay(TimeSpan.FromSeconds(2), cancellationToken); } if (byondResponse == null) From 57c307c8c30542274e29b640255c36898015f310 Mon Sep 17 00:00:00 2001 From: Dominion Date: Sat, 27 May 2023 18:15:44 -0400 Subject: [PATCH 29/45] Fix a bad if case --- src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs index 2c5d0ba569..8cb6f8f674 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs @@ -272,7 +272,7 @@ namespace Tgstation.Server.Host.Components.Watchdog { using (await SemaphoreSlimContext.Lock(synchronizationSemaphore, cancellationToken)) { - if (Status == WatchdogStatus.Offline) + if (Status != WatchdogStatus.Online) return new MessageContent { Text = "TGS: Server offline!", From 637c07d354615752a36e11d55a9c89e90412882b Mon Sep 17 00:00:00 2001 From: Dominion Date: Sat, 27 May 2023 18:19:30 -0400 Subject: [PATCH 30/45] Short circuit topic retries if killed or rebooted --- .../Components/Session/SessionController.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Components/Session/SessionController.cs b/src/Tgstation.Server.Host/Components/Session/SessionController.cs index 9c35318f29..0f3b5fd847 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionController.cs @@ -930,7 +930,10 @@ namespace Tgstation.Server.Host.Components.Session Logger.LogWarning(ex, "SendTopic exception!{retryDetails}", priority ? $" {i} attempts remaining." : String.Empty); if (i > 0) - await asyncDelayer.Delay(TimeSpan.FromSeconds(2), cancellationToken); + { + var delayTask = asyncDelayer.Delay(TimeSpan.FromSeconds(2), cancellationToken); + await Task.WhenAny(killedOrRebootedTask, delayTask); + } } if (byondResponse == null) From 202729caf1f09238a682011841567e615984a6fa Mon Sep 17 00:00:00 2001 From: Dominion Date: Sat, 27 May 2023 18:21:08 -0400 Subject: [PATCH 31/45] Fix non-priority topic calls getting delayed on failure --- .../Components/Session/SessionController.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Components/Session/SessionController.cs b/src/Tgstation.Server.Host/Components/Session/SessionController.cs index 0f3b5fd847..e3da5a1ea7 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionController.cs @@ -929,7 +929,7 @@ namespace Tgstation.Server.Host.Components.Session { Logger.LogWarning(ex, "SendTopic exception!{retryDetails}", priority ? $" {i} attempts remaining." : String.Empty); - if (i > 0) + if (priority && i > 0) { var delayTask = asyncDelayer.Delay(TimeSpan.FromSeconds(2), cancellationToken); await Task.WhenAny(killedOrRebootedTask, delayTask); From 5886e939daa24af5864de6fe9f27d9bc345873d3 Mon Sep 17 00:00:00 2001 From: Dominion Date: Sun, 28 May 2023 08:44:29 -0400 Subject: [PATCH 32/45] Fix possible exception when a provider queues a message and then disconnects --- .../Components/Chat/ChatManager.cs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs b/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs index 7ad410ea8a..292dcc6517 100644 --- a/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs +++ b/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs @@ -660,10 +660,19 @@ namespace Tgstation.Server.Host.Components.Chat // important, otherwise we could end up processing during shutdown cancellationToken.ThrowIfCancellationRequested(); - providerId = providers + var providerIdNullable = providers .Where(x => x.Value == provider) - .Select(x => x.Key) - .First(); + .Select(x => (long?)x.Key) + .FirstOrDefault(); + + if (!providerIdNullable.HasValue) + { + // possible to have a message queued and then the provider immediately disconnects + logger.LogDebug("Unable to process command \"{command}\" due to provider disconnecting", message.Content); + return; + } + + providerId = providerIdNullable.Value; mappedChannel = mappedChannels .Where(x => x.Value.ProviderId == providerId && x.Value.ProviderChannelId == providerChannelId) .Select(x => (KeyValuePair?)x) From a8f644ea1b14d26cfead93e8cab50d61ee952077 Mon Sep 17 00:00:00 2001 From: Dominion Date: Sun, 28 May 2023 08:46:57 -0400 Subject: [PATCH 33/45] Fix a bad semicolon --- src/Tgstation.Server.Host/Components/Instance.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Instance.cs b/src/Tgstation.Server.Host/Components/Instance.cs index 5c5ec4d692..95ca246c42 100644 --- a/src/Tgstation.Server.Host/Components/Instance.cs +++ b/src/Tgstation.Server.Host/Components/Instance.cs @@ -207,8 +207,7 @@ namespace Tgstation.Server.Host.Components Configuration.StopAsync(cancellationToken), ByondManager.StopAsync(cancellationToken), Chat.StopAsync(cancellationToken), - dmbFactory.StopAsync(cancellationToken)) - ; + dmbFactory.StopAsync(cancellationToken)); } } From 7599a4d988f824f4fe529512ecb6f6a494db2883 Mon Sep 17 00:00:00 2001 From: Dominion Date: Sun, 28 May 2023 08:51:05 -0400 Subject: [PATCH 34/45] Ignore FileNotFoundExceptions when deleting directories --- src/Tgstation.Server.Host/IO/DefaultIOManager.cs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/Tgstation.Server.Host/IO/DefaultIOManager.cs b/src/Tgstation.Server.Host/IO/DefaultIOManager.cs index d3d36804e3..85dd4ce31d 100644 --- a/src/Tgstation.Server.Host/IO/DefaultIOManager.cs +++ b/src/Tgstation.Server.Host/IO/DefaultIOManager.cs @@ -52,8 +52,15 @@ namespace Tgstation.Server.Host.IO foreach (var file in dir.EnumerateFiles()) { cancellationToken.ThrowIfCancellationRequested(); - file.Attributes = FileAttributes.Normal; - file.Delete(); + try + { + file.Attributes = FileAttributes.Normal; + file.Delete(); + } + catch (FileNotFoundException) + { + // has happened before with .dyn.rsc.lk + } } cancellationToken.ThrowIfCancellationRequested(); From ccd0f00a30afbbd196196e62308a840aaaec2e47 Mon Sep 17 00:00:00 2001 From: Dominion Date: Sun, 28 May 2023 08:57:31 -0400 Subject: [PATCH 35/45] Some QoL improvements to debugging error log test failures --- tests/Tgstation.Server.Tests/Live/HardFailLogger.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/Tgstation.Server.Tests/Live/HardFailLogger.cs b/tests/Tgstation.Server.Tests/Live/HardFailLogger.cs index 5f5e1e1713..f67aa83f72 100644 --- a/tests/Tgstation.Server.Tests/Live/HardFailLogger.cs +++ b/tests/Tgstation.Server.Tests/Live/HardFailLogger.cs @@ -27,7 +27,10 @@ namespace Tgstation.Server.Tests.Live && !logMessage.StartsWith("Error disconnecting connection ") && !(logMessage.StartsWith("An exception occurred while iterating over the results of a query for context type") && (exception is TaskCanceledException || exception?.InnerException is TaskCanceledException))) || (logLevel == LogLevel.Critical && logMessage != "DropDatabase configuration option set! Dropping any existing database...")) - failureSink(new AssertFailedException("TGS logged an error!")); + { + failureSink(new AssertFailedException("TGS logged an unexpected error!")); + Console.WriteLine("UNEXPECTED ERROR OCCURS NEAR HERE"); + } } } } From 252196ccedc749e68f5d4bb613a7e9c9431e2616 Mon Sep 17 00:00:00 2001 From: Dominion Date: Sun, 28 May 2023 10:58:32 -0400 Subject: [PATCH 36/45] Run these continuations asynchronously --- tests/Tgstation.Server.Tests/Live/HardFailLoggerProvider.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Tgstation.Server.Tests/Live/HardFailLoggerProvider.cs b/tests/Tgstation.Server.Tests/Live/HardFailLoggerProvider.cs index e72ea557db..59a677abd8 100644 --- a/tests/Tgstation.Server.Tests/Live/HardFailLoggerProvider.cs +++ b/tests/Tgstation.Server.Tests/Live/HardFailLoggerProvider.cs @@ -10,7 +10,7 @@ namespace Tgstation.Server.Tests.Live public static Task FailureSource => failureSink.Task; - static readonly TaskCompletionSource failureSink = new TaskCompletionSource(); + static readonly TaskCompletionSource failureSink = new (TaskCreationOptions.RunContinuationsAsynchronously); public ILogger CreateLogger(string categoryName) => new HardFailLogger(ex => { From 401bb866a4c24e46ebc957dc952c85e186b28691 Mon Sep 17 00:00:00 2001 From: Dominion Date: Sun, 28 May 2023 11:18:17 -0400 Subject: [PATCH 37/45] More DummyChatProvider fixes --- .../Live/DummyChatProvider.cs | 128 +++++++++++++----- 1 file changed, 92 insertions(+), 36 deletions(-) diff --git a/tests/Tgstation.Server.Tests/Live/DummyChatProvider.cs b/tests/Tgstation.Server.Tests/Live/DummyChatProvider.cs index 8ecce8d2fa..6eb24a0c0f 100644 --- a/tests/Tgstation.Server.Tests/Live/DummyChatProvider.cs +++ b/tests/Tgstation.Server.Tests/Live/DummyChatProvider.cs @@ -9,6 +9,8 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; +using Newtonsoft.Json; + using Tgstation.Server.Host.Components.Chat; using Tgstation.Server.Host.Components.Chat.Commands; using Tgstation.Server.Host.Components.Chat.Providers; @@ -34,6 +36,7 @@ namespace Tgstation.Server.Tests.Live readonly ICryptographySuite cryptographySuite; readonly CancellationTokenSource randomMessageCts; readonly Task randomMessageTask; + readonly Dictionary knownChannels; bool connectedOnce; bool connected; @@ -85,8 +88,9 @@ namespace Tgstation.Server.Tests.Live logger.LogTrace("Base channel ID {baseChannelId}", channelIdAllocator); - this.randomMessageCts = new CancellationTokenSource(); - this.randomMessageTask = RandomMessageLoop(this.randomMessageCts.Token); + knownChannels = new Dictionary(); + randomMessageCts = new CancellationTokenSource(); + randomMessageTask = RandomMessageLoop(this.randomMessageCts.Token); } public override async ValueTask DisposeAsync() @@ -105,9 +109,7 @@ namespace Tgstation.Server.Tests.Live Logger.LogTrace("SendMessage"); - Assert.AreNotEqual(0UL, channelId); - var condition = channelId <= channelIdAllocator || channelId >= (Int32.MaxValue / 2); - Assert.IsTrue(condition); + Assert.IsTrue(knownChannels.ContainsKey(channelId)); cancellationToken.ThrowIfCancellationRequested(); @@ -131,8 +133,7 @@ namespace Tgstation.Server.Tests.Live Logger.LogTrace("SendUpdateMessage"); - Assert.AreNotEqual(0UL, channelId); - Assert.IsTrue(channelId <= channelIdAllocator); + Assert.IsTrue(knownChannels.ContainsKey(channelId)); cancellationToken.ThrowIfCancellationRequested(); @@ -171,7 +172,6 @@ namespace Tgstation.Server.Tests.Live Logger.LogTrace("DisconnectImpl"); cancellationToken.ThrowIfCancellationRequested(); connected = false; - channelIdAllocator = 0; if (random.Next(0, 100) > 70) throw new Exception("Random disconnection failure!"); @@ -196,19 +196,43 @@ namespace Tgstation.Server.Tests.Live channel, new List { - new ChannelRepresentation - { - IsAdminChannel = channel.IsAdminChannel.Value, - ConnectionName = $"Connection_{channelIdAllocator + 1}", - EmbedsSupported = ChatBot.Provider.Value != Api.Models.ChatProvider.Irc, - FriendlyName = $"(Friendly) Channel_ID_{channelIdAllocator + 1}", - IsPrivateChannel = false, - RealId = ++channelIdAllocator, - Tag = channel.Tag, - } + CreateChannel(channel) })))); } + private ChannelRepresentation CreateChannel(ChatChannel channel) + { + + ulong channelId; + if (channel.DiscordChannelId.HasValue) + channelId = channel.DiscordChannelId.Value; + else + channelId = (ulong)channel.IrcChannel.GetHashCode(); + + var entry = new ChannelRepresentation + { + IsAdminChannel = channel.IsAdminChannel.Value, + ConnectionName = $"Connection_{channelId}", + EmbedsSupported = ChatBot.Provider.Value != Api.Models.ChatProvider.Irc, + FriendlyName = $"(Friendly) Channel_ID_{channelId}", + IsPrivateChannel = false, + RealId = channelId, + Tag = channel.Tag, + }; + + knownChannels.Remove(channelId); + knownChannels.Add(channelId, entry); + + return CloneChannel(entry); + } + + ChannelRepresentation CloneChannel(ChannelRepresentation channel) + { + return JsonConvert.DeserializeObject( + JsonConvert.SerializeObject( + channel)); + } + async Task RandomMessageLoop(CancellationToken cancellationToken) { Logger.LogTrace("RandomMessageLoop"); @@ -222,34 +246,66 @@ namespace Tgstation.Server.Tests.Live // %5 chance to disconnect randomly if (enableRandomDisconnections != 0 && random.Next(0, 100) > 95) - { connected = false; - channelIdAllocator = 0; - } if (!connected) continue; - if (channelIdAllocator >= Int32.MaxValue / 2) - Assert.Fail("Too many channels have been allocated!"); - var isPm = channelIdAllocator == 0 || random.Next(0, 100) > 80; - var idRandomizer = isPm ? channelIdAllocator + (Int32.MaxValue / 2) : channelIdAllocator; - var realId = (ulong)random.Next(isPm ? Int32.MaxValue / 2 : 1, (int)idRandomizer); + ChannelRepresentation channel; var username = $"RandomUser{i}"; + if (isPm) + // 50% chance to be a new user + if (random.Next(0, 100) > 50) + { + var enumerator = knownChannels + .Where(x => x.Value.IsPrivateChannel) + .ToList(); + if (enumerator.Count == 0) + continue; + + var index = random.Next(0, enumerator.Count); + channel = enumerator[index].Value; + username = channel.ConnectionName.Substring(0, channel.ConnectionName.Length - 11); + } + else + { + ulong channelId; + do + { + channelId = ++channelIdAllocator; + } + while (knownChannels.ContainsKey(channelId)); + + channel = new ChannelRepresentation + { + RealId = channelId, + IsPrivateChannel = true, + ConnectionName = $"{username}_Connection", + FriendlyName = $"{username}_Channel", + EmbedsSupported = ChatBot.Provider.Value != Api.Models.ChatProvider.Irc, + + // isAdmin and Tag populated by manager + }; + + knownChannels.Add(channelId, channel); + } + else + { + var enumerator = knownChannels + .Where(x => !x.Value.IsPrivateChannel) + .ToList(); + if (enumerator.Count == 0) + continue; + + var index = random.Next(0, enumerator.Count); + channel = enumerator[index].Value; + } + var sender = new ChatUser { - Channel = new ChannelRepresentation - { - RealId = realId, - IsPrivateChannel = isPm, - ConnectionName = isPm ? $"{username}_Connection" : $"Connection_{realId}", - FriendlyName = isPm ? $"{username}_Channel" : $"(Friendly) Channel_ID_{realId}", - EmbedsSupported = ChatBot.Provider.Value != Api.Models.ChatProvider.Irc, - - // isAdmin and Tag populated by manager - }, + Channel = CloneChannel(channel), FriendlyName = username, RealId = i + 50000, Mention = $"@{username}", From 74a9a0392c4e83659403e342fed3866bc3bc4896 Mon Sep 17 00:00:00 2001 From: Dominion Date: Sun, 28 May 2023 13:10:27 -0400 Subject: [PATCH 38/45] Fix a NullReferenceException in ByondCommand --- .../Components/Chat/Commands/ByondCommand.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Components/Chat/Commands/ByondCommand.cs b/src/Tgstation.Server.Host/Components/Chat/Commands/ByondCommand.cs index 756dfcd525..e24c20a699 100644 --- a/src/Tgstation.Server.Host/Components/Chat/Commands/ByondCommand.cs +++ b/src/Tgstation.Server.Host/Components/Chat/Commands/ByondCommand.cs @@ -61,7 +61,7 @@ namespace Tgstation.Server.Host.Components.Chat.Commands }); return Task.FromResult(new MessageContent { - Text = watchdog.ActiveCompileJob.ByondVersion, + Text = watchdog.ActiveCompileJob?.ByondVersion ?? "None!", }); } } From 3b22b86c228422880acc47fb190e4ac02fc88735 Mon Sep 17 00:00:00 2001 From: Dominion Date: Sun, 28 May 2023 13:10:38 -0400 Subject: [PATCH 39/45] Fix a NullReferenceException in WatchdogBase --- .../Components/Watchdog/WatchdogBase.cs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs index 8cb6f8f674..725f99fb3a 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs @@ -272,17 +272,15 @@ namespace Tgstation.Server.Host.Components.Watchdog { using (await SemaphoreSlimContext.Lock(synchronizationSemaphore, cancellationToken)) { - if (Status != WatchdogStatus.Online) + var commandObject = new ChatCommand(sender, commandName, arguments); + var command = new TopicParameters(commandObject); + var activeServer = GetActiveController(); + if (Status != WatchdogStatus.Online || activeServer == null) return new MessageContent { Text = "TGS: Server offline!", }; - var commandObject = new ChatCommand(sender, commandName, arguments); - - var command = new TopicParameters(commandObject); - - var activeServer = GetActiveController(); var commandResult = await activeServer.SendCommand(command, cancellationToken); if (commandResult == null) From c3992971cb5050e4ae79d0c4b273b54b878f542a Mon Sep 17 00:00:00 2001 From: Dominion Date: Sun, 28 May 2023 13:17:51 -0400 Subject: [PATCH 40/45] Write console line before log failing --- tests/Tgstation.Server.Tests/Live/HardFailLogger.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Tgstation.Server.Tests/Live/HardFailLogger.cs b/tests/Tgstation.Server.Tests/Live/HardFailLogger.cs index f67aa83f72..1f66ee01f6 100644 --- a/tests/Tgstation.Server.Tests/Live/HardFailLogger.cs +++ b/tests/Tgstation.Server.Tests/Live/HardFailLogger.cs @@ -28,8 +28,8 @@ namespace Tgstation.Server.Tests.Live && !(logMessage.StartsWith("An exception occurred while iterating over the results of a query for context type") && (exception is TaskCanceledException || exception?.InnerException is TaskCanceledException))) || (logLevel == LogLevel.Critical && logMessage != "DropDatabase configuration option set! Dropping any existing database...")) { - failureSink(new AssertFailedException("TGS logged an unexpected error!")); Console.WriteLine("UNEXPECTED ERROR OCCURS NEAR HERE"); + failureSink(new AssertFailedException("TGS logged an unexpected error!")); } } } From 05832bfee57d2a992e3966710cd1d4f7acf7ef37 Mon Sep 17 00:00:00 2001 From: Dominion Date: Sun, 28 May 2023 13:18:21 -0400 Subject: [PATCH 41/45] Reverse ordering of sort for initial test chat connection jobs --- tests/Tgstation.Server.Tests/Live/Instance/ChatTest.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/ChatTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/ChatTest.cs index e53f5d54bc..bad4fee0b0 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/ChatTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/ChatTest.cs @@ -97,7 +97,7 @@ namespace Tgstation.Server.Tests.Live.Instance var jobs = await JobsClient.List(null, cancellationToken); var reconnectJob = jobs .Where(x => x.StartedAt >= beforeChatBotEnabled && x.Description.Contains(updatedBot.Name)) - .OrderByDescending(x => x.StartedAt) + .OrderBy(x => x.StartedAt) .FirstOrDefault(); Assert.IsNotNull(reconnectJob); @@ -196,7 +196,7 @@ namespace Tgstation.Server.Tests.Live.Instance var jobs = await JobsClient.List(null, cancellationToken); var reconnectJob = jobs .Where(x => x.StartedAt >= beforeChatBotEnabled && x.Description.Contains(updatedBot.Name)) - .OrderByDescending(x => x.StartedAt) + .OrderBy(x => x.StartedAt) .FirstOrDefault(); Assert.IsNotNull(reconnectJob); From 8f603302397fa1c83fdbdc148387153c47d9f4f2 Mon Sep 17 00:00:00 2001 From: Dominion Date: Sun, 28 May 2023 20:04:50 -0400 Subject: [PATCH 42/45] Fix rogue exceptions flying in CompileJob cleanup --- .../Components/Deployment/DmbFactory.cs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs b/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs index 1e01982bee..f57222c8b2 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs @@ -412,7 +412,20 @@ namespace Tgstation.Server.Host.Components.Deployment var deleteTask = DeleteCompileJobContent(job.DirectoryName.ToString(), cleanupCts.Token); var otherTask = cleanupTask; - await Task.WhenAll(otherTask, deleteTask, deploymentJob); + + async Task WrapThrowableTasks() + { + try + { + await Task.WhenAll(deleteTask, deploymentJob); + } + catch (Exception ex) + { + logger.LogWarning(ex, "Error cleaning up compile job {jobGuid}!", job.DirectoryName); + } + } + + await Task.WhenAll(otherTask, WrapThrowableTasks()); } lock (jobLockCounts) From 1f56ee7b90f5027c4eefbaadfcf8bdbac342f0e9 Mon Sep 17 00:00:00 2001 From: Dominion Date: Sun, 28 May 2023 21:34:29 -0400 Subject: [PATCH 43/45] Reduce log spam in live tests. This is slowing GitHub actions to an ABSOLUTE FUCKING GARBAGE pace (4hrs of interop for a 7 minute run) --- tests/DMAPI/LongRunning/Config.dm | 2 +- tests/DMAPI/LongRunning/Test.dm | 19 ++++++++++++++++--- .../Live/Instance/WatchdogTest.cs | 11 ++++++++++- 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/tests/DMAPI/LongRunning/Config.dm b/tests/DMAPI/LongRunning/Config.dm index a7dfd4936a..af95587445 100644 --- a/tests/DMAPI/LongRunning/Config.dm +++ b/tests/DMAPI/LongRunning/Config.dm @@ -4,7 +4,7 @@ #define TGS_WRITE_GLOBAL(Name, Value) global.##Name = ##Value #define TGS_PROTECT_DATUM(Path) #define TGS_WORLD_ANNOUNCE(message) world << ##message -#define TGS_INFO_LOG(message) world.log << "Info: [##message]" +#define TGS_INFO_LOG(message) TgsInfo(##message) #define TGS_WARNING_LOG(message) world.log << "Warn: [##message]" #define TGS_ERROR_LOG(message) TgsError(##message) #define TGS_NOTIFY_ADMINS(event) diff --git a/tests/DMAPI/LongRunning/Test.dm b/tests/DMAPI/LongRunning/Test.dm index 762add090c..de5cd5831d 100644 --- a/tests/DMAPI/LongRunning/Test.dm +++ b/tests/DMAPI/LongRunning/Test.dm @@ -54,7 +54,10 @@ CheckBridgeLimits() /world/Topic(T, Addr, Master, Keys) - log << "Topic: [T]" + if(findtext(T, "tgs_integration_test_tactics3") == 0) + log << "Topic: [T]" + else + log << "tgs_integration_test_tactics3 " . = HandleTopic(T) log << "Response: [.]" @@ -210,10 +213,18 @@ var/run_bridge_test return response var/lastTgsError +var/suppress_bridge_spam = FALSE + +/proc/TgsInfo(message) + if(suppress_bridge_spam && findtext(message, "Export: http://127.0.0.1:") != 0) + return + world.log << "Info: [message]" /proc/TgsError(message) - world.log << "Err: [message]" lastTgsError = message + if(suppress_bridge_spam && findtext(message, "Failed bridge request: http://127.0.0.1:") != 0) + return + world.log << "Err: [message]" /proc/create_payload(size) var/builder = list() @@ -230,7 +241,9 @@ var/lastTgsError /proc/BridgeWithoutChunking(command, list/data) var/datum/tgs_api/v5/api = TGS_READ_GLOBAL(tgs) var/bridge_request = api.CreateBridgeRequest(command, data) - return api.PerformBridgeRequest(bridge_request) + suppress_bridge_spam = TRUE + . = api.PerformBridgeRequest(bridge_request) + suppress_bridge_spam = FALSE /proc/CheckBridgeLimitsImpl() sleep(30) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index 8ed4bf6850..e8fc623199 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -519,7 +519,8 @@ namespace Tgstation.Server.Tests.Live.Instance TopicResponse topicRequestResult = null; try { - topicRequestResult = await TopicClient.SendTopic( + System.Console.WriteLine($"Topic limit test S:{payloadSize}..."); + topicRequestResult = await TopicClientNoLogger.SendTopic( IPAddress.Loopback, $"tgs_integration_test_tactics3={TopicClient.SanitizeString(JsonConvert.SerializeObject(topic, DMApiConstants.SerializerSettings))}", TestLiveServer.DDPort, @@ -883,6 +884,14 @@ namespace Tgstation.Server.Tests.Live.Instance DisconnectTimeout = TimeSpan.FromSeconds(30) }, new Logger(DummyChatProvider.CreateLoggerFactoryForLogger(loggerFactory.CreateLogger("WatchdogTest.TopicClient"), out var mockLoggerFactory))); + public static readonly TopicClient TopicClientNoLogger = new(new SocketParameters + { + SendTimeout = TimeSpan.FromSeconds(30), + ReceiveTimeout = TimeSpan.FromSeconds(30), + ConnectTimeout = TimeSpan.FromSeconds(30), + DisconnectTimeout = TimeSpan.FromSeconds(30) + }); + public async Task TellWorldToReboot(CancellationToken cancellationToken) { var daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); From d837cc5812c9ff74cd6ca99151454fc86c3b157e Mon Sep 17 00:00:00 2001 From: Dominion Date: Mon, 29 May 2023 02:49:44 -0400 Subject: [PATCH 44/45] This log line does not make sense in its current form --- src/Tgstation.Server.Host/Swarm/SwarmService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Swarm/SwarmService.cs b/src/Tgstation.Server.Host/Swarm/SwarmService.cs index ebda13992f..2e977abb38 100644 --- a/src/Tgstation.Server.Host/Swarm/SwarmService.cs +++ b/src/Tgstation.Server.Host/Swarm/SwarmService.cs @@ -826,7 +826,7 @@ namespace Tgstation.Server.Host.Swarm updateCommitTcs = new TaskCompletionSource(); } - logger.LogDebug("Prepared for update to version {version}", version); + logger.LogDebug("Local node prepared for update to version {version}", version); } catch (Exception ex) { From 8129bc12777c829d61dd58ad788eed77d2a63b8b Mon Sep 17 00:00:00 2001 From: Dominion Date: Mon, 29 May 2023 03:31:54 -0400 Subject: [PATCH 45/45] Attempting to debug a random-ass pipeline error https://github.com/tgstation/tgstation-server/actions/runs/5107521897/jobs/9180568543 https://gist.github.com/Cyberboss/ec68324347401bfd077828848de1e462 This run failed because either the exact same GitHub API request returned different results or the configuration changed underneath us. I want to know which it is. --- .../Configuration/UpdatesConfiguration.cs | 6 ++++ .../Core/ServerUpdater.cs | 15 +++++++-- .../Live/LiveTestingServer.cs | 5 ++- .../Live/TestLiveServer.cs | 32 +++++++++++++++++++ 4 files changed, 54 insertions(+), 4 deletions(-) diff --git a/src/Tgstation.Server.Host/Configuration/UpdatesConfiguration.cs b/src/Tgstation.Server.Host/Configuration/UpdatesConfiguration.cs index 406560f458..fae04d8684 100644 --- a/src/Tgstation.Server.Host/Configuration/UpdatesConfiguration.cs +++ b/src/Tgstation.Server.Host/Configuration/UpdatesConfiguration.cs @@ -39,5 +39,11 @@ /// Asset package containing the new assembly in zip form. /// public string UpdatePackageAssetName { get; set; } = DefaultUpdatePackageAssetName; + + /// + /// Dump all retrieved releases from the GitHub API when a requested release is not found. + /// + /// This is an internal config and may be adjusted or removed without a version change. + public bool DumpReleasesOnNotFound { get; set; } } } diff --git a/src/Tgstation.Server.Host/Core/ServerUpdater.cs b/src/Tgstation.Server.Host/Core/ServerUpdater.cs index e552db745a..adcb902fd7 100644 --- a/src/Tgstation.Server.Host/Core/ServerUpdater.cs +++ b/src/Tgstation.Server.Host/Core/ServerUpdater.cs @@ -108,13 +108,16 @@ namespace Tgstation.Server.Host.Core logger.LogTrace("Received {releaseCount} total releases from GitHub", releases.Count); - releases = releases + var filteredReleases = releases .Where(x => x.TagName.StartsWith(updatesConfiguration.GitTagPrefix, StringComparison.InvariantCulture)) .ToList(); - logger.LogTrace("Filtered to {releaseCount} releases matching a TGS version", releases.Count); + logger.LogTrace( + "Filtered to {releaseCount} releases matching the configured tag prefix of \"{tagPrefix}\"", + filteredReleases.Count, + updatesConfiguration.GitTagPrefix); - foreach (var release in releases) + foreach (var release in filteredReleases) if (Version.TryParse( release.TagName.Replace( updatesConfiguration.GitTagPrefix, String.Empty, StringComparison.Ordinal), @@ -149,6 +152,12 @@ namespace Tgstation.Server.Host.Core else logger.LogDebug("Unparsable release tag: {releaseTag}", release.TagName); + if (updatesConfiguration.DumpReleasesOnNotFound) + logger.LogInformation( + "Found releases:{newline}\t{releases}", + Environment.NewLine, + String.Join($"{Environment.NewLine}\t", releases.Select(x => x.TagName).OrderBy(x => x))); + return ServerUpdateResult.ReleaseMissing; } diff --git a/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs b/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs index a03ff12abe..c6fcd0ea82 100644 --- a/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs +++ b/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs @@ -46,7 +46,7 @@ namespace Tgstation.Server.Tests.Live SerilogContextHelper.AddSwarmNodeIdentifierToTemplate(); } - public LiveTestingServer(SwarmConfiguration swarmConfiguration, bool enableOAuth, ushort port = 5010) + public LiveTestingServer(SwarmConfiguration swarmConfiguration, bool enableOAuth, ushort port = 5010, bool dumpOnMissingUpdate = true) { Directory = Environment.GetEnvironmentVariable("TGS_TEST_TEMP_DIRECTORY"); if (string.IsNullOrWhiteSpace(Directory)) @@ -102,6 +102,9 @@ namespace Tgstation.Server.Tests.Live "General:ByondTopicTimeout=3000" }; + if (dumpOnMissingUpdate) + args.Add("Updates:DumpReleasesOnNotFound=true"); + swarmArgs = new List(); if (swarmConfiguration != null) { diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index 9fe1dd22ed..b3a7596c5f 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -215,6 +215,38 @@ namespace Tgstation.Server.Tests.Live Assert.IsTrue(server.RestartRequested, "Server not requesting restart!"); } + [TestMethod] + public async Task TestUpdateBadVersion() + { + using var server = new LiveTestingServer(null, false, dumpOnMissingUpdate: false); + using var serverCts = new CancellationTokenSource(); + var cancellationToken = serverCts.Token; + var serverTask = server.Run(cancellationToken); + try + { + var testUpdateVersion = new Version(5, 11, 20); + using var adminClient = await CreateAdminClient(server.Url, cancellationToken); + await ApiAssert.ThrowsException(() => adminClient.Administration.Update(new ServerUpdateRequest + { + NewVersion = testUpdateVersion + }, cancellationToken), ErrorCode.ResourceNotPresent); + } + finally + { + serverCts.Cancel(); + try + { + await serverTask; + } + catch (OperationCanceledException) { } + catch (AggregateException ex) + { + if (ex.InnerException is NotSupportedException notSupportedException) + Assert.Inconclusive(notSupportedException.Message); + } + } + } + [TestMethod] public async Task TestOneServerSwarmUpdate() {