diff --git a/.github/workflows/ci-suite.yml b/.github/workflows/ci-suite.yml index 217b2f4f01..cc126e5b20 100644 --- a/.github/workflows/ci-suite.yml +++ b/.github/workflows/ci-suite.yml @@ -311,8 +311,6 @@ jobs: name: Windows Live Tests needs: dmapi-build if: "!(cancelled() || failure()) && needs.dmapi-build.result == 'success'" - env: - TGS_TEST_DUMP_API_SPEC: yes strategy: fail-fast: false matrix: @@ -329,6 +327,10 @@ jobs: - name: Upgrade NPM run: npm install -g npm + - name: Set TGS_TEST_DUMP_API_SPEC + if: ${{ matrix.configuration == 'Release' && matrix.watchdog-type == 'System' && matrix.database-type == 'SqlServer' }} + run: echo "TGS_TEST_DUMP_API_SPEC=yes" >> $Env:GITHUB_ENV + - name: Set General__UseBasicWatchdog if: ${{ matrix.watchdog-type == 'Basic' }} run: echo "General__UseBasicWatchdog=true" >> $Env:GITHUB_ENV diff --git a/build/Version.props b/build/Version.props index b7a3d895b6..2bc53dd4cc 100644 --- a/build/Version.props +++ b/build/Version.props @@ -9,7 +9,7 @@ 10.4.1 11.4.2 6.4.4 - 5.6.0 + 5.6.1 1.2.2 1.2.1 1.0.1 diff --git a/src/DMAPI/tgs/v5/__interop_version.dm b/src/DMAPI/tgs/v5/__interop_version.dm index 6ef7c86ef7..5d3d491a73 100644 --- a/src/DMAPI/tgs/v5/__interop_version.dm +++ b/src/DMAPI/tgs/v5/__interop_version.dm @@ -1 +1 @@ -"5.6.0" +"5.6.1" diff --git a/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs b/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs index 99829dac91..fcb93c8477 100644 --- a/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs +++ b/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs @@ -662,10 +662,19 @@ namespace Tgstation.Server.Host.Components.Chat // important, otherwise we could end up processing during shutdown cancellationToken.ThrowIfCancellationRequested(); - providerId = providers + var providerIdNullable = providers .Where(x => x.Value == provider) - .Select(x => x.Key) - .First(); + .Select(x => (long?)x.Key) + .FirstOrDefault(); + + if (!providerIdNullable.HasValue) + { + // possible to have a message queued and then the provider immediately disconnects + logger.LogDebug("Unable to process command \"{command}\" due to provider disconnecting", message.Content); + return; + } + + providerId = providerIdNullable.Value; mappedChannel = mappedChannels .Where(x => x.Value.ProviderId == providerId && x.Value.ProviderChannelId == providerChannelId) .Select(x => (KeyValuePair?)x) @@ -675,7 +684,7 @@ namespace Tgstation.Server.Host.Components.Chat .Any(); } - if (!recursed && !mappedChannel.HasValue && hasChannelZero) + if (!recursed && !mappedChannel.HasValue && !message.User.Channel.IsPrivateChannel && hasChannelZero) { logger.LogInformation("Receieved message from unmapped channel whose provider contains ID 0. Remapping..."); await RemapProvider(provider, cancellationToken); @@ -721,6 +730,9 @@ namespace Tgstation.Server.Host.Components.Chat "Error mapping message: Provider ID: {providerId}, Channel Real ID: {realId}", providerId, message.User.Channel.RealId); + logger.LogTrace("message: {messageJson}", JsonConvert.SerializeObject(message)); + lock (mappedChannels) + logger.LogTrace("mappedChannels: {mappedChannelsJson}", JsonConvert.SerializeObject(mappedChannels)); await SendMessage( new List { @@ -731,8 +743,7 @@ namespace Tgstation.Server.Host.Components.Chat { Text = "TGS: Processing error, check logs!", }, - cancellationToken) - ; + cancellationToken); return; } diff --git a/src/Tgstation.Server.Host/Components/Chat/Commands/ByondCommand.cs b/src/Tgstation.Server.Host/Components/Chat/Commands/ByondCommand.cs index 756dfcd525..e24c20a699 100644 --- a/src/Tgstation.Server.Host/Components/Chat/Commands/ByondCommand.cs +++ b/src/Tgstation.Server.Host/Components/Chat/Commands/ByondCommand.cs @@ -61,7 +61,7 @@ namespace Tgstation.Server.Host.Components.Chat.Commands }); return Task.FromResult(new MessageContent { - Text = watchdog.ActiveCompileJob.ByondVersion, + Text = watchdog.ActiveCompileJob?.ByondVersion ?? "None!", }); } } diff --git a/src/Tgstation.Server.Host/Components/Chat/Providers/DiscordProvider.cs b/src/Tgstation.Server.Host/Components/Chat/Providers/DiscordProvider.cs index b15ccc02ef..be488af4bb 100644 --- a/src/Tgstation.Server.Host/Components/Chat/Providers/DiscordProvider.cs +++ b/src/Tgstation.Server.Host/Components/Chat/Providers/DiscordProvider.cs @@ -611,15 +611,18 @@ namespace Tgstation.Server.Host.Components.Chat.Providers { await Task.WhenAny(gatewayReadyTcs.Task, localGatewayTask); - if (localGatewayTask.IsCompleted || cancellationToken.IsCancellationRequested) + cancellationToken.ThrowIfCancellationRequested(); + if (localGatewayTask.IsCompleted) throw new JobException(ErrorCode.ChatCannotConnectProvider); var userClient = serviceProvider.GetRequiredService(); using var localCombinedCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, gatewayCancellationToken); - var currentUserResult = await userClient.GetCurrentUserAsync(localCombinedCts.Token); + var localCombinedCancellationToken = localCombinedCts.Token; + var currentUserResult = await userClient.GetCurrentUserAsync(localCombinedCancellationToken); if (!currentUserResult.IsSuccess) { + localCombinedCancellationToken.ThrowIfCancellationRequested(); Logger.LogWarning("Unable to retrieve current user: {result}", currentUserResult.LogFormat()); throw new JobException(ErrorCode.ChatCannotConnectProvider); } diff --git a/src/Tgstation.Server.Host/Components/Chat/Providers/IrcProvider.cs b/src/Tgstation.Server.Host/Components/Chat/Providers/IrcProvider.cs index 179105d248..08d4b7a1ce 100644 --- a/src/Tgstation.Server.Host/Components/Chat/Providers/IrcProvider.cs +++ b/src/Tgstation.Server.Host/Components/Chat/Providers/IrcProvider.cs @@ -455,11 +455,7 @@ namespace Tgstation.Server.Host.Components.Chat.Providers Logger.LogTrace("Connection established!"); } - catch (OperationCanceledException) - { - throw; - } - catch (Exception e) + catch (Exception e) when (e is not OperationCanceledException) { throw new JobException(ErrorCode.ChatCannotConnectProvider, e); } diff --git a/src/Tgstation.Server.Host/Components/Chat/Providers/Provider.cs b/src/Tgstation.Server.Host/Components/Chat/Providers/Provider.cs index e13f46ade8..f9ef3396c6 100644 --- a/src/Tgstation.Server.Host/Components/Chat/Providers/Provider.cs +++ b/src/Tgstation.Server.Host/Components/Chat/Providers/Provider.cs @@ -312,16 +312,18 @@ namespace Tgstation.Server.Host.Components.Chat.Providers await jobManager.WaitForJobCompletion(job, null, cancellationToken, default); } } - catch (OperationCanceledException) + catch (OperationCanceledException e) { - break; + Logger.LogTrace(e, "ReconnectionLoop cancelled"); } catch (Exception e) { Logger.LogError(e, "Error reconnecting!"); } } - while (true); + while (!cancellationToken.IsCancellationRequested); + + Logger.LogTrace("ReconnectionLoop exiting..."); } } } diff --git a/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs b/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs index 1e01982bee..f57222c8b2 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs @@ -412,7 +412,20 @@ namespace Tgstation.Server.Host.Components.Deployment var deleteTask = DeleteCompileJobContent(job.DirectoryName.ToString(), cleanupCts.Token); var otherTask = cleanupTask; - await Task.WhenAll(otherTask, deleteTask, deploymentJob); + + async Task WrapThrowableTasks() + { + try + { + await Task.WhenAll(deleteTask, deploymentJob); + } + catch (Exception ex) + { + logger.LogWarning(ex, "Error cleaning up compile job {jobGuid}!", job.DirectoryName); + } + } + + await Task.WhenAll(otherTask, WrapThrowableTasks()); } lock (jobLockCounts) diff --git a/src/Tgstation.Server.Host/Components/Instance.cs b/src/Tgstation.Server.Host/Components/Instance.cs index 5c5ec4d692..95ca246c42 100644 --- a/src/Tgstation.Server.Host/Components/Instance.cs +++ b/src/Tgstation.Server.Host/Components/Instance.cs @@ -207,8 +207,7 @@ namespace Tgstation.Server.Host.Components Configuration.StopAsync(cancellationToken), ByondManager.StopAsync(cancellationToken), Chat.StopAsync(cancellationToken), - dmbFactory.StopAsync(cancellationToken)) - ; + dmbFactory.StopAsync(cancellationToken)); } } diff --git a/src/Tgstation.Server.Host/Components/Interop/Topic/TopicParameters.cs b/src/Tgstation.Server.Host/Components/Interop/Topic/TopicParameters.cs index 6e3e604d7c..d4373d244d 100644 --- a/src/Tgstation.Server.Host/Components/Interop/Topic/TopicParameters.cs +++ b/src/Tgstation.Server.Host/Components/Interop/Topic/TopicParameters.cs @@ -1,5 +1,7 @@ using System; +using Newtonsoft.Json; + using Tgstation.Server.Host.Components.Session; namespace Tgstation.Server.Host.Components.Interop.Topic @@ -55,6 +57,25 @@ namespace Tgstation.Server.Host.Components.Interop.Topic /// public ChunkData Chunk { get; } + /// + /// Whether or not the constitute a priority request. + /// + [JsonIgnore] + public bool IsPriority => CommandType switch + { + TopicCommandType.EventNotification + or TopicCommandType.ChangePort + or TopicCommandType.ChangeRebootState + or TopicCommandType.InstanceRenamed + or TopicCommandType.ChatChannelsUpdate + or TopicCommandType.ServerRestarted => true, + TopicCommandType.ChatCommand + or TopicCommandType.Heartbeat + or TopicCommandType.ReceiveChunk => false, + TopicCommandType.SendChunk => throw new InvalidOperationException("SendChunk topic priority should be based on the original TopicParameters!"), + _ => throw new InvalidOperationException($"Invalid value for {nameof(CommandType)}: {CommandType}"), + }; + /// /// Initializes a new instance of the class. /// diff --git a/src/Tgstation.Server.Host/Components/Session/SessionController.cs b/src/Tgstation.Server.Host/Components/Session/SessionController.cs index a831a37679..e3da5a1ea7 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionController.cs @@ -123,6 +123,11 @@ namespace Tgstation.Server.Host.Components.Session /// readonly IChatManager chat; + /// + /// The for the . + /// + readonly IAsyncDelayer asyncDelayer; + /// /// for port updates and . /// @@ -212,7 +217,12 @@ namespace Tgstation.Server.Host.Components.Session this.chatTrackingContext = chatTrackingContext ?? throw new ArgumentNullException(nameof(chatTrackingContext)); if (bridgeRegistrar == null) throw new ArgumentNullException(nameof(bridgeRegistrar)); + this.chat = chat ?? throw new ArgumentNullException(nameof(chat)); + if (assemblyInformationProvider == null) + throw new ArgumentNullException(nameof(assemblyInformationProvider)); + + this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer)); portClosedForReboot = false; disposed = false; @@ -760,8 +770,9 @@ namespace Tgstation.Server.Host.Components.Session var fullCommandString = GenerateQueryString(parameters, out var json); Logger.LogTrace("Topic request: {json}", json); var fullCommandByteCount = Encoding.UTF8.GetByteCount(fullCommandString); + var topicPriority = parameters.IsPriority; if (fullCommandByteCount <= DMApiConstants.MaximumTopicRequestLength) - return await SendRawTopic(fullCommandString, cancellationToken); + return await SendRawTopic(fullCommandString, topicPriority, cancellationToken); var interopChunkingVersion = new Version(5, 6, 0); if (ReattachInformation.Dmb.CompileJob.DMApiVersion < interopChunkingVersion) @@ -849,7 +860,7 @@ namespace Tgstation.Server.Host.Components.Session foreach (var chunkCommandString in chunkQueryStrings) { - combinedResponse = await SendRawTopic(chunkCommandString, cancellationToken); + combinedResponse = await SendRawTopic(chunkCommandString, topicPriority, cancellationToken); if (LogRequestIssue(chunkCommandString == chunkQueryStrings.Last())) return null; } @@ -861,7 +872,7 @@ namespace Tgstation.Server.Host.Components.Session foreach (var missingChunkIndex in combinedResponse.InteropResponse.MissingChunks) { var chunkCommandString = chunkQueryStrings[(int)missingChunkIndex]; - combinedResponse = await SendRawTopic(chunkCommandString, cancellationToken); + combinedResponse = await SendRawTopic(chunkCommandString, topicPriority, cancellationToken); if (LogRequestIssue(missingChunkIndex == lastIndex)) return null; } @@ -891,22 +902,53 @@ namespace Tgstation.Server.Host.Components.Session /// Send a given to DreamDaemon's /world/Topic. /// /// The sanitized topic query string to send. + /// If this is a priority message. If so, the topic will make 5 attempts to send unless BYOND reboots or exits. /// The for the operation. /// A resulting in the of the topic request. - async Task SendRawTopic(string queryString, CancellationToken cancellationToken) + async Task SendRawTopic(string queryString, bool priority, CancellationToken cancellationToken) { var targetPort = ReattachInformation.Port; - global::Byond.TopicSender.TopicResponse byondResponse; - try + var killedOrRebootedTask = Task.WhenAny(Lifetime, OnReboot); + global::Byond.TopicSender.TopicResponse byondResponse = null; + var firstSend = true; + + const int PrioritySendAttempts = 5; + for (var i = PrioritySendAttempts - 1; i >= 0 && (priority || firstSend); --i) + try + { + firstSend = false; + if (!killedOrRebootedTask.IsCompleted) + byondResponse = await byondTopicSender.SendTopic( + new IPEndPoint(IPAddress.Loopback, targetPort), + queryString, + cancellationToken); + + break; + } + catch (Exception ex) + { + Logger.LogWarning(ex, "SendTopic exception!{retryDetails}", priority ? $" {i} attempts remaining." : String.Empty); + + if (priority && i > 0) + { + var delayTask = asyncDelayer.Delay(TimeSpan.FromSeconds(2), cancellationToken); + await Task.WhenAny(killedOrRebootedTask, delayTask); + } + } + + if (byondResponse == null) { - byondResponse = await byondTopicSender.SendTopic( - new IPEndPoint(IPAddress.Loopback, targetPort), - queryString, - cancellationToken); - } - catch (Exception ex) - { - Logger.LogWarning(ex, "SendTopic exception!"); + if (priority) + if (killedOrRebootedTask.IsCompleted) + Logger.LogWarning( + "Unable to send priority topic \"{queryString}\" DreamDaemon {stateClearAction}!", + queryString, + Lifetime.IsCompleted ? "process ended" : "rebooted"); + else + Logger.LogError( + "Unable to send priority topic \"{queryString}\"!", + queryString); + return null; } diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs index 5d5f8c998b..aa6b1f1425 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs @@ -272,17 +272,15 @@ namespace Tgstation.Server.Host.Components.Watchdog { using (await SemaphoreSlimContext.Lock(synchronizationSemaphore, cancellationToken)) { - if (Status == WatchdogStatus.Offline) + var commandObject = new ChatCommand(sender, commandName, arguments); + var command = new TopicParameters(commandObject); + var activeServer = GetActiveController(); + if (Status != WatchdogStatus.Online || activeServer == null) return new MessageContent { Text = "TGS: Server offline!", }; - var commandObject = new ChatCommand(sender, commandName, arguments); - - var command = new TopicParameters(commandObject); - - var activeServer = GetActiveController(); var commandResult = await activeServer.SendCommand(command, cancellationToken); if (commandResult == null) diff --git a/src/Tgstation.Server.Host/Configuration/UpdatesConfiguration.cs b/src/Tgstation.Server.Host/Configuration/UpdatesConfiguration.cs index 406560f458..fae04d8684 100644 --- a/src/Tgstation.Server.Host/Configuration/UpdatesConfiguration.cs +++ b/src/Tgstation.Server.Host/Configuration/UpdatesConfiguration.cs @@ -39,5 +39,11 @@ /// Asset package containing the new assembly in zip form. /// public string UpdatePackageAssetName { get; set; } = DefaultUpdatePackageAssetName; + + /// + /// Dump all retrieved releases from the GitHub API when a requested release is not found. + /// + /// This is an internal config and may be adjusted or removed without a version change. + public bool DumpReleasesOnNotFound { get; set; } } } diff --git a/src/Tgstation.Server.Host/Core/ServerUpdater.cs b/src/Tgstation.Server.Host/Core/ServerUpdater.cs index e552db745a..adcb902fd7 100644 --- a/src/Tgstation.Server.Host/Core/ServerUpdater.cs +++ b/src/Tgstation.Server.Host/Core/ServerUpdater.cs @@ -108,13 +108,16 @@ namespace Tgstation.Server.Host.Core logger.LogTrace("Received {releaseCount} total releases from GitHub", releases.Count); - releases = releases + var filteredReleases = releases .Where(x => x.TagName.StartsWith(updatesConfiguration.GitTagPrefix, StringComparison.InvariantCulture)) .ToList(); - logger.LogTrace("Filtered to {releaseCount} releases matching a TGS version", releases.Count); + logger.LogTrace( + "Filtered to {releaseCount} releases matching the configured tag prefix of \"{tagPrefix}\"", + filteredReleases.Count, + updatesConfiguration.GitTagPrefix); - foreach (var release in releases) + foreach (var release in filteredReleases) if (Version.TryParse( release.TagName.Replace( updatesConfiguration.GitTagPrefix, String.Empty, StringComparison.Ordinal), @@ -149,6 +152,12 @@ namespace Tgstation.Server.Host.Core else logger.LogDebug("Unparsable release tag: {releaseTag}", release.TagName); + if (updatesConfiguration.DumpReleasesOnNotFound) + logger.LogInformation( + "Found releases:{newline}\t{releases}", + Environment.NewLine, + String.Join($"{Environment.NewLine}\t", releases.Select(x => x.TagName).OrderBy(x => x))); + return ServerUpdateResult.ReleaseMissing; } diff --git a/src/Tgstation.Server.Host/Extensions/ServiceCollectionExtensions.cs b/src/Tgstation.Server.Host/Extensions/ServiceCollectionExtensions.cs index 4fcbc5d32e..4980d732d8 100644 --- a/src/Tgstation.Server.Host/Extensions/ServiceCollectionExtensions.cs +++ b/src/Tgstation.Server.Host/Extensions/ServiceCollectionExtensions.cs @@ -5,6 +5,7 @@ using System.Globalization; using Elastic.CommonSchema.Serilog; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Logging; using Serilog; @@ -27,6 +28,11 @@ namespace Tgstation.Server.Host.Extensions /// static Type chatProviderFactoryType = typeof(ProviderFactory); + /// + /// A for an additional to use. + /// + static ServiceDescriptor additionalLoggerProvider; + /// /// Change the used as an implementation for calls to . /// @@ -36,6 +42,17 @@ namespace Tgstation.Server.Host.Extensions chatProviderFactoryType = typeof(TProviderFactory); } + /// + /// Add an additional to s that call . + /// + /// The of to add. + public static void UseAdditionalLoggerProvider() where TLoggerProvider : class, ILoggerProvider + { + if (additionalLoggerProvider != null) + throw new InvalidOperationException("Cannot have multiple additionalLoggerProviders!"); + additionalLoggerProvider = ServiceDescriptor.Singleton(); + } + /// /// Adds a implementation to the given . /// @@ -133,6 +150,9 @@ namespace Tgstation.Server.Host.Extensions if (Debugger.IsAttached) builder.AddDebug(); + + if (additionalLoggerProvider != null) + builder.Services.TryAddEnumerable(additionalLoggerProvider); }); } } diff --git a/src/Tgstation.Server.Host/IO/DefaultIOManager.cs b/src/Tgstation.Server.Host/IO/DefaultIOManager.cs index d3d36804e3..85dd4ce31d 100644 --- a/src/Tgstation.Server.Host/IO/DefaultIOManager.cs +++ b/src/Tgstation.Server.Host/IO/DefaultIOManager.cs @@ -52,8 +52,15 @@ namespace Tgstation.Server.Host.IO foreach (var file in dir.EnumerateFiles()) { cancellationToken.ThrowIfCancellationRequested(); - file.Attributes = FileAttributes.Normal; - file.Delete(); + try + { + file.Attributes = FileAttributes.Normal; + file.Delete(); + } + catch (FileNotFoundException) + { + // has happened before with .dyn.rsc.lk + } } cancellationToken.ThrowIfCancellationRequested(); diff --git a/src/Tgstation.Server.Host/Jobs/JobManager.cs b/src/Tgstation.Server.Host/Jobs/JobManager.cs index dbf9352ea7..88828baca2 100644 --- a/src/Tgstation.Server.Host/Jobs/JobManager.cs +++ b/src/Tgstation.Server.Host/Jobs/JobManager.cs @@ -185,16 +185,11 @@ namespace Tgstation.Server.Host.Jobs JobHandler handler; lock (addCancelLock) { - try - { - handler = CheckGetJob(job); - } - catch (InvalidOperationException) - { - // this is fine - return null; - } + lock (synchronizationLock) + if (!jobs.TryGetValue(job.Id.Value, out handler)) + return null; + logger.LogDebug("Cancelling job ID {jobId}...", job.Id.Value); handler.Cancel(); // this will ensure the db update is only done once } @@ -267,21 +262,6 @@ namespace Tgstation.Server.Host.Jobs activationTcs.SetResult(instanceCoreProvider); } - /// - /// Gets the for a given if it exists. - /// - /// The to get the for. - /// The . - JobHandler CheckGetJob(Job job) - { - lock (synchronizationLock) - { - if (!jobs.TryGetValue(job.Id.Value, out JobHandler jobHandler)) - throw new InvalidOperationException("Job not running!"); - return jobHandler; - } - } - /// /// Runner for s. /// diff --git a/src/Tgstation.Server.Host/Swarm/SwarmService.cs b/src/Tgstation.Server.Host/Swarm/SwarmService.cs index ebda13992f..2e977abb38 100644 --- a/src/Tgstation.Server.Host/Swarm/SwarmService.cs +++ b/src/Tgstation.Server.Host/Swarm/SwarmService.cs @@ -826,7 +826,7 @@ namespace Tgstation.Server.Host.Swarm updateCommitTcs = new TaskCompletionSource(); } - logger.LogDebug("Prepared for update to version {version}", version); + logger.LogDebug("Local node prepared for update to version {version}", version); } catch (Exception ex) { diff --git a/tests/DMAPI/LongRunning/Config.dm b/tests/DMAPI/LongRunning/Config.dm index a7dfd4936a..af95587445 100644 --- a/tests/DMAPI/LongRunning/Config.dm +++ b/tests/DMAPI/LongRunning/Config.dm @@ -4,7 +4,7 @@ #define TGS_WRITE_GLOBAL(Name, Value) global.##Name = ##Value #define TGS_PROTECT_DATUM(Path) #define TGS_WORLD_ANNOUNCE(message) world << ##message -#define TGS_INFO_LOG(message) world.log << "Info: [##message]" +#define TGS_INFO_LOG(message) TgsInfo(##message) #define TGS_WARNING_LOG(message) world.log << "Warn: [##message]" #define TGS_ERROR_LOG(message) TgsError(##message) #define TGS_NOTIFY_ADMINS(event) diff --git a/tests/DMAPI/LongRunning/Test.dm b/tests/DMAPI/LongRunning/Test.dm index 762add090c..de5cd5831d 100644 --- a/tests/DMAPI/LongRunning/Test.dm +++ b/tests/DMAPI/LongRunning/Test.dm @@ -54,7 +54,10 @@ CheckBridgeLimits() /world/Topic(T, Addr, Master, Keys) - log << "Topic: [T]" + if(findtext(T, "tgs_integration_test_tactics3") == 0) + log << "Topic: [T]" + else + log << "tgs_integration_test_tactics3 " . = HandleTopic(T) log << "Response: [.]" @@ -210,10 +213,18 @@ var/run_bridge_test return response var/lastTgsError +var/suppress_bridge_spam = FALSE + +/proc/TgsInfo(message) + if(suppress_bridge_spam && findtext(message, "Export: http://127.0.0.1:") != 0) + return + world.log << "Info: [message]" /proc/TgsError(message) - world.log << "Err: [message]" lastTgsError = message + if(suppress_bridge_spam && findtext(message, "Failed bridge request: http://127.0.0.1:") != 0) + return + world.log << "Err: [message]" /proc/create_payload(size) var/builder = list() @@ -230,7 +241,9 @@ var/lastTgsError /proc/BridgeWithoutChunking(command, list/data) var/datum/tgs_api/v5/api = TGS_READ_GLOBAL(tgs) var/bridge_request = api.CreateBridgeRequest(command, data) - return api.PerformBridgeRequest(bridge_request) + suppress_bridge_spam = TRUE + . = api.PerformBridgeRequest(bridge_request) + suppress_bridge_spam = FALSE /proc/CheckBridgeLimitsImpl() sleep(30) diff --git a/tests/Tgstation.Server.Tests/Live/DummyChatProvider.cs b/tests/Tgstation.Server.Tests/Live/DummyChatProvider.cs index 331931cef6..6eb24a0c0f 100644 --- a/tests/Tgstation.Server.Tests/Live/DummyChatProvider.cs +++ b/tests/Tgstation.Server.Tests/Live/DummyChatProvider.cs @@ -9,6 +9,8 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; +using Newtonsoft.Json; + using Tgstation.Server.Host.Components.Chat; using Tgstation.Server.Host.Components.Chat.Commands; using Tgstation.Server.Host.Components.Chat.Providers; @@ -34,13 +36,14 @@ namespace Tgstation.Server.Tests.Live readonly ICryptographySuite cryptographySuite; readonly CancellationTokenSource randomMessageCts; readonly Task randomMessageTask; + readonly Dictionary knownChannels; bool connectedOnce; bool connected; ulong channelIdAllocator; - static ILoggerFactory CreateLoggerFactoryForLogger(ILogger logger, out Mock mockLoggerFactory) + public static ILoggerFactory CreateLoggerFactoryForLogger(ILogger logger, out Mock mockLoggerFactory) { mockLoggerFactory = new Mock(); mockLoggerFactory.Setup(x => x.CreateLogger(It.IsAny())).Returns(() => @@ -83,12 +86,11 @@ namespace Tgstation.Server.Tests.Live this.commands = commands ?? throw new ArgumentNullException(nameof(commands)); this.random = random ?? throw new ArgumentNullException(nameof(random)); - // this could be random but there's no point - channelIdAllocator = 100000; logger.LogTrace("Base channel ID {baseChannelId}", channelIdAllocator); - this.randomMessageCts = new CancellationTokenSource(); - this.randomMessageTask = RandomMessageLoop(this.randomMessageCts.Token); + knownChannels = new Dictionary(); + randomMessageCts = new CancellationTokenSource(); + randomMessageTask = RandomMessageLoop(this.randomMessageCts.Token); } public override async ValueTask DisposeAsync() @@ -107,8 +109,7 @@ namespace Tgstation.Server.Tests.Live Logger.LogTrace("SendMessage"); - Assert.AreNotEqual(0UL, channelId); - Assert.IsTrue(channelId <= channelIdAllocator); + Assert.IsTrue(knownChannels.ContainsKey(channelId)); cancellationToken.ThrowIfCancellationRequested(); @@ -132,8 +133,7 @@ namespace Tgstation.Server.Tests.Live Logger.LogTrace("SendUpdateMessage"); - Assert.AreNotEqual(0UL, channelId); - Assert.IsTrue(channelId <= channelIdAllocator); + Assert.IsTrue(knownChannels.ContainsKey(channelId)); cancellationToken.ThrowIfCancellationRequested(); @@ -196,19 +196,43 @@ namespace Tgstation.Server.Tests.Live channel, new List { - new ChannelRepresentation - { - IsAdminChannel = channel.IsAdminChannel.Value, - ConnectionName = $"Connection_{channelIdAllocator + 1}", - EmbedsSupported = ChatBot.Provider.Value != Api.Models.ChatProvider.Irc, - FriendlyName = $"(Friendly) Channel_ID_{channelIdAllocator + 1}", - IsPrivateChannel = false, - RealId = ++channelIdAllocator, - Tag = channel.Tag, - } + CreateChannel(channel) })))); } + private ChannelRepresentation CreateChannel(ChatChannel channel) + { + + ulong channelId; + if (channel.DiscordChannelId.HasValue) + channelId = channel.DiscordChannelId.Value; + else + channelId = (ulong)channel.IrcChannel.GetHashCode(); + + var entry = new ChannelRepresentation + { + IsAdminChannel = channel.IsAdminChannel.Value, + ConnectionName = $"Connection_{channelId}", + EmbedsSupported = ChatBot.Provider.Value != Api.Models.ChatProvider.Irc, + FriendlyName = $"(Friendly) Channel_ID_{channelId}", + IsPrivateChannel = false, + RealId = channelId, + Tag = channel.Tag, + }; + + knownChannels.Remove(channelId); + knownChannels.Add(channelId, entry); + + return CloneChannel(entry); + } + + ChannelRepresentation CloneChannel(ChannelRepresentation channel) + { + return JsonConvert.DeserializeObject( + JsonConvert.SerializeObject( + channel)); + } + async Task RandomMessageLoop(CancellationToken cancellationToken) { Logger.LogTrace("RandomMessageLoop"); @@ -220,35 +244,68 @@ namespace Tgstation.Server.Tests.Live var delay = random.Next(0, 10000); await Task.Delay(delay, cancellationToken); - if (!connected) - continue; - // %5 chance to disconnect randomly if (enableRandomDisconnections != 0 && random.Next(0, 100) > 95) connected = false; - if (channelIdAllocator >= Int32.MaxValue / 2) - Assert.Fail("Too many channels have been allocated!"); + if (!connected) + continue; - var isPm = channelIdAllocator == 0 || random.Next(0, 100) > 20; - var realId = (ulong)random.Next(1, (int)channelIdAllocator); - - if (isPm) - realId += Int32.MaxValue / 2; + var isPm = channelIdAllocator == 0 || random.Next(0, 100) > 80; + ChannelRepresentation channel; var username = $"RandomUser{i}"; + if (isPm) + // 50% chance to be a new user + if (random.Next(0, 100) > 50) + { + var enumerator = knownChannels + .Where(x => x.Value.IsPrivateChannel) + .ToList(); + if (enumerator.Count == 0) + continue; + + var index = random.Next(0, enumerator.Count); + channel = enumerator[index].Value; + username = channel.ConnectionName.Substring(0, channel.ConnectionName.Length - 11); + } + else + { + ulong channelId; + do + { + channelId = ++channelIdAllocator; + } + while (knownChannels.ContainsKey(channelId)); + + channel = new ChannelRepresentation + { + RealId = channelId, + IsPrivateChannel = true, + ConnectionName = $"{username}_Connection", + FriendlyName = $"{username}_Channel", + EmbedsSupported = ChatBot.Provider.Value != Api.Models.ChatProvider.Irc, + + // isAdmin and Tag populated by manager + }; + + knownChannels.Add(channelId, channel); + } + else + { + var enumerator = knownChannels + .Where(x => !x.Value.IsPrivateChannel) + .ToList(); + if (enumerator.Count == 0) + continue; + + var index = random.Next(0, enumerator.Count); + channel = enumerator[index].Value; + } + var sender = new ChatUser { - Channel = new ChannelRepresentation - { - RealId = realId, - IsPrivateChannel = isPm, - ConnectionName = isPm ? $"{username}_Connection" : $"Connection_{realId}", - FriendlyName = isPm ? $"{username}_Channel" : $"(Friendly) Channel_ID_{realId}", - EmbedsSupported = ChatBot.Provider.Value != Api.Models.ChatProvider.Irc, - - // isAdmin and Tag populated by manager - }, + Channel = CloneChannel(channel), FriendlyName = username, RealId = i + 50000, Mention = $"@{username}", diff --git a/tests/Tgstation.Server.Tests/Live/DummyChatProviderFactory.cs b/tests/Tgstation.Server.Tests/Live/DummyChatProviderFactory.cs index 55162d85c2..3f6fd02275 100644 --- a/tests/Tgstation.Server.Tests/Live/DummyChatProviderFactory.cs +++ b/tests/Tgstation.Server.Tests/Live/DummyChatProviderFactory.cs @@ -47,7 +47,15 @@ namespace Tgstation.Server.Tests.Live commandFactory.SetWatchdog(Mock.Of()); commands = commandFactory.GenerateCommands(); - var baseRng = new Random(22475); + const bool UseUnseededProviderRng = false; + + var randomSeed = UseUnseededProviderRng + ? new Random().Next() + : 22475; + + logger.LogInformation("Random seed: {0}", randomSeed); + + var baseRng = new Random(randomSeed); seededRng = new Dictionary{ { ChatProvider.Irc, new Random(baseRng.Next()) }, { ChatProvider.Discord, new Random(baseRng.Next()) }, diff --git a/tests/Tgstation.Server.Tests/Live/HardFailLogger.cs b/tests/Tgstation.Server.Tests/Live/HardFailLogger.cs new file mode 100644 index 0000000000..1f66ee01f6 --- /dev/null +++ b/tests/Tgstation.Server.Tests/Live/HardFailLogger.cs @@ -0,0 +1,36 @@ +using System; +using System.Text.RegularExpressions; +using System.Threading.Tasks; + +using Microsoft.Extensions.Logging; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace Tgstation.Server.Tests.Live +{ + sealed class HardFailLogger : ILogger + { + readonly Action failureSink; + + public HardFailLogger(Action failureSink) + { + this.failureSink = failureSink ?? throw new ArgumentNullException(nameof(failureSink)); + } + + public IDisposable BeginScope(TState state) => Task.CompletedTask; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func formatter) + { + var logMessage = formatter(state, exception); + if ((logLevel == LogLevel.Error + && !logMessage.StartsWith("Error disconnecting connection ") + && !(logMessage.StartsWith("An exception occurred while iterating over the results of a query for context type") && (exception is TaskCanceledException || exception?.InnerException is TaskCanceledException))) + || (logLevel == LogLevel.Critical && logMessage != "DropDatabase configuration option set! Dropping any existing database...")) + { + Console.WriteLine("UNEXPECTED ERROR OCCURS NEAR HERE"); + failureSink(new AssertFailedException("TGS logged an unexpected error!")); + } + } + } +} diff --git a/tests/Tgstation.Server.Tests/Live/HardFailLoggerProvider.cs b/tests/Tgstation.Server.Tests/Live/HardFailLoggerProvider.cs new file mode 100644 index 0000000000..59a677abd8 --- /dev/null +++ b/tests/Tgstation.Server.Tests/Live/HardFailLoggerProvider.cs @@ -0,0 +1,23 @@ +using System.Threading.Tasks; + +using Microsoft.Extensions.Logging; + +namespace Tgstation.Server.Tests.Live +{ + sealed class HardFailLoggerProvider : ILoggerProvider + { + public static bool BlockFails { get; set; } + + public static Task FailureSource => failureSink.Task; + + static readonly TaskCompletionSource failureSink = new (TaskCreationOptions.RunContinuationsAsynchronously); + + public ILogger CreateLogger(string categoryName) => new HardFailLogger(ex => + { + if (!BlockFails) + failureSink.TrySetException(ex); + }); + + public void Dispose() { } + } +} diff --git a/tests/Tgstation.Server.Tests/Live/Instance/ChatTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/ChatTest.cs index 38d8d458b7..f90c1dc116 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/ChatTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/ChatTest.cs @@ -97,7 +97,7 @@ namespace Tgstation.Server.Tests.Live.Instance var jobs = await JobsClient.List(null, cancellationToken); var reconnectJob = jobs .Where(x => x.StartedAt >= beforeChatBotEnabled && x.Description.Contains(updatedBot.Name)) - .OrderByDescending(x => x.StartedAt) + .OrderBy(x => x.StartedAt) .FirstOrDefault(); Assert.IsNotNull(reconnectJob); @@ -198,7 +198,7 @@ namespace Tgstation.Server.Tests.Live.Instance var jobs = await JobsClient.List(null, cancellationToken); var reconnectJob = jobs .Where(x => x.StartedAt >= beforeChatBotEnabled && x.Description.Contains(updatedBot.Name)) - .OrderByDescending(x => x.StartedAt) + .OrderBy(x => x.StartedAt) .FirstOrDefault(); Assert.IsNotNull(reconnectJob); diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index 78b287a641..e8fc623199 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -519,7 +519,8 @@ namespace Tgstation.Server.Tests.Live.Instance TopicResponse topicRequestResult = null; try { - topicRequestResult = await TopicClient.SendTopic( + System.Console.WriteLine($"Topic limit test S:{payloadSize}..."); + topicRequestResult = await TopicClientNoLogger.SendTopic( IPAddress.Loopback, $"tgs_integration_test_tactics3={TopicClient.SanitizeString(JsonConvert.SerializeObject(topic, DMApiConstants.SerializerSettings))}", TestLiveServer.DDPort, @@ -869,7 +870,21 @@ namespace Tgstation.Server.Tests.Live.Instance return ddProc != null; } + static readonly ILoggerFactory loggerFactory = LoggerFactory.Create(builder => + { + builder.AddConsole(); + builder.SetMinimumLevel(LogLevel.Trace); + }); + public static readonly TopicClient TopicClient = new(new SocketParameters + { + SendTimeout = TimeSpan.FromSeconds(30), + ReceiveTimeout = TimeSpan.FromSeconds(30), + ConnectTimeout = TimeSpan.FromSeconds(30), + DisconnectTimeout = TimeSpan.FromSeconds(30) + }, new Logger(DummyChatProvider.CreateLoggerFactoryForLogger(loggerFactory.CreateLogger("WatchdogTest.TopicClient"), out var mockLoggerFactory))); + + public static readonly TopicClient TopicClientNoLogger = new(new SocketParameters { SendTimeout = TimeSpan.FromSeconds(30), ReceiveTimeout = TimeSpan.FromSeconds(30), diff --git a/tests/Tgstation.Server.Tests/Live/InstanceManagerTest.cs b/tests/Tgstation.Server.Tests/Live/InstanceManagerTest.cs index 99125b5606..4dcefae0bc 100644 --- a/tests/Tgstation.Server.Tests/Live/InstanceManagerTest.cs +++ b/tests/Tgstation.Server.Tests/Live/InstanceManagerTest.cs @@ -38,9 +38,20 @@ namespace Tgstation.Server.Tests.Live this.testRootPath = testRootPath ?? throw new ArgumentNullException(nameof(testRootPath)); } - public Task CreateTestInstance(CancellationToken cancellationToken) => instanceManagerClient.CreateOrAttach(new InstanceCreateRequest + public async Task CreateTestInstance(CancellationToken cancellationToken) { - Name = TestInstanceName, + var instance = await CreateTestInstanceStub("LiveTestsInstance", cancellationToken); + return await instanceManagerClient.Update(new InstanceUpdateRequest + { + Id = instance.Id, + Online = true, + ConfigurationType = ConfigurationType.HostWrite + }, cancellationToken); + } + + Task CreateTestInstanceStub(string name, CancellationToken cancellationToken) => instanceManagerClient.CreateOrAttach(new InstanceCreateRequest + { + Name = name, Path = Path.Combine(testRootPath, Guid.NewGuid().ToString()), Online = true, ChatBotLimit = 2 @@ -57,9 +68,9 @@ namespace Tgstation.Server.Tests.Live Online = response.Online }; - public async Task RunPreInstanceTest(CancellationToken cancellationToken) + public async Task RunPreTest(CancellationToken cancellationToken) { - var firstTest = await CreateTestInstance(cancellationToken); + var firstTest = await CreateTestInstanceStub(TestInstanceName, cancellationToken); //instances always start offline Assert.AreEqual(false, firstTest.Online); //check it exists @@ -161,8 +172,14 @@ namespace Tgstation.Server.Tests.Live Assert.IsTrue(Directory.Exists(firstTest.Path)); await RegressionTest1256(cancellationToken); + firstTest = await instanceManagerClient.Update(new InstanceUpdateRequest + { + Id = firstTest.Id, + Online = false, + }, cancellationToken); - return firstTest; + Assert.AreEqual(false, firstTest.Online); + await instanceManagerClient.Detach(firstTest, cancellationToken); } async Task RegressionTest1256(CancellationToken cancellationToken) @@ -197,10 +214,10 @@ namespace Tgstation.Server.Tests.Live Assert.AreEqual(3, paginated.TotalPages); } - public async Task RunPostTest(CancellationToken cancellationToken) + public async Task RunPostTest(Api.Models.Instance instance, CancellationToken cancellationToken) { var instances = await instanceManagerClient.List(null, cancellationToken); - var firstTest = instances.Single(x => x.Name == TestInstanceName); + var firstTest = instances.Single(x => x.Name == instance.Name); var instanceClient = instanceManagerClient.CreateClient(firstTest); Assert.IsTrue(firstTest.Accessible); diff --git a/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs b/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs index 4508ee9fc1..c6fcd0ea82 100644 --- a/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs +++ b/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs @@ -46,7 +46,7 @@ namespace Tgstation.Server.Tests.Live SerilogContextHelper.AddSwarmNodeIdentifierToTemplate(); } - public LiveTestingServer(SwarmConfiguration swarmConfiguration, bool enableOAuth, ushort port = 5010) + public LiveTestingServer(SwarmConfiguration swarmConfiguration, bool enableOAuth, ushort port = 5010, bool dumpOnMissingUpdate = true) { Directory = Environment.GetEnvironmentVariable("TGS_TEST_TEMP_DIRECTORY"); if (string.IsNullOrWhiteSpace(Directory)) @@ -86,6 +86,7 @@ namespace Tgstation.Server.Tests.Live args = new List() { string.Format(CultureInfo.InvariantCulture, "Database:DropDatabase={0}", true), // Replaced after first Run + string.Format(CultureInfo.InvariantCulture, "General:ConfigVersion={0}", GeneralConfiguration.CurrentConfigVersion), string.Format(CultureInfo.InvariantCulture, "General:ApiPort={0}", port), string.Format(CultureInfo.InvariantCulture, "Database:DatabaseType={0}", DatabaseType), string.Format(CultureInfo.InvariantCulture, "Database:ConnectionString={0}", connectionString), @@ -101,6 +102,9 @@ namespace Tgstation.Server.Tests.Live "General:ByondTopicTimeout=3000" }; + if (dumpOnMissingUpdate) + args.Add("Updates:DumpReleasesOnNotFound=true"); + swarmArgs = new List(); if (swarmConfiguration != null) { diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index e39d36650e..b3a7596c5f 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -215,6 +215,38 @@ namespace Tgstation.Server.Tests.Live Assert.IsTrue(server.RestartRequested, "Server not requesting restart!"); } + [TestMethod] + public async Task TestUpdateBadVersion() + { + using var server = new LiveTestingServer(null, false, dumpOnMissingUpdate: false); + using var serverCts = new CancellationTokenSource(); + var cancellationToken = serverCts.Token; + var serverTask = server.Run(cancellationToken); + try + { + var testUpdateVersion = new Version(5, 11, 20); + using var adminClient = await CreateAdminClient(server.Url, cancellationToken); + await ApiAssert.ThrowsException(() => adminClient.Administration.Update(new ServerUpdateRequest + { + NewVersion = testUpdateVersion + }, cancellationToken), ErrorCode.ResourceNotPresent); + } + finally + { + serverCts.Cancel(); + try + { + await serverTask; + } + catch (OperationCanceledException) { } + catch (AggregateException ex) + { + if (ex.InnerException is NotSupportedException notSupportedException) + Assert.Inconclusive(notSupportedException.Message); + } + } + } + [TestMethod] public async Task TestOneServerSwarmUpdate() { @@ -697,6 +729,45 @@ namespace Tgstation.Server.Tests.Live [TestMethod] public async Task TestStandardTgsOperation() + { + const int MaximumTestMinutes = 30; + using var hardCancellationTokenSource = new CancellationTokenSource(TimeSpan.FromMinutes(MaximumTestMinutes)); + var hardCancellationToken = hardCancellationTokenSource.Token; + + ServiceCollectionExtensions.UseAdditionalLoggerProvider(); + + var failureTask = HardFailLoggerProvider.FailureSource; + var internalTask = TestTgsInternal(hardCancellationToken); + await Task.WhenAny( + internalTask, + failureTask); + + if (!internalTask.IsCompleted) + { + hardCancellationTokenSource.Cancel(); + try + { + await failureTask; + } + finally + { + try + { + await internalTask; + } + catch (OperationCanceledException) + { + Console.WriteLine("TEST CANCELLED!"); + } + catch (Exception ex) + { + Console.WriteLine($"ADDITIONAL TEST ERROR: {ex}"); + } + } + } + } + + async Task TestTgsInternal(CancellationToken hardCancellationToken) { var discordConnectionString = Environment.GetEnvironmentVariable("TGS_TEST_DISCORD_TOKEN"); var ircConnectionString = Environment.GetEnvironmentVariable("TGS_TEST_IRC_CONNECTION_STRING"); @@ -734,9 +805,6 @@ namespace Tgstation.Server.Tests.Live using var server = new LiveTestingServer(null, true); - const int MaximumTestMinutes = 180; - using var hardTimeoutCancellationTokenSource = new CancellationTokenSource(TimeSpan.FromMinutes(MaximumTestMinutes)); - var hardCancellationToken = hardTimeoutCancellationTokenSource.Token; using var serverCts = CancellationTokenSource.CreateLinkedTokenSource(hardCancellationToken); var cancellationToken = serverCts.Token; @@ -786,7 +854,9 @@ namespace Tgstation.Server.Tests.Live var rootTest = FailFast(RawRequestTests.Run(clientFactory, adminClient, cancellationToken)); var adminTest = FailFast(new AdministrationTest(adminClient.Administration).Run(cancellationToken)); var usersTest = FailFast(new UsersTest(adminClient).Run(cancellationToken)); - instance = await new InstanceManagerTest(adminClient, server.Directory).RunPreInstanceTest(cancellationToken); + var instanceMangagerTest = new InstanceManagerTest(adminClient, server.Directory); + var instancesTest = FailFast(instanceMangagerTest.RunPreTest(cancellationToken)); + instance = await instanceMangagerTest.CreateTestInstance(cancellationToken); Assert.IsTrue(Directory.Exists(instance.Path)); var instanceClient = adminClient.Instances.CreateClient(instance); @@ -794,7 +864,7 @@ namespace Tgstation.Server.Tests.Live var instanceTests = FailFast(new InstanceTest(instanceClient, adminClient.Instances, GetInstanceManager(), (ushort)server.Url.Port).RunTests(cancellationToken)); - await Task.WhenAll(rootTest, adminTest, instanceTests, usersTest); + await Task.WhenAll(rootTest, adminTest, instancesTest, instanceTests, usersTest); await adminClient.Administration.Restart(cancellationToken); } @@ -816,12 +886,13 @@ namespace Tgstation.Server.Tests.Live // http bind test https://github.com/tgstation/tgstation-server/issues/1065 if (new PlatformIdentifier().IsWindows) { - using var blockingSocket = new Socket(SocketType.Stream, ProtocolType.Tcp); - blockingSocket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ExclusiveAddressUse, true); - blockingSocket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, false); - blockingSocket.Bind(new IPEndPoint(IPAddress.Any, server.Url.Port)); + HardFailLoggerProvider.BlockFails = true; try { + using var blockingSocket = new Socket(SocketType.Stream, ProtocolType.Tcp); + blockingSocket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ExclusiveAddressUse, true); + blockingSocket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, false); + blockingSocket.Bind(new IPEndPoint(IPAddress.Any, server.Url.Port)); // bind test run await server.Run(cancellationToken); Assert.Fail("Expected server task to end with a SocketException"); @@ -830,6 +901,10 @@ namespace Tgstation.Server.Tests.Live { Assert.AreEqual(ex.SocketErrorCode, SocketError.AddressAlreadyInUse); } + finally + { + HardFailLoggerProvider.BlockFails = false; + } } await Task.WhenAny(serverTask, Task.Delay(TimeSpan.FromMinutes(1), cancellationToken)); @@ -999,7 +1074,7 @@ namespace Tgstation.Server.Tests.Live await new ChatTest(instanceClient.ChatBots, adminClient.Instances, instanceClient.Jobs, instance).RunPostTest(cancellationToken); await repoTest; - await new InstanceManagerTest(adminClient, server.Directory).RunPostTest(cancellationToken); + await new InstanceManagerTest(adminClient, server.Directory).RunPostTest(instance, cancellationToken); } } catch (ApiException ex)