diff --git a/.github/workflows/ci-suite.yml b/.github/workflows/ci-suite.yml index 55a3e49cd8..e2e0fa100b 100644 --- a/.github/workflows/ci-suite.yml +++ b/.github/workflows/ci-suite.yml @@ -513,7 +513,7 @@ jobs: deploy-http: name: Deploy HTTP API - needs: [linux-unit-tests, linux-integration-tests, windows-unit-tests, windows-integration-test] + needs: [upload-code-coverage, validate-openapi-spec] runs-on: windows-latest if: github.event_name == 'push' && github.ref == 'master' && contains(github.event.head_commit.message, '[APIDeploy]') steps: @@ -556,7 +556,7 @@ jobs: deploy-dm: name: Deploy DreamMaker API - needs: [linux-unit-tests, linux-integration-tests, windows-unit-tests, windows-integration-test] + needs: [upload-code-coverage, validate-openapi-spec] runs-on: windows-latest if: github.event_name == 'push' && github.ref == 'master' && contains(github.event.head_commit.message, '[DMDeploy]') steps: @@ -597,7 +597,7 @@ jobs: deploy-client: name: Deploy Nuget Packages - needs: [linux-unit-tests, linux-integration-tests, windows-unit-tests, windows-integration-test] + needs: [upload-code-coverage, validate-openapi-spec] runs-on: ubuntu-latest if: github.event_name == 'push' && github.ref == 'master' && contains(github.event.head_commit.message, '[NugetDeploy]') steps: diff --git a/README.md b/README.md index b5aa0d90df..09032179ea 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # tgstation-server v4: -![Test Suite](https://github.com/tgstation/tgstation-server/workflows/Test%20Suite/badge.svg) [![Build Status](https://travis-ci.org/tgstation/tgstation-server.svg?branch=master)](https://travis-ci.org/tgstation/tgstation-server) [![codecov](https://codecov.io/gh/tgstation/tgstation-server/branch/master/graph/badge.svg)](https://codecov.io/gh/tgstation/tgstation-server) +![CI](https://github.com/tgstation/tgstation-server/workflows/CI/badge.svg) [![Build Status](https://travis-ci.org/tgstation/tgstation-server.svg?branch=master)](https://travis-ci.org/tgstation/tgstation-server) [![codecov](https://codecov.io/gh/tgstation/tgstation-server/branch/master/graph/badge.svg)](https://codecov.io/gh/tgstation/tgstation-server) [![GitHub license](https://img.shields.io/github/license/tgstation/tgstation-server.svg)](LICENSE) [![Average time to resolve an issue](http://isitmaintained.com/badge/resolution/tgstation/tgstation-server.svg)](http://isitmaintained.com/project/tgstation/tgstation-server "Average time to resolve an issue") [![NuGet version](https://img.shields.io/nuget/v/Tgstation.Server.Api.svg)](https://www.nuget.org/packages/Tgstation.Server.Api) [![NuGet version](https://img.shields.io/nuget/v/Tgstation.Server.Client.svg)](https://www.nuget.org/packages/Tgstation.Server.Client) diff --git a/src/Tgstation.Server.Host/Components/Chat/Providers/DiscordProvider.cs b/src/Tgstation.Server.Host/Components/Chat/Providers/DiscordProvider.cs index e41de6012a..6ba2897ba5 100644 --- a/src/Tgstation.Server.Host/Components/Chat/Providers/DiscordProvider.cs +++ b/src/Tgstation.Server.Host/Components/Chat/Providers/DiscordProvider.cs @@ -131,13 +131,13 @@ namespace Tgstation.Server.Host.Components.Chat.Providers Mention = NormalizeMentions(e.Author.Mention) } }; + EnqueueMessage(result); } /// protected override async Task Connect(CancellationToken cancellationToken) { - Logger.LogTrace("Connecting..."); try { await client.LoginAsync(TokenType.Bot, BotToken, true).ConfigureAwait(false); @@ -150,14 +150,22 @@ namespace Tgstation.Server.Host.Components.Chat.Providers Logger.LogTrace("Started."); var channelsAvailable = new TaskCompletionSource(); - client.Ready += () => + Task ReadyCallback() { channelsAvailable.TrySetResult(null); return Task.CompletedTask; - }; - using (cancellationToken.Register(() => channelsAvailable.SetCanceled())) - await channelsAvailable.Task.ConfigureAwait(false); - Logger.LogDebug("Connection established!"); + } + + client.Ready += ReadyCallback; + try + { + using (cancellationToken.Register(() => channelsAvailable.SetCanceled())) + await channelsAvailable.Task.ConfigureAwait(false); + } + finally + { + client.Ready -= ReadyCallback; + } } catch (OperationCanceledException) { @@ -172,13 +180,6 @@ namespace Tgstation.Server.Host.Components.Chat.Providers /// protected override async Task DisconnectImpl(CancellationToken cancellationToken) { - Logger.LogTrace("Disconnecting..."); - if (!Connected) - { - Logger.LogTrace("Already disconnected not doing disconnection attempt!"); - return; - } - try { cancellationToken.ThrowIfCancellationRequested(); @@ -254,7 +255,8 @@ namespace Tgstation.Server.Host.Components.Chat.Providers var channel = client.GetChannel(channelId) as IMessageChannel; await (channel?.SendMessageAsync(message, false, null, new RequestOptions { - CancelToken = cancellationToken + CancelToken = cancellationToken, + Timeout = 10000 // prevent stupid long hold ups from this }) ?? Task.CompletedTask).ConfigureAwait(false); } catch (OperationCanceledException) diff --git a/src/Tgstation.Server.Host/Components/Chat/Providers/IrcProvider.cs b/src/Tgstation.Server.Host/Components/Chat/Providers/IrcProvider.cs index dc4c2f1f50..7db05d816d 100644 --- a/src/Tgstation.Server.Host/Components/Chat/Providers/IrcProvider.cs +++ b/src/Tgstation.Server.Host/Components/Chat/Providers/IrcProvider.cs @@ -20,6 +20,9 @@ namespace Tgstation.Server.Host.Components.Chat.Providers /// sealed class IrcProvider : Provider { + /// + /// Number of seconds used for several IRC related timeouts. + /// const int TimeoutSeconds = 5; /// @@ -149,7 +152,9 @@ namespace Tgstation.Server.Host.Components.Chat.Providers public override async ValueTask DisposeAsync() { await base.DisposeAsync().ConfigureAwait(false); - await HardDisconnect().ConfigureAwait(false); + + // DCT: None available + await HardDisconnect(default).ConfigureAwait(false); } /// @@ -239,6 +244,7 @@ namespace Tgstation.Server.Host.Components.Chat.Providers cancellationToken.ThrowIfCancellationRequested(); + Logger.LogTrace("Authenticating ({0})...", passwordType); switch (passwordType) { case IrcPasswordType.Server: @@ -259,24 +265,50 @@ namespace Tgstation.Server.Host.Components.Chat.Providers } cancellationToken.ThrowIfCancellationRequested(); - client.Listen(false); + Logger.LogTrace("Processing initial messages..."); + await NonBlockingListen(cancellationToken).ConfigureAwait(false); - listenTask = Task.Factory.StartNew(() => + var nickCheckCompleteTcs = new TaskCompletionSource(); + using (cancellationToken.Register(() => nickCheckCompleteTcs.TrySetCanceled())) { - while (!disconnecting && client.IsConnected && client.Nickname != nickname) + listenTask = Task.Factory.StartNew( + async () => { - client.ListenOnce(true); - if (disconnecting || !client.IsConnected) - break; - client.Listen(false); + Logger.LogTrace("Entering nick check loop"); + while (!disconnecting && client.IsConnected && client.Nickname != nickname) + { + client.ListenOnce(true); + if (disconnecting || !client.IsConnected) + break; + await NonBlockingListen(cancellationToken).ConfigureAwait(false); - // ensure we have the correct nick - if (client.GetIrcUser(nickname) == null) - client.RfcNick(nickname); - } + // ensure we have the correct nick + if (client.GetIrcUser(nickname) == null) + client.RfcNick(nickname); + } - client.Listen(); - }, cancellationToken, TaskCreationOptions.LongRunning, TaskScheduler.Current); + nickCheckCompleteTcs.TrySetResult(null); + + Logger.LogTrace("Starting blocking listen..."); + try + { + client.Listen(); + } + catch (Exception ex) + { + Logger.LogWarning("IRC Listen Error: {0}", ex); + } + + Logger.LogTrace("Exiting listening task..."); + }, + cancellationToken, + TaskCreationOptions.LongRunning, + TaskScheduler.Current); + + await nickCheckCompleteTcs.Task.ConfigureAwait(false); + } + + Logger.LogTrace("Connection established!"); } catch (OperationCanceledException) { @@ -288,6 +320,28 @@ namespace Tgstation.Server.Host.Components.Chat.Providers } } + /// + /// Perform a non-blocking . + /// + /// The for the operation. + /// A representing the running operation. + Task NonBlockingListen(CancellationToken cancellationToken) => Task.Factory.StartNew( + () => + { + try + { + client.Listen(false); + } + catch (Exception ex) + { + Logger.LogWarning("IRC Listen Error: {0}", ex); + } + }, + cancellationToken, + TaskCreationOptions.None, + TaskScheduler.Current) + .WithToken(cancellationToken); + /// /// Run SASL authentication on . /// @@ -297,6 +351,8 @@ namespace Tgstation.Server.Host.Components.Chat.Providers { client.WriteLine("CAP REQ :sasl", Priority.Critical); // needs to be put in the buffer before anything else cancellationToken.ThrowIfCancellationRequested(); + + Logger.LogTrace("Logging in..."); client.Login(nickname, nickname, 0, nickname); cancellationToken.ThrowIfCancellationRequested(); @@ -312,69 +368,72 @@ namespace Tgstation.Server.Host.Components.Chat.Providers recievedPlus = true; } + Logger.LogTrace("Performing handshake..."); client.OnReadLine += AuthenticationDelegate; - try { - using (var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken)) - { - timeoutCts.CancelAfter(TimeSpan.FromSeconds(TimeoutSeconds)); - var timeoutToken = timeoutCts.Token; + using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeoutCts.CancelAfter(TimeSpan.FromSeconds(TimeoutSeconds)); + var timeoutToken = timeoutCts.Token; - var listenTimeSpan = TimeSpan.FromMilliseconds(10); - for (; !recievedAck; - await asyncDelayer.Delay(listenTimeSpan, timeoutToken).ConfigureAwait(false)) - client.Listen(false); + var listenTimeSpan = TimeSpan.FromMilliseconds(10); + for (; !recievedAck; + await asyncDelayer.Delay(listenTimeSpan, timeoutToken).ConfigureAwait(false)) + await NonBlockingListen(cancellationToken).ConfigureAwait(false); - client.WriteLine("AUTHENTICATE PLAIN", Priority.Critical); - timeoutToken.ThrowIfCancellationRequested(); + client.WriteLine("AUTHENTICATE PLAIN", Priority.Critical); + timeoutToken.ThrowIfCancellationRequested(); - for (; !recievedPlus; - await asyncDelayer.Delay(listenTimeSpan, timeoutToken).ConfigureAwait(false)) - client.Listen(false); - } - - cancellationToken.ThrowIfCancellationRequested(); - - // Stolen! https://github.com/znc/znc/blob/1e697580155d5a38f8b5a377f3b1d94aaa979539/modules/sasl.cpp#L196 - var authString = String.Format( - CultureInfo.InvariantCulture, - "{0}{1}{0}{1}{2}", - nickname, - '\0', - password); - var b64 = Convert.ToBase64String(Encoding.UTF8.GetBytes(authString)); - var authLine = $"AUTHENTICATE {b64}"; - client.WriteLine(authLine, Priority.Critical); - - cancellationToken.ThrowIfCancellationRequested(); - client.WriteLine("CAP END", Priority.Critical); + for (; !recievedPlus; + await asyncDelayer.Delay(listenTimeSpan, timeoutToken).ConfigureAwait(false)) + await NonBlockingListen(cancellationToken).ConfigureAwait(false); } finally { client.OnReadLine -= AuthenticationDelegate; } + + cancellationToken.ThrowIfCancellationRequested(); + + // Stolen! https://github.com/znc/znc/blob/1e697580155d5a38f8b5a377f3b1d94aaa979539/modules/sasl.cpp#L196 + Logger.LogTrace("Sending credentials..."); + var authString = String.Format( + CultureInfo.InvariantCulture, + "{0}{1}{0}{1}{2}", + nickname, + '\0', + password); + var b64 = Convert.ToBase64String(Encoding.UTF8.GetBytes(authString)); + var authLine = $"AUTHENTICATE {b64}"; + client.WriteLine(authLine, Priority.Critical); + cancellationToken.ThrowIfCancellationRequested(); + + Logger.LogTrace("Finishing authentication..."); + client.WriteLine("CAP END", Priority.Critical); } /// protected override async Task DisconnectImpl(CancellationToken cancellationToken) { - if (!Connected) - return; try { - await Task.Factory.StartNew(() => - { - try + await Task.Factory.StartNew( + () => { - client.RfcQuit("Mr. Stark, I don't feel so good...", Priority.Critical); // priocritical otherwise it wont go through - } - catch (Exception e) - { - Logger.LogWarning("Error quitting IRC: {0}", e); - } - }, cancellationToken, TaskCreationOptions.LongRunning, TaskScheduler.Current).ConfigureAwait(false); - await HardDisconnect().ConfigureAwait(false); + try + { + client.RfcQuit("Mr. Stark, I don't feel so good...", Priority.Critical); // priocritical otherwise it wont go through + } + catch (Exception e) + { + Logger.LogWarning("Error quitting IRC: {0}", e); + } + }, + cancellationToken, + TaskCreationOptions.LongRunning, + TaskScheduler.Current) + .ConfigureAwait(false); + await HardDisconnect(cancellationToken).ConfigureAwait(false); } catch (OperationCanceledException) { @@ -386,16 +445,42 @@ namespace Tgstation.Server.Host.Components.Chat.Providers } } - async Task HardDisconnect() + async Task HardDisconnect(CancellationToken cancellationToken) { if (!Connected) + { + Logger.LogTrace("Not hard disconnecting, already offline"); return; + } + + Logger.LogTrace("Hard disconnect"); disconnecting = true; - client.Disconnect(); - if(listenTask != null) - await listenTask.ConfigureAwait(false); + // This call blocks permanently randomly sometimes + // Frankly I don't give a shit + var disconnectTask = Task.Factory.StartNew( + () => + { + try + { + client.Disconnect(); + } + catch (Exception e) + { + Logger.LogWarning("Error disconnecting IRC: {0}", e); + } + }, + cancellationToken, + TaskCreationOptions.None, + TaskScheduler.Current); + + await Task.WhenAny( + Task.WhenAll( + disconnectTask, + listenTask ?? Task.CompletedTask), + asyncDelayer.Delay(TimeSpan.FromSeconds(TimeoutSeconds), cancellationToken)) + .ConfigureAwait(false); } /// diff --git a/src/Tgstation.Server.Host/Components/Chat/Providers/Provider.cs b/src/Tgstation.Server.Host/Components/Chat/Providers/Provider.cs index ea03fea802..744e68867d 100644 --- a/src/Tgstation.Server.Host/Components/Chat/Providers/Provider.cs +++ b/src/Tgstation.Server.Host/Components/Chat/Providers/Provider.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using Tgstation.Server.Api.Rights; +using Tgstation.Server.Host.Extensions; using Tgstation.Server.Host.Jobs; using Tgstation.Server.Host.Models; @@ -119,8 +120,12 @@ namespace Tgstation.Server.Host.Components.Chat.Providers /// public async Task Disconnect(CancellationToken cancellationToken) { - if(Connected) + if (Connected) + { await DisconnectImpl(cancellationToken).ConfigureAwait(false); + Logger.LogTrace("Disconnected"); + } + await StopReconnectionTimer().ConfigureAwait(false); } @@ -130,16 +135,17 @@ namespace Tgstation.Server.Host.Components.Chat.Providers /// public async Task NextMessage(CancellationToken cancellationToken) { - var cancelTcs = new TaskCompletionSource(); - using (cancellationToken.Register(() => cancelTcs.SetCanceled())) - await Task.WhenAny(nextMessage.Task, cancelTcs.Task).ConfigureAwait(false); - cancellationToken.ThrowIfCancellationRequested(); - lock (messageQueue) + while (true) { - var result = messageQueue.Dequeue(); - if (messageQueue.Count == 0) - nextMessage = new TaskCompletionSource(); - return result; + await nextMessage.Task.WithToken(cancellationToken).ConfigureAwait(false); + lock (messageQueue) + if (messageQueue.Count > 0) + { + var result = messageQueue.Dequeue(); + if (messageQueue.Count == 0) + nextMessage = new TaskCompletionSource(); + return result; + } } } @@ -211,8 +217,17 @@ namespace Tgstation.Server.Host.Components.Chat.Providers job, async (core, databaseContextFactory, paramJob, progressReporter, jobCancellationToken) => { - await DisconnectImpl(jobCancellationToken).ConfigureAwait(false); + if (Connected) + { + Logger.LogTrace("Disconnecting..."); + await DisconnectImpl(jobCancellationToken).ConfigureAwait(false); + } + else + Logger.LogTrace("Already disconnected not doing disconnection attempt!"); + + Logger.LogTrace("Connecting..."); await Connect(jobCancellationToken).ConfigureAwait(false); + Logger.LogTrace("Connected successfully"); EnqueueMessage(null); }, cancellationToken) diff --git a/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs b/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs index 1c63870f1b..1d5c26b26d 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs @@ -1,4 +1,4 @@ -using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; using System; using System.Collections.Generic; @@ -202,26 +202,32 @@ namespace Tgstation.Server.Host.Components.Deployment if (compileJob == null) throw new ArgumentNullException(nameof(compileJob)); - // ensure we have the entire compile job tree + // ensure we have the entire metadata tree logger.LogTrace("Loading compile job {0}...", compileJob.Id); await databaseContextFactory.UseContext( async db => compileJob = await db .CompileJobs .AsQueryable() .Where(x => x.Id == compileJob.Id) - .Include(x => x.Job).ThenInclude(x => x.StartedBy) - .Include(x => x.RevisionInformation).ThenInclude(x => x.PrimaryTestMerge).ThenInclude(x => x.MergedBy) - .Include(x => x.RevisionInformation).ThenInclude(x => x.ActiveTestMerges).ThenInclude(x => x.TestMerge).ThenInclude(x => x.MergedBy) + .Include(x => x.Job) + .ThenInclude(x => x.StartedBy) + .Include(x => x.RevisionInformation) + .ThenInclude(x => x.PrimaryTestMerge) + .ThenInclude(x => x.MergedBy) + .Include(x => x.RevisionInformation) + .ThenInclude(x => x.ActiveTestMerges) + .ThenInclude(x => x.TestMerge) + .ThenInclude(x => x.MergedBy) .FirstAsync(cancellationToken) .ConfigureAwait(false)) .ConfigureAwait(false); // can't wait to see that query if (!compileJob.Job.StoppedAt.HasValue) { - // This happens if we're told to load the compile job that is currently finished up - // It can constitute an API violation if it's returned by the DreamDaemonController so just set it here - // Bit of a hack, but it should work out to be the same value - logger.LogTrace("Setting missing StoppedAt for CompileJob job..."); + // This happens when we're told to load the compile job that is currently finished up + // It constitutes an API violation if it's returned by the DreamDaemonController so just set it here + // Bit of a hack, but it works out to be nearly if not the same value that's put in the DB + logger.LogTrace("Setting missing StoppedAt for CompileJob.Job #{0}...", compileJob.Job.Id); compileJob.Job.StoppedAt = DateTimeOffset.Now; } @@ -284,9 +290,9 @@ namespace Tgstation.Server.Host.Components.Deployment else jobLockCounts[compileJob.Id] = ++value; - logger.LogTrace("Compile job {0} lock count now: {1}", compileJob.Id, value); - providerSubmitted = true; + + logger.LogTrace("Compile job {0} lock count now: {1}", compileJob.Id, value); return newProvider; } } diff --git a/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs b/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs index 89667e14ec..434cb9a781 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs @@ -1,4 +1,4 @@ -using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; using Octokit; using System; @@ -633,6 +633,7 @@ namespace Tgstation.Server.Host.Components.Deployment // The difficulty with compile jobs is they have a two part commit await databaseContext.Save(cancellationToken).ConfigureAwait(false); + logger.LogTrace("Created CompileJob {0}", compileJob.Id); try { await compileJobConsumer.LoadCompileJob(compileJob, cancellationToken).ConfigureAwait(false); diff --git a/src/Tgstation.Server.Host/Components/Instance.cs b/src/Tgstation.Server.Host/Components/Instance.cs index 8826d73929..1553263b9f 100644 --- a/src/Tgstation.Server.Host/Components/Instance.cs +++ b/src/Tgstation.Server.Host/Components/Instance.cs @@ -471,7 +471,7 @@ namespace Tgstation.Server.Host.Components // race condition, just quit if (timerTask != null) { - logger.LogDebug("Aborting auto update interval change due to race condition!"); + logger.LogWarning("Aborting auto update interval change due to race condition!"); return; } diff --git a/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs b/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs index e211b982d2..b682e3682f 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs @@ -1,4 +1,4 @@ -using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging; using System; using System.Globalization; using System.Linq; @@ -116,10 +116,11 @@ namespace Tgstation.Server.Host.Components.Session /// Check if a given can be bound to. /// /// The port number to test. - static void PortBindTest(ushort port) + void PortBindTest(ushort port) { try { + logger.LogTrace("Bind test: {0}", port); SocketExtensions.BindTest(port, false); } catch (SocketException ex) when (ex.SocketErrorCode == SocketError.AddressAlreadyInUse) diff --git a/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs index 9ccd92a28c..810a3dc740 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs @@ -159,12 +159,12 @@ namespace Tgstation.Server.Host.Components.Watchdog protected override async Task DisposeAndNullControllersImpl() { var disposeTask = Server?.DisposeAsync(); + gracefulRebootRequired = false; if (!disposeTask.HasValue) return; await disposeTask.Value.ConfigureAwait(false); Server = null; - gracefulRebootRequired = false; } /// diff --git a/src/Tgstation.Server.Host/Controllers/InstanceController.cs b/src/Tgstation.Server.Host/Controllers/InstanceController.cs index 13a29097d5..87229c6c41 100644 --- a/src/Tgstation.Server.Host/Controllers/InstanceController.cs +++ b/src/Tgstation.Server.Host/Controllers/InstanceController.cs @@ -1,4 +1,4 @@ -using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -56,11 +56,6 @@ namespace Tgstation.Server.Host.Controllers /// readonly IIOManager ioManager; - /// - /// The for the - /// - readonly IAssemblyInformationProvider assemblyInformationProvider; - /// /// The for the /// @@ -79,7 +74,6 @@ namespace Tgstation.Server.Host.Controllers /// The value of /// The value of /// The value of - /// The value of /// The value of /// The containing the value of . /// The for the @@ -89,7 +83,6 @@ namespace Tgstation.Server.Host.Controllers IJobManager jobManager, IInstanceManager instanceManager, IIOManager ioManager, - IAssemblyInformationProvider assemblyInformationProvider, IPlatformIdentifier platformIdentifier, IOptions generalConfigurationOptions, ILogger logger) @@ -102,11 +95,51 @@ namespace Tgstation.Server.Host.Controllers this.jobManager = jobManager ?? throw new ArgumentNullException(nameof(jobManager)); this.instanceManager = instanceManager ?? throw new ArgumentNullException(nameof(instanceManager)); this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); - this.assemblyInformationProvider = assemblyInformationProvider ?? throw new ArgumentNullException(nameof(assemblyInformationProvider)); this.platformIdentifier = platformIdentifier ?? throw new ArgumentNullException(nameof(platformIdentifier)); generalConfiguration = generalConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(generalConfigurationOptions)); } + Models.Instance CreateDefaultInstance(Api.Models.Instance initialSettings) + => new Models.Instance + { + ConfigurationType = initialSettings.ConfigurationType ?? ConfigurationType.Disallowed, + DreamDaemonSettings = new DreamDaemonSettings + { + AllowWebClient = false, + AutoStart = false, + Port = 1337, + SecurityLevel = DreamDaemonSecurity.Safe, + StartupTimeout = 60, + HeartbeatSeconds = 60, + TopicRequestTimeout = generalConfiguration.ByondTopicTimeout + }, + DreamMakerSettings = new DreamMakerSettings + { + ApiValidationPort = 1339, + ApiValidationSecurityLevel = DreamDaemonSecurity.Safe, + RequireDMApiValidation = true + }, + Name = initialSettings.Name, + Online = false, + Path = initialSettings.Path, + AutoUpdateInterval = initialSettings.AutoUpdateInterval ?? 0, + ChatBotLimit = initialSettings.ChatBotLimit ?? Models.Instance.DefaultChatBotLimit, + RepositorySettings = new RepositorySettings + { + CommitterEmail = "tgstation-server@users.noreply.github.com", + CommitterName = "tgstation-server", + PushTestMergeCommits = false, + ShowTestMergeCommitters = false, + AutoUpdatesKeepTestMerges = false, + AutoUpdatesSynchronize = false, + PostTestMergeComment = false + }, + InstanceUsers = new List // give this user full privileges on the instance + { + InstanceAdminUser(null) + } + }; + string NormalizePath(string path) { if (path == null) @@ -243,45 +276,7 @@ namespace Tgstation.Server.Host.Controllers else attached = true; - var newInstance = new Models.Instance - { - ConfigurationType = model.ConfigurationType ?? ConfigurationType.Disallowed, - DreamDaemonSettings = new DreamDaemonSettings - { - AllowWebClient = false, - AutoStart = false, - Port = 1337, - SecurityLevel = DreamDaemonSecurity.Safe, - StartupTimeout = 60, - HeartbeatSeconds = 60, - TopicRequestTimeout = generalConfiguration.ByondTopicTimeout - }, - DreamMakerSettings = new DreamMakerSettings - { - ApiValidationPort = 1339, - ApiValidationSecurityLevel = DreamDaemonSecurity.Safe, - RequireDMApiValidation = true - }, - Name = model.Name, - Online = false, - Path = model.Path, - AutoUpdateInterval = model.AutoUpdateInterval ?? 0, - ChatBotLimit = model.ChatBotLimit ?? Models.Instance.DefaultChatBotLimit, - RepositorySettings = new RepositorySettings - { - CommitterEmail = "tgstation-server@users.noreply.github.com", - CommitterName = assemblyInformationProvider.VersionPrefix, - PushTestMergeCommits = false, - ShowTestMergeCommitters = false, - AutoUpdatesKeepTestMerges = false, - AutoUpdatesSynchronize = false, - PostTestMergeComment = false - }, - InstanceUsers = new List // give this user full privileges on the instance - { - InstanceAdminUser(null) - } - }; + var newInstance = CreateDefaultInstance(model); DatabaseContext.Instances.Add(newInstance); try diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index 5093e1083c..c1f5852419 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -1,4 +1,4 @@ -using Cyberboss.AspNetCore.AsyncInitializer; +using Cyberboss.AspNetCore.AsyncInitializer; using Microsoft.AspNetCore.Authentication.JwtBearer; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Cors.Infrastructure; @@ -98,6 +98,10 @@ namespace Tgstation.Server.Host.Core // enable options which give us config reloading services.AddOptions(); + // Set the timeout for IHostedService.StopAsync + services.Configure( + opts => opts.ShutdownTimeout = TimeSpan.FromMilliseconds(postSetupServices.GeneralConfiguration.RestartTimeout)); + static LogEventLevel? ConvertSeriLogLevel(LogLevel logLevel) => logLevel switch { @@ -441,4 +445,4 @@ namespace Tgstation.Server.Host.Core // 404 anything that gets this far } } -} \ No newline at end of file +} diff --git a/src/Tgstation.Server.Host/Database/Migrations/20200422010115_MYAddCompileJobDMApiVersion.cs b/src/Tgstation.Server.Host/Database/Migrations/20200422010115_MYAddCompileJobDMApiVersion.cs index 16972b87d9..cb14d3256e 100644 --- a/src/Tgstation.Server.Host/Database/Migrations/20200422010115_MYAddCompileJobDMApiVersion.cs +++ b/src/Tgstation.Server.Host/Database/Migrations/20200422010115_MYAddCompileJobDMApiVersion.cs @@ -1,4 +1,4 @@ -using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Migrations; using System; namespace Tgstation.Server.Host.Database.Migrations @@ -55,8 +55,7 @@ namespace Tgstation.Server.Host.Database.Migrations migrationBuilder.AddColumn( name: "ServerCommandsJson", table: "ReattachInformations", - nullable: false, - defaultValue: "server_commands.tgs.json"); + nullable: false); } } } diff --git a/src/Tgstation.Server.Host/Extensions/ServiceCollectionExtensions.cs b/src/Tgstation.Server.Host/Extensions/ServiceCollectionExtensions.cs index b6c6e9e5d1..e98ff88694 100644 --- a/src/Tgstation.Server.Host/Extensions/ServiceCollectionExtensions.cs +++ b/src/Tgstation.Server.Host/Extensions/ServiceCollectionExtensions.cs @@ -1,4 +1,4 @@ -using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Serilog; diff --git a/src/Tgstation.Server.Host/Jobs/JobManager.cs b/src/Tgstation.Server.Host/Jobs/JobManager.cs index d59736c8c9..2442c0d87e 100644 --- a/src/Tgstation.Server.Host/Jobs/JobManager.cs +++ b/src/Tgstation.Server.Host/Jobs/JobManager.cs @@ -111,6 +111,7 @@ namespace Tgstation.Server.Host.Jobs await activationTcs.Task.WithToken(cancellationToken).ConfigureAwait(false); + logger.LogTrace("Starting job..."); await operation( instanceCoreProvider.Value.GetInstance(oldJob.Instance), databaseContextFactory, @@ -201,7 +202,7 @@ namespace Tgstation.Server.Host.Jobs await databaseContext.Save(cancellationToken).ConfigureAwait(false); - logger.LogDebug("Starting job {0}: {1}...", job.Id, job.Description); + logger.LogDebug("Registering job {0}: {1}...", job.Id, job.Description); var jobHandler = new JobHandler(jobCancellationToken => RunJob(job, operation, jobCancellationToken)); try { @@ -283,15 +284,12 @@ namespace Tgstation.Server.Host.Jobs await databaseContextFactory.UseContext(async databaseContext => { if (user == null) - { user = await databaseContext.Users.GetTgsUser(cancellationToken).ConfigureAwait(false); - databaseContext.Users.Attach(user); - } var updatedJob = new Job { Id = job.Id }; - databaseContext.Jobs.Attach(job); + databaseContext.Jobs.Attach(updatedJob); var attachedUser = new User { Id = user.Id }; - databaseContext.Users.Attach(user); + databaseContext.Users.Attach(attachedUser); updatedJob.CancelledBy = attachedUser; // let either startup or cancellation set job.cancelled diff --git a/src/Tgstation.Server.Host/Server.cs b/src/Tgstation.Server.Host/Server.cs index 2d5dc1bd80..d62ecccc87 100644 --- a/src/Tgstation.Server.Host/Server.cs +++ b/src/Tgstation.Server.Host/Server.cs @@ -1,4 +1,4 @@ -using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -290,29 +290,29 @@ namespace Tgstation.Server.Host } if (exception == null) - using (var cts = new CancellationTokenSource()) - { - logger.LogInformation("Restarting server..."); - var cancellationToken = cts.Token; - var eventsTask = Task.WhenAll(restartHandlers.Select(x => x.HandleRestart(newVersion, cancellationToken)).ToList()); + { + logger.LogInformation("Restarting server..."); + using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(generalConfiguration.RestartTimeout)); + var cancellationToken = cts.Token; + var eventsTask = Task.WhenAll( + restartHandlers.Select( + x => x.HandleRestart(newVersion, cancellationToken)) + .ToList()); - var expiryTask = Task.Delay(TimeSpan.FromMilliseconds(generalConfiguration.RestartTimeout)); - await Task.WhenAny(eventsTask, expiryTask).ConfigureAwait(false); - logger.LogTrace("Joining restart handlers..."); - cts.Cancel(); - try - { - await eventsTask.ConfigureAwait(false); - } - catch (OperationCanceledException) - { - logger.LogError("Restart timeout hit! Existing DreamDaemon processes will be lost and must be killed manually before being restarted with TGS!"); - } - catch (Exception e) - { - logger.LogError("Restart handlers error! Exception: {0}", e); - } + logger.LogTrace("Joining restart handlers..."); + try + { + await eventsTask.ConfigureAwait(false); } + catch (OperationCanceledException) + { + logger.LogError("Restart timeout hit! Existing DreamDaemon processes will be lost and must be killed manually before being restarted with TGS!"); + } + catch (Exception e) + { + logger.LogError("Restart handlers error! Exception: {0}", e); + } + } logger.LogTrace("Stopping host..."); cancellationTokenSource.Cancel(); diff --git a/src/Tgstation.Server.Host/System/Process.cs b/src/Tgstation.Server.Host/System/Process.cs index 5f2c3cf5a6..1800b77563 100644 --- a/src/Tgstation.Server.Host/System/Process.cs +++ b/src/Tgstation.Server.Host/System/Process.cs @@ -1,4 +1,4 @@ -using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging; using System; using System.Diagnostics; using System.Text; @@ -10,6 +10,11 @@ namespace Tgstation.Server.Host.System /// sealed class Process : IProcess { + /// + /// Maximum time to wait in a call to . + /// + const int MaximumWaitMilliseconds = 30000; + /// public int Id { get; } @@ -31,6 +36,11 @@ namespace Tgstation.Server.Host.System readonly global::System.Diagnostics.Process handle; + /// + /// A so that we can complete if the becomes unresponsive. + /// + readonly TaskCompletionSource emergencyLifetimeTcs; + readonly StringBuilder outputStringBuilder; readonly StringBuilder errorStringBuilder; readonly StringBuilder combinedStringBuilder; @@ -65,6 +75,7 @@ namespace Tgstation.Server.Host.System this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + emergencyLifetimeTcs = new TaskCompletionSource(); Lifetime = WrapLifetimeTask(lifetime ?? throw new ArgumentNullException(nameof(lifetime))); Id = handle.Id; @@ -92,9 +103,12 @@ namespace Tgstation.Server.Host.System async Task WrapLifetimeTask(Task lifetimeTask) { - var result = await lifetimeTask.ConfigureAwait(false); - logger.LogTrace("PID {0} ended with code {1}", Id, result); - return result; + await Task.WhenAny(lifetimeTask, emergencyLifetimeTcs.Task).ConfigureAwait(false); + if (lifetimeTask.IsCompleted) + return await lifetimeTask.ConfigureAwait(false); + + logger.LogTrace("Using exit code -1 for hung PID {0}.", Id); + return -1; } /// @@ -130,7 +144,15 @@ namespace Tgstation.Server.Host.System { logger.LogTrace("Terminating PID {0}...", Id); handle.Kill(); - handle.WaitForExit(); + if (!handle.WaitForExit(MaximumWaitMilliseconds)) + { + logger.LogError( + "PID {0} hasn't exited in {1} seconds! This may cause issues with port reuse.", + Id, + TimeSpan.FromMilliseconds(MaximumWaitMilliseconds).TotalSeconds); + + emergencyLifetimeTcs.TrySetResult(null); + } } catch (Exception e) { diff --git a/src/Tgstation.Server.Host/System/ProcessExecutor.cs b/src/Tgstation.Server.Host/System/ProcessExecutor.cs index 9284a0703e..8abefc2cc7 100644 --- a/src/Tgstation.Server.Host/System/ProcessExecutor.cs +++ b/src/Tgstation.Server.Host/System/ProcessExecutor.cs @@ -1,4 +1,4 @@ -using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging; using System; using System.Text; using System.Threading.Tasks; @@ -28,7 +28,7 @@ namespace Tgstation.Server.Host.System /// /// The to attach the for /// A new resulting in the exit code of - static Task AttachExitHandler(global::System.Diagnostics.Process handle) + Task AttachExitHandler(global::System.Diagnostics.Process handle) { handle.EnableRaisingEvents = true; var tcs = new TaskCompletionSource(); @@ -45,7 +45,8 @@ namespace Tgstation.Server.Host.System } // Try because this can be invoked twice for weird reasons - tcs.TrySetResult(exitCode); + if (tcs.TrySetResult(exitCode)) + logger.LogTrace("Process exit event completed"); }; return tcs.Task; diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index 0222a5eb92..ba065d5aa6 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -1,4 +1,4 @@ - + @@ -69,7 +69,7 @@ - + diff --git a/tests/DMAPI/BasicOperation/Test.dm b/tests/DMAPI/BasicOperation/Test.dm index 066c9eceac..0fc07d72f9 100644 --- a/tests/DMAPI/BasicOperation/Test.dm +++ b/tests/DMAPI/BasicOperation/Test.dm @@ -21,7 +21,7 @@ text2file("DMAPI version [TGS_DMAPI_VERSION] does not match active API version [active_version.raw_parameter]", "test_fail_reason.txt") world.log << "sleep2" - sleep(50) + sleep(150) world.log << "Terminating..." world.TgsEndProcess() diff --git a/tests/Tgstation.Server.Tests/Instance/DeploymentTest.cs b/tests/Tgstation.Server.Tests/Instance/DeploymentTest.cs index 7772e32343..578dffa9a5 100644 --- a/tests/Tgstation.Server.Tests/Instance/DeploymentTest.cs +++ b/tests/Tgstation.Server.Tests/Instance/DeploymentTest.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Threading; using System.Threading.Tasks; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -36,7 +36,8 @@ namespace Tgstation.Server.Tests.Instance if (!new PlatformIdentifier().IsWindows) await dreamMakerClient.Update(new DreamMaker { - ProjectName = "tests/DMAPI/ApiFree/api_free" + ProjectName = "tests/DMAPI/ApiFree/api_free", + ApiValidationPort = IntegrationTest.DMPort }, cancellationToken); var updatedDD = await dreamDaemonClient.Update(new DreamDaemon diff --git a/tests/Tgstation.Server.Tests/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Instance/WatchdogTest.cs index 9ae2620165..4b2e5e817d 100644 --- a/tests/Tgstation.Server.Tests/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Instance/WatchdogTest.cs @@ -36,7 +36,8 @@ namespace Tgstation.Server.Tests.Instance var initialSettings = await instanceClient.DreamDaemon.Update(new DreamDaemon { StartupTimeout = 30, - HeartbeatSeconds = 0 + HeartbeatSeconds = 0, + Port = IntegrationTest.DDPort }, cancellationToken); await ApiAssert.ThrowsException(() => instanceClient.DreamDaemon.Update(new DreamDaemon @@ -87,7 +88,7 @@ namespace Tgstation.Server.Tests.Instance KillDD(false); var job = await WaitForJob(await dumpTask, 10, true, null, cancellationToken); Assert.IsTrue(job.ErrorCode == ErrorCode.DreamDaemonOffline || job.ErrorCode == ErrorCode.GCoreFailure); - await Task.Delay(TimeSpan.FromSeconds(10), cancellationToken); + await Task.Delay(TimeSpan.FromSeconds(20), cancellationToken); var ddStatus = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.AreEqual(WatchdogStatus.Online, ddStatus.Status.Value); @@ -145,7 +146,7 @@ namespace Tgstation.Server.Tests.Instance { blockSocket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ExclusiveAddressUse, true); blockSocket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, false); - blockSocket.Bind(new IPEndPoint(IPAddress.Any, 1337)); + blockSocket.Bind(new IPEndPoint(IPAddress.Any, IntegrationTest.DDPort)); startJob = await instanceClient.DreamDaemon.Start(cancellationToken).ConfigureAwait(false); await WaitForJob(startJob, 20, true, ErrorCode.DreamDaemonPortInUse, cancellationToken); @@ -160,7 +161,7 @@ namespace Tgstation.Server.Tests.Instance Assert.AreEqual(false, daemonStatus.SoftRestart); Assert.AreEqual(false, daemonStatus.SoftShutdown); - await GracefulWatchdogShutdown(30, cancellationToken); + await GracefulWatchdogShutdown(60, cancellationToken); daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.AreEqual(WatchdogStatus.Offline, daemonStatus.Status.Value); @@ -170,7 +171,7 @@ namespace Tgstation.Server.Tests.Instance async Task RunHeartbeatTest(CancellationToken cancellationToken) { - global::System.Console.WriteLine("TEST: WATCHDOG HEARTBEAT TEST"); + System.Console.WriteLine("TEST: WATCHDOG HEARTBEAT TEST"); // enable heartbeats await instanceClient.DreamDaemon.Update(new DreamDaemon { @@ -210,7 +211,7 @@ namespace Tgstation.Server.Tests.Instance await Task.WhenAny(ourProcessHandler.Lifetime, Task.Delay(TimeSpan.FromSeconds(20))); - var timeout = 10; + var timeout = 20; do { await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken).ConfigureAwait(false); @@ -260,9 +261,8 @@ namespace Tgstation.Server.Tests.Instance Assert.AreNotEqual(initialCompileJob.Id, newerCompileJob.Id); Assert.AreEqual(DreamDaemonSecurity.Ultrasafe, newerCompileJob.MinimumSecurityLevel); - await TellWorldToReboot(cancellationToken); + daemonStatus = await TellWorldToReboot(cancellationToken); - daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.AreNotEqual(initialCompileJob.Id, daemonStatus.ActiveCompileJob.Id); Assert.IsNull(daemonStatus.StagedCompileJob); @@ -301,9 +301,8 @@ namespace Tgstation.Server.Tests.Instance Assert.AreNotEqual(initialCompileJob.Id, newerCompileJob.Id); Assert.AreEqual(DreamDaemonSecurity.Ultrasafe, newerCompileJob.MinimumSecurityLevel); - await TellWorldToReboot(cancellationToken); + daemonStatus = await TellWorldToReboot(cancellationToken); - daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.AreNotEqual(initialCompileJob.Id, daemonStatus.ActiveCompileJob.Id); Assert.IsNull(daemonStatus.StagedCompileJob); @@ -354,9 +353,8 @@ namespace Tgstation.Server.Tests.Instance Assert.AreEqual(true, daemonStatus.SoftRestart); - await TellWorldToReboot(cancellationToken); + daemonStatus = await TellWorldToReboot(cancellationToken); - daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.AreEqual(versionToInstall, daemonStatus.ActiveCompileJob.ByondVersion); Assert.IsNull(daemonStatus.StagedCompileJob); @@ -379,6 +377,7 @@ namespace Tgstation.Server.Tests.Instance var daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value); + Assert.AreEqual(IntegrationTest.DDPort, daemonStatus.CurrentPort); // The measure we use to test dream daemon startup doesn't work on linux currently if (new PlatformIdentifier().IsWindows) @@ -422,28 +421,45 @@ namespace Tgstation.Server.Tests.Instance return ddProc != null; } - async Task TellWorldToReboot(CancellationToken cancellationToken) + async Task TellWorldToReboot(CancellationToken cancellationToken) { + var daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); + var initialCompileJob = daemonStatus.ActiveCompileJob; + var bts = new TopicClient(new SocketParameters { - SendTimeout = TimeSpan.FromSeconds(15), - ReceiveTimeout = TimeSpan.FromSeconds(15), - ConnectTimeout = TimeSpan.FromSeconds(15), - DisconnectTimeout = TimeSpan.FromSeconds(15) + SendTimeout = TimeSpan.FromSeconds(30), + ReceiveTimeout = TimeSpan.FromSeconds(30), + ConnectTimeout = TimeSpan.FromSeconds(30), + DisconnectTimeout = TimeSpan.FromSeconds(30) }); try { - global::System.Console.WriteLine("TEST: Sending world reboot topic..."); - var result = await bts.SendTopic(IPAddress.Loopback, "tgs_integration_test_special_tactics=1", 1337, cancellationToken); + System.Console.WriteLine("TEST: Sending world reboot topic..."); + var result = await bts.SendTopic(IPAddress.Loopback, "tgs_integration_test_special_tactics=1", IntegrationTest.DDPort, cancellationToken); Assert.AreEqual("ack", result.StringData); - await Task.Delay(20000, cancellationToken); + using (var tempCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken)) + using (tempCts.Token.Register(() => System.Console.WriteLine("TEST ERROR: Timeout in TellWorldToReboot!"))) + { + tempCts.CancelAfter(TimeSpan.FromMinutes(1)); + var tempToken = tempCts.Token; + + do + { + await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken).ConfigureAwait(false); + daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); + } + while (initialCompileJob.Id == daemonStatus.ActiveCompileJob.Id && !tempToken.IsCancellationRequested); + } } catch (OperationCanceledException) { throw; } + + return daemonStatus; } async Task DeployTestDme(string dmeName, DreamDaemonSecurity deploymentSecurity, bool requireApi, CancellationToken cancellationToken) diff --git a/tests/Tgstation.Server.Tests/IntegrationTest.cs b/tests/Tgstation.Server.Tests/IntegrationTest.cs index 77fc00eae6..496fa12d7a 100644 --- a/tests/Tgstation.Server.Tests/IntegrationTest.cs +++ b/tests/Tgstation.Server.Tests/IntegrationTest.cs @@ -199,7 +199,11 @@ namespace Tgstation.Server.Tests } using var server = new TestingServer(); - using var serverCts = new CancellationTokenSource(); + + const int MaximumTestMinutes = 20; + using var hardTimeoutCancellationTokenSource = new CancellationTokenSource(TimeSpan.FromMinutes(MaximumTestMinutes)); + var hardCancellationToken = hardTimeoutCancellationTokenSource.Token; + using var serverCts = CancellationTokenSource.CreateLinkedTokenSource(hardCancellationToken); var cancellationToken = serverCts.Token; TerminateAllDDs(); @@ -259,8 +263,9 @@ namespace Tgstation.Server.Tests Assert.IsTrue(serverTask.IsCompleted); // http bind test https://github.com/tgstation/tgstation-server/issues/1065 - using (var blockingSocket = new Socket(SocketType.Stream, ProtocolType.Tcp)) + if (new PlatformIdentifier().IsWindows) { + using var blockingSocket = new Socket(SocketType.Stream, ProtocolType.Tcp); blockingSocket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ExclusiveAddressUse, true); blockingSocket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, false); blockingSocket.Bind(new IPEndPoint(IPAddress.Any, server.Url.Port)); @@ -343,12 +348,12 @@ namespace Tgstation.Server.Tests .ToList(); } - Assert.AreEqual(1, jobs.Count); - - var launchJob = jobs.Single(); - Assert.IsTrue(launchJob.StartedAt.Value >= preStartupTime); - - await new JobsRequiredTest(instanceClient.Jobs).WaitForJob(launchJob, 40, false, null, cancellationToken); + var jrt = new JobsRequiredTest(instanceClient.Jobs); + foreach (var job in jobs) + { + Assert.IsTrue(job.StartedAt.Value >= preStartupTime); + await jrt.WaitForJob(job, 40, false, null, cancellationToken); + } var dd = await instanceClient.DreamDaemon.Read(cancellationToken); @@ -374,12 +379,9 @@ namespace Tgstation.Server.Tests finally { serverCts.Cancel(); - - // Give the test 1 minute to cleanup - using var hardTimeoutCancellationTokenSource = new CancellationTokenSource(TimeSpan.FromMinutes(1)); try { - await serverTask.WithToken(hardTimeoutCancellationTokenSource.Token).ConfigureAwait(false); + await serverTask.WithToken(hardCancellationToken).ConfigureAwait(false); } catch (OperationCanceledException) { } @@ -390,6 +392,33 @@ namespace Tgstation.Server.Tests await serverTask; } + public static ushort DDPort = FreeTcpPort(); + public static ushort DMPort = GetDMPort(); + + static ushort GetDMPort() + { + ushort result; + do + { + result = FreeTcpPort(); + } while (result == DDPort); + return result; + } + + static ushort FreeTcpPort() + { + var l = new TcpListener(IPAddress.Loopback, 0); + l.Start(); + try + { + return (ushort)((IPEndPoint)l.LocalEndpoint).Port; + } + finally + { + l.Stop(); + } + } + [TestMethod] public async Task TestScriptExecution() {