From 300f5d994a59b370406d4b8521c17da0c8d279ab Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Wed, 13 May 2020 18:02:59 -0400 Subject: [PATCH 01/41] Improve a query --- .../Components/Deployment/DmbFactory.cs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs b/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs index fd368e5ef3..685ff6a4bc 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs @@ -260,7 +260,13 @@ namespace Tgstation.Server.Host.Components.Deployment // find the uids of locked directories await databaseContextFactory.UseContext(async db => { - jobUidsToNotErase = await db.CompileJobs.Where(x => x.Job.Instance.Id == instance.Id && jobIdsToSkip.Contains(x.Id)).Select(x => x.DirectoryName.Value.ToString().ToUpperInvariant()).ToListAsync(cancellationToken).ConfigureAwait(false); + jobUidsToNotErase = (await db.CompileJobs.Where( + x => x.Job.Instance.Id == instance.Id && jobIdsToSkip.Contains(x.Id)) + .Select(x => x.DirectoryName.Value) + .ToListAsync(cancellationToken) + .ConfigureAwait(false)) + .Select(x => x.ToString()) + .ToList(); }).ConfigureAwait(false); // add the other exemption From 1bba9a26741f236fac0f477908aa77bbbf646fb7 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Wed, 13 May 2020 18:06:53 -0400 Subject: [PATCH 02/41] Add logging --- src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs b/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs index 685ff6a4bc..966ee57bc7 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs @@ -265,7 +265,7 @@ namespace Tgstation.Server.Host.Components.Deployment .Select(x => x.DirectoryName.Value) .ToListAsync(cancellationToken) .ConfigureAwait(false)) - .Select(x => x.ToString()) + .Select(x => x.ToString().ToUpperInvariant()) .ToList(); }).ConfigureAwait(false); @@ -273,6 +273,8 @@ namespace Tgstation.Server.Host.Components.Deployment if (exceptThisOne != null) jobUidsToNotErase.Add(exceptThisOne.DirectoryName.Value.ToString().ToUpperInvariant()); + logger.LogTrace("We will not clean the following directories: {0}", String.Join(", ", jobUidsToNotErase)); + // cleanup var gameDirectory = ioManager.ResolvePath(); await ioManager.CreateDirectory(gameDirectory, cancellationToken).ConfigureAwait(false); From d6be521865a39df297ebb0bb7cfb5e8619904035 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Wed, 13 May 2020 18:07:37 -0400 Subject: [PATCH 03/41] Add a MISSING AWAIT WTF --- src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs b/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs index 966ee57bc7..e485015cca 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs @@ -302,7 +302,7 @@ namespace Tgstation.Server.Host.Components.Deployment if (deleting > 0) { logger.LogDebug("Cleaning unused game folders: {0}...", String.Join(", ", directories)); - await Task.WhenAll().ConfigureAwait(false); + await Task.WhenAll(tasks).ConfigureAwait(false); } } #pragma warning restore CA1506 From e3096ef6ca6cff9392e54b2ab1ef712797a38731 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Wed, 13 May 2020 18:13:55 -0400 Subject: [PATCH 04/41] Correct an error message description --- src/Tgstation.Server.Api/Models/ErrorCode.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Api/Models/ErrorCode.cs b/src/Tgstation.Server.Api/Models/ErrorCode.cs index 19c999d4f2..9de878067c 100644 --- a/src/Tgstation.Server.Api/Models/ErrorCode.cs +++ b/src/Tgstation.Server.Api/Models/ErrorCode.cs @@ -415,7 +415,7 @@ namespace Tgstation.Server.Api.Models /// /// Attempted to start the watchdog with a corrupted . /// - [Description("Cannot launch with active compile job as it is corrupted!")] + [Description("Cannot launch active compile job as it is missing or corrupted!")] WatchdogCompileJobCorrupted, /// From 4c884c21605c9057381f9fd7bb1c0bff0d997c1e Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 14 May 2020 12:19:57 -0400 Subject: [PATCH 05/41] Add shell script tests --- .../Tgstation.Server.Tests/IntegrationTest.cs | 28 ++++++++++++++++--- .../Tgstation.Server.Tests.csproj | 9 ++++++ tests/Tgstation.Server.Tests/test.bat | 3 ++ tests/Tgstation.Server.Tests/test.sh | 3 ++ 4 files changed, 39 insertions(+), 4 deletions(-) create mode 100644 tests/Tgstation.Server.Tests/test.bat create mode 100755 tests/Tgstation.Server.Tests/test.sh diff --git a/tests/Tgstation.Server.Tests/IntegrationTest.cs b/tests/Tgstation.Server.Tests/IntegrationTest.cs index 2b0e580be7..7258ca32f3 100644 --- a/tests/Tgstation.Server.Tests/IntegrationTest.cs +++ b/tests/Tgstation.Server.Tests/IntegrationTest.cs @@ -16,7 +16,8 @@ using Tgstation.Server.Api; using Tgstation.Server.Api.Models; using Tgstation.Server.Client; using Tgstation.Server.Host; -using Tgstation.Server.Host.Components.Chat.Providers; +using Tgstation.Server.Host.Extensions; +using Tgstation.Server.Host.System; using Tgstation.Server.Tests.Instance; namespace Tgstation.Server.Tests @@ -28,7 +29,7 @@ namespace Tgstation.Server.Tests readonly IServerClientFactory clientFactory = new ServerClientFactory(new ProductHeaderValue(Assembly.GetExecutingAssembly().GetName().Name, Assembly.GetExecutingAssembly().GetName().Version.ToString())); [TestMethod] - public async Task TestServerUpdate() + public async Task TestUpdateProtocol() { using var server = new TestingServer(); @@ -101,13 +102,13 @@ namespace Tgstation.Server.Tests static void TerminateAllDDs() { - foreach (var proc in Process.GetProcessesByName("DreamDaemon")) + foreach (var proc in System.Diagnostics.Process.GetProcessesByName("DreamDaemon")) using (proc) proc.Kill(); } [TestMethod] - public async Task TestFullStandardOperation() + public async Task TestServer() { using var server = new TestingServer(); using var serverCts = new CancellationTokenSource(); @@ -262,5 +263,24 @@ namespace Tgstation.Server.Tests TerminateAllDDs(); } } + + [TestMethod] + public async Task TestScriptExecution() + { + var platformIdentifier = new PlatformIdentifier(); + var processExecutor = new ProcessExecutor( + Mock.Of(), + Mock.Of>(), + LoggerFactory.Create(x => { })); + + using var process = processExecutor.LaunchProcess("test." + platformIdentifier.ScriptFileExtension, ".", String.Empty, true, true, true); + using var cts = new CancellationTokenSource(); + //cts.CancelAfter(3000); + var exitCode = await process.Lifetime.WithToken(cts.Token); + + Assert.AreEqual(0, exitCode); + Assert.AreEqual(String.Empty, process.GetErrorOutput().Trim()); + Assert.AreEqual("Hello World!", process.GetStandardOutput().Trim()); + } } } diff --git a/tests/Tgstation.Server.Tests/Tgstation.Server.Tests.csproj b/tests/Tgstation.Server.Tests/Tgstation.Server.Tests.csproj index 67ce0f1a7d..48036e1b73 100644 --- a/tests/Tgstation.Server.Tests/Tgstation.Server.Tests.csproj +++ b/tests/Tgstation.Server.Tests/Tgstation.Server.Tests.csproj @@ -26,4 +26,13 @@ + + + Always + + + Always + + + diff --git a/tests/Tgstation.Server.Tests/test.bat b/tests/Tgstation.Server.Tests/test.bat new file mode 100644 index 0000000000..7ac29630a7 --- /dev/null +++ b/tests/Tgstation.Server.Tests/test.bat @@ -0,0 +1,3 @@ +@echo off + +echo Hello World! diff --git a/tests/Tgstation.Server.Tests/test.sh b/tests/Tgstation.Server.Tests/test.sh new file mode 100755 index 0000000000..982e2cd73f --- /dev/null +++ b/tests/Tgstation.Server.Tests/test.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +echo Hello World! From 5950a84434dbc65fdb6e07b557aa13de5ed1e1e8 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 14 May 2020 12:20:13 -0400 Subject: [PATCH 06/41] Fix IProcess.GetStandardOutput() --- src/Tgstation.Server.Host/System/Process.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/System/Process.cs b/src/Tgstation.Server.Host/System/Process.cs index 07b8d5fbe3..a722179341 100644 --- a/src/Tgstation.Server.Host/System/Process.cs +++ b/src/Tgstation.Server.Host/System/Process.cs @@ -117,7 +117,7 @@ namespace Tgstation.Server.Host.System { if (outputStringBuilder == null) throw new InvalidOperationException("Output reading was not enabled!"); - return errorStringBuilder.ToString().TrimStart(Environment.NewLine.ToCharArray()); + return outputStringBuilder.ToString().TrimStart(Environment.NewLine.ToCharArray()); } /// From cf09941d1370333ee46a809cfd8e1ce59e6e7524 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 14 May 2020 12:21:33 -0400 Subject: [PATCH 07/41] ProcessExecutor cleanup - Lifetime will not complete until out/err streams finish reading - Added warning when requesting read support with !noShellExecute --- .../System/ProcessExecutor.cs | 42 ++++++++++++++++++- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/src/Tgstation.Server.Host/System/ProcessExecutor.cs b/src/Tgstation.Server.Host/System/ProcessExecutor.cs index 9e66aedf85..79363f0fe7 100644 --- a/src/Tgstation.Server.Host/System/ProcessExecutor.cs +++ b/src/Tgstation.Server.Host/System/ProcessExecutor.cs @@ -104,7 +104,14 @@ namespace Tgstation.Server.Host.System /// public IProcess LaunchProcess(string fileName, string workingDirectory, string arguments, bool readOutput, bool readError, bool noShellExecute) { - logger.LogDebug("Launching process in {0}: {1} {2}", workingDirectory, fileName, arguments); + if (!noShellExecute && (readOutput || readError)) + { + logger.LogWarning("CODE ERROR: Requesting output/error reading requires noShellExecute to be true! Setting it now..."); + + noShellExecute = true; + } + + logger.LogDebug("{3}aunching process in {0}: {1} {2}", workingDirectory, fileName, arguments, noShellExecute ? "L" : "Shell l"); var handle = new global::System.Diagnostics.Process(); try { @@ -112,9 +119,12 @@ namespace Tgstation.Server.Host.System handle.StartInfo.Arguments = arguments; handle.StartInfo.WorkingDirectory = workingDirectory; - handle.StartInfo.UseShellExecute = !(noShellExecute || readOutput || readError); + handle.StartInfo.UseShellExecute = !noShellExecute; StringBuilder outputStringBuilder = null, errorStringBuilder = null, combinedStringBuilder = null; + + TaskCompletionSource outputReadTcs = null; + TaskCompletionSource errorReadTcs = null; if (readOutput || readError) { combinedStringBuilder = new StringBuilder(); @@ -122,8 +132,15 @@ namespace Tgstation.Server.Host.System { outputStringBuilder = new StringBuilder(); handle.StartInfo.RedirectStandardOutput = true; + outputReadTcs = new TaskCompletionSource(); handle.OutputDataReceived += (sender, e) => { + if (e.Data == null) + { + outputReadTcs.SetResult(null); + return; + } + combinedStringBuilder.Append(Environment.NewLine); combinedStringBuilder.Append(e.Data); outputStringBuilder.Append(Environment.NewLine); @@ -135,8 +152,15 @@ namespace Tgstation.Server.Host.System { errorStringBuilder = new StringBuilder(); handle.StartInfo.RedirectStandardError = true; + errorReadTcs = new TaskCompletionSource(); handle.ErrorDataReceived += (sender, e) => { + if (e.Data == null) + { + errorReadTcs.SetResult(null); + return; + } + combinedStringBuilder.Append(Environment.NewLine); combinedStringBuilder.Append(e.Data); errorStringBuilder.Append(Environment.NewLine); @@ -148,16 +172,30 @@ namespace Tgstation.Server.Host.System var lifetimeTask = AttachExitHandler(handle); handle.Start(); + + static async Task AddToLifetimeTask(Task originalTask, TaskCompletionSource tcs) + { + var exitCode = await originalTask.ConfigureAwait(false); + await tcs.Task.ConfigureAwait(false); + return exitCode; + } + try { if (readOutput) + { handle.BeginOutputReadLine(); + lifetimeTask = AddToLifetimeTask(lifetimeTask, outputReadTcs); + } } catch (InvalidOperationException) { } try { if (readError) + { handle.BeginErrorReadLine(); + lifetimeTask = AddToLifetimeTask(lifetimeTask, errorReadTcs); + } } catch (InvalidOperationException) { } From e96296448baa28b99736406b896a79be3c91cdd7 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 14 May 2020 12:30:31 -0400 Subject: [PATCH 08/41] This test can now run on sqlite --- tests/Tgstation.Server.Tests/IntegrationTest.cs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/Tgstation.Server.Tests/IntegrationTest.cs b/tests/Tgstation.Server.Tests/IntegrationTest.cs index 7258ca32f3..20c2d76df0 100644 --- a/tests/Tgstation.Server.Tests/IntegrationTest.cs +++ b/tests/Tgstation.Server.Tests/IntegrationTest.cs @@ -32,10 +32,6 @@ namespace Tgstation.Server.Tests public async Task TestUpdateProtocol() { using var server = new TestingServer(); - - if (server.DatabaseType == "Sqlite") - Assert.Inconclusive("Cannot run this test on SQLite yet!"); - using var serverCts = new CancellationTokenSource(); var cancellationToken = serverCts.Token; var serverTask = server.Run(cancellationToken); @@ -275,7 +271,7 @@ namespace Tgstation.Server.Tests using var process = processExecutor.LaunchProcess("test." + platformIdentifier.ScriptFileExtension, ".", String.Empty, true, true, true); using var cts = new CancellationTokenSource(); - //cts.CancelAfter(3000); + cts.CancelAfter(3000); var exitCode = await process.Lifetime.WithToken(cts.Token); Assert.AreEqual(0, exitCode); From fe33421c6f0f3a4f9e7eaa197aa920cba57a96ab Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 14 May 2020 13:30:28 -0400 Subject: [PATCH 09/41] Hard throw with bad noShellExecute calls --- .../Components/Byond/WindowsByondInstaller.cs | 5 ++++- .../Components/Deployment/DreamMaker.cs | 1 + .../Session/SessionControllerFactory.cs | 6 +++++- .../Components/StaticFiles/Configuration.cs | 6 +++++- .../System/ProcessExecutor.cs | 21 +++++++++++++------ 5 files changed, 30 insertions(+), 9 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Byond/WindowsByondInstaller.cs b/src/Tgstation.Server.Host/Components/Byond/WindowsByondInstaller.cs index c57ae5c352..eb249c99c7 100644 --- a/src/Tgstation.Server.Host/Components/Byond/WindowsByondInstaller.cs +++ b/src/Tgstation.Server.Host/Components/Byond/WindowsByondInstaller.cs @@ -119,7 +119,10 @@ namespace Tgstation.Server.Host.Components.Byond IProcess directXInstaller; try { - directXInstaller = processExecutor.LaunchProcess(IOManager.ConcatPath(rbdx, "DXSETUP.exe"), rbdx, "/silent", noShellExecute: true); + directXInstaller = processExecutor.LaunchProcess( + IOManager.ConcatPath(rbdx, "DXSETUP.exe"), + rbdx, "/silent", + noShellExecute: true); } catch (Exception e) { diff --git a/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs b/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs index 8cceeb1dd4..7ac55b1dc9 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs @@ -244,6 +244,7 @@ namespace Tgstation.Server.Host.Components.Deployment ADirectoryName)), $"-clean {job.DmeName}.{DmeExtension}", true, + true, true); int exitCode; using (cancellationToken.Register(() => dm.Terminate())) diff --git a/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs b/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs index aef22ee773..d01de4bbc8 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs @@ -213,7 +213,11 @@ namespace Tgstation.Server.Host.Components.Session var noShellExecute = !platformIdentifier.IsWindows; // launch dd - var process = processExecutor.LaunchProcess(byondLock.DreamDaemonPath, basePath, arguments, noShellExecute: noShellExecute); + var process = processExecutor.LaunchProcess( + byondLock.DreamDaemonPath, + basePath, + arguments, + noShellExecute: noShellExecute); try { networkPromptReaper.RegisterProcess(process); diff --git a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs index c2ce055042..27a335f896 100644 --- a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs +++ b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs @@ -456,7 +456,11 @@ namespace Tgstation.Server.Host.Components.StaticFiles var resolvedScriptsDir = ioManager.ResolvePath(EventScriptsSubdirectory); foreach (var I in files.Select(x => ioManager.GetFileName(x)).Where(x => x.StartsWith(scriptName, StringComparison.Ordinal))) - using (var script = processExecutor.LaunchProcess(ioManager.ConcatPath(resolvedScriptsDir, I), resolvedScriptsDir, String.Join(' ', parameters), noShellExecute: true)) + using (var script = processExecutor.LaunchProcess( + ioManager.ConcatPath(resolvedScriptsDir, I), + resolvedScriptsDir, + String.Join(' ', parameters), + noShellExecute: true)) using (cancellationToken.Register(() => script.Terminate())) { var exitCode = await script.Lifetime.ConfigureAwait(false); diff --git a/src/Tgstation.Server.Host/System/ProcessExecutor.cs b/src/Tgstation.Server.Host/System/ProcessExecutor.cs index 79363f0fe7..0c323d3b23 100644 --- a/src/Tgstation.Server.Host/System/ProcessExecutor.cs +++ b/src/Tgstation.Server.Host/System/ProcessExecutor.cs @@ -102,14 +102,23 @@ namespace Tgstation.Server.Host.System } /// - public IProcess LaunchProcess(string fileName, string workingDirectory, string arguments, bool readOutput, bool readError, bool noShellExecute) + public IProcess LaunchProcess( + string fileName, + string workingDirectory, + string arguments, + bool readOutput, + bool readError, + bool noShellExecute) { - if (!noShellExecute && (readOutput || readError)) - { - logger.LogWarning("CODE ERROR: Requesting output/error reading requires noShellExecute to be true! Setting it now..."); + if (fileName == null) + throw new ArgumentNullException(nameof(fileName)); + if (workingDirectory == null) + throw new ArgumentNullException(nameof(workingDirectory)); + if (arguments == null) + throw new ArgumentNullException(nameof(arguments)); - noShellExecute = true; - } + if (!noShellExecute && (readOutput || readError)) + throw new InvalidOperationException("Requesting output/error reading requires noShellExecute to be true!"); logger.LogDebug("{3}aunching process in {0}: {1} {2}", workingDirectory, fileName, arguments, noShellExecute ? "L" : "Shell l"); var handle = new global::System.Diagnostics.Process(); From 159ecd98f11b1c59dd198ba1ed74620f8f2104fb Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 14 May 2020 13:32:28 -0400 Subject: [PATCH 10/41] Log event script output --- .../Components/StaticFiles/Configuration.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs index 27a335f896..41e6fa1743 100644 --- a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs +++ b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs @@ -460,10 +460,14 @@ namespace Tgstation.Server.Host.Components.StaticFiles ioManager.ConcatPath(resolvedScriptsDir, I), resolvedScriptsDir, String.Join(' ', parameters), - noShellExecute: true)) + true, + true, + true)) using (cancellationToken.Register(() => script.Terminate())) { var exitCode = await script.Lifetime.ConfigureAwait(false); + var scriptOutput = script.GetCombinedOutput(); + logger.LogInformation("{0} Output:{1}{2}", I, Environment.NewLine, scriptOutput); cancellationToken.ThrowIfCancellationRequested(); if (exitCode != 0) return false; From 7274e0c01dbd85f73dde3ce1782400527ee8f4c2 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 14 May 2020 13:46:47 -0400 Subject: [PATCH 11/41] Add a warning about the BYOND pager --- tests/Tgstation.Server.Tests/IntegrationTest.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/Tgstation.Server.Tests/IntegrationTest.cs b/tests/Tgstation.Server.Tests/IntegrationTest.cs index 20c2d76df0..fd6fb1ce63 100644 --- a/tests/Tgstation.Server.Tests/IntegrationTest.cs +++ b/tests/Tgstation.Server.Tests/IntegrationTest.cs @@ -106,6 +106,14 @@ namespace Tgstation.Server.Tests [TestMethod] public async Task TestServer() { + var procs = System.Diagnostics.Process.GetProcessesByName("byond"); + if(procs.Any()) + { + foreach (var proc in procs) + proc.Dispose(); + Assert.Inconclusive("Cannot run server test because DreamDaemon will not start headless while the BYOND pager is running!"); + } + using var server = new TestingServer(); using var serverCts = new CancellationTokenSource(); var cancellationToken = serverCts.Token; From 3e3f48d96df95679254c198e9f0c440e859c7851 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 14 May 2020 13:51:52 -0400 Subject: [PATCH 12/41] Fix a log message --- src/Tgstation.Server.Host/System/ProcessExecutor.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/System/ProcessExecutor.cs b/src/Tgstation.Server.Host/System/ProcessExecutor.cs index 0c323d3b23..08fefc73df 100644 --- a/src/Tgstation.Server.Host/System/ProcessExecutor.cs +++ b/src/Tgstation.Server.Host/System/ProcessExecutor.cs @@ -120,7 +120,12 @@ namespace Tgstation.Server.Host.System if (!noShellExecute && (readOutput || readError)) throw new InvalidOperationException("Requesting output/error reading requires noShellExecute to be true!"); - logger.LogDebug("{3}aunching process in {0}: {1} {2}", workingDirectory, fileName, arguments, noShellExecute ? "L" : "Shell l"); + logger.LogDebug( + "{0}aunching process in {1}: {2} {3}", + noShellExecute ? "L" : "Shell l", + workingDirectory, + fileName, + arguments); var handle = new global::System.Diagnostics.Process(); try { From 642db732f4823444e033d06fe2130488c70a7539 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 14 May 2020 13:58:39 -0400 Subject: [PATCH 13/41] Switch event handler return code for JobExceptions --- .../Components/EventConsumer.cs | 7 +++---- .../Components/IEventConsumer.cs | 4 ++-- .../Components/Repository/Repository.cs | 12 ++++-------- .../Components/StaticFiles/Configuration.cs | 10 ++++------ .../Components/Watchdog/WatchdogBase.cs | 12 +++--------- 5 files changed, 16 insertions(+), 29 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/EventConsumer.cs b/src/Tgstation.Server.Host/Components/EventConsumer.cs index e204cbcf8f..1cebdd1d2b 100644 --- a/src/Tgstation.Server.Host/Components/EventConsumer.cs +++ b/src/Tgstation.Server.Host/Components/EventConsumer.cs @@ -30,14 +30,13 @@ namespace Tgstation.Server.Host.Components } /// - public async Task HandleEvent(EventType eventType, IEnumerable parameters, CancellationToken cancellationToken) + public async Task HandleEvent(EventType eventType, IEnumerable parameters, CancellationToken cancellationToken) { if (watchdog == null) throw new InvalidOperationException("EventConsumer used without watchdog set!"); - if (!await configuration.HandleEvent(eventType, parameters, cancellationToken).ConfigureAwait(false)) - return false; - return await watchdog.HandleEvent(eventType, parameters, cancellationToken).ConfigureAwait(false); + await configuration.HandleEvent(eventType, parameters, cancellationToken).ConfigureAwait(false); + await watchdog.HandleEvent(eventType, parameters, cancellationToken).ConfigureAwait(false); } /// diff --git a/src/Tgstation.Server.Host/Components/IEventConsumer.cs b/src/Tgstation.Server.Host/Components/IEventConsumer.cs index aa30d18827..f4dd616bd8 100644 --- a/src/Tgstation.Server.Host/Components/IEventConsumer.cs +++ b/src/Tgstation.Server.Host/Components/IEventConsumer.cs @@ -15,7 +15,7 @@ namespace Tgstation.Server.Host.Components /// The /// The parameters for /// The for the operation - /// A resulting in if more should run, otherwise - Task HandleEvent(EventType eventType, IEnumerable parameters, CancellationToken cancellationToken); + /// A representing the running operation. + Task HandleEvent(EventType eventType, IEnumerable parameters, CancellationToken cancellationToken); } } diff --git a/src/Tgstation.Server.Host/Components/Repository/Repository.cs b/src/Tgstation.Server.Host/Components/Repository/Repository.cs index 43bf3baa27..4a6aef276e 100644 --- a/src/Tgstation.Server.Host/Components/Repository/Repository.cs +++ b/src/Tgstation.Server.Host/Components/Repository/Repository.cs @@ -163,8 +163,8 @@ namespace Tgstation.Server.Host.Components.Repository if (remote.EndsWith(item, StringComparison.OrdinalIgnoreCase)) remote = remote.Substring(0, remote.LastIndexOf(item, StringComparison.OrdinalIgnoreCase)); var splits = remote.Split('/'); - name = splits[splits.Length - 1]; - owner = splits[splits.Length - 2].Split('.')[0]; + name = splits.Last(); + owner = splits[^2].Split('.').First(); logger.LogTrace("GetRepositoryOwnerName({0}) => {1} / {2}", remote, owner, name); } @@ -622,18 +622,14 @@ namespace Tgstation.Server.Host.Components.Repository cancellationToken.ThrowIfCancellationRequested(); try { - if (!await eventConsumer.HandleEvent( + await eventConsumer.HandleEvent( EventType.RepoPreSynchronize, new List { ioMananger.ResolvePath() }, cancellationToken) - .ConfigureAwait(false)) - { - logger.LogDebug("Aborted synchronize due to event handler response!"); - return false; - } + .ConfigureAwait(false); } finally { diff --git a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs index 41e6fa1743..bc9450b40d 100644 --- a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs +++ b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs @@ -11,6 +11,7 @@ using System.Threading.Tasks; using Tgstation.Server.Api.Models; using Tgstation.Server.Host.Core; using Tgstation.Server.Host.IO; +using Tgstation.Server.Host.Jobs; using Tgstation.Server.Host.Security; using Tgstation.Server.Host.System; @@ -442,12 +443,12 @@ namespace Tgstation.Server.Host.Components.StaticFiles public Task StopAsync(CancellationToken cancellationToken) => EnsureDirectories(cancellationToken); /// - public async Task HandleEvent(EventType eventType, IEnumerable parameters, CancellationToken cancellationToken) + public async Task HandleEvent(EventType eventType, IEnumerable parameters, CancellationToken cancellationToken) { await EnsureDirectories(cancellationToken).ConfigureAwait(false); if (!EventTypeScriptFileNameMap.TryGetValue(eventType, out var scriptName)) - return true; + return; // always execute in serial using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken).ConfigureAwait(false)) @@ -467,14 +468,11 @@ namespace Tgstation.Server.Host.Components.StaticFiles { var exitCode = await script.Lifetime.ConfigureAwait(false); var scriptOutput = script.GetCombinedOutput(); - logger.LogInformation("{0} Output:{1}{2}", I, Environment.NewLine, scriptOutput); cancellationToken.ThrowIfCancellationRequested(); if (exitCode != 0) - return false; + throw new JobException($"Script {I} exited with code {exitCode}:{Environment.NewLine}{scriptOutput}"); } } - - return true; } /// diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs index abbbb4598d..3f026305ce 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs @@ -727,19 +727,15 @@ namespace Tgstation.Server.Host.Components.Watchdog } /// - public async Task HandleEvent(EventType eventType, IEnumerable parameters, CancellationToken cancellationToken) + public async Task HandleEvent(EventType eventType, IEnumerable parameters, CancellationToken cancellationToken) { - if (!Running) - return true; - - var notification = new EventNotification(eventType, parameters); - var activeServer = GetActiveController(); // Server may have ended if (activeServer == null) - return true; + return; + var notification = new EventNotification(eventType, parameters); var result = await activeServer.SendCommand( new TopicParameters(notification), cancellationToken) @@ -762,8 +758,6 @@ namespace Tgstation.Server.Host.Components.Watchdog .Select(nullableChannelId => nullableChannelId.Value), cancellationToken))) .ConfigureAwait(false); - - return true; } /// From b24ab492e43dac87913da5b74ffeae617fd6b204 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 14 May 2020 13:59:46 -0400 Subject: [PATCH 14/41] Micro optimization --- .../Components/StaticFiles/Configuration.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs index bc9450b40d..17ebbe5a5e 100644 --- a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs +++ b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs @@ -467,8 +467,8 @@ namespace Tgstation.Server.Host.Components.StaticFiles using (cancellationToken.Register(() => script.Terminate())) { var exitCode = await script.Lifetime.ConfigureAwait(false); - var scriptOutput = script.GetCombinedOutput(); cancellationToken.ThrowIfCancellationRequested(); + var scriptOutput = script.GetCombinedOutput(); if (exitCode != 0) throw new JobException($"Script {I} exited with code {exitCode}:{Environment.NewLine}{scriptOutput}"); } From 53928f10c6ec05903e8e801a4c2d5bef155dedd8 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 14 May 2020 14:07:22 -0400 Subject: [PATCH 15/41] Fix chat shutdown exception --- src/Tgstation.Server.Host/Components/Chat/ChatManager.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs b/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs index 2782f9aba1..94e01c8af5 100644 --- a/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs +++ b/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs @@ -646,7 +646,8 @@ namespace Tgstation.Server.Host.Components.Chat public async Task StopAsync(CancellationToken cancellationToken) { handlerCts.Cancel(); - await chatHandler.ConfigureAwait(false); + if (chatHandler != null) + await chatHandler.ConfigureAwait(false); await Task.WhenAll(providers.Select(x => x.Value).Select(x => x.Disconnect(cancellationToken))).ConfigureAwait(false); } From 6bf8bf65f3eedc609414186127ed188c087487a7 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 14 May 2020 14:28:58 -0400 Subject: [PATCH 16/41] Add support for detecting the BYOND pager on Windows --- .../Components/Session/SessionControllerFactory.cs | 12 ++++++++++++ src/Tgstation.Server.Host/System/IProcessExecutor.cs | 7 +++++++ src/Tgstation.Server.Host/System/ProcessExecutor.cs | 11 +++++++++++ 3 files changed, 30 insertions(+) diff --git a/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs b/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs index d01de4bbc8..f315c9f335 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs @@ -15,6 +15,7 @@ using Tgstation.Server.Host.Components.Interop; using Tgstation.Server.Host.Components.Interop.Bridge; using Tgstation.Server.Host.Core; using Tgstation.Server.Host.IO; +using Tgstation.Server.Host.Jobs; using Tgstation.Server.Host.Security; using Tgstation.Server.Host.System; @@ -192,6 +193,8 @@ namespace Tgstation.Server.Host.Components.Session if (launchParameters.SecurityLevel == DreamDaemonSecurity.Trusted) await byondLock.TrustDmbPath(ioManager.ConcatPath(basePath, dmbProvider.DmbName), cancellationToken).ConfigureAwait(false); + CheckPagerIsNotRunning(); + var accessIdentifier = cryptographySuite.GetSecureString(); // set command line options @@ -375,5 +378,14 @@ namespace Tgstation.Server.Host.Components.Session securityLevel, apiValidateOnly); } + + /// + /// Make sure the BYOND pager is not running. + /// + void CheckPagerIsNotRunning() + { + if (platformIdentifier.IsWindows && processExecutor.IsProcessWithNameRunning("byond")) + throw new JobException("Cannot start DreamDaemon headless with the BYOND pager running!"); + } } } diff --git a/src/Tgstation.Server.Host/System/IProcessExecutor.cs b/src/Tgstation.Server.Host/System/IProcessExecutor.cs index 3d9a001b9e..6b732915dd 100644 --- a/src/Tgstation.Server.Host/System/IProcessExecutor.cs +++ b/src/Tgstation.Server.Host/System/IProcessExecutor.cs @@ -23,5 +23,12 @@ /// The /// The represented by on success, on failure IProcess GetProcess(int id); + + /// + /// Check if a with a given is running. + /// + /// The name of the process without the extension. + /// if the process is running, otherwise. + bool IsProcessWithNameRunning(string name); } } diff --git a/src/Tgstation.Server.Host/System/ProcessExecutor.cs b/src/Tgstation.Server.Host/System/ProcessExecutor.cs index 08fefc73df..772c754131 100644 --- a/src/Tgstation.Server.Host/System/ProcessExecutor.cs +++ b/src/Tgstation.Server.Host/System/ProcessExecutor.cs @@ -1,5 +1,6 @@ using Microsoft.Extensions.Logging; using System; +using System.Linq; using System.Text; using System.Threading.Tasks; @@ -228,5 +229,15 @@ namespace Tgstation.Server.Host.System throw; } } + + /// + public bool IsProcessWithNameRunning(string name) + { + var procs = global::System.Diagnostics.Process.GetProcessesByName(name); + foreach (var proc in procs) + proc.Dispose(); + + return procs.Any(); + } } } From 2833cbb796827fe44d2b35976d0391787a1dbd22 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 14 May 2020 14:40:33 -0400 Subject: [PATCH 17/41] Fix the DMAPI --- src/DMAPI/tgs.dm | 2 +- src/DMAPI/tgs/core/datum.dm | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/DMAPI/tgs.dm b/src/DMAPI/tgs.dm index 9f84fdd3cb..e164a4ec96 100644 --- a/src/DMAPI/tgs.dm +++ b/src/DMAPI/tgs.dm @@ -1,6 +1,6 @@ //tgstation-server DMAPI -#define TGS_DMAPI_VERSION "5.1.0" +#define TGS_DMAPI_VERSION "5.1.1" //All functions and datums outside this document are subject to change with any version and should not be relied on diff --git a/src/DMAPI/tgs/core/datum.dm b/src/DMAPI/tgs/core/datum.dm index 3abf5f284f..ccc75fe344 100644 --- a/src/DMAPI/tgs/core/datum.dm +++ b/src/DMAPI/tgs/core/datum.dm @@ -6,7 +6,7 @@ TGS_DEFINE_AND_SET_GLOBAL(tgs, null) /datum/tgs_api/New(datum/tgs_event_handler/event_handler, datum/tgs_version/version) . = ..() - src.event_handler = version + src.event_handler = event_handler src.version = version /datum/tgs_api/latest From 1755acf1b085beaa8cf185b0fd01d92251a3a938 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 14 May 2020 14:42:43 -0400 Subject: [PATCH 18/41] TGS side DMAPI version bump --- build/Version.props | 2 +- src/Tgstation.Server.Host/Components/Interop/DMApiConstants.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build/Version.props b/build/Version.props index ef659a9af2..744f273aa1 100644 --- a/build/Version.props +++ b/build/Version.props @@ -5,7 +5,7 @@ 4.2.0 6.2.0 6.1.0 - 5.1.0 + 5.1.1 0.4.0 1.1.0 diff --git a/src/Tgstation.Server.Host/Components/Interop/DMApiConstants.cs b/src/Tgstation.Server.Host/Components/Interop/DMApiConstants.cs index f14967cf94..f7f4820dc7 100644 --- a/src/Tgstation.Server.Host/Components/Interop/DMApiConstants.cs +++ b/src/Tgstation.Server.Host/Components/Interop/DMApiConstants.cs @@ -33,7 +33,7 @@ namespace Tgstation.Server.Host.Components.Interop /// /// The DMAPI being used. /// - public static readonly Version Version = new Version(5, 1, 0); + public static readonly Version Version = new Version(5, 1, 1); /// /// for use when communicating with the DMAPI. From 93e4c12a2b3269940007546f70782806b294586a Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 14 May 2020 14:43:15 -0400 Subject: [PATCH 19/41] Version bump to 4.2.1 --- build/Version.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/Version.props b/build/Version.props index 744f273aa1..b169916252 100644 --- a/build/Version.props +++ b/build/Version.props @@ -2,7 +2,7 @@ - 4.2.0 + 4.2.1 6.2.0 6.1.0 5.1.1 From 83ef6372175ae491894656ab21f66d7a59ea5dc9 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 14 May 2020 15:17:13 -0400 Subject: [PATCH 20/41] Fix here too --- src/DMAPI/tgs/v5/api.dm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DMAPI/tgs/v5/api.dm b/src/DMAPI/tgs/v5/api.dm index f0c38a7c24..eedefb2877 100644 --- a/src/DMAPI/tgs/v5/api.dm +++ b/src/DMAPI/tgs/v5/api.dm @@ -16,7 +16,7 @@ var/list/chat_channels /datum/tgs_api/v5/ApiVersion() - return new /datum/tgs_version("5.1.0") + return new /datum/tgs_version("5.1.1") /datum/tgs_api/v5/OnWorldNew(minimum_required_security_level) server_port = world.params[DMAPI5_PARAM_SERVER_PORT] From b8f04d81ce6f06304179b7f3a59373cf367c09a3 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 14 May 2020 15:40:37 -0400 Subject: [PATCH 21/41] Build time optimization --- tests/Tgstation.Server.Tests/Tgstation.Server.Tests.csproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Tgstation.Server.Tests/Tgstation.Server.Tests.csproj b/tests/Tgstation.Server.Tests/Tgstation.Server.Tests.csproj index 48036e1b73..ff83e2f565 100644 --- a/tests/Tgstation.Server.Tests/Tgstation.Server.Tests.csproj +++ b/tests/Tgstation.Server.Tests/Tgstation.Server.Tests.csproj @@ -28,10 +28,10 @@ - Always + PreserveNewest - Always + PreserveNewest From e53404ea0498d6df6b5f05a06d072dcbe8e38cac Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 14 May 2020 15:58:54 -0400 Subject: [PATCH 22/41] Add better test topic logging --- tests/DMAPI/BasicOperation/Test.dm | 5 +++++ tests/DMAPI/LongRunning/Test.dm | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/tests/DMAPI/BasicOperation/Test.dm b/tests/DMAPI/BasicOperation/Test.dm index 5f9b79e2f9..21ba9e6226 100644 --- a/tests/DMAPI/BasicOperation/Test.dm +++ b/tests/DMAPI/BasicOperation/Test.dm @@ -26,6 +26,11 @@ world.log << "You really shouldn't be able to read this" /world/Topic(T, Addr, Master, Keys) + world.log << "Topic: [T]" + . = HandleTopic(T) + world.log << "Response: [.]" + +/world/proc/HandleTopic(T) TGS_TOPIC /world/Reboot(reason) diff --git a/tests/DMAPI/LongRunning/Test.dm b/tests/DMAPI/LongRunning/Test.dm index 4c8bf5c7d1..631dcfccae 100644 --- a/tests/DMAPI/LongRunning/Test.dm +++ b/tests/DMAPI/LongRunning/Test.dm @@ -12,6 +12,11 @@ world.TgsInitializationComplete() /world/Topic(T, Addr, Master, Keys) + world.log << "Topic: [T]" + . = HandleTopic(T) + world.log << "Response: [.]" + +/world/proc/HandleTopic(T) TGS_TOPIC world.sleep_offline = FALSE From 43e8c0d793939bde9e1a6469dcc5d7366171c833 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 14 May 2020 16:35:16 -0400 Subject: [PATCH 23/41] Improve process of creating migrations Also stops msbuild retriggers --- .../DesignTimeDbContextFactoryHelpers.cs | 35 ++++++------------- .../Design/MySqlDesignTimeDbContextFactory.cs | 5 ++- .../SqlServerDesignTimeDbContextFactory.cs | 5 ++- .../SqliteDesignTimeDbContextFactory.cs | 5 ++- .../Tgstation.Server.Host.csproj | 13 ++----- 5 files changed, 25 insertions(+), 38 deletions(-) diff --git a/src/Tgstation.Server.Host/Database/Design/DesignTimeDbContextFactoryHelpers.cs b/src/Tgstation.Server.Host/Database/Design/DesignTimeDbContextFactoryHelpers.cs index 68fae14333..9699c86a16 100644 --- a/src/Tgstation.Server.Host/Database/Design/DesignTimeDbContextFactoryHelpers.cs +++ b/src/Tgstation.Server.Host/Database/Design/DesignTimeDbContextFactoryHelpers.cs @@ -1,8 +1,5 @@ -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Options; +using Microsoft.Extensions.Options; using Tgstation.Server.Host.Configuration; -using Tgstation.Server.Host.IO; -using Tgstation.Server.Host.System; namespace Tgstation.Server.Host.Database.Design { @@ -11,31 +8,21 @@ namespace Tgstation.Server.Host.Database.Design /// static class DesignTimeDbContextFactoryHelpers { - /// - /// Path to the json file to use for migrations configuration - /// - const string RootJson = "appsettings.json"; - - /// - /// Path to the development json file to use for migrations configuration - /// - const string DevJson = "appsettings.Development.json"; - /// /// Get the for the /// + /// The . + /// The . /// The for the - public static IOptions GetDbContextOptions() + public static IOptions GetDbContextOptions(DatabaseType databaseType, string connectionString) { - var builder = new ConfigurationBuilder(); - var assemblyInfoProvider = new AssemblyInformationProvider(); - var ioManager = new DefaultIOManager(); - builder.SetBasePath(ioManager.GetDirectoryName(assemblyInfoProvider.Path)); - builder.AddJsonFile(RootJson); - builder.AddJsonFile(DevJson); - var configuration = builder.Build(); - var dbConfig = configuration.GetSection(DatabaseConfiguration.Section).Get(); - dbConfig.DesignTime = true; + var dbConfig = new DatabaseConfiguration + { + DesignTime = true, + DatabaseType = databaseType, + ConnectionString = connectionString + }; + return Options.Create(dbConfig); } } diff --git a/src/Tgstation.Server.Host/Database/Design/MySqlDesignTimeDbContextFactory.cs b/src/Tgstation.Server.Host/Database/Design/MySqlDesignTimeDbContextFactory.cs index 4ac39129ac..6496f55009 100644 --- a/src/Tgstation.Server.Host/Database/Design/MySqlDesignTimeDbContextFactory.cs +++ b/src/Tgstation.Server.Host/Database/Design/MySqlDesignTimeDbContextFactory.cs @@ -2,6 +2,7 @@ using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Design; using Microsoft.Extensions.Logging; +using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.Models; using Tgstation.Server.Host.Security; using Tgstation.Server.Host.System; @@ -17,7 +18,9 @@ namespace Tgstation.Server.Host.Database.Design using var loggerFactory = new LoggerFactory(); return new MySqlDatabaseContext( new DbContextOptions(), - DesignTimeDbContextFactoryHelpers.GetDbContextOptions(), + DesignTimeDbContextFactoryHelpers.GetDbContextOptions( + DatabaseType.MariaDB, + "Server=127.0.0.1;User Id=root;Password=fake;Database=TGS_Design"), new DatabaseSeeder( new CryptographySuite( new PasswordHasher()), diff --git a/src/Tgstation.Server.Host/Database/Design/SqlServerDesignTimeDbContextFactory.cs b/src/Tgstation.Server.Host/Database/Design/SqlServerDesignTimeDbContextFactory.cs index dc56d54d4b..483fc08b2b 100644 --- a/src/Tgstation.Server.Host/Database/Design/SqlServerDesignTimeDbContextFactory.cs +++ b/src/Tgstation.Server.Host/Database/Design/SqlServerDesignTimeDbContextFactory.cs @@ -2,6 +2,7 @@ using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Design; using Microsoft.Extensions.Logging; +using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.Models; using Tgstation.Server.Host.Security; using Tgstation.Server.Host.System; @@ -17,7 +18,9 @@ namespace Tgstation.Server.Host.Database.Design using var loggerFactory = new LoggerFactory(); return new SqlServerDatabaseContext( new DbContextOptions(), - DesignTimeDbContextFactoryHelpers.GetDbContextOptions(), + DesignTimeDbContextFactoryHelpers.GetDbContextOptions( + DatabaseType.SqlServer, + "Data Source=fake;Initial Catalog=TGS_Design;Integrated Security=True;Application Name=tgstation-server"), new DatabaseSeeder( new CryptographySuite( new PasswordHasher()), diff --git a/src/Tgstation.Server.Host/Database/Design/SqliteDesignTimeDbContextFactory.cs b/src/Tgstation.Server.Host/Database/Design/SqliteDesignTimeDbContextFactory.cs index 65789a71c6..09e506711f 100644 --- a/src/Tgstation.Server.Host/Database/Design/SqliteDesignTimeDbContextFactory.cs +++ b/src/Tgstation.Server.Host/Database/Design/SqliteDesignTimeDbContextFactory.cs @@ -2,6 +2,7 @@ using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Design; using Microsoft.Extensions.Logging; +using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.Models; using Tgstation.Server.Host.Security; using Tgstation.Server.Host.System; @@ -17,7 +18,9 @@ namespace Tgstation.Server.Host.Database.Design using var loggerFactory = new LoggerFactory(); return new SqliteDatabaseContext( new DbContextOptions(), - DesignTimeDbContextFactoryHelpers.GetDbContextOptions(), + DesignTimeDbContextFactoryHelpers.GetDbContextOptions( + DatabaseType.Sqlite, + "Data Source=tgs_design.sqlite3;Mode=ReadWriteCreate"), new DatabaseSeeder( new CryptographySuite( new PasswordHasher()), diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index 4db31fc889..4a3e6018cc 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -112,19 +112,10 @@ - - - - PreserveNewest - - - PreserveNewest - - - + - Always + PreserveNewest PreserveNewest From ecde933fd4f9bbfe5d926b58bafb4c200c3096de Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 14 May 2020 16:39:44 -0400 Subject: [PATCH 24/41] Log aborted topic requests --- .../Components/Session/SessionController.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Tgstation.Server.Host/Components/Session/SessionController.cs b/src/Tgstation.Server.Host/Components/Session/SessionController.cs index 51f33bf3df..0cc61fe497 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionController.cs @@ -560,6 +560,7 @@ namespace Tgstation.Server.Host.Components.Session } catch (OperationCanceledException e) { + logger.LogTrace("Topic request aborted!"); cancellationToken.ThrowIfCancellationRequested(); caughtException = e; } From 9e5703ff45e6cd43211f75c874788dd4303c8d06 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 14 May 2020 16:41:51 -0400 Subject: [PATCH 25/41] Fix topic request failure logging --- .../Components/Session/SessionController.cs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Session/SessionController.cs b/src/Tgstation.Server.Host/Components/Session/SessionController.cs index 0cc61fe497..1b3cc62281 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionController.cs @@ -522,7 +522,6 @@ namespace Tgstation.Server.Host.Components.Session var json = JsonConvert.SerializeObject(parameters, DMApiConstants.SerializerSettings); logger.LogTrace("Topic request: {0}", json); - Exception caughtException; try { var commandString = String.Format(CultureInfo.InvariantCulture, @@ -558,20 +557,16 @@ namespace Tgstation.Server.Host.Components.Session return new CombinedTopicResponse(topicResponse, interopResponse); } - catch (OperationCanceledException e) + catch (OperationCanceledException) { logger.LogTrace("Topic request aborted!"); cancellationToken.ThrowIfCancellationRequested(); - caughtException = e; } catch (Exception e) { - caughtException = e; + logger.LogWarning("Send command exception:{0}{1}", Environment.NewLine, e); } - if (caughtException == null) - logger.LogWarning("Send command exception:{0}{1}", Environment.NewLine, caughtException.Message); - return null; } From d97f36d173a6452571b5791c87869445c79fc6b1 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 14 May 2020 17:09:58 -0400 Subject: [PATCH 26/41] Update to BYOND.TopicSender 4.0.1 --- src/Tgstation.Server.Host/Tgstation.Server.Host.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index 4a3e6018cc..57d99ca5df 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -48,7 +48,7 @@ - + From 672c35ba72976204dd145d017c3b14681caed366 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 14 May 2020 17:18:39 -0400 Subject: [PATCH 27/41] Add successful heartbeat tests. + Tests topic calls --- .../Instance/WatchdogTest.cs | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/tests/Tgstation.Server.Tests/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Instance/WatchdogTest.cs index 268f7a17b3..b82751d5fa 100644 --- a/tests/Tgstation.Server.Tests/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Instance/WatchdogTest.cs @@ -89,21 +89,26 @@ namespace Tgstation.Server.Tests.Instance await WaitForJob(startJob, 10, false, cancellationToken); - await instanceClient.DreamDaemon.Update(new DreamDaemon - { - SoftShutdown = true - }, cancellationToken); - // lock on to DD and pause it so it can't heartbeat var ddProcs = System.Diagnostics.Process.GetProcessesByName("DreamDaemon").ToList(); if (ddProcs.Count != 1) Assert.Inconclusive($"Incorrect number of DD processes: {ddProcs.Count}"); - var pid = ddProcs.Single().Id; + using var ddProc = ddProcs.Single(); + + // Ensure it's responding to heartbeats + await Task.Delay(6000); + Assert.IsFalse(ddProc.HasExited); + + await instanceClient.DreamDaemon.Update(new DreamDaemon + { + SoftShutdown = true + }, cancellationToken); + using var ourProcessHandler = new ProcessExecutor( new PlatformIdentifier().IsWindows ? (IProcessSuspender)new WindowsProcessSuspender(Mock.Of>()) : new PosixProcessSuspender(Mock.Of>()), Mock.Of>(), - LoggerFactory.Create(x => { })).GetProcess(pid); + LoggerFactory.Create(x => { })).GetProcess(ddProc.Id); ourProcessHandler.Suspend(); await Task.WhenAny(ourProcessHandler.Lifetime, Task.Delay(TimeSpan.FromSeconds(20))); From 70bb1621cd0b4ea98da3fe7007feaa030c3a0f9b Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 14 May 2020 18:01:44 -0400 Subject: [PATCH 28/41] Fix DMAPI command defines --- src/DMAPI/tgs/v5/_defines.dm | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/DMAPI/tgs/v5/_defines.dm b/src/DMAPI/tgs/v5/_defines.dm index 4abd059677..2baf3e12d7 100644 --- a/src/DMAPI/tgs/v5/_defines.dm +++ b/src/DMAPI/tgs/v5/_defines.dm @@ -60,10 +60,10 @@ #define DMAPI5_TOPIC_COMMAND_CHANGE_PORT 2 #define DMAPI5_TOPIC_COMMAND_CHANGE_REBOOT_STATE 3 #define DMAPI5_TOPIC_COMMAND_INSTANCE_RENAMED 4 -#define DMAPI5_TOPIC_COMMAND_CHAT_CHANNELS_UPDATE 4 -#define DMAPI5_TOPIC_COMMAND_SERVER_PORT_UPDATE 5 -#define DMAPI5_TOPIC_COMMAND_HEARTBEAT 6 -#define DMAPI5_TOPIC_COMMAND_WATCHDOG_REATTACH 7 +#define DMAPI5_TOPIC_COMMAND_CHAT_CHANNELS_UPDATE 5 +#define DMAPI5_TOPIC_COMMAND_SERVER_PORT_UPDATE 6 +#define DMAPI5_TOPIC_COMMAND_HEARTBEAT 7 +#define DMAPI5_TOPIC_COMMAND_WATCHDOG_REATTACH 8 #define DMAPI5_TOPIC_PARAMETER_COMMAND_TYPE "commandType" #define DMAPI5_TOPIC_PARAMETER_CHAT_COMMAND "chatCommand" From 3f8649259095ba914fa6efbb5b65c7991a9006d8 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 14 May 2020 18:02:00 -0400 Subject: [PATCH 29/41] More testing --- .../Instance/WatchdogTest.cs | 27 ++++++++--- .../Tgstation.Server.Tests/IntegrationTest.cs | 45 +++++++++++++++---- 2 files changed, 57 insertions(+), 15 deletions(-) diff --git a/tests/Tgstation.Server.Tests/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Instance/WatchdogTest.cs index b82751d5fa..7111026da4 100644 --- a/tests/Tgstation.Server.Tests/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Instance/WatchdogTest.cs @@ -42,11 +42,15 @@ namespace Tgstation.Server.Tests.Instance }, cancellationToken), ErrorCode.DreamDaemonDoubleSoft); await RunBasicTest(cancellationToken); - await RunHeartbeatTest(cancellationToken); // await RunLongRunningTestThenUpdate(cancellationToken); // await RunLongRunningTestThenUpdateWithByondVersionSwitch(cancellationToken); + // Remove this deploy when the above tests are reenabled + await DeployTestDme("LongRunning/long_running_test", DreamDaemonSecurity.Trusted, cancellationToken); + + await RunHeartbeatTest(cancellationToken); + await StartAndLeaveRunning(cancellationToken); } @@ -95,9 +99,16 @@ namespace Tgstation.Server.Tests.Instance Assert.Inconclusive($"Incorrect number of DD processes: {ddProcs.Count}"); using var ddProc = ddProcs.Single(); - + using var ourProcessHandler = new ProcessExecutor( + new PlatformIdentifier().IsWindows + ? (IProcessSuspender)new WindowsProcessSuspender(Mock.Of>()) + : new PosixProcessSuspender(Mock.Of>()), + Mock.Of>(), + LoggerFactory.Create(x => { })) + .GetProcess(ddProc.Id); + // Ensure it's responding to heartbeats - await Task.Delay(6000); + await Task.WhenAny(Task.Delay(20000), ourProcessHandler.Lifetime); Assert.IsFalse(ddProc.HasExited); await instanceClient.DreamDaemon.Update(new DreamDaemon @@ -105,10 +116,6 @@ namespace Tgstation.Server.Tests.Instance SoftShutdown = true }, cancellationToken); - using var ourProcessHandler = new ProcessExecutor( - new PlatformIdentifier().IsWindows ? (IProcessSuspender)new WindowsProcessSuspender(Mock.Of>()) : new PosixProcessSuspender(Mock.Of>()), - Mock.Of>(), - LoggerFactory.Create(x => { })).GetProcess(ddProc.Id); ourProcessHandler.Suspend(); await Task.WhenAny(ourProcessHandler.Lifetime, Task.Delay(TimeSpan.FromSeconds(20))); @@ -126,6 +133,12 @@ namespace Tgstation.Server.Tests.Instance Assert.Fail("DreamDaemon didn't shutdown within the timeout!"); } while (timeout > 0); + + // disable heartbeats + await instanceClient.DreamDaemon.Update(new DreamDaemon + { + HeartbeatSeconds = 0, + }, cancellationToken); } async Task RunLongRunningTestThenUpdate(CancellationToken cancellationToken) diff --git a/tests/Tgstation.Server.Tests/IntegrationTest.cs b/tests/Tgstation.Server.Tests/IntegrationTest.cs index fd6fb1ce63..0934e7e8dd 100644 --- a/tests/Tgstation.Server.Tests/IntegrationTest.cs +++ b/tests/Tgstation.Server.Tests/IntegrationTest.cs @@ -196,20 +196,35 @@ namespace Tgstation.Server.Tests await Task.WhenAny(serverTask, Task.Delay(30000, cancellationToken)); Assert.IsTrue(serverTask.IsCompleted); + var preStartupTime = DateTimeOffset.Now; + serverTask = server.Run(cancellationToken); using (var adminClient = await CreateAdminClient()) { var instanceClient = adminClient.Instances.CreateClient(instance); - // reattach job var jobs = await instanceClient.Jobs.ListActive(cancellationToken); - if (jobs.Any()) + if (!jobs.Any()) { - Assert.AreEqual(1, jobs.Count); + var entities = await instanceClient.Jobs.List(cancellationToken); + var getTasks = entities + .Select(e => instanceClient.Jobs.GetId(e, cancellationToken)) + .ToList(); - await new JobsRequiredTest(instanceClient.Jobs).WaitForJob(jobs.Single(), 40, false, cancellationToken); + await Task.WhenAll(getTasks); + jobs = getTasks + .Select(x => x.Result) + .Where(x => x.StartedAt.Value > preStartupTime) + .ToList(); } + Assert.AreEqual(1, jobs.Count); + + var reattachJob = jobs.Single(); + Assert.IsTrue(reattachJob.StartedAt.Value >= preStartupTime); + + await new JobsRequiredTest(instanceClient.Jobs).WaitForJob(reattachJob, 40, false, cancellationToken); + var dd = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.IsTrue(dd.Running.Value); @@ -225,20 +240,34 @@ namespace Tgstation.Server.Tests await Task.WhenAny(serverTask, Task.Delay(30000, cancellationToken)); Assert.IsTrue(serverTask.IsCompleted); + preStartupTime = DateTimeOffset.Now; serverTask = server.Run(cancellationToken); using (var adminClient = await CreateAdminClient()) { var instanceClient = adminClient.Instances.CreateClient(instance); - // launch job var jobs = await instanceClient.Jobs.ListActive(cancellationToken); - if (jobs.Any()) + if (!jobs.Any()) { - Assert.AreEqual(1, jobs.Count); + var entities = await instanceClient.Jobs.List(cancellationToken); + var getTasks = entities + .Select(e => instanceClient.Jobs.GetId(e, cancellationToken)) + .ToList(); - await new JobsRequiredTest(instanceClient.Jobs).WaitForJob(jobs.Single(), 40, false, cancellationToken); + await Task.WhenAll(getTasks); + jobs = getTasks + .Select(x => x.Result) + .Where(x => x.StartedAt.Value > preStartupTime) + .ToList(); } + Assert.AreEqual(1, jobs.Count); + + var launchJob = jobs.Single(); + Assert.IsTrue(launchJob.StartedAt.Value >= preStartupTime); + + await new JobsRequiredTest(instanceClient.Jobs).WaitForJob(launchJob, 40, false, cancellationToken); + var dd = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.IsTrue(dd.Running.Value); From f993f52a85a071d8808c8b9f6c80e58cbfe293fc Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 14 May 2020 20:15:38 -0400 Subject: [PATCH 30/41] Remove unused using --- tests/Tgstation.Server.Tests/IntegrationTest.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/Tgstation.Server.Tests/IntegrationTest.cs b/tests/Tgstation.Server.Tests/IntegrationTest.cs index 0934e7e8dd..63e61fe328 100644 --- a/tests/Tgstation.Server.Tests/IntegrationTest.cs +++ b/tests/Tgstation.Server.Tests/IntegrationTest.cs @@ -1,4 +1,3 @@ -using Discord.WebSocket; using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; From d98ac1dbfbafbeb0c3f7d10c4e2a7bcafd025c5e Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 14 May 2020 20:16:10 -0400 Subject: [PATCH 31/41] Better log difference between BYOND topic timeout and abort --- .../Components/Session/SessionController.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Components/Session/SessionController.cs b/src/Tgstation.Server.Host/Components/Session/SessionController.cs index 1b3cc62281..0e8e1736bf 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionController.cs @@ -559,7 +559,11 @@ namespace Tgstation.Server.Host.Components.Session } catch (OperationCanceledException) { - logger.LogTrace("Topic request aborted!"); + logger.LogTrace( + "Topic request {0}!", + cancellationToken.IsCancellationRequested + ? "aborted" + : "timed out"); cancellationToken.ThrowIfCancellationRequested(); } catch (Exception e) From f694dc57444af9579fba151cb3abe7730c8df156 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 14 May 2020 20:16:18 -0400 Subject: [PATCH 32/41] BYOND.TopicSender V5 --- src/Tgstation.Server.Host/Tgstation.Server.Host.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index 57d99ca5df..8de83ec181 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -48,7 +48,7 @@ - + From 7ba22b4d841f2defafaeea6318d811f7587fd5db Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 14 May 2020 20:27:37 -0400 Subject: [PATCH 33/41] Adjust a log message slightly --- src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs b/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs index e485015cca..a157d75e30 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs @@ -118,7 +118,7 @@ namespace Tgstation.Server.Host.Components.Deployment if (!jobLockCounts.TryGetValue(job.Id, out var currentVal) || currentVal == 1) { jobLockCounts.Remove(job.Id); - logger.LogDebug("Cleaning compile job {0} => {1}", job.Id, job.DirectoryName); + logger.LogDebug("Cleaning lock-free compile job {0} => {1}", job.Id, job.DirectoryName); cleanupTask = HandleCleanup(); } else From 9987e4b5a81fea91610af5c91ebe9a1ec0c35d4b Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 14 May 2020 21:07:26 -0400 Subject: [PATCH 34/41] Remove extra space in deployment message --- src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs b/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs index 7ac55b1dc9..3c2b21bb43 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs @@ -382,7 +382,7 @@ namespace Tgstation.Server.Host.Components.Deployment await chatManager.SendUpdateMessage( String.Format( CultureInfo.InvariantCulture, - "*Deployment Triggered*{0}Revision: {1}{2}{3}{0} BYOND Version: {4}.{5}", + "*Deployment Triggered*{0}Revision: {1}{2}{3}{0}BYOND Version: {4}.{5}", Environment.NewLine, commitInsert, testmergeInsert, From 34e4584f700a107ce2fd9d39f83f5cb15fbcd1de Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 14 May 2020 21:15:57 -0400 Subject: [PATCH 35/41] Fix Live game directory being cleaned at instance startup --- .../Components/Deployment/DmbFactory.cs | 14 +++++--------- .../Components/Deployment/IDmbFactory.cs | 5 ++--- .../Deployment/WindowsSwappableDmbProvider.cs | 2 +- src/Tgstation.Server.Host/Components/Instance.cs | 8 +------- 4 files changed, 9 insertions(+), 20 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs b/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs index a157d75e30..62a03990dd 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs @@ -247,7 +247,7 @@ namespace Tgstation.Server.Host.Components.Deployment /// #pragma warning disable CA1506 // TODO: Decomplexify - public async Task CleanUnusedCompileJobs(CompileJob exceptThisOne, CancellationToken cancellationToken) + public async Task CleanUnusedCompileJobs(CancellationToken cancellationToken) { List jobIdsToSkip; @@ -265,13 +265,11 @@ namespace Tgstation.Server.Host.Components.Deployment .Select(x => x.DirectoryName.Value) .ToListAsync(cancellationToken) .ConfigureAwait(false)) - .Select(x => x.ToString().ToUpperInvariant()) + .Select(x => x.ToString()) .ToList(); }).ConfigureAwait(false); - // add the other exemption - if (exceptThisOne != null) - jobUidsToNotErase.Add(exceptThisOne.DirectoryName.Value.ToString().ToUpperInvariant()); + jobUidsToNotErase.Add(WindowsSwappableDmbProvider.LiveGameDirectory); logger.LogTrace("We will not clean the following directories: {0}", String.Join(", ", jobUidsToNotErase)); @@ -283,8 +281,9 @@ namespace Tgstation.Server.Host.Components.Deployment var tasks = directories.Select(async x => { var nameOnly = ioManager.GetFileName(x); - if (jobUidsToNotErase.Contains(nameOnly.ToUpperInvariant())) + if (jobUidsToNotErase.Contains(nameOnly)) return; + logger.LogDebug("Cleaning unused game folder: {0}...", nameOnly); try { ++deleting; @@ -300,10 +299,7 @@ namespace Tgstation.Server.Host.Components.Deployment } }).ToList(); if (deleting > 0) - { - logger.LogDebug("Cleaning unused game folders: {0}...", String.Join(", ", directories)); await Task.WhenAll(tasks).ConfigureAwait(false); - } } #pragma warning restore CA1506 } diff --git a/src/Tgstation.Server.Host/Components/Deployment/IDmbFactory.cs b/src/Tgstation.Server.Host/Components/Deployment/IDmbFactory.cs index 266f3b41e8..4f5059c2be 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/IDmbFactory.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/IDmbFactory.cs @@ -37,11 +37,10 @@ namespace Tgstation.Server.Host.Components.Deployment Task FromCompileJob(CompileJob compileJob, CancellationToken cancellationToken); /// - /// Deletes all compile jobs that are inactive in the Game folder + /// Deletes all compile jobs that are inactive in the Game folder. /// - /// An optional compile job to not delete /// The for the operation /// A representing the running operation - Task CleanUnusedCompileJobs(CompileJob exceptThisOne, CancellationToken cancellationToken); + Task CleanUnusedCompileJobs(CancellationToken cancellationToken); } } diff --git a/src/Tgstation.Server.Host/Components/Deployment/WindowsSwappableDmbProvider.cs b/src/Tgstation.Server.Host/Components/Deployment/WindowsSwappableDmbProvider.cs index add8f65690..597333c54c 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/WindowsSwappableDmbProvider.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/WindowsSwappableDmbProvider.cs @@ -14,7 +14,7 @@ namespace Tgstation.Server.Host.Components.Deployment /// /// The directory where the is symlinked to. /// - const string LiveGameDirectory = "Live"; + public const string LiveGameDirectory = "Live"; /// public string DmbName => baseProvider.DmbName; diff --git a/src/Tgstation.Server.Host/Components/Instance.cs b/src/Tgstation.Server.Host/Components/Instance.cs index 0e23c4cca1..87814d36c6 100644 --- a/src/Tgstation.Server.Host/Components/Instance.cs +++ b/src/Tgstation.Server.Host/Components/Instance.cs @@ -15,7 +15,6 @@ using Tgstation.Server.Host.Components.Repository; using Tgstation.Server.Host.Components.Watchdog; using Tgstation.Server.Host.Core; using Tgstation.Server.Host.Database; -using Tgstation.Server.Host.Extensions; using Tgstation.Server.Host.Jobs; using Tgstation.Server.Host.Models; @@ -623,12 +622,7 @@ namespace Tgstation.Server.Host.Components // dependent on so many things, its just safer this way await Watchdog.StartAsync(cancellationToken).ConfigureAwait(false); - CompileJob latestCompileJob = null; - await databaseContextFactory.UseContext(async db => - { - latestCompileJob = await db.MostRecentCompletedCompileJobOrDefault(metadata, cancellationToken).ConfigureAwait(false); - }).ConfigureAwait(false); - await dmbFactory.CleanUnusedCompileJobs(latestCompileJob, cancellationToken).ConfigureAwait(false); + await dmbFactory.CleanUnusedCompileJobs(cancellationToken).ConfigureAwait(false); } /// From 4b9b72ca0479f00c37c5804e4f220e5786beb8d9 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 14 May 2020 21:49:21 -0400 Subject: [PATCH 36/41] Improve release notes --- tools/ReleaseNotes/Program.cs | 135 ++++++++++++++++++++-------------- 1 file changed, 80 insertions(+), 55 deletions(-) diff --git a/tools/ReleaseNotes/Program.cs b/tools/ReleaseNotes/Program.cs index 7d12f64ff2..e4354a06e2 100644 --- a/tools/ReleaseNotes/Program.cs +++ b/tools/ReleaseNotes/Program.cs @@ -35,6 +35,10 @@ namespace ReleaseNotes var doNotCloseMilestone = args.Length >= 2 && args[1].ToUpperInvariant() == "--NO-CLOSE"; + string limitComponent = null; + if (args.Length > 1 && !doNotCloseMilestone) + limitComponent = args[1]; + const string ReleaseNotesEnvVar = "TGS4_RELEASE_NOTES_TOKEN"; var githubToken = Environment.GetEnvironmentVariable(ReleaseNotesEnvVar); if (String.IsNullOrWhiteSpace(githubToken) && !doNotCloseMilestone) @@ -73,7 +77,7 @@ namespace ReleaseNotes Task milestoneTask = null; var milestoneTaskLock = new object(); - var releaseDictionary = new Dictionary>(); + var releaseDictionary = new Dictionary>>(StringComparer.OrdinalIgnoreCase); var authorizedUsers = new Dictionary>(); bool postControlPanelMessage = false; @@ -94,74 +98,84 @@ namespace ReleaseNotes if (milestoneTask == null) milestoneTask = GetMilestone(); - if (!fullPR.Merged) - return; + // if (!fullPR.Merged) + //return; async Task BuildNotesFromComment(string comment, User user) { + async Task CommitNotes(string component, List notes) + { + Task authTask; + TaskCompletionSource ourTcs = null; + lock (authorizedUsers) + { + if (!authorizedUsers.TryGetValue(user.Id, out authTask)) + { + ourTcs = new TaskCompletionSource(); + authTask = ourTcs.Task; + authorizedUsers.Add(user.Id, authTask); + } + } + + if (ourTcs != null) + try + { + //check if the user has access + var perm = String.IsNullOrWhiteSpace(githubToken) + ? PermissionLevel.Write + : (await client.Repository.Collaborator.ReviewPermission(RepoOwner, RepoName, user.Login).ConfigureAwait(false)).Permission; + ourTcs.SetResult(perm == PermissionLevel.Write || perm == PermissionLevel.Admin); + } + catch + { + ourTcs.SetResult(false); + throw; + } + + var authorized = await authTask.ConfigureAwait(false); + if (!authorized) + return; + + lock (releaseDictionary) + { + foreach (var I in notes) + Console.WriteLine(component + " #" + fullPR.Number + " - " + I + " (@" + user.Login + ")"); + + var tupleSelector = notes.Select(note => Tuple.Create(note, fullPR.Number)); + if (releaseDictionary.TryGetValue(component, out var currentValues)) + currentValues.AddRange(tupleSelector); + else + releaseDictionary.Add(component, tupleSelector.ToList()); + } + } + var commentSplits = comment.Split('\n'); - var notesOpen = false; + string targetComponent = null; var notes = new List(); foreach (var line in commentSplits) { var trimmedLine = line.Trim(); - if (!notesOpen) + if (targetComponent == null) { - notesOpen = trimmedLine.StartsWith(":cl:", StringComparison.Ordinal); + if (trimmedLine.StartsWith(":cl:", StringComparison.Ordinal)) + { + targetComponent = trimmedLine.Substring(4).Trim(); + if (targetComponent.Length == 0) + targetComponent = "Core"; + } continue; } if (trimmedLine.StartsWith("/:cl:", StringComparison.Ordinal)) { - notesOpen = false; + await CommitNotes(targetComponent, notes); + targetComponent = null; + notes.Clear(); continue; } if (trimmedLine.Length == 0) continue; notes.Add(trimmedLine); } - if (notesOpen || notes.Count == 0) - return; - - Task authTask; - TaskCompletionSource ourTcs = null; - lock (authorizedUsers) - { - if (!authorizedUsers.TryGetValue(user.Id, out authTask)) - { - ourTcs = new TaskCompletionSource(); - authTask = ourTcs.Task; - authorizedUsers.Add(user.Id, authTask); - } - } - - if (ourTcs != null) - try - { - //check if the user has access - var perm = String.IsNullOrWhiteSpace(githubToken) - ? PermissionLevel.Write - : (await client.Repository.Collaborator.ReviewPermission(RepoOwner, RepoName, user.Login).ConfigureAwait(false)).Permission; - ourTcs.SetResult(perm == PermissionLevel.Write || perm == PermissionLevel.Admin); - } - catch - { - ourTcs.SetResult(false); - throw; - } - - var authorized = await authTask.ConfigureAwait(false); - if (!authorized) - return; - - lock (releaseDictionary) - { - foreach (var I in notes) - Console.WriteLine("#" + fullPR.Number + " - " + I + " (@" + user.Login + ")"); - if (releaseDictionary.TryGetValue(fullPR.Number, out var currentValues)) - currentValues.AddRange(notes); - else - releaseDictionary.Add(fullPR.Number, notes); - } } var comments = await client.Issue.Comment.GetAllForIssue(RepoOwner, RepoName, fullPR.Number).ConfigureAwait(false); @@ -205,7 +219,7 @@ namespace ReleaseNotes //trim away all the lines that don't start with # string keepThisRelease; - if (version.Build == 0) + if (version.Build <= 1) keepThisRelease = "# "; else keepThisRelease = "## "; @@ -294,17 +308,28 @@ namespace ReleaseNotes } foreach (var I in releaseDictionary.OrderBy(kvp => kvp.Key)) - foreach (var note in I.Value) + { + if (limitComponent != null && I.Key != limitComponent) + continue; + + newNotes.Append(Environment.NewLine); + newNotes.Append("#### "); + newNotes.Append(I.Key); + + + foreach (var noteTuple in I.Value) { newNotes.Append(Environment.NewLine); newNotes.Append("- "); - newNotes.Append(note); + newNotes.Append(noteTuple.Item1); newNotes.Append(" (#"); - newNotes.Append(I.Key); + newNotes.Append(noteTuple.Item2); newNotes.Append(')'); } - newNotes.Append(Environment.NewLine); + newNotes.Append(Environment.NewLine); + } + newNotes.Append(Environment.NewLine); if (version != new Version(4, 1, 0)) From 89873d89e747d9e22f1f644b76f22646048d1c5e Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 14 May 2020 22:16:50 -0400 Subject: [PATCH 37/41] Even better release notes generation --- build/prep_deployment.ps1 | 6 ++++-- tools/ReleaseNotes/Program.cs | 12 ++++-------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/build/prep_deployment.ps1 b/build/prep_deployment.ps1 index 2f50c5b2b7..358b72d8c6 100644 --- a/build/prep_deployment.ps1 +++ b/build/prep_deployment.ps1 @@ -1,5 +1,7 @@ $bf = $env:APPVEYOR_BUILD_FOLDER -[XML]$versionXML = Get-Content "$bf/build/Version.props" +$propsPath = "$bf/build/Version.props" + +[XML]$versionXML = Get-Content $propsPath $env:TGSVersion = $versionXML.Project.PropertyGroup.TgsCoreVersion $env:APIVersion = $versionXML.Project.PropertyGroup.TgsApiVersion $env:DMVersion = $versionXML.Project.PropertyGroup.TgsDmapiVersion @@ -11,7 +13,7 @@ if (($env:CONFIGURATION -match "Release") -And ($env:APPVEYOR_REPO_BRANCH -match $env:TGSDeploy = "Do it." Write-Host "Generating release notes..." - dotnet run -p "$bf/tools/ReleaseNotes" $env:TGSVersion + dotnet run -p "$bf/tools/ReleaseNotes" $env:TGSVersion $propsPath $env:TGSDraftNotes = !($?) $releaseNotesPath = "$bf/release_notes.md" Write-Host "Reading release notes from $releaseNotesPath..." diff --git a/tools/ReleaseNotes/Program.cs b/tools/ReleaseNotes/Program.cs index e4354a06e2..5e205c6b8a 100644 --- a/tools/ReleaseNotes/Program.cs +++ b/tools/ReleaseNotes/Program.cs @@ -1,7 +1,6 @@ using Octokit; using System; using System.Collections.Generic; -using System.Globalization; using System.IO; using System.Linq; using System.Text; @@ -33,11 +32,11 @@ namespace ReleaseNotes return 2; } - var doNotCloseMilestone = args.Length >= 2 && args[1].ToUpperInvariant() == "--NO-CLOSE"; + var doNotCloseMilestone = args.Length > 1 && args[1].ToUpperInvariant() == "--NO-CLOSE"; - string limitComponent = null; + string propsPath = "../../../../../build/Version.props"; if (args.Length > 1 && !doNotCloseMilestone) - limitComponent = args[1]; + propsPath = args[1]; const string ReleaseNotesEnvVar = "TGS4_RELEASE_NOTES_TOKEN"; var githubToken = Environment.GetEnvironmentVariable(ReleaseNotesEnvVar); @@ -237,7 +236,7 @@ namespace ReleaseNotes switch (releasingSuite) { case 4: - var doc = XDocument.Load("../../../../../build/Version.props"); + var doc = XDocument.Load(propsPath); var project = doc.Root; var xmlNamespace = project.GetDefaultNamespace(); var versionsPropertyGroup = project.Elements().First(); @@ -309,9 +308,6 @@ namespace ReleaseNotes foreach (var I in releaseDictionary.OrderBy(kvp => kvp.Key)) { - if (limitComponent != null && I.Key != limitComponent) - continue; - newNotes.Append(Environment.NewLine); newNotes.Append("#### "); newNotes.Append(I.Key); From 7342cf1ea739fb80b3a3b37d8e16b177e728c07c Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 14 May 2020 22:27:43 -0400 Subject: [PATCH 38/41] Log DD output if possible --- .../Session/SessionControllerFactory.cs | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs b/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs index f315c9f335..92a76a973b 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs @@ -84,6 +84,11 @@ namespace Tgstation.Server.Host.Components.Session /// readonly ILoggerFactory loggerFactory; + /// + /// The for the + /// + readonly ILogger logger; + /// /// The for the /// @@ -121,6 +126,7 @@ namespace Tgstation.Server.Host.Components.Session /// The value of . /// The value of . /// The value of + /// The value of . public SessionControllerFactory( IProcessExecutor processExecutor, IByondManager byond, @@ -134,6 +140,7 @@ namespace Tgstation.Server.Host.Components.Session IBridgeRegistrar bridgeRegistrar, IServerPortProvider serverPortProvider, ILoggerFactory loggerFactory, + ILogger logger, Api.Models.Instance instance) { this.processExecutor = processExecutor ?? throw new ArgumentNullException(nameof(processExecutor)); @@ -149,6 +156,7 @@ namespace Tgstation.Server.Host.Components.Session this.bridgeRegistrar = bridgeRegistrar ?? throw new ArgumentNullException(nameof(bridgeRegistrar)); this.serverPortProvider = serverPortProvider ?? throw new ArgumentNullException(nameof(serverPortProvider)); this.loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); } /// @@ -220,7 +228,20 @@ namespace Tgstation.Server.Host.Components.Session byondLock.DreamDaemonPath, basePath, arguments, + noShellExecute, + noShellExecute, noShellExecute: noShellExecute); + + if (noShellExecute) + { + // Log DD output + _ = process.Lifetime.ContinueWith( + x => logger.LogTrace( + "DreamDaemon Output:{0}{1}", + Environment.NewLine, process.GetCombinedOutput()), + TaskScheduler.Current); + } + try { networkPromptReaper.RegisterProcess(process); From e621f3599ea370916d654f66eab8291ce402375e Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 14 May 2020 22:32:53 -0400 Subject: [PATCH 39/41] Add missing change --- src/Tgstation.Server.Host/Components/InstanceFactory.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Tgstation.Server.Host/Components/InstanceFactory.cs b/src/Tgstation.Server.Host/Components/InstanceFactory.cs index d9991f0044..7b841e1270 100644 --- a/src/Tgstation.Server.Host/Components/InstanceFactory.cs +++ b/src/Tgstation.Server.Host/Components/InstanceFactory.cs @@ -235,6 +235,7 @@ namespace Tgstation.Server.Host.Components bridgeRegistrar, serverPortProvider, loggerFactory, + loggerFactory.CreateLogger(), metadata.CloneMetadata()); var dmbFactory = new DmbFactory(databaseContextFactory, gameIoManager, loggerFactory.CreateLogger(), metadata.CloneMetadata()); From 7f09ffe2d46e5977c448ebccd584c73854c66067 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 14 May 2020 23:08:54 -0400 Subject: [PATCH 40/41] Change BYOND version used under test --- tests/Tgstation.Server.Tests/Instance/ByondTest.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/Tgstation.Server.Tests/Instance/ByondTest.cs b/tests/Tgstation.Server.Tests/Instance/ByondTest.cs index 6c6de168bd..b83876aa00 100644 --- a/tests/Tgstation.Server.Tests/Instance/ByondTest.cs +++ b/tests/Tgstation.Server.Tests/Instance/ByondTest.cs @@ -25,7 +25,7 @@ namespace Tgstation.Server.Tests.Instance public async Task Run(CancellationToken cancellationToken) { await TestNoVersion(cancellationToken).ConfigureAwait(false); - await TestInstall511(cancellationToken).ConfigureAwait(false); + await TestInstallStable(cancellationToken).ConfigureAwait(false); await TestInstallFakeVersion(cancellationToken).ConfigureAwait(false); } @@ -40,11 +40,11 @@ namespace Tgstation.Server.Tests.Instance await WaitForJob(test.InstallJob, 60, true, cancellationToken).ConfigureAwait(false); } - async Task TestInstall511(CancellationToken cancellationToken) + async Task TestInstallStable(CancellationToken cancellationToken) { var newModel = new Api.Models.Byond { - Version = new Version(511, 1385) + Version = new Version(513, 1514) }; var test = await byondClient.SetActiveVersion(newModel, cancellationToken).ConfigureAwait(false); Assert.IsNotNull(test.InstallJob); From c88c4bbc3b0d52d9d7636fb232734c6c658c41ea Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 14 May 2020 23:21:43 -0400 Subject: [PATCH 41/41] Fix test assert --- tests/Tgstation.Server.Tests/Instance/ByondTest.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Tgstation.Server.Tests/Instance/ByondTest.cs b/tests/Tgstation.Server.Tests/Instance/ByondTest.cs index b83876aa00..e19a840a3e 100644 --- a/tests/Tgstation.Server.Tests/Instance/ByondTest.cs +++ b/tests/Tgstation.Server.Tests/Instance/ByondTest.cs @@ -57,7 +57,7 @@ namespace Tgstation.Server.Tests.Instance if (new PlatformIdentifier().IsWindows) dreamMaker += ".exe"; - var dreamMakerDir = Path.Combine(metadata.Path, "Byond", "511.1385", "byond", "bin"); + var dreamMakerDir = Path.Combine(metadata.Path, "Byond", newModel.Version.ToString(), "byond", "bin"); Assert.IsTrue(Directory.Exists(dreamMakerDir), $"Directory {dreamMakerDir} does not exist!"); Assert.IsTrue(File.Exists(Path.Combine(dreamMakerDir, dreamMaker)), $"Missing DreamMaker executable! Dir contents: {String.Join(", ", Directory.GetFileSystemEntries(dreamMakerDir))}");