diff --git a/README.md b/README.md index c8b6b54c27..77c2286a73 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,8 @@ Create an `appsettings.Production.yml` file next to `appsettings.yml`. This will - `Session:LowPriorityDeploymentProcesses `: Boolean controlling if DreamMaker and API validation DreamDaemon instances get set to below normal priority processes. +- `Session:DoNotUseDDExe`: Boolean controlling if dd.exe, the CLI DreamDaemon for Windows released in BYOND 515.1598, is NOT used (by default it is when available). + - `FileLogging:Directory`: Override the default directory where server logs are stored. Default is C:/ProgramData/tgstation-server/logs on Windows, /usr/share/tgstation-server/logs otherwise - `FileLogging:LogLevel`: Can be one of `Trace`, `Debug`, `Information`, `Warning`, `Error`, or `Critical`. Restricts what is put into the log files. Currently `Debug` is reccommended for help with error reporting. diff --git a/build/Version.props b/build/Version.props index 42dc20d5d1..0ea9bdf9ac 100644 --- a/build/Version.props +++ b/build/Version.props @@ -4,7 +4,7 @@ 5.13.0 - 4.6.0 + 4.7.0 9.11.0 6.0.0 11.0.0 diff --git a/src/Tgstation.Server.Host/Components/Byond/WindowsByondInstaller.cs b/src/Tgstation.Server.Host/Components/Byond/WindowsByondInstaller.cs index ea78d2f5bd..c2dbc34076 100644 --- a/src/Tgstation.Server.Host/Components/Byond/WindowsByondInstaller.cs +++ b/src/Tgstation.Server.Host/Components/Byond/WindowsByondInstaller.cs @@ -70,6 +70,11 @@ namespace Tgstation.Server.Host.Components.Byond /// readonly GeneralConfiguration generalConfiguration; + /// + /// The for the . + /// + readonly SessionConfiguration sessionConfiguration; + /// /// The for the . /// @@ -85,6 +90,7 @@ namespace Tgstation.Server.Host.Components.Byond /// /// The value of . /// The containing the value of . + /// The containing the value of . /// The for the . /// The for the . /// The for the . @@ -93,11 +99,13 @@ namespace Tgstation.Server.Host.Components.Byond IIOManager ioManager, IFileDownloader fileDownloader, IOptions generalConfigurationOptions, + IOptions sessionConfigurationOptions, ILogger logger) : base(ioManager, fileDownloader, logger) { this.processExecutor = processExecutor ?? throw new ArgumentNullException(nameof(processExecutor)); generalConfiguration = generalConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(generalConfigurationOptions)); + sessionConfiguration = sessionConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(sessionConfigurationOptions)); PathToUserByondFolder = IOManager.ResolvePath(IOManager.ConcatPath(Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments), "BYOND")); @@ -113,7 +121,7 @@ namespace Tgstation.Server.Host.Components.Byond { ArgumentNullException.ThrowIfNull(version); - supportsCli = version >= DDExeVersion; + supportsCli = !sessionConfiguration.DoNotUseDDExe && version >= DDExeVersion; supportsMapThreads = version >= MapThreadsVersion; return supportsCli ? "dd.exe" : "dreamdaemon.exe"; } diff --git a/src/Tgstation.Server.Host/Configuration/SessionConfiguration.cs b/src/Tgstation.Server.Host/Configuration/SessionConfiguration.cs index 9d0a18f656..6260b9aae0 100644 --- a/src/Tgstation.Server.Host/Configuration/SessionConfiguration.cs +++ b/src/Tgstation.Server.Host/Configuration/SessionConfiguration.cs @@ -24,5 +24,10 @@ /// If the deployment DreamMaker and DreamDaemon instances are set to be below normal priority processes. /// public bool LowPriorityDeploymentProcesses { get; set; } + + /// + /// If dd.exe should not be used if it is available. + /// + public bool DoNotUseDDExe { get; set; } } } diff --git a/src/Tgstation.Server.Host/ServerFactory.cs b/src/Tgstation.Server.Host/ServerFactory.cs index 01c1c752b2..20e179b98b 100644 --- a/src/Tgstation.Server.Host/ServerFactory.cs +++ b/src/Tgstation.Server.Host/ServerFactory.cs @@ -117,6 +117,9 @@ namespace Tgstation.Server.Host kestrelOptions.ListenAnyIP( serverPortProvider.HttpApiPort, listenOptions => listenOptions.Protocols = HttpProtocols.Http1AndHttp2); + + // with 515 we lost the ability to test this effectively. Just bump it slightly above the default and let the existing limit hold us back + kestrelOptions.Limits.MaxRequestLineSize = 8400; }) .UseIIS() .UseIISIntegration() diff --git a/src/Tgstation.Server.Host/appsettings.yml b/src/Tgstation.Server.Host/appsettings.yml index 98e98062a8..9c3f10eb12 100644 --- a/src/Tgstation.Server.Host/appsettings.yml +++ b/src/Tgstation.Server.Host/appsettings.yml @@ -16,6 +16,7 @@ General: Session: HighPriorityLiveDreamDaemon: false LowPriorityDeploymentProcesses: true + DoNotUseDDExe: false FileLogging: Directory: Disable: false diff --git a/tests/DMAPI/LongRunning/Test.dm b/tests/DMAPI/LongRunning/Test.dm index 840e994c2c..268d8015a7 100644 --- a/tests/DMAPI/LongRunning/Test.dm +++ b/tests/DMAPI/LongRunning/Test.dm @@ -248,14 +248,6 @@ var/suppress_bridge_spam = FALSE set waitfor = FALSE CheckBridgeLimitsImpl() - -/proc/BridgeWithoutChunking(command, list/data) - var/datum/tgs_api/v5/api = TGS_READ_GLOBAL(tgs) - var/bridge_request = api.CreateBridgeRequest(command, data) - suppress_bridge_spam = TRUE - . = api.PerformBridgeRequest(bridge_request) - suppress_bridge_spam = FALSE - /proc/CheckBridgeLimitsImpl() sleep(30) @@ -264,33 +256,30 @@ var/suppress_bridge_spam = FALSE var/old_ai = api.access_identifier api.access_identifier = "tgs_integration_test" - // Always send chat messages because they can have extremely large payloads with the text - - // bisecting request test - var/base = 1 - var/nextPow = 0 - var/lastI = 0 - var/i lastTgsError = null - for(i = 1; ; i = base + (2 ** nextPow)) - var/payload = create_payload(i) - var/list/result = BridgeWithoutChunking(0, list("chatMessage" = list("text" = "payload:[payload]"))) + var/limit = 8198 // DMAPI5_BRIDGE_REQUEST_LIMIT - if(!result || lastTgsError || result["integrationHack"] != "ok") - lastTgsError = null - if(i == lastI + 1) - break - i = lastI - base = lastI - nextPow = 0 - continue + // Always send chat messages because they can have extremely large payloads with the text + var/base_bridge_request = api.CreateBridgeRequest(0, list("chatMessage" = list("text" = "payload:"))) - lastI = i - ++nextPow + // In 515 overloaded bridge requests started BUG-ing because of the HTTP 414 response it gets on errors + // so now we can only test that the limit is valid + // It's fine, chunking will handle the rest + var/payload_size = limit - length(base_bridge_request) + + var/payload = create_payload(payload_size) + var/bridge_request = api.CreateBridgeRequest(0, list("chatMessage" = list("text" = "payload:[payload]"))) - // DMAPI5_BRIDGE_REQUEST_LIMIT - var/limit = 8198 + var/list/result + try + result = api.PerformBridgeRequest(bridge_request) + catch(var/exception/e) + result = null + + if(!result || lastTgsError || result["integrationHack"] != "ok") + FailTest("Failed bridge request limit test!") + return // this actually gets doubled because it's in two fields for backwards compatibility, but that's fine var/list/final_result = api.Bridge(0, list("chatMessage" = list("text" = "done:[create_payload(limit * 3)]"))) diff --git a/tests/Tgstation.Server.Tests/Live/Instance/ByondTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/ByondTest.cs index dafc3b03e7..332c7196e6 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/ByondTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/ByondTest.cs @@ -158,6 +158,8 @@ namespace Tgstation.Server.Tests.Live.Instance { var generalConfigOptionsMock = new Mock>(); generalConfigOptionsMock.SetupGet(x => x.Value).Returns(new GeneralConfiguration()); + var sessionConfigOptionsMock = new Mock>(); + sessionConfigOptionsMock.SetupGet(x => x.Value).Returns(new SessionConfiguration()); var assemblyInformationProvider = new AssemblyInformationProvider(); @@ -167,6 +169,7 @@ namespace Tgstation.Server.Tests.Live.Instance Mock.Of(), fileDownloader, generalConfigOptionsMock.Object, + sessionConfigOptionsMock.Object, Mock.Of>()) : new PosixByondInstaller( Mock.Of(), diff --git a/tests/Tgstation.Server.Tests/Live/Instance/DeploymentTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/DeploymentTest.cs index e4171c29d5..b770494875 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/DeploymentTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/DeploymentTest.cs @@ -57,11 +57,11 @@ namespace Tgstation.Server.Tests.Live.Instance ? "dd" : "DreamDaemon"; - var allProcesses = System.Diagnostics.Process.GetProcessesByName(ddProcessName); - if (allProcesses.Length == 0) + var allProcesses = TestLiveServer.GetAllDDProcesses(); + if (allProcesses.Count == 0) continue; - if (allProcesses.Length > 1) + if (allProcesses.Count > 1) Assert.Fail("Multiple DreamDaemon-like processes running!"); using var process = allProcesses[0]; diff --git a/tests/Tgstation.Server.Tests/Live/Instance/TestBridgeHandler.cs b/tests/Tgstation.Server.Tests/Live/Instance/TestBridgeHandler.cs index c3828ebb6e..dacd8bdd98 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/TestBridgeHandler.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/TestBridgeHandler.cs @@ -45,6 +45,7 @@ namespace Tgstation.Server.Tests.Live.Instance { try { + Logger.LogTrace("Bridge request received"); Assert.AreEqual(DMApiParameters.AccessIdentifier, parameters.AccessIdentifier); if (parameters.CommandType == BridgeCommandType.Chunk) return await ProcessChunk( @@ -84,8 +85,9 @@ namespace Tgstation.Server.Tests.Live.Instance } Assert.AreEqual("payload", coreMessage); - lastBridgeRequestSize = $"http://127.0.0.1:{serverPort}/Bridge?data=".Length + HttpUtility.UrlEncode( - JsonConvert.SerializeObject(parameters, DMApiConstants.SerializerSettings)).Length; + var serializedRequest = JsonConvert.SerializeObject(parameters, DMApiConstants.SerializerSettings); + var actualLastRequest = $"http://127.0.0.1:{serverPort}/Bridge?data=" + HttpUtility.UrlEncode(serializedRequest); + lastBridgeRequestSize = actualLastRequest.Length; return new BridgeResponseHack { IntegrationHack = "ok" diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index 1ae928dbe9..8e1f889570 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -12,7 +12,6 @@ using System.Globalization; using System.IO; using System.Linq; using System.Net; -using System.Net.NetworkInformation; using System.Net.Sockets; using System.Runtime.InteropServices; using System.Threading; @@ -399,7 +398,7 @@ namespace Tgstation.Server.Tests.Live.Instance CheckDDPriority(); // lock on to DD and pause it so it can't health check - var ddProcs = System.Diagnostics.Process.GetProcessesByName("DreamDaemon").Where(x => !x.HasExited).ToList(); + var ddProcs = TestLiveServer.GetAllDDProcesses().Where(x => !x.HasExited).ToList(); if (ddProcs.Count != 1) Assert.Fail($"Incorrect number of DD processes: {ddProcs.Count}"); @@ -748,11 +747,11 @@ namespace Tgstation.Server.Tests.Live.Instance ? "dd" : "DreamDaemon"; - var allProcesses = System.Diagnostics.Process.GetProcessesByName(ddProcessName); - if (allProcesses.Length == 0) + var allProcesses = TestLiveServer.GetAllDDProcesses().Where(x => !x.HasExited).ToList(); + if (allProcesses.Count == 0) Assert.Fail("Expected DreamDaemon to be running here"); - if (allProcesses.Length > 1) + if (allProcesses.Count > 1) Assert.Fail("Multiple DreamDaemon-like processes running!"); using var process = allProcesses[0]; @@ -954,7 +953,7 @@ namespace Tgstation.Server.Tests.Live.Instance static bool KillDD(bool require) { - var ddProcs = System.Diagnostics.Process.GetProcessesByName("DreamDaemon").Where(x => !x.HasExited).ToList(); + var ddProcs = TestLiveServer.GetAllDDProcesses().Where(x => !x.HasExited).ToList(); if (require && ddProcs.Count == 0 || ddProcs.Count > 1) Assert.Fail($"Incorrect number of DD processes: {ddProcs.Count}"); @@ -1091,7 +1090,7 @@ namespace Tgstation.Server.Tests.Live.Instance return; var daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); - if (daemonStatus.Status != WatchdogStatus.Offline) + if (daemonStatus.Status != WatchdogStatus.Offline || !daemonStatus.LogOutput.Value) return; var outerLogsDir = Path.Combine(instanceClient.Metadata.Path, "Diagnostics", "DreamDaemonLogs"); diff --git a/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs b/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs index 6d932735e8..bd51817853 100644 --- a/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs +++ b/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs @@ -49,8 +49,9 @@ namespace Tgstation.Server.Tests.Live SerilogContextHelper.AddSwarmNodeIdentifierToTemplate(); } - public LiveTestingServer(SwarmConfiguration swarmConfiguration, bool enableOAuth, ushort port = 5010) + public LiveTestingServer(SwarmConfiguration swarmConfiguration, bool enableOAuth, ushort port = 15010, bool disableDDExe = false) { + Assert.IsTrue(port >= 10000); // for testing bridge request limit Directory = Environment.GetEnvironmentVariable("TGS_TEST_TEMP_DIRECTORY"); if (string.IsNullOrWhiteSpace(Directory)) { @@ -113,6 +114,7 @@ namespace Tgstation.Server.Tests.Live "General:ByondTopicTimeout=3000", $"Session:HighPriorityLiveDreamDaemon={HighPriorityDreamDaemon}", $"Session:LowPriorityDeploymentProcesses={LowPriorityDeployments}", + $"Session:DoNotUseDDExe={disableDDExe}", }; swarmArgs = new List(); @@ -151,7 +153,7 @@ namespace Tgstation.Server.Tests.Live for (int i = 0; i < 5; ++i) try { - System.IO.Directory.Delete(Directory, true); + //System.IO.Directory.Delete(Directory, true); } catch { diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index 8d17868781..5ba9b47b0b 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Data.SqlClient; using System.Diagnostics; using System.IO; @@ -51,9 +52,19 @@ namespace Tgstation.Server.Tests.Live readonly IServerClientFactory clientFactory = new ServerClientFactory(new ProductHeaderValue(Assembly.GetExecutingAssembly().GetName().Name, Assembly.GetExecutingAssembly().GetName().Version.ToString())); + public static List GetAllDDProcesses() + { + var result = new List(); + result.AddRange(System.Diagnostics.Process.GetProcessesByName("DreamDaemon")); + if(new PlatformIdentifier().IsWindows) + result.AddRange(System.Diagnostics.Process.GetProcessesByName("dd")); + + return result; + } + static void TerminateAllDDs() { - foreach (var proc in System.Diagnostics.Process.GetProcessesByName("DreamDaemon")) + foreach (var proc in GetAllDDProcesses()) using (proc) proc.Kill(); } @@ -368,13 +379,13 @@ namespace Tgstation.Server.Tests.Live const string PrivateKey = "adlfj73ywifhks7iwrgfegjs"; - var controllerAddress = new Uri("http://localhost:5011"); + var controllerAddress = new Uri("http://localhost:15011"); using (var controller = new LiveTestingServer(new SwarmConfiguration { Address = controllerAddress, Identifier = "controller", PrivateKey = PrivateKey - }, false, 5011)) + }, false, 15011)) { using var serverCts = new CancellationTokenSource(); serverCts.CancelAfter(TimeSpan.FromHours(3)); @@ -393,7 +404,7 @@ namespace Tgstation.Server.Tests.Live Assert.AreEqual(1, serverInformation.SwarmServers.Count); var controller = serverInformation.SwarmServers.SingleOrDefault(x => x.Identifier == "controller"); Assert.IsNotNull(controller); - Assert.AreEqual(controller.Address, new Uri("http://localhost:5011")); + Assert.AreEqual(controller.Address, new Uri("http://localhost:15011")); Assert.IsTrue(controller.Controller); } @@ -454,28 +465,28 @@ namespace Tgstation.Server.Tests.Live const string PrivateKey = "adlfj73ywifhks7iwrgfegjs"; - var controllerAddress = new Uri("http://localhost:5011"); + var controllerAddress = new Uri("http://localhost:15011"); using (var controller = new LiveTestingServer(new SwarmConfiguration { Address = controllerAddress, Identifier = "controller", PrivateKey = PrivateKey - }, false, 5011)) + }, false, 15011)) { using var node1 = new LiveTestingServer(new SwarmConfiguration { - Address = new Uri("http://localhost:5012"), + Address = new Uri("http://localhost:15012"), ControllerAddress = controllerAddress, Identifier = "node1", PrivateKey = PrivateKey - }, false, 5012); + }, false, 15012); using var node2 = new LiveTestingServer(new SwarmConfiguration { - Address = new Uri("http://localhost:5013"), + Address = new Uri("http://localhost:15013"), ControllerAddress = controllerAddress, Identifier = "node2", PrivateKey = PrivateKey - }, false, 5013); + }, false, 15013); using var serverCts = new CancellationTokenSource(); var cancellationToken = serverCts.Token; var serverTask = Task.WhenAll( @@ -509,17 +520,17 @@ namespace Tgstation.Server.Tests.Live var node1 = serverInformation.SwarmServers.SingleOrDefault(x => x.Identifier == "node1"); Assert.IsNotNull(node1); - Assert.AreEqual(node1.Address, new Uri("http://localhost:5012")); + Assert.AreEqual(node1.Address, new Uri("http://localhost:15012")); Assert.IsFalse(node1.Controller); var node2 = serverInformation.SwarmServers.SingleOrDefault(x => x.Identifier == "node2"); Assert.IsNotNull(node2); - Assert.AreEqual(node2.Address, new Uri("http://localhost:5013")); + Assert.AreEqual(node2.Address, new Uri("http://localhost:15013")); Assert.IsFalse(node2.Controller); var controller = serverInformation.SwarmServers.SingleOrDefault(x => x.Identifier == "controller"); Assert.IsNotNull(controller); - Assert.AreEqual(controller.Address, new Uri("http://localhost:5011")); + Assert.AreEqual(controller.Address, new Uri("http://localhost:15011")); Assert.IsTrue(controller.Controller); } @@ -718,29 +729,29 @@ namespace Tgstation.Server.Tests.Live const string PrivateKey = "adlfj73ywifhks7iwrgfegjs"; - var controllerAddress = new Uri("http://localhost:5011"); + var controllerAddress = new Uri("http://localhost:15011"); using (var controller = new LiveTestingServer(new SwarmConfiguration { Address = controllerAddress, Identifier = "controller", PrivateKey = PrivateKey, UpdateRequiredNodeCount = 2, - }, false, 5011)) + }, false, 15011)) { using var node1 = new LiveTestingServer(new SwarmConfiguration { - Address = new Uri("http://localhost:5012"), + Address = new Uri("http://localhost:15012"), ControllerAddress = controllerAddress, Identifier = "node1", PrivateKey = PrivateKey - }, false, 5012); + }, false, 15012); using var node2 = new LiveTestingServer(new SwarmConfiguration { - Address = new Uri("http://localhost:5013"), + Address = new Uri("http://localhost:15013"), ControllerAddress = controllerAddress, Identifier = "node2", PrivateKey = PrivateKey - }, false, 5013); + }, false, 15013); using var serverCts = new CancellationTokenSource(); var cancellationToken = serverCts.Token; @@ -778,17 +789,17 @@ namespace Tgstation.Server.Tests.Live var node1 = serverInformation.SwarmServers.SingleOrDefault(x => x.Identifier == "node1"); Assert.IsNotNull(node1); - Assert.AreEqual(node1.Address, new Uri("http://localhost:5012")); + Assert.AreEqual(node1.Address, new Uri("http://localhost:15012")); Assert.IsFalse(node1.Controller); var node2 = serverInformation.SwarmServers.SingleOrDefault(x => x.Identifier == "node2"); Assert.IsNotNull(node2); - Assert.AreEqual(node2.Address, new Uri("http://localhost:5013")); + Assert.AreEqual(node2.Address, new Uri("http://localhost:15013")); Assert.IsFalse(node2.Controller); var controller = serverInformation.SwarmServers.SingleOrDefault(x => x.Identifier == "controller"); Assert.IsNotNull(controller); - Assert.AreEqual(controller.Address, new Uri("http://localhost:5011")); + Assert.AreEqual(controller.Address, new Uri("http://localhost:15011")); Assert.IsTrue(controller.Controller); } diff --git a/tests/Tgstation.Server.Tests/TestVersions.cs b/tests/Tgstation.Server.Tests/TestVersions.cs index 21fd76bd58..2b202da4af 100644 --- a/tests/Tgstation.Server.Tests/TestVersions.cs +++ b/tests/Tgstation.Server.Tests/TestVersions.cs @@ -93,6 +93,8 @@ namespace Tgstation.Server.Tests { var mockGeneralConfigurationOptions = new Mock>(); mockGeneralConfigurationOptions.SetupGet(x => x.Value).Returns(new GeneralConfiguration()); + var mockSessionConfigurationOptions = new Mock>(); + mockSessionConfigurationOptions.SetupGet(x => x.Value).Returns(new SessionConfiguration()); // windows only BYOND but can be checked on any system var init1 = CachingFileDownloader.InitializeByondVersion( @@ -112,6 +114,7 @@ namespace Tgstation.Server.Tests Mock.Of(), new CachingFileDownloader(Mock.Of>()), mockGeneralConfigurationOptions.Object, + mockSessionConfigurationOptions.Object, Mock.Of>()); const string ArchiveEntryPath = "byond/bin/dd.exe"; @@ -137,6 +140,8 @@ namespace Tgstation.Server.Tests { SkipAddingByondFirewallException = true, }); + var mockSessionConfigurationOptions = new Mock>(); + mockSessionConfigurationOptions.SetupGet(x => x.Value).Returns(new SessionConfiguration()); using var loggerFactory = LoggerFactory.Create(builder => { @@ -164,6 +169,7 @@ namespace Tgstation.Server.Tests Mock.Of(), fileDownloader, mockGeneralConfigurationOptions.Object, + mockSessionConfigurationOptions.Object, loggerFactory.CreateLogger()) : new PosixByondInstaller( Mock.Of(),