diff --git a/build/Version.props b/build/Version.props index 4aed1e7d1c..3b58bdbe15 100644 --- a/build/Version.props +++ b/build/Version.props @@ -3,7 +3,7 @@ - 5.12.3 + 5.12.4 4.6.0 9.10.2 10.4.1 diff --git a/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs b/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs index 4d1a37fe96..8470210425 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs @@ -814,8 +814,6 @@ namespace Tgstation.Server.Host.Components.Deployment using (var provider = new TemporaryDmbProvider(ioManager.ResolvePath(job.DirectoryName.ToString()), String.Concat(job.DmeName, DmbExtension), job)) await using (var controller = await sessionControllerFactory.LaunchNew(provider, byondLock, launchParameters, true, cancellationToken)) { - controller.AdjustPriority(false); - var launchResult = await controller.LaunchResult; if (launchResult.StartupTime.HasValue) diff --git a/src/Tgstation.Server.Host/Components/Session/SessionController.cs b/src/Tgstation.Server.Host/Components/Session/SessionController.cs index e3da5a1ea7..164fb6e218 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionController.cs @@ -29,6 +29,11 @@ namespace Tgstation.Server.Host.Components.Session /// sealed class SessionController : Chunker, ISessionController, IBridgeHandler, IChannelSink { + /// + /// If calls to should be trace logged. + /// + internal static bool LogTopicRequests { get; set; } = true; + /// public DMApiParameters DMApiParameters => ReattachInformation; @@ -365,7 +370,7 @@ namespace Tgstation.Server.Host.Components.Session void LogCombinedResponse() { - if (combinedResponse != null) + if (LogTopicRequests && combinedResponse != null) Logger.LogTrace("Topic response: {topicString}", combinedResponse.ByondTopicResponse.StringData ?? "(NO STRING DATA)"); } @@ -768,7 +773,8 @@ namespace Tgstation.Server.Host.Components.Session parameters.AccessIdentifier = ReattachInformation.AccessIdentifier; var fullCommandString = GenerateQueryString(parameters, out var json); - Logger.LogTrace("Topic request: {json}", json); + if (LogTopicRequests) + Logger.LogTrace("Topic request: {json}", json); var fullCommandByteCount = Encoding.UTF8.GetByteCount(fullCommandString); var topicPriority = parameters.IsPriority; if (fullCommandByteCount <= DMApiConstants.MaximumTopicRequestLength) diff --git a/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs b/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs index 99426d3bb7..e1a9f00833 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs @@ -507,9 +507,7 @@ namespace Tgstation.Server.Host.Components.Session VisibilityWord(launchParameters.Visibility.Value), !byondLock.SupportsCli ? $" -logself -log {logFilePath}" - : !platformIdentifier.IsWindows // Just use stdout on if CLI is supported - ? " -logself" - : String.Empty, // Windows doesn't output anything to dd.exe if -logself is set? + : String.Empty, // DD doesn't output anything if -logself is set??? launchParameters.StartProfiler.Value ? " -profile" : String.Empty, diff --git a/src/Tgstation.Server.Host/Core/ServerUpdater.cs b/src/Tgstation.Server.Host/Core/ServerUpdater.cs index adcb902fd7..86623be571 100644 --- a/src/Tgstation.Server.Host/Core/ServerUpdater.cs +++ b/src/Tgstation.Server.Host/Core/ServerUpdater.cs @@ -87,78 +87,18 @@ namespace Tgstation.Server.Host.Core } /// - public async Task BeginUpdate(ISwarmService swarmService, Version newVersion, CancellationToken cancellationToken) + public async Task BeginUpdate(ISwarmService swarmService, Version version, CancellationToken cancellationToken) { if (swarmService == null) throw new ArgumentNullException(nameof(swarmService)); - if (newVersion == null) - throw new ArgumentNullException(nameof(newVersion)); + if (version == null) + throw new ArgumentNullException(nameof(version)); if (!swarmService.ExpectedNumberOfNodesConnected) return ServerUpdateResult.SwarmIntegrityCheckFailed; - logger.LogDebug("Looking for GitHub releases version {version}...", newVersion); - var gitHubClient = gitHubClientFactory.CreateClient(); - var releases = await gitHubClient - .Repository - .Release - .GetAll(updatesConfiguration.GitHubRepositoryId) - .WithToken(cancellationToken); - - logger.LogTrace("Received {releaseCount} total releases from GitHub", releases.Count); - - var filteredReleases = releases - .Where(x => x.TagName.StartsWith(updatesConfiguration.GitTagPrefix, StringComparison.InvariantCulture)) - .ToList(); - - logger.LogTrace( - "Filtered to {releaseCount} releases matching the configured tag prefix of \"{tagPrefix}\"", - filteredReleases.Count, - updatesConfiguration.GitTagPrefix); - - foreach (var release in filteredReleases) - if (Version.TryParse( - release.TagName.Replace( - updatesConfiguration.GitTagPrefix, String.Empty, StringComparison.Ordinal), - out var version)) - { - if (version == newVersion) - { - var asset = release.Assets.Where(x => x.Name.Equals(updatesConfiguration.UpdatePackageAssetName, StringComparison.Ordinal)).FirstOrDefault(); - if (asset == default) - continue; - - serverUpdateOperation = new ServerUpdateOperation - { - TargetVersion = version, - UpdateZipUrl = new Uri(asset.BrowserDownloadUrl), - SwarmService = swarmService, - }; - - try - { - if (!serverControl.TryStartUpdate(this, version)) - return ServerUpdateResult.UpdateInProgress; - } - finally - { - serverUpdateOperation = null; - } - - return ServerUpdateResult.Started; - } - } - else - logger.LogDebug("Unparsable release tag: {releaseTag}", release.TagName); - - if (updatesConfiguration.DumpReleasesOnNotFound) - logger.LogInformation( - "Found releases:{newline}\t{releases}", - Environment.NewLine, - String.Join($"{Environment.NewLine}\t", releases.Select(x => x.TagName).OrderBy(x => x))); - - return ServerUpdateResult.ReleaseMissing; + return await BeginUpdateImpl(swarmService, version, false, cancellationToken); } /// @@ -272,5 +212,84 @@ namespace Tgstation.Server.Host.Core return false; } + + /// + /// Start the process of downloading and applying an update to a new server version. Doesn't perform argument checking. + /// + /// The to use to coordinate the update. + /// The TGS to update to. + /// If this is a recursive call. + /// The for the operation. + /// A resulting in the . + async Task BeginUpdateImpl(ISwarmService swarmService, Version newVersion, bool recursed, CancellationToken cancellationToken) + { + logger.LogDebug("Looking for GitHub releases version {version}...", newVersion); + var gitHubClient = gitHubClientFactory.CreateClient(); + var releases = await gitHubClient + .Repository + .Release + .GetAll(updatesConfiguration.GitHubRepositoryId) + .WithToken(cancellationToken); + + logger.LogTrace("Received {releaseCount} total releases from GitHub", releases.Count); + + var filteredReleases = releases + .Where(x => x.TagName.StartsWith(updatesConfiguration.GitTagPrefix, StringComparison.InvariantCulture)) + .ToList(); + + logger.LogTrace( + "Filtered to {releaseCount} releases matching the configured tag prefix of \"{tagPrefix}\"", + filteredReleases.Count, + updatesConfiguration.GitTagPrefix); + + foreach (var release in filteredReleases) + if (Version.TryParse( + release.TagName.Replace( + updatesConfiguration.GitTagPrefix, String.Empty, StringComparison.Ordinal), + out var version)) + { + if (version == newVersion) + { + var asset = release.Assets.Where(x => x.Name.Equals(updatesConfiguration.UpdatePackageAssetName, StringComparison.Ordinal)).FirstOrDefault(); + if (asset == default) + continue; + + serverUpdateOperation = new ServerUpdateOperation + { + TargetVersion = version, + UpdateZipUrl = new Uri(asset.BrowserDownloadUrl), + SwarmService = swarmService, + }; + + try + { + if (!serverControl.TryStartUpdate(this, version)) + return ServerUpdateResult.UpdateInProgress; + } + finally + { + serverUpdateOperation = null; + } + + return ServerUpdateResult.Started; + } + } + else + logger.LogDebug("Unparsable release tag: {releaseTag}", release.TagName); + + if (updatesConfiguration.DumpReleasesOnNotFound) + logger.LogInformation( + "Found releases:{newline}\t{releases}", + Environment.NewLine, + String.Join($"{Environment.NewLine}\t", releases.Select(x => x.TagName).OrderBy(x => x))); + + if (!recursed) + { + logger.LogWarning("We didn't find the requested release, but GitHub has been known to just not give full results when querying all releases. We'll try one more time."); + return await BeginUpdateImpl(swarmService, newVersion, true, cancellationToken); + } + + return ServerUpdateResult.ReleaseMissing; + } } } diff --git a/tests/DMAPI/test_setup.dm b/tests/DMAPI/test_setup.dm index 514921d668..18b35fbe9a 100644 --- a/tests/DMAPI/test_setup.dm +++ b/tests/DMAPI/test_setup.dm @@ -1,4 +1,5 @@ /world/New() + log << "Starting test..." text2file("SUCCESS", "test_success.txt") world.RunTest() diff --git a/tests/Tgstation.Server.Tests/Live/DummyChatProvider.cs b/tests/Tgstation.Server.Tests/Live/DummyChatProvider.cs index 6eb24a0c0f..972e7acc75 100644 --- a/tests/Tgstation.Server.Tests/Live/DummyChatProvider.cs +++ b/tests/Tgstation.Server.Tests/Live/DummyChatProvider.cs @@ -86,8 +86,6 @@ namespace Tgstation.Server.Tests.Live this.commands = commands ?? throw new ArgumentNullException(nameof(commands)); this.random = random ?? throw new ArgumentNullException(nameof(random)); - logger.LogTrace("Base channel ID {baseChannelId}", channelIdAllocator); - knownChannels = new Dictionary(); randomMessageCts = new CancellationTokenSource(); randomMessageTask = RandomMessageLoop(this.randomMessageCts.Token); @@ -95,7 +93,6 @@ namespace Tgstation.Server.Tests.Live public override async ValueTask DisposeAsync() { - Logger.LogTrace("DisposeAsync Child"); this.randomMessageCts.Cancel(); this.randomMessageCts.Dispose(); await this.randomMessageTask; @@ -107,8 +104,6 @@ namespace Tgstation.Server.Tests.Live if (message == null) throw new ArgumentNullException(nameof(message)); - Logger.LogTrace("SendMessage"); - Assert.IsTrue(knownChannels.ContainsKey(channelId)); cancellationToken.ThrowIfCancellationRequested(); @@ -131,8 +126,6 @@ namespace Tgstation.Server.Tests.Live if (gitHubRepo == null) throw new ArgumentNullException(nameof(gitHubRepo)); - Logger.LogTrace("SendUpdateMessage"); - Assert.IsTrue(knownChannels.ContainsKey(channelId)); cancellationToken.ThrowIfCancellationRequested(); @@ -155,7 +148,6 @@ namespace Tgstation.Server.Tests.Live protected override Task Connect(CancellationToken cancellationToken) { - Logger.LogTrace("Connect"); cancellationToken.ThrowIfCancellationRequested(); // 30% chance to fail AFTER initial connection @@ -169,7 +161,6 @@ namespace Tgstation.Server.Tests.Live protected override Task DisconnectImpl(CancellationToken cancellationToken) { - Logger.LogTrace("DisconnectImpl"); cancellationToken.ThrowIfCancellationRequested(); connected = false; @@ -181,7 +172,6 @@ namespace Tgstation.Server.Tests.Live protected override Task>> MapChannelsImpl(IEnumerable channels, CancellationToken cancellationToken) { channels = channels.ToList(); - Logger.LogTrace("MapChannels: [{channels}]", String.Join(", ", channels.Select(channel => channel.IrcChannel ?? channel.DiscordChannelId?.ToString() ?? throw new InvalidOperationException("BAD CHANNEL")))); cancellationToken.ThrowIfCancellationRequested(); @@ -202,7 +192,6 @@ namespace Tgstation.Server.Tests.Live private ChannelRepresentation CreateChannel(ChatChannel channel) { - ulong channelId; if (channel.DiscordChannelId.HasValue) channelId = channel.DiscordChannelId.Value; @@ -235,7 +224,6 @@ namespace Tgstation.Server.Tests.Live async Task RandomMessageLoop(CancellationToken cancellationToken) { - Logger.LogTrace("RandomMessageLoop"); try { for (var i = 0UL; !cancellationToken.IsCancellationRequested; ++i) @@ -337,11 +325,7 @@ namespace Tgstation.Server.Tests.Live content = $"{content} {commands.ElementAt(random.Next(0, commands.Count)).Name}"; // 40% chance to attempt a custom chat command in long_running_test else - // equal chance for each - if (random.Next(0, 100) > 50) - content = $"{content} embeds_test"; - else - content = $"{content} response_overload_test"; + content = $"{content} embeds_test"; // NEVER send the response_overload_test, it causes so much havoc in CI and we test it manually EnqueueMessage(new Message { @@ -353,7 +337,6 @@ namespace Tgstation.Server.Tests.Live } catch (OperationCanceledException) { - Logger.LogTrace("RandomMessageLoop cancelled"); } } } diff --git a/tests/Tgstation.Server.Tests/Live/Instance/DeploymentTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/DeploymentTest.cs index b56d30efd1..e4171c29d5 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/DeploymentTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/DeploymentTest.cs @@ -20,13 +20,16 @@ namespace Tgstation.Server.Tests.Live.Instance readonly IDreamDaemonClient dreamDaemonClient; readonly IInstanceClient instanceClient; + readonly bool lowPriorityDeployments; + Task vpTest; - public DeploymentTest(IInstanceClient instanceClient, IJobsClient jobsClient) : base(jobsClient) + public DeploymentTest(IInstanceClient instanceClient, IJobsClient jobsClient, bool lowPriorityDeployments) : base(jobsClient) { this.instanceClient = instanceClient ?? throw new ArgumentNullException(nameof(instanceClient)); dreamMakerClient = instanceClient.DreamMaker; dreamDaemonClient = instanceClient.DreamDaemon; + this.lowPriorityDeployments = lowPriorityDeployments; } public async Task RunPreRepoClone(CancellationToken cancellationToken) @@ -34,7 +37,9 @@ namespace Tgstation.Server.Tests.Live.Instance Assert.IsNull(vpTest); vpTest = TestVisibilityPermission(cancellationToken); var deployJob = await dreamMakerClient.Compile(cancellationToken); - deployJob = await WaitForJob(deployJob, 30, true, null, cancellationToken); + var deploymentJobWaitTask = WaitForJob(deployJob, 30, true, null, cancellationToken); + await CheckDreamDaemonPriority(deploymentJobWaitTask, cancellationToken); + deployJob = await deploymentJobWaitTask; Assert.IsTrue(deployJob.ErrorCode == ErrorCode.RepoCloning || deployJob.ErrorCode == ErrorCode.RepoMissing); var dmSettings = await dreamMakerClient.Read(cancellationToken); @@ -42,6 +47,65 @@ namespace Tgstation.Server.Tests.Live.Instance Assert.AreEqual(null, dmSettings.ProjectName); } + async Task CheckDreamDaemonPriority(Task deploymentJobWaitTask, CancellationToken cancellationToken) + { + // this doesn't check dm's priority, but it really should + + while (!deploymentJobWaitTask.IsCompleted) + { + var ddProcessName = new PlatformIdentifier().IsWindows && ByondTest.TestVersion >= new Version(515, 1598) + ? "dd" + : "DreamDaemon"; + + var allProcesses = System.Diagnostics.Process.GetProcessesByName(ddProcessName); + if (allProcesses.Length == 0) + continue; + + if (allProcesses.Length > 1) + Assert.Fail("Multiple DreamDaemon-like processes running!"); + + using var process = allProcesses[0]; + + int processId; + try + { + processId = process.Id; + } + catch + { + return; // vOv + } + + bool good = false; + while (!process.HasExited) + { + // we need to constantly reacquire the handle to invalidate caches + using var localProcess = System.Diagnostics.Process.GetProcessById(processId); + if (lowPriorityDeployments) + { + if (localProcess.PriorityClass == System.Diagnostics.ProcessPriorityClass.BelowNormal) + { + good = true; + break; + } + else + Assert.AreEqual(System.Diagnostics.ProcessPriorityClass.Normal, localProcess.PriorityClass); + } + else + { + good = true; + Assert.AreEqual(System.Diagnostics.ProcessPriorityClass.Normal, localProcess.PriorityClass, "DreamDaemon's process priority changed when it shouldn't have!"); + await Task.Delay(1, cancellationToken); + } + } + + if (!good) + Assert.Fail("Did not detect DreamDaemon lowering its process priority!"); + + break; + } + } + public async Task RunPostRepoClone(Task byondTask, CancellationToken cancellationToken) { Assert.IsNotNull(vpTest); @@ -80,6 +144,11 @@ namespace Tgstation.Server.Tests.Live.Instance } var deployJobTask = CompileAfterByondInstall(); + var deployJob = await deployJobTask; + var deploymentJobWaitTask = WaitForJob(deployJob, 40, true, ErrorCode.DreamMakerNeverValidated, cancellationToken); + + await CheckDreamDaemonPriority(deploymentJobWaitTask, cancellationToken); + await Task.WhenAll( ApiAssert.ThrowsException(() => dreamDaemonClient.Update(new DreamDaemonRequest { @@ -89,11 +158,7 @@ namespace Tgstation.Server.Tests.Live.Instance { ApiValidationPort = TestLiveServer.DDPort }, cancellationToken), ErrorCode.PortNotAvailable), - deployJobTask); - - var deployJob = await deployJobTask; - - await WaitForJob(deployJob, 40, true, ErrorCode.DreamMakerNeverValidated, cancellationToken); + deploymentJobWaitTask); const string FailProject = "tests/DMAPI/BuildFail/build_fail"; var updated = await dreamMakerClient.Update(new DreamMakerRequest diff --git a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs index fd502926a9..0eecae4997 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs @@ -23,13 +23,13 @@ namespace Tgstation.Server.Tests.Live.Instance this.serverPort = serverPort; } - public async Task RunTests(CancellationToken cancellationToken) + public async Task RunTests(CancellationToken cancellationToken, bool highPrioDD, bool lowPrioDeployment) { var byondTest = new ByondTest(instanceClient.Byond, instanceClient.Jobs, instanceClient.Metadata); var chatTest = new ChatTest(instanceClient.ChatBots, instanceManagerClient, instanceClient.Jobs, instanceClient.Metadata); var configTest = new ConfigurationTest(instanceClient.Configuration, instanceClient.Metadata); var repoTest = new RepositoryTest(instanceClient.Repository, instanceClient.Jobs); - var dmTest = new DeploymentTest(instanceClient, instanceClient.Jobs); + var dmTest = new DeploymentTest(instanceClient, instanceClient.Jobs, lowPrioDeployment); var byondTask = byondTest.Run(cancellationToken, out var firstInstall); var chatTask = chatTest.RunPreWatchdog(cancellationToken); @@ -45,7 +45,7 @@ namespace Tgstation.Server.Tests.Live.Instance await dmTask; await byondTask; - await new WatchdogTest(instanceClient, instanceManager, serverPort).Run(cancellationToken); + await new WatchdogTest(instanceClient, instanceManager, serverPort, highPrioDD).Run(cancellationToken); } } } diff --git a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs index e8fc623199..12aa55e81a 100644 --- a/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs @@ -26,12 +26,12 @@ using Tgstation.Server.Client.Components; using Tgstation.Server.Host.Components; using Tgstation.Server.Host.Components.Chat; using Tgstation.Server.Host.Components.Interop; +using Tgstation.Server.Host.Components.Session; using Tgstation.Server.Host.Components.Watchdog; using Tgstation.Server.Host.Controllers; using Tgstation.Server.Host.Extensions; using Tgstation.Server.Host.IO; using Tgstation.Server.Host.System; -using Tgstation.Server.Tests.Live; namespace Tgstation.Server.Tests.Live.Instance { @@ -40,15 +40,17 @@ namespace Tgstation.Server.Tests.Live.Instance readonly IInstanceClient instanceClient; readonly InstanceManager instanceManager; readonly ushort serverPort; + readonly bool highPrioDD; bool ranTimeoutTest = false; - public WatchdogTest(IInstanceClient instanceClient, InstanceManager instanceManager, ushort serverPort) + public WatchdogTest(IInstanceClient instanceClient, InstanceManager instanceManager, ushort serverPort, bool highPrioDD) : base(instanceClient.Jobs) { this.instanceClient = instanceClient ?? throw new ArgumentNullException(nameof(instanceClient)); this.instanceManager = instanceManager ?? throw new ArgumentNullException(nameof(instanceManager)); this.serverPort = serverPort; + this.highPrioDD = highPrioDD; } public async Task Run(CancellationToken cancellationToken) @@ -61,7 +63,8 @@ namespace Tgstation.Server.Tests.Live.Instance { StartupTimeout = 15, HeartbeatSeconds = 0, - Port = TestLiveServer.DDPort + Port = TestLiveServer.DDPort, + LogOutput = false, }, cancellationToken), ApiAssert.ThrowsException(() => instanceClient.DreamDaemon.Update(new DreamDaemonRequest { @@ -101,9 +104,11 @@ namespace Tgstation.Server.Tests.Live.Instance var deleteJobTask = TestDeleteByondInstallErrorCasesAndQueing(cancellationToken); + SessionController.LogTopicRequests = false; await WhiteBoxChatCommandTest(cancellationToken); await SendChatOverloadCommand(cancellationToken); await ValidateTopicLimits(cancellationToken); + SessionController.LogTopicRequests = true; // This one fucks with the access_identifer, run it in isolation await WhiteBoxValidateBridgeRequestLimitAndTestChunking(cancellationToken); @@ -264,7 +269,9 @@ namespace Tgstation.Server.Tests.Live.Instance await WaitForJob(startJob, 40, false, null, cancellationToken); daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); + Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value); + CheckDDPriority(cancellationToken); Assert.AreEqual(false, daemonStatus.SoftRestart); Assert.AreEqual(false, daemonStatus.SoftShutdown); Assert.AreEqual(string.Empty, daemonStatus.AdditionalParameters); @@ -321,7 +328,9 @@ namespace Tgstation.Server.Tests.Live.Instance await WaitForJob(startJob, 40, false, null, cancellationToken); daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); + Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value); + CheckDDPriority(cancellationToken); Assert.AreEqual(false, daemonStatus.SoftRestart); Assert.AreEqual(false, daemonStatus.SoftShutdown); @@ -330,11 +339,12 @@ namespace Tgstation.Server.Tests.Live.Instance daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.AreEqual(WatchdogStatus.Offline, daemonStatus.Status.Value); - await CheckDMApiFail(daemonStatus.ActiveCompileJob, cancellationToken); + await CheckDMApiFail(daemonStatus.ActiveCompileJob, cancellationToken, false); daemonStatus = await instanceClient.DreamDaemon.Update(new DreamDaemonRequest { - AdditionalParameters = string.Empty + AdditionalParameters = string.Empty, + LogOutput = true, }, cancellationToken); Assert.AreEqual(string.Empty, daemonStatus.AdditionalParameters); } @@ -353,6 +363,8 @@ namespace Tgstation.Server.Tests.Live.Instance await WaitForJob(startJob, 40, false, null, cancellationToken); + CheckDDPriority(cancellationToken); + // lock on to DD and pause it so it can't heartbeat var ddProcs = System.Diagnostics.Process.GetProcessesByName("DreamDaemon").Where(x => !x.HasExited).ToList(); if (ddProcs.Count != 1) @@ -676,6 +688,28 @@ namespace Tgstation.Server.Tests.Live.Instance Assert.AreEqual("Footer text", embedsResponse.Embed.Footer?.Text); } + void CheckDDPriority(CancellationToken cancellationToken) + { + var ddProcessName = new PlatformIdentifier().IsWindows && ByondTest.TestVersion >= new Version(515, 1598) + ? "dd" + : "DreamDaemon"; + + var allProcesses = System.Diagnostics.Process.GetProcessesByName(ddProcessName); + if (allProcesses.Length == 0) + Assert.Fail("Expected DreamDaemon to be running here"); + + if (allProcesses.Length > 1) + Assert.Fail("Multiple DreamDaemon-like processes running!"); + + using var process = allProcesses[0]; + + Assert.AreEqual( + highPrioDD + ? System.Diagnostics.ProcessPriorityClass.AboveNormal + : System.Diagnostics.ProcessPriorityClass.Normal, + process.PriorityClass); + } + async Task RunLongRunningTestThenUpdate(CancellationToken cancellationToken) { System.Console.WriteLine("TEST: WATCHDOG LONG RUNNING WITH UPDATE TEST"); @@ -697,6 +731,7 @@ namespace Tgstation.Server.Tests.Live.Instance daemonStatus = await DeployTestDme(DmeName, DreamDaemonSecurity.Safe, true, cancellationToken); Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value); + CheckDDPriority(cancellationToken); Assert.AreEqual(initialCompileJob.Id, daemonStatus.ActiveCompileJob.Id); var newerCompileJob = daemonStatus.StagedCompileJob; @@ -737,7 +772,9 @@ namespace Tgstation.Server.Tests.Live.Instance daemonStatus = await DeployTestDme("LongRunning/long_running_test_copy", DreamDaemonSecurity.Safe, true, cancellationToken); + Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value); + CheckDDPriority(cancellationToken); Assert.AreEqual(initialCompileJob.Id, daemonStatus.ActiveCompileJob.Id); var newerCompileJob = daemonStatus.StagedCompileJob; @@ -774,6 +811,8 @@ namespace Tgstation.Server.Tests.Live.Instance await WaitForJob(startJob, 70, false, null, cancellationToken); + CheckDDPriority(cancellationToken); + var byondInstallJobTask = instanceClient.Byond.SetActiveVersion( new ByondVersionRequest { @@ -828,7 +867,9 @@ namespace Tgstation.Server.Tests.Live.Instance await WaitForJob(startJob, 40, false, null, cancellationToken); var daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); + Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value); + CheckDDPriority(cancellationToken); Assert.AreEqual(TestLiveServer.DDPort, daemonStatus.CurrentPort); // Try killing the DD process to ensure it gets set to the restoring state @@ -985,7 +1026,7 @@ namespace Tgstation.Server.Tests.Live.Instance while (timeout > 0); } - async Task CheckDMApiFail(CompileJobResponse compileJob, CancellationToken cancellationToken) + async Task CheckDMApiFail(CompileJobResponse compileJob, CancellationToken cancellationToken, bool checkLogs = true) { var gameDir = Path.Combine(instanceClient.Metadata.Path, "Game", compileJob.DirectoryName.Value.ToString(), Path.GetDirectoryName(compileJob.DmeName)); var failFile = Path.Combine(gameDir, "test_fail_reason.txt"); @@ -993,11 +1034,29 @@ namespace Tgstation.Server.Tests.Live.Instance { var successFile = Path.Combine(gameDir, "test_success.txt"); Assert.IsTrue(File.Exists(successFile)); - return; + } + else + { + var text = await File.ReadAllTextAsync(failFile, cancellationToken); + Assert.Fail(text); } - var text = await File.ReadAllTextAsync(failFile, cancellationToken); - Assert.Fail(text); + if (!checkLogs) + return; + + var daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); + if (daemonStatus.Status != WatchdogStatus.Offline) + return; + + var outerLogsDir = Path.Combine(instanceClient.Metadata.Path, "Diagnostics", "DreamDaemonLogs"); + var logsDir = new DirectoryInfo(outerLogsDir).GetDirectories().OrderByDescending(x => x.CreationTime).FirstOrDefault(); + Assert.IsNotNull(logsDir); + + var logfile = logsDir.GetFiles().OrderByDescending(x => x.CreationTime).FirstOrDefault(); + Assert.IsNotNull(logfile); + + var logtext = await File.ReadAllTextAsync(logfile.FullName, cancellationToken); + Assert.IsFalse(String.IsNullOrWhiteSpace(logtext)); } } } diff --git a/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs b/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs index c6fcd0ea82..6a66269efb 100644 --- a/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs +++ b/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs @@ -32,6 +32,9 @@ namespace Tgstation.Server.Tests.Live public bool DumpOpenApiSpecpath { get; } + public bool HighPriorityDreamDaemon { get; } + public bool LowPriorityDeployments { get; } + public bool RestartRequested => RealServer.RestartRequested; readonly List args; @@ -72,34 +75,45 @@ namespace Tgstation.Server.Tests.Live var gitHubAccessToken = Environment.GetEnvironmentVariable("TGS_TEST_GITHUB_TOKEN"); var dumpOpenAPISpecPathEnvVar = Environment.GetEnvironmentVariable("TGS_TEST_DUMP_API_SPEC"); - if (string.IsNullOrEmpty(DatabaseType)) + if (String.IsNullOrEmpty(DatabaseType)) Assert.Inconclusive("No database type configured in env var TGS_TEST_DATABASE_TYPE!"); - if (string.IsNullOrEmpty(connectionString)) + if (String.IsNullOrEmpty(connectionString)) Assert.Inconclusive("No connection string configured in env var TGS_TEST_CONNECTION_STRING!"); - if (string.IsNullOrEmpty(gitHubAccessToken)) + if (String.IsNullOrEmpty(gitHubAccessToken)) Console.WriteLine("WARNING: No GitHub access token configured, test may fail due to rate limits!"); - DumpOpenApiSpecpath = !string.IsNullOrEmpty(dumpOpenAPISpecPathEnvVar); + DumpOpenApiSpecpath = !String.IsNullOrEmpty(dumpOpenAPISpecPathEnvVar); + + // neither of these should really matter but it's better that we test them + // high prio DD might help with some topic flakiness actually + // github doesn't allow nicing on linux though + var runningInGitHubActions = !String.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("GITHUB_RUN_ID")); + var windows = new Host.System.PlatformIdentifier().IsWindows; + var nicingAllowed = windows || !runningInGitHubActions; + HighPriorityDreamDaemon = nicingAllowed; + LowPriorityDeployments = nicingAllowed; args = new List() { - string.Format(CultureInfo.InvariantCulture, "Database:DropDatabase={0}", true), // Replaced after first Run - string.Format(CultureInfo.InvariantCulture, "General:ConfigVersion={0}", GeneralConfiguration.CurrentConfigVersion), - string.Format(CultureInfo.InvariantCulture, "General:ApiPort={0}", port), - string.Format(CultureInfo.InvariantCulture, "Database:DatabaseType={0}", DatabaseType), - string.Format(CultureInfo.InvariantCulture, "Database:ConnectionString={0}", connectionString), - string.Format(CultureInfo.InvariantCulture, "General:SetupWizardMode={0}", SetupWizardMode.Never), - string.Format(CultureInfo.InvariantCulture, "General:MinimumPasswordLength={0}", 10), - string.Format(CultureInfo.InvariantCulture, "General:InstanceLimit={0}", 11), - string.Format(CultureInfo.InvariantCulture, "General:UserLimit={0}", 150), - string.Format(CultureInfo.InvariantCulture, "General:UserGroupLimit={0}", 47), - string.Format(CultureInfo.InvariantCulture, "General:HostApiDocumentation={0}", DumpOpenApiSpecpath), - string.Format(CultureInfo.InvariantCulture, "FileLogging:Directory={0}", Path.Combine(Directory, "Logs")), - string.Format(CultureInfo.InvariantCulture, "FileLogging:LogLevel={0}", "Trace"), - string.Format(CultureInfo.InvariantCulture, "General:ValidInstancePaths:0={0}", Directory), - "General:ByondTopicTimeout=3000" + String.Format(CultureInfo.InvariantCulture, "Database:DropDatabase={0}", true), // Replaced after first Run + String.Format(CultureInfo.InvariantCulture, "General:ConfigVersion={0}", GeneralConfiguration.CurrentConfigVersion), + String.Format(CultureInfo.InvariantCulture, "General:ApiPort={0}", port), + String.Format(CultureInfo.InvariantCulture, "Database:DatabaseType={0}", DatabaseType), + String.Format(CultureInfo.InvariantCulture, "Database:ConnectionString={0}", connectionString), + String.Format(CultureInfo.InvariantCulture, "General:SetupWizardMode={0}", SetupWizardMode.Never), + String.Format(CultureInfo.InvariantCulture, "General:MinimumPasswordLength={0}", 10), + String.Format(CultureInfo.InvariantCulture, "General:InstanceLimit={0}", 11), + String.Format(CultureInfo.InvariantCulture, "General:UserLimit={0}", 150), + String.Format(CultureInfo.InvariantCulture, "General:UserGroupLimit={0}", 47), + String.Format(CultureInfo.InvariantCulture, "General:HostApiDocumentation={0}", DumpOpenApiSpecpath), + String.Format(CultureInfo.InvariantCulture, "FileLogging:Directory={0}", Path.Combine(Directory, "Logs")), + String.Format(CultureInfo.InvariantCulture, "FileLogging:LogLevel={0}", "Trace"), + String.Format(CultureInfo.InvariantCulture, "General:ValidInstancePaths:0={0}", Directory), + "General:ByondTopicTimeout=3000", + $"Session:HighPriorityLiveDreamDaemon={HighPriorityDreamDaemon}", + $"Session:LowPriorityDeploymentProcesses={LowPriorityDeployments}", }; if (dumpOnMissingUpdate) diff --git a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs index 1ade1e1c5a..5b10b5b6d6 100644 --- a/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs +++ b/tests/Tgstation.Server.Tests/Live/TestLiveServer.cs @@ -730,6 +730,22 @@ namespace Tgstation.Server.Tests.Live [TestMethod] public async Task TestStandardTgsOperation() { + using(var currentProcess = System.Diagnostics.Process.GetCurrentProcess()) + { + var currentPriorityClass = currentProcess.PriorityClass; + if (currentPriorityClass != ProcessPriorityClass.Normal) + { + // attempt to adjust it + Console.WriteLine($"TEST PROCESS PRIORITY: Attempting to normalize process priority from {currentPriorityClass}..."); + currentProcess.PriorityClass = ProcessPriorityClass.Normal; + } + } + + using (var currentProcess = System.Diagnostics.Process.GetCurrentProcess()) + { + Assert.AreEqual(ProcessPriorityClass.Normal, currentProcess.PriorityClass); + } + const int MaximumTestMinutes = 30; using var hardCancellationTokenSource = new CancellationTokenSource(TimeSpan.FromMinutes(MaximumTestMinutes)); var hardCancellationToken = hardCancellationTokenSource.Token; @@ -864,7 +880,7 @@ namespace Tgstation.Server.Tests.Live Assert.IsTrue(Directory.Exists(instanceClient.Metadata.Path)); - var instanceTests = FailFast(new InstanceTest(instanceClient, adminClient.Instances, GetInstanceManager(), (ushort)server.Url.Port).RunTests(cancellationToken)); + var instanceTests = FailFast(new InstanceTest(instanceClient, adminClient.Instances, GetInstanceManager(), (ushort)server.Url.Port).RunTests(cancellationToken, server.HighPriorityDreamDaemon, server.LowPriorityDeployments)); await Task.WhenAll(rootTest, adminTest, instancesTest, instanceTests, usersTest); @@ -1020,7 +1036,7 @@ namespace Tgstation.Server.Tests.Live Assert.AreEqual(WatchdogStatus.Online, dd.Status.Value); var compileJob = await instanceClient.DreamMaker.Compile(cancellationToken); - var wdt = new WatchdogTest(instanceClient, GetInstanceManager(), (ushort)server.Url.Port); + var wdt = new WatchdogTest(instanceClient, GetInstanceManager(), (ushort)server.Url.Port, server.HighPriorityDreamDaemon); await wdt.WaitForJob(compileJob, 30, false, null, cancellationToken); dd = await instanceClient.DreamDaemon.Read(cancellationToken); @@ -1067,7 +1083,7 @@ namespace Tgstation.Server.Tests.Live Assert.AreEqual(WatchdogStatus.Online, currentDD.Status); Assert.AreEqual(expectedStaged, currentDD.StagedCompileJob.Job.Id.Value); - var wdt = new WatchdogTest(instanceClient, GetInstanceManager(), (ushort)server.Url.Port); + var wdt = new WatchdogTest(instanceClient, GetInstanceManager(), (ushort)server.Url.Port, server.HighPriorityDreamDaemon); currentDD = await wdt.TellWorldToReboot(cancellationToken); Assert.AreEqual(expectedStaged, currentDD.ActiveCompileJob.Job.Id.Value); Assert.IsNull(currentDD.StagedCompileJob);