From d6bf75b43919db1598883c9cd4775658c80519e1 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 9 Jul 2020 11:24:15 -0400 Subject: [PATCH] Lots of test improvements - More deployment tests - More job error code asserts --- tests/DMAPI/BuildFail/Test.dm | 2 + tests/DMAPI/BuildFail/build_fail.dme | 15 +++++ .../Instance/ByondTest.cs | 6 +- .../Instance/DeploymentTest.cs | 63 +++++++++++++++++++ .../Instance/InstanceTest.cs | 8 ++- .../Instance/JobsRequiredTest.cs | 7 ++- .../Instance/RepositoryTest.cs | 10 +-- .../Instance/WatchdogTest.cs | 21 +++---- .../Tgstation.Server.Tests/IntegrationTest.cs | 4 +- tests/Tgstation.Server.Tests/TestingServer.cs | 1 + tgstation-server.sln | 7 +++ 11 files changed, 118 insertions(+), 26 deletions(-) create mode 100644 tests/DMAPI/BuildFail/Test.dm create mode 100644 tests/DMAPI/BuildFail/build_fail.dme create mode 100644 tests/Tgstation.Server.Tests/Instance/DeploymentTest.cs diff --git a/tests/DMAPI/BuildFail/Test.dm b/tests/DMAPI/BuildFail/Test.dm new file mode 100644 index 0000000000..9dc47be24a --- /dev/null +++ b/tests/DMAPI/BuildFail/Test.dm @@ -0,0 +1,2 @@ +/world/New() + this isn't valid DM diff --git a/tests/DMAPI/BuildFail/build_fail.dme b/tests/DMAPI/BuildFail/build_fail.dme new file mode 100644 index 0000000000..444e2491d8 --- /dev/null +++ b/tests/DMAPI/BuildFail/build_fail.dme @@ -0,0 +1,15 @@ +// Hand crafted DME, will not work if saved with DreamMaker + +// BEGIN_INTERNALS +// END_INTERNALS + +// BEGIN_FILE_DIR +#define FILE_DIR . +// END_FILE_DIR + +// BEGIN_PREFERENCES +// END_PREFERENCES + +// BEGIN_INCLUDE +#include "Test.dm" +// END_INCLUDE diff --git a/tests/Tgstation.Server.Tests/Instance/ByondTest.cs b/tests/Tgstation.Server.Tests/Instance/ByondTest.cs index 0402e2eaa7..9266e01da6 100644 --- a/tests/Tgstation.Server.Tests/Instance/ByondTest.cs +++ b/tests/Tgstation.Server.Tests/Instance/ByondTest.cs @@ -47,7 +47,7 @@ namespace Tgstation.Server.Tests.Instance }; var test = await byondClient.SetActiveVersion(newModel, cancellationToken).ConfigureAwait(false); Assert.IsNotNull(test.InstallJob); - await WaitForJob(test.InstallJob, 60, true, cancellationToken).ConfigureAwait(false); + await WaitForJob(test.InstallJob, 60, true, ErrorCode.ByondDownloadFail, cancellationToken).ConfigureAwait(false); } async Task TestInstallStable(CancellationToken cancellationToken) @@ -59,7 +59,7 @@ namespace Tgstation.Server.Tests.Instance var test = await byondClient.SetActiveVersion(newModel, cancellationToken).ConfigureAwait(false); Assert.IsNotNull(test.InstallJob); Assert.IsNull(test.Version); - await WaitForJob(test.InstallJob, 60, false, cancellationToken).ConfigureAwait(false); + await WaitForJob(test.InstallJob, 60, false, null, cancellationToken).ConfigureAwait(false); var currentShit = await byondClient.ActiveVersion(cancellationToken).ConfigureAwait(false); Assert.AreEqual(newModel.Version.Semver(), currentShit.Version); @@ -105,7 +105,7 @@ namespace Tgstation.Server.Tests.Instance }, cancellationToken).ConfigureAwait(false); Assert.IsNotNull(test.InstallJob); - await WaitForJob(test.InstallJob, 60, false, cancellationToken).ConfigureAwait(false); + await WaitForJob(test.InstallJob, 60, false, null, cancellationToken).ConfigureAwait(false); var newSettings = await byondClient.ActiveVersion(cancellationToken); Assert.AreEqual(new Version(TestVersion.Major, TestVersion.Minor, 1), newSettings.Version); diff --git a/tests/Tgstation.Server.Tests/Instance/DeploymentTest.cs b/tests/Tgstation.Server.Tests/Instance/DeploymentTest.cs new file mode 100644 index 0000000000..861fab942b --- /dev/null +++ b/tests/Tgstation.Server.Tests/Instance/DeploymentTest.cs @@ -0,0 +1,63 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Tgstation.Server.Api.Models; +using Tgstation.Server.Client.Components; + +namespace Tgstation.Server.Tests.Instance +{ + sealed class DeploymentTest : JobsRequiredTest + { + readonly IDreamMakerClient dreamMakerClient; + readonly IDreamDaemonClient dreamDaemonClient; + + public DeploymentTest(IDreamMakerClient dreamMakerClient, IDreamDaemonClient dreamDaemonClient, IJobsClient jobsClient) : base(jobsClient) + { + this.dreamMakerClient = dreamMakerClient ?? throw new ArgumentNullException(nameof(dreamMakerClient)); + this.dreamDaemonClient = dreamDaemonClient ?? throw new ArgumentNullException(nameof(dreamDaemonClient)); + } + + public async Task Run(Task repositoryTask, CancellationToken cancellationToken) + { + Assert.IsFalse(repositoryTask.IsCompleted); + var deployJob = await dreamMakerClient.Compile(cancellationToken); + deployJob = await WaitForJob(deployJob, 30, true, null, cancellationToken); + Assert.IsTrue(deployJob.ErrorCode == ErrorCode.RepoCloning || deployJob.ErrorCode == ErrorCode.RepoMissing); + + var dmSettings = await dreamMakerClient.Read(cancellationToken); + Assert.AreEqual(true, dmSettings.RequireDMApiValidation); + Assert.AreEqual(null, dmSettings.ProjectName); + + await repositoryTask; + + // by alphabetization rules, it should discover api_free here + var updatedDD = await dreamDaemonClient.Update(new DreamDaemon + { + StartupTimeout = 5 + }, cancellationToken); + Assert.AreEqual(5, updatedDD.StartupTimeout); + deployJob = await dreamMakerClient.Compile(cancellationToken); + await WaitForJob(deployJob, 30, true, ErrorCode.DreamMakerNeverValidated, cancellationToken); + + const string FailProject = "tests/DMAPI/BuildFail/build_fail"; + var updated = await dreamMakerClient.Update(new DreamMaker + { + ProjectName = FailProject + }, cancellationToken); + + Assert.AreEqual(FailProject, updated.ProjectName); + + deployJob = await dreamMakerClient.Compile(cancellationToken); + await WaitForJob(deployJob, 30, true, ErrorCode.DreamMakerExitCode, cancellationToken); + + await dreamMakerClient.Update(new DreamMaker + { + ProjectName = "tests/DMAPI/ThisDoesntExist/this_doesnt_exist" + }, cancellationToken); + + deployJob = await dreamMakerClient.Compile(cancellationToken); + await WaitForJob(deployJob, 30, true, ErrorCode.DreamMakerMissingDme, cancellationToken); + } + } +} diff --git a/tests/Tgstation.Server.Tests/Instance/InstanceTest.cs b/tests/Tgstation.Server.Tests/Instance/InstanceTest.cs index 18b16d36d5..77bacd7da6 100644 --- a/tests/Tgstation.Server.Tests/Instance/InstanceTest.cs +++ b/tests/Tgstation.Server.Tests/Instance/InstanceTest.cs @@ -23,15 +23,17 @@ namespace Tgstation.Server.Tests.Instance var chatTest = new ChatTest(instanceClient.ChatBots, instanceManagerClient, instanceClient.Metadata.CloneMetadata()); var configTest = new ConfigurationTest(instanceClient.Configuration, instanceClient.Metadata); var repoTest = new RepositoryTest(instanceClient.Repository, instanceClient.Jobs); + var dmTest = new DeploymentTest(instanceClient.DreamMaker, instanceClient.DreamDaemon, instanceClient.Jobs); - var repoTests = repoTest.RunPreWatchdog(cancellationToken); var byondTests = byondTest.Run(cancellationToken); + var repoTests = repoTest.RunPreWatchdog(cancellationToken); var chatTests = chatTest.RunPreWatchdog(cancellationToken); - await configTest.Run(cancellationToken).ConfigureAwait(false); await byondTests.ConfigureAwait(false); + await dmTest.Run(repoTests, cancellationToken); + + await configTest.Run(cancellationToken).ConfigureAwait(false); await chatTests.ConfigureAwait(false); await repoTests; - await new WatchdogTest(instanceClient).Run(cancellationToken); } } diff --git a/tests/Tgstation.Server.Tests/Instance/JobsRequiredTest.cs b/tests/Tgstation.Server.Tests/Instance/JobsRequiredTest.cs index bb3cb43f45..258e69de57 100644 --- a/tests/Tgstation.Server.Tests/Instance/JobsRequiredTest.cs +++ b/tests/Tgstation.Server.Tests/Instance/JobsRequiredTest.cs @@ -18,7 +18,7 @@ namespace Tgstation.Server.Tests.Instance this.JobsClient = jobsClient; } - public async Task WaitForJob(Job originalJob, int timeout, bool expectFailure, CancellationToken cancellationToken) + public async Task WaitForJob(Job originalJob, int timeout, bool expectFailure, ErrorCode? expectedCode, CancellationToken cancellationToken) { var job = originalJob; do @@ -38,6 +38,9 @@ namespace Tgstation.Server.Tests.Instance if (expectFailure ^ job.ExceptionDetails != null) Assert.Fail(job.ExceptionDetails ?? $"Expected job \"{job.Id}\" \"{job.Description}\" to fail but it didn't"); + if (expectedCode.HasValue) + Assert.AreEqual(expectedCode.Value, job.ErrorCode, job.ExceptionDetails); + return job; } @@ -62,7 +65,7 @@ namespace Tgstation.Server.Tests.Instance Assert.Fail(job.ExceptionDetails); await JobsClient.Cancel(job, cancellationToken); - return await WaitForJob(job, timeout, false, cancellationToken).ConfigureAwait(false); + return await WaitForJob(job, timeout, false, null, cancellationToken).ConfigureAwait(false); } } } diff --git a/tests/Tgstation.Server.Tests/Instance/RepositoryTest.cs b/tests/Tgstation.Server.Tests/Instance/RepositoryTest.cs index b236ddf342..9c77a811fb 100644 --- a/tests/Tgstation.Server.Tests/Instance/RepositoryTest.cs +++ b/tests/Tgstation.Server.Tests/Instance/RepositoryTest.cs @@ -58,7 +58,7 @@ namespace Tgstation.Server.Tests.Instance clone = await repositoryClient.Clone(initalRepo, cancellationToken).ConfigureAwait(false); - await WaitForJob(clone.ActiveJob, 180, false, cancellationToken).ConfigureAwait(false); + await WaitForJob(clone.ActiveJob, 180, false, null, cancellationToken).ConfigureAwait(false); var cloned = await repositoryClient.Read(cancellationToken); Assert.AreEqual(Origin, cloned.Origin); @@ -124,7 +124,7 @@ namespace Tgstation.Server.Tests.Instance var checkingOut = await repositoryClient.Update(updated, cancellationToken); Assert.IsNotNull(checkingOut.ActiveJob); - await WaitForJob(checkingOut.ActiveJob, 30, expectFailure, cancellationToken); + await WaitForJob(checkingOut.ActiveJob, 30, expectFailure, null, cancellationToken); var result = await repositoryClient.Read(cancellationToken); if (!expectFailure) if (isRef) @@ -153,7 +153,7 @@ namespace Tgstation.Server.Tests.Instance Assert.IsNotNull(numberOnlyMerging.ActiveJob); Assert.IsTrue(numberOnlyMerging.ActiveJob.Description.Contains(prNumber.ToString())); - await WaitForJob(numberOnlyMerging.ActiveJob, 20, false, cancellationToken); + await WaitForJob(numberOnlyMerging.ActiveJob, 20, false, null,cancellationToken); var withMerge = await repositoryClient.Read(cancellationToken); Assert.AreEqual(repository.Reference, withMerge.Reference); @@ -187,7 +187,7 @@ namespace Tgstation.Server.Tests.Instance var mergingAgain = await repositoryClient.Update(withMerge, cancellationToken); Assert.IsNotNull(mergingAgain.ActiveJob); - await WaitForJob(mergingAgain.ActiveJob, 30, false, cancellationToken); + await WaitForJob(mergingAgain.ActiveJob, 30, false, null, cancellationToken); var final = await repositoryClient.Read(cancellationToken); Assert.AreEqual("asdffdsa", final.RevisionInformation.PrimaryTestMerge.Comment); @@ -202,7 +202,7 @@ namespace Tgstation.Server.Tests.Instance var deleting = await repositoryClient.Delete(cancellationToken); Assert.IsNotNull(deleting.ActiveJob); - await WaitForJob(deleting.ActiveJob, 60, false, cancellationToken).ConfigureAwait(false); + await WaitForJob(deleting.ActiveJob, 60, false, null, cancellationToken).ConfigureAwait(false); var deleted = await repositoryClient.Read(cancellationToken).ConfigureAwait(false); Assert.IsNull(deleted.Origin); diff --git a/tests/Tgstation.Server.Tests/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Instance/WatchdogTest.cs index 4a7da7395d..f56686b601 100644 --- a/tests/Tgstation.Server.Tests/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Instance/WatchdogTest.cs @@ -73,7 +73,7 @@ namespace Tgstation.Server.Tests.Instance { System.Console.WriteLine("TEST: WATCHDOG DUMP TESTS"); var dumpJob = await instanceClient.DreamDaemon.CreateDump(cancellationToken); - await WaitForJob(dumpJob, 3000, false, cancellationToken); + await WaitForJob(dumpJob, 3000, false, null, cancellationToken); var dumpFiles = Directory.GetFiles(Path.Combine( instanceClient.Metadata.Path, "Diagnostics", "ProcessDumps"), "*.dmp"); @@ -84,8 +84,7 @@ namespace Tgstation.Server.Tests.Instance var dumpTask = instanceClient.DreamDaemon.CreateDump(cancellationToken); while (!dumpTask.IsCompleted) KillDD(false); - var result = await WaitForJob(await dumpTask, 5, true, cancellationToken); - Assert.AreEqual(ErrorCode.DreamDaemonOffline, result.ErrorCode); + await WaitForJob(await dumpTask, 5, true, ErrorCode.DreamDaemonOffline, cancellationToken); await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken); var ddStatus = await instanceClient.DreamDaemon.Read(cancellationToken); @@ -105,7 +104,7 @@ namespace Tgstation.Server.Tests.Instance var startJob = await instanceClient.DreamDaemon.Start(cancellationToken).ConfigureAwait(false); - await WaitForJob(startJob, 10, false, cancellationToken); + await WaitForJob(startJob, 10, false, null, cancellationToken); daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value); @@ -140,7 +139,7 @@ namespace Tgstation.Server.Tests.Instance var startJob = await instanceClient.DreamDaemon.Start(cancellationToken).ConfigureAwait(false); - await WaitForJob(startJob, 10, false, cancellationToken); + await WaitForJob(startJob, 10, false, null, cancellationToken); daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value); @@ -166,7 +165,7 @@ namespace Tgstation.Server.Tests.Instance var startJob = await instanceClient.DreamDaemon.Start(cancellationToken).ConfigureAwait(false); - await WaitForJob(startJob, 10, false, cancellationToken); + await WaitForJob(startJob, 10, false, null, cancellationToken); // lock on to DD and pause it so it can't heartbeat var ddProcs = System.Diagnostics.Process.GetProcessesByName("DreamDaemon").ToList(); @@ -234,7 +233,7 @@ namespace Tgstation.Server.Tests.Instance var startJob = await instanceClient.DreamDaemon.Start(cancellationToken).ConfigureAwait(false); - await WaitForJob(startJob, 10, false, cancellationToken); + await WaitForJob(startJob, 10, false, null, cancellationToken); daemonStatus = await DeployTestDme(DmeName, DreamDaemonSecurity.Safe, true, cancellationToken); @@ -275,7 +274,7 @@ namespace Tgstation.Server.Tests.Instance var startJob = await instanceClient.DreamDaemon.Start(cancellationToken).ConfigureAwait(false); - await WaitForJob(startJob, 10, false, cancellationToken); + await WaitForJob(startJob, 10, false, null, cancellationToken); daemonStatus = await DeployTestDme(DmeName + "_copy", DreamDaemonSecurity.Safe, true, cancellationToken); @@ -313,7 +312,7 @@ namespace Tgstation.Server.Tests.Instance var startJob = await instanceClient.DreamDaemon.Start(cancellationToken).ConfigureAwait(false); - await WaitForJob(startJob, 40, false, cancellationToken); + await WaitForJob(startJob, 40, false, null, cancellationToken); var byondInstallJobTask = instanceClient.Byond.SetActiveVersion( new Api.Models.Byond @@ -362,7 +361,7 @@ namespace Tgstation.Server.Tests.Instance var startJob = await instanceClient.DreamDaemon.Start(cancellationToken).ConfigureAwait(false); - await WaitForJob(startJob, 40, false, cancellationToken); + await WaitForJob(startJob, 40, false, null, cancellationToken); var daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value); @@ -443,7 +442,7 @@ namespace Tgstation.Server.Tests.Instance var compileJobJob = await instanceClient.DreamMaker.Compile(cancellationToken); - await WaitForJob(compileJobJob, 90, false, cancellationToken); + await WaitForJob(compileJobJob, 90, false, null, cancellationToken); return await instanceClient.DreamDaemon.Read(cancellationToken); } diff --git a/tests/Tgstation.Server.Tests/IntegrationTest.cs b/tests/Tgstation.Server.Tests/IntegrationTest.cs index 1d524bdddb..6800d2ead8 100644 --- a/tests/Tgstation.Server.Tests/IntegrationTest.cs +++ b/tests/Tgstation.Server.Tests/IntegrationTest.cs @@ -218,7 +218,7 @@ namespace Tgstation.Server.Tests var reattachJob = jobs.Single(); Assert.IsTrue(reattachJob.StartedAt.Value >= preStartupTime); - await new JobsRequiredTest(instanceClient.Jobs).WaitForJob(reattachJob, 40, false, cancellationToken); + await new JobsRequiredTest(instanceClient.Jobs).WaitForJob(reattachJob, 40, false, null, cancellationToken); var dd = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.AreEqual(WatchdogStatus.Online, dd.Status.Value); @@ -261,7 +261,7 @@ namespace Tgstation.Server.Tests var launchJob = jobs.Single(); Assert.IsTrue(launchJob.StartedAt.Value >= preStartupTime); - await new JobsRequiredTest(instanceClient.Jobs).WaitForJob(launchJob, 40, false, cancellationToken); + await new JobsRequiredTest(instanceClient.Jobs).WaitForJob(launchJob, 40, false, null, cancellationToken); var dd = await instanceClient.DreamDaemon.Read(cancellationToken); diff --git a/tests/Tgstation.Server.Tests/TestingServer.cs b/tests/Tgstation.Server.Tests/TestingServer.cs index aad3a9e24f..5bcf4c5857 100644 --- a/tests/Tgstation.Server.Tests/TestingServer.cs +++ b/tests/Tgstation.Server.Tests/TestingServer.cs @@ -78,6 +78,7 @@ namespace Tgstation.Server.Tests String.Format(CultureInfo.InvariantCulture, "General:MinimumPasswordLength={0}", 10), String.Format(CultureInfo.InvariantCulture, "General:InstanceLimit={0}", 11), String.Format(CultureInfo.InvariantCulture, "General:UserLimit={0}", 150), + "General:UseBasicWatchdog=true", String.Format(CultureInfo.InvariantCulture, "FileLogging:Directory={0}", Path.Combine(Directory, "Logs")), String.Format(CultureInfo.InvariantCulture, "General:ValidInstancePaths:0={0}", Directory), "General:ByondTopicTimeout=3000" diff --git a/tgstation-server.sln b/tgstation-server.sln index 4c6a9078e0..39cb8ff54e 100644 --- a/tgstation-server.sln +++ b/tgstation-server.sln @@ -161,6 +161,12 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "BasicOperation", "BasicOper tests\DMAPI\BasicOperation\Test.dm = tests\DMAPI\BasicOperation\Test.dm EndProjectSection EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "BuildFail", "BuildFail", "{103C61AB-67D6-46FE-AA47-CC633B88EE0F}" + ProjectSection(SolutionItems) = preProject + tests\DMAPI\BuildFail\build_fail.dme = tests\DMAPI\BuildFail\build_fail.dme + tests\DMAPI\BuildFail\Test.dm = tests\DMAPI\BuildFail\Test.dm + EndProjectSection +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -300,6 +306,7 @@ Global {EB1DDE8C-CA6F-4BE3-947B-597CA8EABEA5} = {82066812-6C73-4360-943B-B23F2F491261} {7B8FC2AF-1B64-4A89-8480-F0FE360DC9EC} = {82066812-6C73-4360-943B-B23F2F491261} {F32B9514-AAD9-429D-841A-ED810FC2598C} = {82066812-6C73-4360-943B-B23F2F491261} + {103C61AB-67D6-46FE-AA47-CC633B88EE0F} = {82066812-6C73-4360-943B-B23F2F491261} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {DFD36C95-3E49-41C7-ACDB-86BAF5B18A79}