Merge branch 'V6' into BetterSig

.github/CONTRIBUTING.md

@@ -51,6 +51,9 @@ In order to run the integration tests you must have the following environment va
 	- `TGS_TEST_DISCORD_CHANNEL`: To a valid discord channel ID that the above bot can access.
 	- `TGS_TEST_IRC_CONNECTION_STRING`: To a valid IRC connection string. See the code for [IrcConnectionStringBuilder](../src/Tgstation.Server.Api/Models/IrcConnectionStringBuilder.cs) for details.
 	- `TGS_TEST_IRC_CHANNEL`: To a valid IRC channel accessible with the above connection.
+- (Optional) `TGS_TEST_OD_ENGINE_VERSION`: Specify the full git commit SHA of the [OpenDream](https://github.com/OpenDreamProject/OpenDream) version to use in the main integration test, the default is the current HEAD of the default branch.
+- (Optional) `TGS_TEST_OD_GIT_DIRECTORY`: Path to a local [OpenDream](https://github.com/OpenDreamProject/OpenDream) git repository to use as an upstream for testing.
+- (Optional) `TGS_TEST_OD_EXCLUSIVE`: Set to `true` to enable the quicker integration test that only runs [OpenDream](https://github.com/OpenDreamProject/OpenDream) functionality. This is tested by default in the main integration test.
 
 ### Notes About Forks
 

build/Common.props

@@ -4,6 +4,7 @@
   <PropertyGroup>
     <TgsFrameworkVersion>net$(TgsNetMajorVersion).0</TgsFrameworkVersion>
     <LangVersion>latest</LangVersion>
+    <AccelerateBuildsInVisualStudio>true</AccelerateBuildsInVisualStudio>
     <!-- This is the default but Wix hates it -->
     <!--<DebugType>Portable</DebugType> -->
   </PropertyGroup>

build/TestCommon.props

@@ -13,7 +13,7 @@
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
     <!-- Usage: Hard to say what exactly this is for, but not including it removes the test icon and breaks vstest.console.exe for some reason -->
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.8.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.8.0" Condition="'$(TgsTestNoSdk)' != 'true'" />
     <!-- Usage: Dependency mocking for tests -->
     <!-- Pinned: Be VERY careful about updating https://github.com/moq/moq/issues/1372 -->
     <PackageReference Include="Moq" Version="4.20.69" />

tests/Tgstation.Server.Host.Tests.Signals/Tgstation.Server.Host.Tests.Signals.csproj

@@ -1,4 +1,8 @@
 <Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TgsTestNoSdk>true</TgsTestNoSdk>
+  </PropertyGroup>
+
   <Import Project="../../build/TestCommon.props" />
 
   <PropertyGroup>

tests/Tgstation.Server.Tests/Live/Instance/EngineTest.cs

@@ -82,13 +82,25 @@ namespace Tgstation.Server.Tests.Live.Instance
 			}
 			else if (engineType == EngineType.OpenDream)
 			{
-				var masterBranch = await TestingGitHubService.RealTestClient.Repository.Branch.Get("OpenDreamProject", "OpenDream", "master");
-
-				engineVersion = new EngineVersion
+				var forcedVersion = Environment.GetEnvironmentVariable("TGS_TEST_OD_ENGINE_VERSION");
+				if (!String.IsNullOrWhiteSpace(forcedVersion))
 				{
-					Engine = EngineType.OpenDream,
-					SourceSHA = masterBranch.Commit.Sha,
-				};
+					engineVersion = new EngineVersion
+					{
+						Engine = EngineType.OpenDream,
+						SourceSHA = forcedVersion,
+					};
+				}
+				else
+				{
+					var masterBranch = await TestingGitHubService.RealTestClient.Repository.Branch.Get("OpenDreamProject", "OpenDream", "master");
+
+					engineVersion = new EngineVersion
+					{
+						Engine = EngineType.OpenDream,
+						SourceSHA = masterBranch.Commit.Sha,
+					};
+				}
 			}
 			else
 			{

tests/Tgstation.Server.Tests/Live/Instance/InstanceTest.cs

@@ -23,6 +23,7 @@ using Tgstation.Server.Host.Components.Events;
 using Tgstation.Server.Host.Components.Repository;
 using Tgstation.Server.Host.Configuration;
 using Tgstation.Server.Host.IO;
+using Tgstation.Server.Host.Jobs;
 using Tgstation.Server.Host.System;
 
 namespace Tgstation.Server.Tests.Live.Instance
@@ -78,15 +79,23 @@ namespace Tgstation.Server.Tests.Live.Instance
 
 		public static async ValueTask<IEngineInstallationData> DownloadEngineVersion(
 			EngineVersion compatVersion,
-			IInstanceClient instanceClient,
 			IFileDownloader fileDownloader,
+			Uri openDreamUrl,
 			CancellationToken cancellationToken)
 		{
-			var odRepoDir = Path.GetFullPath(Path.Combine(instanceClient.Metadata.Path, "..", "OpenDreamRepo"));
-			var tmpIOManager = new ResolvingIOManager(new DefaultIOManager(), odRepoDir);
+			var ioManager = new DefaultIOManager();
+			var odRepoDir = ioManager.ConcatPath(
+				Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData),
+				new AssemblyInformationProvider().VersionPrefix,
+				"OpenDreamRepository");
+			var odRepoIoManager = new ResolvingIOManager(ioManager, odRepoDir);
 
 			var mockOptions = new Mock<IOptions<GeneralConfiguration>>();
-			mockOptions.SetupGet(x => x.Value).Returns(new GeneralConfiguration());
+			var genConfig = new GeneralConfiguration
+			{
+				OpenDreamGitUrl = openDreamUrl,
+			};
+			mockOptions.SetupGet(x => x.Value).Returns(genConfig);
 			IEngineInstaller byondInstaller =
 				compatVersion.Engine == EngineType.OpenDream
 				? new OpenDreamInstaller(
@@ -98,20 +107,20 @@ namespace Tgstation.Server.Tests.Live.Instance
 						new LibGit2RepositoryFactory(
 							Mock.Of<ILogger<LibGit2RepositoryFactory>>()),
 						new LibGit2Commands(),
-						tmpIOManager,
+						odRepoIoManager,
 						new NoopEventConsumer(),
 						Mock.Of<IPostWriteHandler>(),
 						Mock.Of<IGitRemoteFeaturesFactory>(),
 						Mock.Of<ILogger<Repository>>(),
 						Mock.Of<ILogger<RepositoryManager>>(),
-						new GeneralConfiguration()),
+						genConfig),
 					mockOptions.Object)
 				: new PlatformIdentifier().IsWindows
 					? new WindowsByondInstaller(
 						Mock.Of<IProcessExecutor>(),
 						Mock.Of<IIOManager>(),
 						fileDownloader,
-						Options.Create(new GeneralConfiguration()),
+						Options.Create(genConfig),
 						Mock.Of<ILogger<WindowsByondInstaller>>())
 					: new PosixByondInstaller(
 						Mock.Of<IPostWriteHandler>(),
@@ -127,6 +136,7 @@ namespace Tgstation.Server.Tests.Live.Instance
 
 		public async Task RunCompatTests(
 			EngineVersion compatVersion,
+			Uri openDreamUrl,
 			IInstanceClient instanceClient,
 			ushort dmPort,
 			ushort ddPort,
@@ -191,7 +201,7 @@ namespace Tgstation.Server.Tests.Live.Instance
 
 			EngineInstallResponse installJob2;
 			await using (var stableBytesMs = await TestingUtils.ExtractMemoryStreamFromInstallationData(
-				await DownloadEngineVersion(compatVersion, instanceClient, fileDownloader, cancellationToken),
+				await DownloadEngineVersion(compatVersion, fileDownloader, openDreamUrl, cancellationToken),
 				cancellationToken))
 			{
 				installJob2 = await instanceClient.Engine.SetActiveVersion(new EngineVersionRequest
@@ -235,9 +245,7 @@ namespace Tgstation.Server.Tests.Live.Instance
 
 			var configSetupTask = new ConfigurationTest(instanceClient.Configuration, instanceClient.Metadata).SetupDMApiTests(true, cancellationToken);
 
-			if (TestingUtils.RunningInGitHubActions
-				|| String.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("TGS_TEST_GITHUB_TOKEN"))
-				|| Environment.MachineName.Equals("CYBERSTATIONXVI", StringComparison.OrdinalIgnoreCase))
+			if (!String.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("TGS_TEST_GITHUB_TOKEN")))
 				await instanceClient.Repository.Update(new RepositoryUpdateRequest
 				{
 					CreateGitHubDeployments = true,

tests/Tgstation.Server.Tests/Live/Instance/WatchdogTest.cs

@@ -20,7 +20,6 @@ using System.Net;
 using System.Net.Sockets;
 using System.Reflection;
 using System.Runtime.InteropServices;
-using System.Runtime.Intrinsics.Arm;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
@@ -41,8 +40,6 @@ using Tgstation.Server.Host.IO;
 using Tgstation.Server.Host.System;
 using Tgstation.Server.Host.Utils;
 
-using static NuGet.Frameworks.FrameworkConstants;
-
 namespace Tgstation.Server.Tests.Live.Instance
 {
 	sealed class WatchdogTest : JobsRequiredTest

tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs

@@ -56,6 +56,8 @@ namespace Tgstation.Server.Tests.Live
 
 		public Uri RootUrl { get; }
 
+		public Uri OpenDreamUrl { get; }
+
 		public string Directory { get; }
 
 		public string UpdatePath { get; }
@@ -123,6 +125,12 @@ namespace Tgstation.Server.Tests.Live
 			HighPriorityDreamDaemon = nicingAllowed;
 			LowPriorityDeployments = nicingAllowed;
 
+			var odGitDir = Environment.GetEnvironmentVariable("TGS_TEST_OD_GIT_DIRECTORY");
+			if (!String.IsNullOrWhiteSpace(odGitDir))
+				OpenDreamUrl = new Uri($"file://{Path.GetFullPath(odGitDir).Replace('\\', '/')}");
+			else
+				OpenDreamUrl = new GeneralConfiguration().OpenDreamGitUrl;
+
 			args = new List<string>()
 			{
 				String.Format(CultureInfo.InvariantCulture, "Database:DropDatabase={0}", true), // Replaced after first Run
@@ -144,6 +152,7 @@ namespace Tgstation.Server.Tests.Live
 				$"Session:HighPriorityLiveDreamDaemon={HighPriorityDreamDaemon}",
 				$"Session:LowPriorityDeploymentProcesses={LowPriorityDeployments}",
 				$"General:SkipAddingByondFirewallException={!TestingUtils.RunningInGitHubActions}",
+				$"General:OpenDreamGitUrl={OpenDreamUrl}",
 			};
 
 			swarmArgs = new List<string>();

tests/Tgstation.Server.Tests/Live/TestLiveServer.cs

@@ -1208,7 +1208,18 @@ namespace Tgstation.Server.Tests.Live
 		}
 
 		[TestMethod]
-		public async Task TestStandardTgsOperation()
+		public Task TestStandardTgsOperation() => TestStandardTgsOperation(false);
+
+		[TestMethod]
+		public Task TestOpenDreamExclusiveTgsOperation()
+		{
+			if (String.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("TGS_TEST_OD_EXCLUSIVE")))
+				Assert.Inconclusive("This test is covered by TestStandardTgsOperation");
+
+			return TestStandardTgsOperation(true);
+		}
+
+		async Task TestStandardTgsOperation(bool openDreamOnly)
 		{
 			using(var currentProcess = System.Diagnostics.Process.GetCurrentProcess())
 			{
@@ -1233,7 +1244,7 @@ namespace Tgstation.Server.Tests.Live
 			ServiceCollectionExtensions.UseAdditionalLoggerProvider<HardFailLoggerProvider>();
 
 			var failureTask = HardFailLoggerProvider.FailureSource;
-			var internalTask = TestTgsInternal(hardCancellationToken);
+			var internalTask = TestTgsInternal(openDreamOnly, hardCancellationToken);
 			await Task.WhenAny(
 				internalTask,
 				failureTask);
@@ -1266,7 +1277,7 @@ namespace Tgstation.Server.Tests.Live
 				await internalTask;
 		}
 
-		async Task TestTgsInternal(CancellationToken hardCancellationToken)
+		async Task TestTgsInternal(bool openDreamOnly, CancellationToken hardCancellationToken)
 		{
 			var discordConnectionString = Environment.GetEnvironmentVariable("TGS_TEST_DISCORD_TOKEN");
 			var ircConnectionString = Environment.GetEnvironmentVariable("TGS_TEST_IRC_CONNECTION_STRING");
@@ -1379,24 +1390,41 @@ namespace Tgstation.Server.Tests.Live
 						}
 					}
 
-					var rootTest = FailFast(RawRequestTests.Run(clientFactory, firstAdminClient, cancellationToken));
-					var adminTest = FailFast(new AdministrationTest(firstAdminClient.Administration).Run(cancellationToken));
-					var usersTest = FailFast(new UsersTest(firstAdminClient).Run(cancellationToken));
+					Task nonInstanceTests;
+					IInstanceClient instanceClient = null;
+					InstanceResponse odInstance, compatInstance;
+					if (!openDreamOnly)
+					{
+						var rootTest = FailFast(RawRequestTests.Run(clientFactory, firstAdminClient, cancellationToken));
+						var adminTest = FailFast(new AdministrationTest(firstAdminClient.Administration).Run(cancellationToken));
+						var usersTest = FailFast(new UsersTest(firstAdminClient).Run(cancellationToken));
 
-					jobsHubTestTask = FailFast(jobsHubTest.Run(cancellationToken));
-					var instanceManagerTest = new InstanceManagerTest(firstAdminClient, server.Directory);
-					var compatInstanceTask = instanceManagerTest.CreateTestInstance("CompatTestsInstance", cancellationToken);
-					var odInstanceTask = instanceManagerTest.CreateTestInstance("OdTestsInstance", cancellationToken);
-					var byondApiCompatInstanceTask = instanceManagerTest.CreateTestInstance("BCAPITestsInstance", cancellationToken);
-					instance = await instanceManagerTest.CreateTestInstance("LiveTestsInstance", cancellationToken);
-					var compatInstance = await compatInstanceTask;
-					var odInstance = await odInstanceTask;
-					var byondApiCompatInstance = await byondApiCompatInstanceTask;
-					var instancesTest = FailFast(instanceManagerTest.RunPreTest(cancellationToken));
-					Assert.IsTrue(Directory.Exists(instance.Path));
-					var instanceClient = firstAdminClient.Instances.CreateClient(instance);
+						jobsHubTestTask = FailFast(jobsHubTest.Run(cancellationToken));
+						var instanceManagerTest = new InstanceManagerTest(firstAdminClient, server.Directory);
+						var compatInstanceTask = instanceManagerTest.CreateTestInstance("CompatTestsInstance", cancellationToken);
+						var odInstanceTask = instanceManagerTest.CreateTestInstance("OdTestsInstance", cancellationToken);
+						var byondApiCompatInstanceTask = instanceManagerTest.CreateTestInstance("BCAPITestsInstance", cancellationToken);
+						instance = await instanceManagerTest.CreateTestInstance("LiveTestsInstance", cancellationToken);
+						compatInstance = await compatInstanceTask;
+						odInstance = await odInstanceTask;
+						var byondApiCompatInstance = await byondApiCompatInstanceTask;
+						var instancesTest = FailFast(instanceManagerTest.RunPreTest(cancellationToken));
+						Assert.IsTrue(Directory.Exists(instance.Path));
+						instanceClient = firstAdminClient.Instances.CreateClient(instance);
 
-					Assert.IsTrue(Directory.Exists(instanceClient.Metadata.Path));
+						Assert.IsTrue(Directory.Exists(instanceClient.Metadata.Path));
+						nonInstanceTests = Task.WhenAll(instancesTest, adminTest, rootTest, usersTest);
+					}
+					else
+					{
+						compatInstance = null;
+						nonInstanceTests = Task.CompletedTask;
+						jobsHubTestTask = null;
+						instance = null;
+						var instanceManagerTest = new InstanceManagerTest(firstAdminClient, server.Directory);
+						var odInstanceTask = instanceManagerTest.CreateTestInstance("OdTestsInstance", cancellationToken);
+						odInstance = await odInstanceTask;
+					}
 
 					var instanceTest = new InstanceTest(
 							firstAdminClient.Instances,
@@ -1418,8 +1446,8 @@ namespace Tgstation.Server.Tests.Live
 										Engine = EngineType.OpenDream,
 										SourceSHA = "f1dc153caf9d84cd1d0056e52286cc0163e3f4d3", // 1b4 verified version
 									},
-									instanceClient,
 									fileDownloader,
+									server.OpenDreamUrl,
 									cancellationToken).AsTask());
 
 							Assert.AreEqual(ErrorCode.OpenDreamTooOld, ex.ErrorCode);
@@ -1427,6 +1455,7 @@ namespace Tgstation.Server.Tests.Live
 							await instanceTest
 								.RunCompatTests(
 									await edgeODVersionTask,
+									server.OpenDreamUrl,
 									firstAdminClient.Instances.CreateClient(odInstance),
 									odDMPort,
 									odDDPort,
@@ -1437,9 +1466,12 @@ namespace Tgstation.Server.Tests.Live
 
 						var odCompatTests = FailFast(ODCompatTests());
 
-						if (testSerialized) // they only have 2 cores, can't handle intense parallelization
+						if (openDreamOnly || testSerialized)
 							await odCompatTests;
 
+						if (openDreamOnly)
+							return;
+
 						var compatTests = FailFast(
 							instanceTest
 								.RunCompatTests(
@@ -1450,6 +1482,7 @@ namespace Tgstation.Server.Tests.Live
 											? new Version(510, 1346)
 											: new Version(512, 1451) // http://www.byond.com/forum/?forum=5&command=search&scope=local&text=resolved%3a512.1451
 									},
+									server.OpenDreamUrl,
 									firstAdminClient.Instances.CreateClient(compatInstance),
 									compatDMPort,
 									compatDDPort,
@@ -1477,7 +1510,10 @@ namespace Tgstation.Server.Tests.Live
 
 					var instanceTests = RunInstanceTests();
 
-					await Task.WhenAll(rootTest, adminTest, instancesTest, instanceTests, usersTest);
+					await Task.WhenAll(nonInstanceTests, instanceTests);
+
+					if (openDreamOnly)
+						return;
 
 					var dd = await instanceClient.DreamDaemon.Read(cancellationToken);
 					Assert.AreEqual(WatchdogStatus.Online, dd.Status.Value);

tests/Tgstation.Server.Tests/TestingUtils.cs

@@ -38,12 +38,12 @@ namespace Tgstation.Server.Tests
 			if (engineInstallationData is ZipStreamEngineInstallationData zipStreamData)
 				return (MemoryStream)zipStreamData.GetType().GetField("zipStream", BindingFlags.NonPublic | BindingFlags.Instance).GetValue(zipStreamData);
 
-			await using var repoData = (RepositoryEngineInstallationData)engineInstallationData;
+			await using var grabby = engineInstallationData;
 			var tempFolder = Path.GetTempFileName();
 			File.Delete(tempFolder);
 			try
 			{
-				await repoData.ExtractToPath(tempFolder, cancellationToken);
+				await engineInstallationData.ExtractToPath(tempFolder, cancellationToken);
 				var resultStream = new FileStream(
 					$"{tempFolder}.zip",
 					FileMode.Create,