Add integration test for server reboots

tests/Tgstation.Server.Tests/Instance/WatchdogTest.cs

@@ -152,8 +152,12 @@ namespace Tgstation.Server.Tests.Instance
 			Assert.IsFalse(daemonStatus.Running.Value);
 		}
 
-		async Task StartAndLeaveRunning(CancellationToken cancellationToken)
+		public async Task StartAndLeaveRunning(CancellationToken cancellationToken)
 		{
+			var dd = await instanceClient.DreamDaemon.Read(cancellationToken);
+			if(dd.ActiveCompileJob == null)
+				await DeployTestDme("LongRunning/long_running_test", DreamDaemonSecurity.Trusted, cancellationToken);
+
 			var startJob = await instanceClient.DreamDaemon.Start(cancellationToken).ConfigureAwait(false);
 
 			await WaitForJob(startJob, 40, false, cancellationToken);

tests/Tgstation.Server.Tests/InstanceManagerTest.cs

@@ -29,7 +29,7 @@ namespace Tgstation.Server.Tests
 			counter = 0;
 		}
 
-		Task<Api.Models.Instance> CreateTestInstance(CancellationToken cancellationToken) => instanceManagerClient.CreateOrAttach(new Api.Models.Instance
+		public Task<Api.Models.Instance> CreateTestInstance(CancellationToken cancellationToken) => instanceManagerClient.CreateOrAttach(new Api.Models.Instance
 		{
 			Name = "TestInstance-" + ++counter,
 			Path = Path.Combine(testRootPath, Guid.NewGuid().ToString()),

tests/Tgstation.Server.Tests/IntegrationTest.cs

@@ -15,7 +15,7 @@ using Tgstation.Server.Api.Models;
 using Tgstation.Server.Client;
 using Tgstation.Server.Host;
 using Tgstation.Server.Host.Components.Chat.Providers;
-using Tgstation.Server.Host.Extensions;
+using Tgstation.Server.Tests.Instance;
 
 namespace Tgstation.Server.Tests
 {
@@ -81,91 +81,80 @@ namespace Tgstation.Server.Tests
 		[TestMethod]
 		public async Task TestServerUpdate()
 		{
-			var updatePathRoot = Path.GetTempFileName();
-			File.Delete(updatePathRoot);
-			Directory.CreateDirectory(updatePathRoot);
+			using var server = new TestingServer(clientFactory);
+
+			if (server.DatabaseType == "Sqlite")
+				Assert.Inconclusive("Cannot run this test on SQLite yet!");
+
+			using var serverCts = new CancellationTokenSource();
+			var cancellationToken = serverCts.Token;
+			var serverTask = server.Run(cancellationToken);
 			try
 			{
-				var updatePath = Path.Combine(updatePathRoot, Guid.NewGuid().ToString());
-				using var server = new TestingServer(clientFactory, updatePath);
+				IServerClient adminClient;
 
-				if (server.DatabaseType == "Sqlite")
-					Assert.Inconclusive("Cannot run this test on SQLite yet!");
-
-				using var serverCts = new CancellationTokenSource();
-				var cancellationToken = serverCts.Token;
-				var serverTask = server.Run(cancellationToken);
-				try
+				var giveUpAt = DateTimeOffset.Now.AddSeconds(60);
+				do
 				{
-					IServerClient adminClient;
-
-					var giveUpAt = DateTimeOffset.Now.AddSeconds(60);
-					do
-					{
-						try
-						{
-							adminClient = await clientFactory.CreateServerClient(server.Url, User.AdminName, User.DefaultAdminPassword).ConfigureAwait(false);
-							break;
-						}
-						catch (HttpRequestException)
-						{
-							//migrating, to be expected
-							if (DateTimeOffset.Now > giveUpAt)
-								throw;
-							await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken);
-						}
-						catch (ServiceUnavailableException)
-						{
-							//migrating, to be expected
-							if (DateTimeOffset.Now > giveUpAt)
-								throw;
-							await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken);
-						}
-					} while (true);
-
-					var testUpdateVersion = new Version(4, 1, 0);
-					using (adminClient)
-						//attempt to update to stable
-						await adminClient.Administration.Update(new Administration
-						{
-							NewVersion = testUpdateVersion
-						}, cancellationToken).ConfigureAwait(false);
-
-					//wait up to 3 minutes for the dl and install
-					await Task.WhenAny(serverTask, Task.Delay(TimeSpan.FromMinutes(3), cancellationToken)).ConfigureAwait(false);
-
-					Assert.IsTrue(serverTask.IsCompleted, "Sever still running!");
-
-					Assert.IsTrue(Directory.Exists(updatePath), "Update directory not present!");
-
-					var updatedAssemblyPath = Path.Combine(updatePath, "Tgstation.Server.Host.dll");
-					Assert.IsTrue(File.Exists(updatedAssemblyPath), "Updated assembly missing!");
-
-					var updatedAssemblyVersion = FileVersionInfo.GetVersionInfo(updatedAssemblyPath);
-					Assert.AreEqual(testUpdateVersion, Version.Parse(updatedAssemblyVersion.FileVersion).Semver());
-				}
-				finally
-				{
-					serverCts.Cancel();
 					try
 					{
-						await serverTask.ConfigureAwait(false);
+						adminClient = await clientFactory.CreateServerClient(server.Url, User.AdminName, User.DefaultAdminPassword).ConfigureAwait(false);
+						break;
 					}
-					catch (OperationCanceledException) { }
-				}
-				Assert.IsTrue(server.RestartRequested, "Server not requesting restart!");
+					catch (HttpRequestException)
+					{
+						//migrating, to be expected
+						if (DateTimeOffset.Now > giveUpAt)
+							throw;
+						await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken);
+					}
+					catch (ServiceUnavailableException)
+					{
+						//migrating, to be expected
+						if (DateTimeOffset.Now > giveUpAt)
+							throw;
+						await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken);
+					}
+				} while (true);
+
+				var testUpdateVersion = new Version(4, 1, 0);
+				using (adminClient)
+					//attempt to update to stable
+					await adminClient.Administration.Update(new Administration
+					{
+						NewVersion = testUpdateVersion
+					}, cancellationToken).ConfigureAwait(false);
+
+				//wait up to 3 minutes for the dl and install
+				await Task.WhenAny(serverTask, Task.Delay(TimeSpan.FromMinutes(3), cancellationToken)).ConfigureAwait(false);
+
+				Assert.IsTrue(serverTask.IsCompleted, "Sever still running!");
+
+				Assert.IsTrue(Directory.Exists(server.UpdatePath), "Update directory not present!");
+
+				var updatedAssemblyPath = Path.Combine(server.UpdatePath, "Tgstation.Server.Host.dll");
+				Assert.IsTrue(File.Exists(updatedAssemblyPath), "Updated assembly missing!");
+
+				var updatedAssemblyVersion = FileVersionInfo.GetVersionInfo(updatedAssemblyPath);
+				Assert.AreEqual(testUpdateVersion, Version.Parse(updatedAssemblyVersion.FileVersion).Semver());
 			}
 			finally
 			{
-				Directory.Delete(updatePathRoot, true);
+				serverCts.Cancel();
+				try
+				{
+					await serverTask.ConfigureAwait(false);
+				}
+				catch (OperationCanceledException) { }
 			}
+			Assert.IsTrue(server.RestartRequested, "Server not requesting restart!");
 		}
 
 		[TestMethod]
 		public async Task TestFullStandardOperation()
 		{
 			RequireDiscordToken();
-			using var server = new TestingServer(clientFactory, null);
+			using var server = new TestingServer(clientFactory);
 			using var serverCts = new CancellationTokenSource();
 			var cancellationToken = serverCts.Token;
 			var serverTask = server.Run(cancellationToken);
@@ -241,5 +230,96 @@ namespace Tgstation.Server.Tests
 				catch (OperationCanceledException) { }
 			}
 		}
+
+		[TestMethod]
+		public async Task TestRebootAndAttach()
+		{
+			using var server = new TestingServer(clientFactory);
+			using var serverCts = new CancellationTokenSource();
+			var cancellationToken = serverCts.Token;
+			var serverTask = server.Run(cancellationToken);
+			try
+			{
+				async Task<IServerClient> CreateAdminClient()
+				{
+					var giveUpAt = DateTimeOffset.Now.AddSeconds(60);
+					do
+					{
+						try
+						{
+							return await clientFactory.CreateServerClient(server.Url, User.AdminName, User.DefaultAdminPassword).ConfigureAwait(false);
+						}
+						catch (HttpRequestException)
+						{
+							//migrating, to be expected
+							if (DateTimeOffset.Now > giveUpAt)
+								throw;
+							await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken);
+						}
+						catch (ServiceUnavailableException)
+						{
+							// migrating, to be expected
+							if (DateTimeOffset.Now > giveUpAt)
+								throw;
+							await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken);
+						}
+					} while (true);
+				}
+
+				Api.Models.Instance instance;
+				using (var adminClient = await CreateAdminClient())
+				{
+					var instanceTest = new InstanceManagerTest(adminClient.Instances, adminClient.Users, server.Directory);
+					instance = await instanceTest.CreateTestInstance(cancellationToken);
+					instance.Online = true;
+					instance = await adminClient.Instances.Update(instance, cancellationToken);
+					var instanceClient = adminClient.Instances.CreateClient(instance);
+					var repoTest = new RepositoryTest(instanceClient.Repository, instanceClient.Jobs);
+					var byondTest = new ByondTest(instanceClient.Byond, instanceClient.Jobs, instance);
+
+					var repoTask = repoTest.RunPreWatchdog(cancellationToken);
+					await byondTest.Run(cancellationToken);
+					await repoTask;
+
+					await new WatchdogTest(instanceClient).StartAndLeaveRunning(cancellationToken);
+
+					await adminClient.Administration.Restart(cancellationToken);
+				}
+
+				await Task.WhenAny(serverTask, Task.Delay(30000, cancellationToken));
+				Assert.IsTrue(serverTask.IsCompleted);
+
+				serverTask = server.Run(cancellationToken);
+				using (var adminClient = await CreateAdminClient())
+				{
+					var instanceClient = adminClient.Instances.CreateClient(instance);
+					var dd = await instanceClient.DreamDaemon.Read(cancellationToken);
+
+					await new RepositoryTest(instanceClient.Repository, instanceClient.Jobs).RunPostWatchdog(cancellationToken);
+
+					Assert.IsTrue(dd.Running.Value);
+				}
+			}
+			catch (Exception ex)
+			{
+				Console.WriteLine($"TEST: ERROR: {ex.GetType()} in flight!");
+				throw;
+			}
+			finally
+			{
+				Console.WriteLine($"TEST: STOPPING SERVER!");
+				serverCts.Cancel();
+				try
+				{
+					Console.WriteLine($"TEST: WAITING FOR SERVER!");
+					await serverTask.ConfigureAwait(false);
+				}
+				catch (OperationCanceledException) { }
+
+				foreach (var proc in System.Diagnostics.Process.GetProcessesByName("DreamDaemon"))
+					using (proc)
+						proc.Kill();
+			}
+		}
 	}
 }

tests/Tgstation.Server.Tests/TestingServer.cs

@@ -3,6 +3,7 @@ using System;
 using System.Collections.Generic;
 using System.Globalization;
 using System.IO;
+using System.Linq;
 using System.Net;
 using System.Net.Http;
 using System.Threading;
@@ -21,17 +22,21 @@ namespace Tgstation.Server.Tests
 
 		public string Directory { get; }
 
+		public string UpdatePath { get; }
+
 		public string DatabaseType { get; }
 
-		public bool RestartRequested => realServer.Result.RestartRequested;
-
-		readonly Task<IServer> realServer;
+		public bool RestartRequested => realServer.RestartRequested;
 
 		readonly IServerClientFactory serverClientFactory;
 
 		readonly bool dumpOpenAPISpecpath;
 
-		public TestingServer(IServerClientFactory serverClientFactory, string updatePath)
+		string[] args;
+
+		IServer realServer;
+
+		public TestingServer(IServerClientFactory serverClientFactory)
 		{
 			this.serverClientFactory = serverClientFactory;
 
@@ -67,10 +72,10 @@ namespace Tgstation.Server.Tests
 
 			var args = new List<string>()
 			{
+				String.Format(CultureInfo.InvariantCulture, "Database:DropDatabase={0}", true),
 				String.Format(CultureInfo.InvariantCulture, "Kestrel:EndPoints:Http:Url={0}", UrlString),
 				String.Format(CultureInfo.InvariantCulture, "Database:DatabaseType={0}", DatabaseType),
 				String.Format(CultureInfo.InvariantCulture, "Database:ConnectionString={0}", connectionString),
-				String.Format(CultureInfo.InvariantCulture, "Database:DropDatabase={0}", true),
 				String.Format(CultureInfo.InvariantCulture, "General:SetupWizardMode={0}", SetupWizardMode.Never),
 				String.Format(CultureInfo.InvariantCulture, "General:MinimumPasswordLength={0}", 10),
 				String.Format(CultureInfo.InvariantCulture, "General:InstanceLimit={0}", 11),
@@ -84,7 +89,8 @@ namespace Tgstation.Server.Tests
 			if (dumpOpenAPISpecpath)
 				Environment.SetEnvironmentVariable("ASPNETCORE_ENVIRONMENT", "Development");
 
-			realServer = Application.CreateDefaultServerFactory().CreateServer(args.ToArray(), updatePath, default);
+			UpdatePath = Path.Combine(Directory, Guid.NewGuid().ToString());
+			this.args = args.ToArray();
 		}
 
 		public void Dispose()
@@ -103,8 +109,18 @@ namespace Tgstation.Server.Tests
 
 		public async Task Run(CancellationToken cancellationToken)
 		{
-			var serverInstance = await realServer.ConfigureAwait(false);
-			Task runTask = serverInstance.Run(cancellationToken);
+			var firstRun = realServer == null;
+			realServer = await Application
+				.CreateDefaultServerFactory()
+				.CreateServer(
+					args,
+					UpdatePath,
+					default);
+
+			if (firstRun)
+				args = args.Skip(1).ToArray();
+
+			Task runTask = realServer.Run(cancellationToken);
 
 			if (dumpOpenAPISpecpath)
 			{