Merge pull request #977 from tgstation/849-FixScheduledUpdate [TGSDeploy]

Fix MySql timeouts

build/Dockerfile

@@ -70,5 +70,6 @@ COPY --from=build /repo/build/tgs.docker.sh tgs.sh
 VOLUME ["/config_data", "/tgs_logs", "/app/lib"]
 
 ENV General__ValidInstancePaths__0 /tgs4_instances
+ENV FileLogging__Directory /tgs_logs
 
 ENTRYPOINT ["./tgs.sh"]

build/Version.props

@@ -2,7 +2,7 @@
   <PropertyGroup>
     <!-- This is the authorative version list -->
     <!-- Integration tests will ensure they match across the board -->
-    <TgsCoreVersion>4.1.4</TgsCoreVersion>
+    <TgsCoreVersion>4.1.5</TgsCoreVersion>
     <TgsApiVersion>6.1.0</TgsApiVersion>
     <TgsClientVersion>6.0.0</TgsClientVersion>
     <TgsDmapiVersion>5.0.0</TgsDmapiVersion>

src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs

@@ -361,7 +361,17 @@ namespace Tgstation.Server.Host.Components.Deployment
 					return result;
 				})));
 
-			await chatManager.SendUpdateMessage(String.Format(CultureInfo.InvariantCulture, "Deploying revision: {0}{1}{2} BYOND Version: {3}", commitInsert, testmergeInsert, remoteCommitInsert, byondLock.Version), cancellationToken).ConfigureAwait(false);
+			await chatManager.SendUpdateMessage(
+				String.Format(
+					CultureInfo.InvariantCulture,
+					"Deploying revision: {0}{1}{2} BYOND Version: {3}.{4}",
+					commitInsert,
+					testmergeInsert,
+					remoteCommitInsert,
+					byondLock.Version.Major,
+					byondLock.Version.Minor),
+				cancellationToken)
+				.ConfigureAwait(false);
 		}
 
 		/// <summary>

src/Tgstation.Server.Host/Components/Watchdog/SessionControllerFactory.cs

@@ -356,9 +356,10 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			var testMerges = dmbProvider
 				.CompileJob
 				.RevisionInformation
-				.ActiveTestMerges
+				.ActiveTestMerges?
 				.Select(x => x.TestMerge)
-				.Select(x => new TestMergeInformation(x, revisionInfo));
+				.Select(x => new TestMergeInformation(x, revisionInfo))
+				?? Enumerable.Empty<TestMergeInformation>();
 
 			return new RuntimeInformation(
 				assemblyInformationProvider,

src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs

@@ -315,8 +315,8 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			Logger.LogTrace("StopMonitor");
 			if (monitorTask == null)
 				return false;
-			monitorCts.Cancel();
 			var wasRunning = !monitorTask.IsCompleted;
+			monitorCts.Cancel();
 			await monitorTask.ConfigureAwait(false);
 			monitorCts.Dispose();
 			monitorTask = null;
@@ -574,6 +574,8 @@ namespace Tgstation.Server.Host.Components.Watchdog
 					await StopMonitor().ConfigureAwait(false);
 
 					var reattachInformation = CreateReattachInformation();
+
+					Logger.LogDebug("Saving reattach information...");
 					await reattachInfoHandler.Save(reattachInformation, cancellationToken).ConfigureAwait(false);
 				}
 

src/Tgstation.Server.Host/Database/MySqlDatabaseContext.cs

@@ -33,16 +33,19 @@ namespace Tgstation.Server.Host.Database
 			};
 			if (stringDeconstructor.Server == "localhost")
 				Logger.LogWarning("MariaDB/MySQL server address is set to 'localhost'! If there are connection issues, try setting it to '127.0.0.1'!");
-			if (!String.IsNullOrEmpty(DatabaseConfiguration.MySqlServerVersion))
-				options.UseMySql(
-					DatabaseConfiguration.ConnectionString,
-					mySqlOptions => mySqlOptions.ServerVersion(
-						Version.Parse(DatabaseConfiguration.MySqlServerVersion),
-						DatabaseConfiguration.DatabaseType == DatabaseType.MariaDB
-							? ServerType.MariaDb
-							: ServerType.MySql));
-			else
-				options.UseMySql(DatabaseConfiguration.ConnectionString);
+			options.UseMySql(
+				DatabaseConfiguration.ConnectionString,
+				mySqlOptions =>
+				{
+					mySqlOptions.EnableRetryOnFailure();
+
+					if (!String.IsNullOrEmpty(DatabaseConfiguration.MySqlServerVersion))
+						mySqlOptions.ServerVersion(
+							Version.Parse(DatabaseConfiguration.MySqlServerVersion),
+							DatabaseConfiguration.DatabaseType == DatabaseType.MariaDB
+								? ServerType.MariaDb
+								: ServerType.MySql);
+				});
 		}
 
 		/// <inheritdoc />

src/Tgstation.Server.Host/Security/AuthenticationContextFactory.cs

@@ -1,9 +1,11 @@
 using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging;
 using System;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using Tgstation.Server.Host.Database;
+using Tgstation.Server.Host.Models;
 
 namespace Tgstation.Server.Host.Security
 {
@@ -23,15 +25,25 @@ namespace Tgstation.Server.Host.Security
 		/// </summary>
 		readonly IIdentityCache identityCache;
 
+		/// <summary>
+		/// The <see cref="ILogger"/> for the <see cref="AuthenticationContextFactory"/>.
+		/// </summary>
+		readonly ILogger<AuthenticationContextFactory> logger;
+
 		/// <summary>
 		/// Construct an <see cref="AuthenticationContextFactory"/>
 		/// </summary>
 		/// <param name="databaseContext">The value of <see cref="databaseContext"/></param>
 		/// <param name="identityCache">The value of <see cref="identityCache"/></param>
-		public AuthenticationContextFactory(IDatabaseContext databaseContext, IIdentityCache identityCache)
+		/// <param name="logger">The value of <see cref="logger"/>.</param>
+		public AuthenticationContextFactory(
+			IDatabaseContext databaseContext,
+			IIdentityCache identityCache,
+			ILogger<AuthenticationContextFactory> logger)
 		{
 			this.databaseContext = databaseContext ?? throw new ArgumentNullException(nameof(databaseContext));
 			this.identityCache = identityCache ?? throw new ArgumentNullException(nameof(identityCache));
+			this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
 		}
 
 		/// <inheritdoc />
@@ -69,13 +81,18 @@ namespace Tgstation.Server.Host.Security
 
 			try
 			{
-				var instanceUser = instanceId.HasValue
-					? await databaseContext.InstanceUsers
+				InstanceUser instanceUser = null;
+				if (instanceId.HasValue)
+				{
+					instanceUser = await databaseContext.InstanceUsers
 						.Where(x => x.UserId == userId && x.InstanceId == instanceId && x.Instance.Online.Value)
 						.Include(x => x.Instance)
 						.FirstOrDefaultAsync(cancellationToken)
-						.ConfigureAwait(false)
-					: null;
+						.ConfigureAwait(false);
+
+					if (instanceUser == null)
+						logger.LogDebug("User {0} does not have permissions on instance {1}!", userId, instanceId.Value);
+				}
 
 				CurrentAuthenticationContext = new AuthenticationContext(systemIdentity, user, instanceUser);
 			}

src/Tgstation.Server.Host/Security/ClaimsInjector.cs

@@ -9,18 +9,12 @@ using System.Threading;
 using System.Threading.Tasks;
 using Tgstation.Server.Api;
 using Tgstation.Server.Api.Rights;
-using Tgstation.Server.Host.Database;
 
 namespace Tgstation.Server.Host.Security
 {
 	/// <inheritdoc />
 	sealed class ClaimsInjector : IClaimsInjector
 	{
-		/// <summary>
-		/// The <see cref="IDatabaseContext"/> for the <see cref="ClaimsInjector"/>
-		/// </summary>
-		readonly IDatabaseContext databaseContext;
-
 		/// <summary>
 		/// The <see cref="IAuthenticationContextFactory"/> for the <see cref="ClaimsInjector"/>
 		/// </summary>
@@ -29,11 +23,9 @@ namespace Tgstation.Server.Host.Security
 		/// <summary>
 		/// Construct a <see cref="ClaimsInjector"/>
 		/// </summary>
-		/// <param name="databaseContext">The value of <see cref="databaseContext"/></param>
 		/// <param name="authenticationContextFactory">The value of <see cref="authenticationContextFactory"/></param>
-		public ClaimsInjector(IDatabaseContext databaseContext, IAuthenticationContextFactory authenticationContextFactory)
+		public ClaimsInjector(IAuthenticationContextFactory authenticationContextFactory)
 		{
-			this.databaseContext = databaseContext ?? throw new ArgumentNullException(nameof(databaseContext));
 			this.authenticationContextFactory = authenticationContextFactory ?? throw new ArgumentNullException(nameof(authenticationContextFactory));
 		}
 
@@ -70,7 +62,12 @@ namespace Tgstation.Server.Host.Security
 			}
 
 			// This populates the CurrentAuthenticationContext field for use by us and subsequent controllers
-			await authenticationContextFactory.CreateAuthenticationContext(userId, apiHeaders.InstanceId, tokenValidatedContext.SecurityToken.ValidFrom, cancellationToken).ConfigureAwait(false);
+			await authenticationContextFactory.CreateAuthenticationContext(
+				userId,
+				apiHeaders.InstanceId,
+				tokenValidatedContext.SecurityToken.ValidFrom,
+				cancellationToken)
+				.ConfigureAwait(false);
 
 			var authenticationContext = authenticationContextFactory.CurrentAuthenticationContext;
 

tests/Tgstation.Server.Tests/Instance/WatchdogTest.cs

@@ -152,8 +152,12 @@ namespace Tgstation.Server.Tests.Instance
 			Assert.IsFalse(daemonStatus.Running.Value);
 		}
 
-		async Task StartAndLeaveRunning(CancellationToken cancellationToken)
+		public async Task StartAndLeaveRunning(CancellationToken cancellationToken)
 		{
+			var dd = await instanceClient.DreamDaemon.Read(cancellationToken);
+			if(dd.ActiveCompileJob == null)
+				await DeployTestDme("LongRunning/long_running_test", DreamDaemonSecurity.Trusted, cancellationToken);
+
 			var startJob = await instanceClient.DreamDaemon.Start(cancellationToken).ConfigureAwait(false);
 
 			await WaitForJob(startJob, 40, false, cancellationToken);

tests/Tgstation.Server.Tests/InstanceManagerTest.cs

@@ -29,7 +29,7 @@ namespace Tgstation.Server.Tests
 			counter = 0;
 		}
 
-		Task<Api.Models.Instance> CreateTestInstance(CancellationToken cancellationToken) => instanceManagerClient.CreateOrAttach(new Api.Models.Instance
+		public Task<Api.Models.Instance> CreateTestInstance(CancellationToken cancellationToken) => instanceManagerClient.CreateOrAttach(new Api.Models.Instance
 		{
 			Name = "TestInstance-" + ++counter,
 			Path = Path.Combine(testRootPath, Guid.NewGuid().ToString()),

tests/Tgstation.Server.Tests/IntegrationTest.cs

@@ -15,7 +15,7 @@ using Tgstation.Server.Api.Models;
 using Tgstation.Server.Client;
 using Tgstation.Server.Host;
 using Tgstation.Server.Host.Components.Chat.Providers;
-using Tgstation.Server.Host.Extensions;
+using Tgstation.Server.Tests.Instance;
 
 namespace Tgstation.Server.Tests
 {
@@ -81,91 +81,80 @@ namespace Tgstation.Server.Tests
 		[TestMethod]
 		public async Task TestServerUpdate()
 		{
-			var updatePathRoot = Path.GetTempFileName();
-			File.Delete(updatePathRoot);
-			Directory.CreateDirectory(updatePathRoot);
+			using var server = new TestingServer(clientFactory);
+
+			if (server.DatabaseType == "Sqlite")
+				Assert.Inconclusive("Cannot run this test on SQLite yet!");
+
+			using var serverCts = new CancellationTokenSource();
+			var cancellationToken = serverCts.Token;
+			var serverTask = server.Run(cancellationToken);
 			try
 			{
-				var updatePath = Path.Combine(updatePathRoot, Guid.NewGuid().ToString());
-				using var server = new TestingServer(clientFactory, updatePath);
+				IServerClient adminClient;
 
-				if (server.DatabaseType == "Sqlite")
-					Assert.Inconclusive("Cannot run this test on SQLite yet!");
-
-				using var serverCts = new CancellationTokenSource();
-				var cancellationToken = serverCts.Token;
-				var serverTask = server.Run(cancellationToken);
-				try
+				var giveUpAt = DateTimeOffset.Now.AddSeconds(60);
+				do
 				{
-					IServerClient adminClient;
-
-					var giveUpAt = DateTimeOffset.Now.AddSeconds(60);
-					do
-					{
-						try
-						{
-							adminClient = await clientFactory.CreateServerClient(server.Url, User.AdminName, User.DefaultAdminPassword).ConfigureAwait(false);
-							break;
-						}
-						catch (HttpRequestException)
-						{
-							//migrating, to be expected
-							if (DateTimeOffset.Now > giveUpAt)
-								throw;
-							await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken);
-						}
-						catch (ServiceUnavailableException)
-						{
-							//migrating, to be expected
-							if (DateTimeOffset.Now > giveUpAt)
-								throw;
-							await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken);
-						}
-					} while (true);
-
-					var testUpdateVersion = new Version(4, 1, 0);
-					using (adminClient)
-						//attempt to update to stable
-						await adminClient.Administration.Update(new Administration
-						{
-							NewVersion = testUpdateVersion
-						}, cancellationToken).ConfigureAwait(false);
-
-					//wait up to 3 minutes for the dl and install
-					await Task.WhenAny(serverTask, Task.Delay(TimeSpan.FromMinutes(3), cancellationToken)).ConfigureAwait(false);
-
-					Assert.IsTrue(serverTask.IsCompleted, "Sever still running!");
-
-					Assert.IsTrue(Directory.Exists(updatePath), "Update directory not present!");
-
-					var updatedAssemblyPath = Path.Combine(updatePath, "Tgstation.Server.Host.dll");
-					Assert.IsTrue(File.Exists(updatedAssemblyPath), "Updated assembly missing!");
-
-					var updatedAssemblyVersion = FileVersionInfo.GetVersionInfo(updatedAssemblyPath);
-					Assert.AreEqual(testUpdateVersion, Version.Parse(updatedAssemblyVersion.FileVersion).Semver());
-				}
-				finally
-				{
-					serverCts.Cancel();
 					try
 					{
-						await serverTask.ConfigureAwait(false);
+						adminClient = await clientFactory.CreateServerClient(server.Url, User.AdminName, User.DefaultAdminPassword).ConfigureAwait(false);
+						break;
 					}
-					catch (OperationCanceledException) { }
-				}
-				Assert.IsTrue(server.RestartRequested, "Server not requesting restart!");
+					catch (HttpRequestException)
+					{
+						//migrating, to be expected
+						if (DateTimeOffset.Now > giveUpAt)
+							throw;
+						await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken);
+					}
+					catch (ServiceUnavailableException)
+					{
+						//migrating, to be expected
+						if (DateTimeOffset.Now > giveUpAt)
+							throw;
+						await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken);
+					}
+				} while (true);
+
+				var testUpdateVersion = new Version(4, 1, 0);
+				using (adminClient)
+					//attempt to update to stable
+					await adminClient.Administration.Update(new Administration
+					{
+						NewVersion = testUpdateVersion
+					}, cancellationToken).ConfigureAwait(false);
+
+				//wait up to 3 minutes for the dl and install
+				await Task.WhenAny(serverTask, Task.Delay(TimeSpan.FromMinutes(3), cancellationToken)).ConfigureAwait(false);
+
+				Assert.IsTrue(serverTask.IsCompleted, "Sever still running!");
+
+				Assert.IsTrue(Directory.Exists(server.UpdatePath), "Update directory not present!");
+
+				var updatedAssemblyPath = Path.Combine(server.UpdatePath, "Tgstation.Server.Host.dll");
+				Assert.IsTrue(File.Exists(updatedAssemblyPath), "Updated assembly missing!");
+
+				var updatedAssemblyVersion = FileVersionInfo.GetVersionInfo(updatedAssemblyPath);
+				Assert.AreEqual(testUpdateVersion, Version.Parse(updatedAssemblyVersion.FileVersion).Semver());
 			}
 			finally
 			{
-				Directory.Delete(updatePathRoot, true);
+				serverCts.Cancel();
+				try
+				{
+					await serverTask.ConfigureAwait(false);
+				}
+				catch (OperationCanceledException) { }
 			}
+			Assert.IsTrue(server.RestartRequested, "Server not requesting restart!");
 		}
 
 		[TestMethod]
 		public async Task TestFullStandardOperation()
 		{
 			RequireDiscordToken();
-			using var server = new TestingServer(clientFactory, null);
+			using var server = new TestingServer(clientFactory);
 			using var serverCts = new CancellationTokenSource();
 			var cancellationToken = serverCts.Token;
 			var serverTask = server.Run(cancellationToken);
@@ -241,5 +230,96 @@ namespace Tgstation.Server.Tests
 				catch (OperationCanceledException) { }
 			}
 		}
+
+		[TestMethod]
+		public async Task TestRebootAndAttach()
+		{
+			using var server = new TestingServer(clientFactory);
+			using var serverCts = new CancellationTokenSource();
+			var cancellationToken = serverCts.Token;
+			var serverTask = server.Run(cancellationToken);
+			try
+			{
+				async Task<IServerClient> CreateAdminClient()
+				{
+					var giveUpAt = DateTimeOffset.Now.AddSeconds(60);
+					do
+					{
+						try
+						{
+							return await clientFactory.CreateServerClient(server.Url, User.AdminName, User.DefaultAdminPassword).ConfigureAwait(false);
+						}
+						catch (HttpRequestException)
+						{
+							//migrating, to be expected
+							if (DateTimeOffset.Now > giveUpAt)
+								throw;
+							await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken);
+						}
+						catch (ServiceUnavailableException)
+						{
+							// migrating, to be expected
+							if (DateTimeOffset.Now > giveUpAt)
+								throw;
+							await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken);
+						}
+					} while (true);
+				}
+
+				Api.Models.Instance instance;
+				using (var adminClient = await CreateAdminClient())
+				{
+					var instanceTest = new InstanceManagerTest(adminClient.Instances, adminClient.Users, server.Directory);
+					instance = await instanceTest.CreateTestInstance(cancellationToken);
+					instance.Online = true;
+					instance = await adminClient.Instances.Update(instance, cancellationToken);
+					var instanceClient = adminClient.Instances.CreateClient(instance);
+					var repoTest = new RepositoryTest(instanceClient.Repository, instanceClient.Jobs);
+					var byondTest = new ByondTest(instanceClient.Byond, instanceClient.Jobs, instance);
+
+					var repoTask = repoTest.RunPreWatchdog(cancellationToken);
+					await byondTest.Run(cancellationToken);
+					await repoTask;
+
+					await new WatchdogTest(instanceClient).StartAndLeaveRunning(cancellationToken);
+
+					await adminClient.Administration.Restart(cancellationToken);
+				}
+
+				await Task.WhenAny(serverTask, Task.Delay(30000, cancellationToken));
+				Assert.IsTrue(serverTask.IsCompleted);
+
+				serverTask = server.Run(cancellationToken);
+				using (var adminClient = await CreateAdminClient())
+				{
+					var instanceClient = adminClient.Instances.CreateClient(instance);
+					var dd = await instanceClient.DreamDaemon.Read(cancellationToken);
+
+					await new RepositoryTest(instanceClient.Repository, instanceClient.Jobs).RunPostWatchdog(cancellationToken);
+
+					Assert.IsTrue(dd.Running.Value);
+				}
+			}
+			catch (Exception ex)
+			{
+				Console.WriteLine($"TEST: ERROR: {ex.GetType()} in flight!");
+				throw;
+			}
+			finally
+			{
+				Console.WriteLine($"TEST: STOPPING SERVER!");
+				serverCts.Cancel();
+				try
+				{
+					Console.WriteLine($"TEST: WAITING FOR SERVER!");
+					await serverTask.ConfigureAwait(false);
+				}
+				catch (OperationCanceledException) { }
+
+				foreach (var proc in System.Diagnostics.Process.GetProcessesByName("DreamDaemon"))
+					using (proc)
+						proc.Kill();
+			}
+		}
 	}
 }

tests/Tgstation.Server.Tests/TestingServer.cs

@@ -3,6 +3,7 @@ using System;
 using System.Collections.Generic;
 using System.Globalization;
 using System.IO;
+using System.Linq;
 using System.Net;
 using System.Net.Http;
 using System.Threading;
@@ -21,17 +22,21 @@ namespace Tgstation.Server.Tests
 
 		public string Directory { get; }
 
+		public string UpdatePath { get; }
+
 		public string DatabaseType { get; }
 
-		public bool RestartRequested => realServer.Result.RestartRequested;
-
-		readonly Task<IServer> realServer;
+		public bool RestartRequested => realServer.RestartRequested;
 
 		readonly IServerClientFactory serverClientFactory;
 
 		readonly bool dumpOpenAPISpecpath;
 
-		public TestingServer(IServerClientFactory serverClientFactory, string updatePath)
+		string[] args;
+
+		IServer realServer;
+
+		public TestingServer(IServerClientFactory serverClientFactory)
 		{
 			this.serverClientFactory = serverClientFactory;
 
@@ -67,10 +72,10 @@ namespace Tgstation.Server.Tests
 
 			var args = new List<string>()
 			{
+				String.Format(CultureInfo.InvariantCulture, "Database:DropDatabase={0}", true),
 				String.Format(CultureInfo.InvariantCulture, "Kestrel:EndPoints:Http:Url={0}", UrlString),
 				String.Format(CultureInfo.InvariantCulture, "Database:DatabaseType={0}", DatabaseType),
 				String.Format(CultureInfo.InvariantCulture, "Database:ConnectionString={0}", connectionString),
-				String.Format(CultureInfo.InvariantCulture, "Database:DropDatabase={0}", true),
 				String.Format(CultureInfo.InvariantCulture, "General:SetupWizardMode={0}", SetupWizardMode.Never),
 				String.Format(CultureInfo.InvariantCulture, "General:MinimumPasswordLength={0}", 10),
 				String.Format(CultureInfo.InvariantCulture, "General:InstanceLimit={0}", 11),
@@ -84,7 +89,8 @@ namespace Tgstation.Server.Tests
 			if (dumpOpenAPISpecpath)
 				Environment.SetEnvironmentVariable("ASPNETCORE_ENVIRONMENT", "Development");
 
-			realServer = Application.CreateDefaultServerFactory().CreateServer(args.ToArray(), updatePath, default);
+			UpdatePath = Path.Combine(Directory, Guid.NewGuid().ToString());
+			this.args = args.ToArray();
 		}
 
 		public void Dispose()
@@ -103,8 +109,18 @@ namespace Tgstation.Server.Tests
 
 		public async Task Run(CancellationToken cancellationToken)
 		{
-			var serverInstance = await realServer.ConfigureAwait(false);
-			Task runTask = serverInstance.Run(cancellationToken);
+			var firstRun = realServer == null;
+			realServer = await Application
+				.CreateDefaultServerFactory()
+				.CreateServer(
+					args,
+					UpdatePath,
+					default);
+
+			if (firstRun)
+				args = args.Skip(1).ToArray();
+
+			Task runTask = realServer.Run(cancellationToken);
 
 			if (dumpOpenAPISpecpath)
 			{