diff --git a/src/Tgstation.Server.Host/Components/InstanceManager.cs b/src/Tgstation.Server.Host/Components/InstanceManager.cs index 044169edc7..094b3d4db1 100644 --- a/src/Tgstation.Server.Host/Components/InstanceManager.cs +++ b/src/Tgstation.Server.Host/Components/InstanceManager.cs @@ -455,13 +455,20 @@ namespace Tgstation.Server.Host.Components /// public async Task StopAsync(CancellationToken cancellationToken) { - await jobManager.StopAsync(cancellationToken).ConfigureAwait(false); - await Task.WhenAll(instances.Select(x => x.Value.Instance.StopAsync(cancellationToken))).ConfigureAwait(false); - await instanceFactory.StopAsync(cancellationToken).ConfigureAwait(false); + try + { + await jobManager.StopAsync(cancellationToken).ConfigureAwait(false); + await Task.WhenAll(instances.Select(x => x.Value.Instance.StopAsync(cancellationToken))).ConfigureAwait(false); + await instanceFactory.StopAsync(cancellationToken).ConfigureAwait(false); - // downgrade the db if necessary - if (downgradeVersion != null) - await databaseContextFactory.UseContext(db => databaseSeeder.Downgrade(db, downgradeVersion, cancellationToken)).ConfigureAwait(false); + // downgrade the db if necessary + if (downgradeVersion != null) + await databaseContextFactory.UseContext(db => databaseSeeder.Downgrade(db, downgradeVersion, cancellationToken)).ConfigureAwait(false); + } + catch (Exception ex) + { + logger.LogError("Instance manager stop exception: {0}", ex); + } } /// diff --git a/src/Tgstation.Server.Host/Components/Session/ISessionController.cs b/src/Tgstation.Server.Host/Components/Session/ISessionController.cs index c85171fb56..778c39ba38 100644 --- a/src/Tgstation.Server.Host/Components/Session/ISessionController.cs +++ b/src/Tgstation.Server.Host/Components/Session/ISessionController.cs @@ -38,11 +38,6 @@ namespace Tgstation.Server.Host.Components.Session /// CompileJob CompileJob { get; } - /// - /// The current port DreamDaemon is listening on - /// - ushort? Port { get; } - /// /// If the port should be rotated off when the world reboots /// diff --git a/src/Tgstation.Server.Host/Components/Session/SessionController.cs b/src/Tgstation.Server.Host/Components/Session/SessionController.cs index e790feb310..e06caba888 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionController.cs @@ -42,26 +42,7 @@ namespace Tgstation.Server.Host.Components.Session public Models.CompileJob CompileJob => reattachInformation.Dmb.CompileJob; /// - public ushort? Port - { - get - { - CheckDisposed(); - if (portClosedForReboot) - return null; - return reattachInformation.Port; - } - } - - /// - public RebootState RebootState - { - get - { - CheckDisposed(); - return reattachInformation.RebootState; - } - } + public RebootState RebootState => reattachInformation.RebootState; /// public Version DMApiVersion { get; private set; } @@ -262,7 +243,10 @@ namespace Tgstation.Server.Host.Components.Session startupTimeout, reattached); - logger.LogDebug("Created session controller. CommsKey: {0}, Port: {1}", reattachInformation.AccessIdentifier, Port); + logger.LogDebug( + "Created session controller. CommsKey: {0}, Port: {1}", + reattachInformation.AccessIdentifier, + reattachInformation.Port); } /// diff --git a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs index a61ba8cea7..12eb031994 100644 --- a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs +++ b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs @@ -1,4 +1,4 @@ -using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging; using System; using System.Collections.Generic; using System.Globalization; @@ -463,7 +463,20 @@ namespace Tgstation.Server.Host.Components.StaticFiles var files = await ioManager.GetFilesWithExtension(EventScriptsSubdirectory, platformIdentifier.ScriptFileExtension, false, cancellationToken).ConfigureAwait(false); var resolvedScriptsDir = ioManager.ResolvePath(EventScriptsSubdirectory); - foreach (var I in files.Select(x => ioManager.GetFileName(x)).Where(x => x.StartsWith(scriptName, StringComparison.Ordinal))) + var scriptFiles = files + .Select(x => ioManager.GetFileName(x)) + .Where(x => x.StartsWith(scriptName, StringComparison.Ordinal)) + .ToList(); + + if (!scriptFiles.Any()) + { + logger.LogTrace("No event scripts starting with \"{0}\" detected", scriptName); + return; + } + + foreach (var I in scriptFiles) + { + logger.LogTrace("Running event script {0}...", I); using (var script = processExecutor.LaunchProcess( ioManager.ConcatPath(resolvedScriptsDir, I), resolvedScriptsDir, @@ -481,6 +494,7 @@ namespace Tgstation.Server.Host.Components.StaticFiles else logger.LogDebug("Script output:{0}{1}", Environment.NewLine, scriptOutput); } + } } } diff --git a/src/Tgstation.Server.Host/Database/Design/DesignTimeDbContextFactoryHelpers.cs b/src/Tgstation.Server.Host/Database/Design/DesignTimeDbContextFactoryHelpers.cs index 4d4f4411dd..36da823aa4 100644 --- a/src/Tgstation.Server.Host/Database/Design/DesignTimeDbContextFactoryHelpers.cs +++ b/src/Tgstation.Server.Host/Database/Design/DesignTimeDbContextFactoryHelpers.cs @@ -1,4 +1,4 @@ -using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Options; using Tgstation.Server.Host.Configuration; @@ -15,17 +15,20 @@ namespace Tgstation.Server.Host.Database.Design /// The to create for. /// The . /// The . + /// The . /// The for the public static DbContextOptions CreateDatabaseContextOptions( DatabaseType databaseType, - string connectionString) + string connectionString, + string serverVersion = null) where TDatabaseContext : DatabaseContext { var dbConfig = new DatabaseConfiguration { DesignTime = true, DatabaseType = databaseType, - ConnectionString = connectionString + ConnectionString = connectionString, + ServerVersion = serverVersion }; var optionsFac = new DbContextOptionsBuilder(); diff --git a/src/Tgstation.Server.Host/Database/Migrations/20200423052709_MYKillJsonsAddDMApiVersion.cs b/src/Tgstation.Server.Host/Database/Migrations/20200423052709_MYKillJsonsAddDMApiVersion.cs index 9c6ed01066..32e03425c8 100644 --- a/src/Tgstation.Server.Host/Database/Migrations/20200423052709_MYKillJsonsAddDMApiVersion.cs +++ b/src/Tgstation.Server.Host/Database/Migrations/20200423052709_MYKillJsonsAddDMApiVersion.cs @@ -1,4 +1,4 @@ -using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Migrations; using System; namespace Tgstation.Server.Host.Database.Migrations @@ -39,19 +39,18 @@ namespace Tgstation.Server.Host.Database.Migrations name: "LaunchSecurityLevel", table: "ReattachInformations"); + // No default values b/c lol mysql migrationBuilder.AddColumn( name: "ChatChannelsJson", table: "ReattachInformations", type: "longtext CHARACTER SET utf8mb4", - nullable: false, - defaultValue: "chat_channels.tgs.json"); + nullable: false); migrationBuilder.AddColumn( name: "ChatCommandsJson", table: "ReattachInformations", type: "longtext CHARACTER SET utf8mb4", - nullable: false, - defaultValue: "chat_commands.tgs.json"); + nullable: false); } } } diff --git a/src/Tgstation.Server.Host/Jobs/JobHandler.cs b/src/Tgstation.Server.Host/Jobs/JobHandler.cs index 86437139d5..c6b923033c 100644 --- a/src/Tgstation.Server.Host/Jobs/JobHandler.cs +++ b/src/Tgstation.Server.Host/Jobs/JobHandler.cs @@ -1,6 +1,7 @@ -using System; +using System; using System.Threading; using System.Threading.Tasks; +using Tgstation.Server.Host.Extensions; namespace Tgstation.Server.Host.Jobs { @@ -47,15 +48,12 @@ namespace Tgstation.Server.Host.Jobs /// /// The for the operation /// A representing the running operation - public async Task Wait(CancellationToken cancellationToken) + public Task Wait(CancellationToken cancellationToken) { if (task == null) throw new InvalidOperationException("Job not started!"); - TaskCompletionSource tcs = new TaskCompletionSource(); - using (cancellationToken.Register(() => tcs.SetCanceled())) - await Task.WhenAny(tcs.Task, task).ConfigureAwait(false); - cancellationToken.ThrowIfCancellationRequested(); + return task.WithToken(cancellationToken); } /// @@ -76,4 +74,4 @@ namespace Tgstation.Server.Host.Jobs } } } -} \ No newline at end of file +} diff --git a/src/Tgstation.Server.Host/Jobs/JobManager.cs b/src/Tgstation.Server.Host/Jobs/JobManager.cs index ced1eb724b..d59736c8c9 100644 --- a/src/Tgstation.Server.Host/Jobs/JobManager.cs +++ b/src/Tgstation.Server.Host/Jobs/JobManager.cs @@ -298,8 +298,14 @@ namespace Tgstation.Server.Host.Jobs await databaseContext.Save(cancellationToken).ConfigureAwait(false); job.CancelledBy = user; }).ConfigureAwait(false); + if (blocking) + { + logger.LogTrace("Waiting on cancelled job #{0}...", job.Id); await handler.Wait(cancellationToken).ConfigureAwait(false); + logger.LogTrace("Done waiting on job #{0}...", job.Id); + } + return job; } @@ -321,8 +327,6 @@ namespace Tgstation.Server.Host.Jobs { if (job == null) throw new ArgumentNullException(nameof(job)); - if (canceller == null) - throw new ArgumentNullException(nameof(canceller)); JobHandler handler; lock (synchronizationLock) { diff --git a/tests/Tgstation.Server.Tests/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Instance/WatchdogTest.cs index 8af1be7c16..9ae2620165 100644 --- a/tests/Tgstation.Server.Tests/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Instance/WatchdogTest.cs @@ -85,8 +85,9 @@ namespace Tgstation.Server.Tests.Instance var dumpTask = instanceClient.DreamDaemon.CreateDump(cancellationToken); while (!dumpTask.IsCompleted) KillDD(false); - await WaitForJob(await dumpTask, 5, true, ErrorCode.DreamDaemonOffline, cancellationToken); - await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken); + var job = await WaitForJob(await dumpTask, 10, true, null, cancellationToken); + Assert.IsTrue(job.ErrorCode == ErrorCode.DreamDaemonOffline || job.ErrorCode == ErrorCode.GCoreFailure); + await Task.Delay(TimeSpan.FromSeconds(10), cancellationToken); var ddStatus = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.AreEqual(WatchdogStatus.Online, ddStatus.Status.Value); @@ -105,7 +106,7 @@ namespace Tgstation.Server.Tests.Instance var startJob = await instanceClient.DreamDaemon.Start(cancellationToken).ConfigureAwait(false); - await WaitForJob(startJob, 10, false, null, cancellationToken); + await WaitForJob(startJob, 20, false, null, cancellationToken); daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value); @@ -147,12 +148,12 @@ namespace Tgstation.Server.Tests.Instance blockSocket.Bind(new IPEndPoint(IPAddress.Any, 1337)); startJob = await instanceClient.DreamDaemon.Start(cancellationToken).ConfigureAwait(false); - await WaitForJob(startJob, 10, true, ErrorCode.DreamDaemonPortInUse, cancellationToken); + await WaitForJob(startJob, 20, true, ErrorCode.DreamDaemonPortInUse, cancellationToken); } startJob = await instanceClient.DreamDaemon.Start(cancellationToken).ConfigureAwait(false); - await WaitForJob(startJob, 10, false, null, cancellationToken); + await WaitForJob(startJob, 20, false, null, cancellationToken); daemonStatus = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.AreEqual(WatchdogStatus.Online, daemonStatus.Status.Value); @@ -178,7 +179,7 @@ namespace Tgstation.Server.Tests.Instance var startJob = await instanceClient.DreamDaemon.Start(cancellationToken).ConfigureAwait(false); - await WaitForJob(startJob, 10, false, null, cancellationToken); + await WaitForJob(startJob, 20, false, null, cancellationToken); // lock on to DD and pause it so it can't heartbeat var ddProcs = System.Diagnostics.Process.GetProcessesByName("DreamDaemon").ToList(); @@ -246,7 +247,7 @@ namespace Tgstation.Server.Tests.Instance var startJob = await instanceClient.DreamDaemon.Start(cancellationToken).ConfigureAwait(false); - await WaitForJob(startJob, 10, false, null, cancellationToken); + await WaitForJob(startJob, 20, false, null, cancellationToken); daemonStatus = await DeployTestDme(DmeName, DreamDaemonSecurity.Safe, true, cancellationToken); @@ -287,7 +288,7 @@ namespace Tgstation.Server.Tests.Instance var startJob = await instanceClient.DreamDaemon.Start(cancellationToken).ConfigureAwait(false); - await WaitForJob(startJob, 10, false, null, cancellationToken); + await WaitForJob(startJob, 20, false, null, cancellationToken); daemonStatus = await DeployTestDme(DmeName + "_copy", DreamDaemonSecurity.Safe, true, cancellationToken); @@ -425,10 +426,10 @@ namespace Tgstation.Server.Tests.Instance { var bts = new TopicClient(new SocketParameters { - SendTimeout = TimeSpan.FromSeconds(7), - ReceiveTimeout = TimeSpan.FromSeconds(7), - ConnectTimeout = TimeSpan.FromSeconds(7), - DisconnectTimeout = TimeSpan.FromSeconds(7) + SendTimeout = TimeSpan.FromSeconds(15), + ReceiveTimeout = TimeSpan.FromSeconds(15), + ConnectTimeout = TimeSpan.FromSeconds(15), + DisconnectTimeout = TimeSpan.FromSeconds(15) }); try diff --git a/tests/Tgstation.Server.Tests/IntegrationTest.cs b/tests/Tgstation.Server.Tests/IntegrationTest.cs index a82f8783e9..77fc00eae6 100644 --- a/tests/Tgstation.Server.Tests/IntegrationTest.cs +++ b/tests/Tgstation.Server.Tests/IntegrationTest.cs @@ -136,21 +136,40 @@ namespace Tgstation.Server.Tests string migrationName = null; DbContext CreateContext() { + string serverVersion = Environment.GetEnvironmentVariable($"{DatabaseConfiguration.Section}__{nameof(DatabaseConfiguration.ServerVersion)}"); + if (String.IsNullOrWhiteSpace(serverVersion)) + serverVersion = null; switch (databaseType) { case DatabaseType.MySql: case DatabaseType.MariaDB: migrationName = nameof(MYInitialCreate); - return new MySqlDatabaseContext(Host.Database.Design.DesignTimeDbContextFactoryHelpers.CreateDatabaseContextOptions(databaseType, connectionString)); + return new MySqlDatabaseContext( + Host.Database.Design.DesignTimeDbContextFactoryHelpers.CreateDatabaseContextOptions( + databaseType, + connectionString, + serverVersion)); case DatabaseType.PostgresSql: migrationName = nameof(PGCreate); - return new PostgresSqlDatabaseContext(Host.Database.Design.DesignTimeDbContextFactoryHelpers.CreateDatabaseContextOptions(databaseType, connectionString)); + return new PostgresSqlDatabaseContext( + Host.Database.Design.DesignTimeDbContextFactoryHelpers.CreateDatabaseContextOptions( + databaseType, + connectionString, + serverVersion)); case DatabaseType.SqlServer: migrationName = nameof(MSInitialCreate); - return new SqlServerDatabaseContext(Host.Database.Design.DesignTimeDbContextFactoryHelpers.CreateDatabaseContextOptions(databaseType, connectionString)); + return new SqlServerDatabaseContext( + Host.Database.Design.DesignTimeDbContextFactoryHelpers.CreateDatabaseContextOptions( + databaseType, + connectionString, + serverVersion)); case DatabaseType.Sqlite: migrationName = nameof(SLRebuild); - return new SqliteDatabaseContext(Host.Database.Design.DesignTimeDbContextFactoryHelpers.CreateDatabaseContextOptions(databaseType, connectionString)); + return new SqliteDatabaseContext( + Host.Database.Design.DesignTimeDbContextFactoryHelpers.CreateDatabaseContextOptions( + databaseType, + connectionString, + serverVersion)); } return null; @@ -180,29 +199,7 @@ namespace Tgstation.Server.Tests } using var server = new TestingServer(); - - using var hardTimeoutCts = new CancellationTokenSource(); - - var maximumTestDuration = new TimeSpan(0, 12, 0); - - hardTimeoutCts.CancelAfter(maximumTestDuration - new TimeSpan(0, 0, 15)); - var hardTimeoutCancellationToken = hardTimeoutCts.Token; - hardTimeoutCancellationToken.Register(() => - { - Console.WriteLine($"[{DateTimeOffset.Now}] TEST TIMEOUT HARD!"); - }); - - using var softTimeoutCts = CancellationTokenSource.CreateLinkedTokenSource(hardTimeoutCancellationToken); - softTimeoutCts.CancelAfter(maximumTestDuration - new TimeSpan(0, 0, 45)); - var softTimeoutCancellationToken = softTimeoutCts.Token; - bool tooLateForSoftTimeout = false; - softTimeoutCancellationToken.Register(() => - { - if (!tooLateForSoftTimeout) - Console.WriteLine($"[{DateTimeOffset.Now}] TEST TIMEOUT SOFT!"); - }); - - using var serverCts = CancellationTokenSource.CreateLinkedTokenSource(softTimeoutCancellationToken); + using var serverCts = new CancellationTokenSource(); var cancellationToken = serverCts.Token; TerminateAllDDs(); @@ -258,7 +255,7 @@ namespace Tgstation.Server.Tests await adminClient.Administration.Restart(cancellationToken); } - await Task.WhenAny(serverTask, Task.Delay(30000, cancellationToken)); + await Task.WhenAny(serverTask, Task.Delay(TimeSpan.FromMinutes(1), cancellationToken)); Assert.IsTrue(serverTask.IsCompleted); // http bind test https://github.com/tgstation/tgstation-server/issues/1065 @@ -278,7 +275,7 @@ namespace Tgstation.Server.Tests } } - await Task.WhenAny(serverTask, Task.Delay(30000, cancellationToken)); + await Task.WhenAny(serverTask, Task.Delay(TimeSpan.FromMinutes(1), cancellationToken)); Assert.IsTrue(serverTask.IsCompleted); var preStartupTime = DateTimeOffset.Now; @@ -303,12 +300,12 @@ namespace Tgstation.Server.Tests .ToList(); } - Assert.AreEqual(1, jobs.Count, $"Why are there multiple active jobs? \"{String.Join("\", \"", jobs.Select(x => x.Description))}\""); - - var reattachJob = jobs.Single(); - Assert.IsTrue(reattachJob.StartedAt.Value >= preStartupTime); - - await new JobsRequiredTest(instanceClient.Jobs).WaitForJob(reattachJob, 40, false, null, cancellationToken); + var jrt = new JobsRequiredTest(instanceClient.Jobs); + foreach (var job in jobs) + { + Assert.IsTrue(job.StartedAt.Value >= preStartupTime); + await jrt.WaitForJob(job, 40, false, null, cancellationToken); + } var dd = await instanceClient.DreamDaemon.Read(cancellationToken); Assert.AreEqual(WatchdogStatus.Online, dd.Status.Value); @@ -322,7 +319,7 @@ namespace Tgstation.Server.Tests await adminClient.Administration.Restart(cancellationToken); } - await Task.WhenAny(serverTask, Task.Delay(30000, cancellationToken)); + await Task.WhenAny(serverTask, Task.Delay(TimeSpan.FromMinutes(1), cancellationToken)); Assert.IsTrue(serverTask.IsCompleted); preStartupTime = DateTimeOffset.Now; @@ -376,18 +373,21 @@ namespace Tgstation.Server.Tests } finally { - tooLateForSoftTimeout = true; serverCts.Cancel(); + + // Give the test 1 minute to cleanup + using var hardTimeoutCancellationTokenSource = new CancellationTokenSource(TimeSpan.FromMinutes(1)); try { - await serverTask.WithToken(hardTimeoutCancellationToken).ConfigureAwait(false); + await serverTask.WithToken(hardTimeoutCancellationTokenSource.Token).ConfigureAwait(false); } catch (OperationCanceledException) { } TerminateAllDDs(); - - hardTimeoutCancellationToken.ThrowIfCancellationRequested(); } + + Assert.IsTrue(serverTask.IsCompleted); + await serverTask; } [TestMethod]