From 9e0e8c27fb6fa1286274838518471115039e2a5f Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Sun, 5 Aug 2018 14:25:57 -0400 Subject: [PATCH 01/36] Various things - Fix DMAPI json field names - Add 2 more compiler statuses - Fix null reference exception in ByondManager.UseExecutables - Clean unused compile jobs on instance startup - Fix file handle overload with CopyDirectory - Fix NullReferenceException with TemporaryDmbProvider - Fix InvalidOperationException with dm.exe invocation --- src/DMAPI/tgs/v4/api.dm | 32 +++++------ src/DMAPI/tgs/v4/commands.dm | 2 +- .../Models/CompilerStatus.cs | 8 +++ .../Components/Byond/ByondManager.cs | 2 +- .../Components/Compiler/DmbFactory.cs | 30 +++++++++++ .../Components/Compiler/DreamMaker.cs | 45 +++++++++++----- .../Components/Compiler/IDmbFactory.cs | 9 ++++ .../Compiler/TemporaryDmbProvider.cs | 6 ++- .../Components/Instance.cs | 11 +++- .../IO/DefaultIOManager.cs | 53 +++++++++++-------- 10 files changed, 142 insertions(+), 56 deletions(-) diff --git a/src/DMAPI/tgs/v4/api.dm b/src/DMAPI/tgs/v4/api.dm index e62de1c25e..5e499e6815 100644 --- a/src/DMAPI/tgs/v4/api.dm +++ b/src/DMAPI/tgs/v4/api.dm @@ -58,36 +58,36 @@ TGS_ERROR_LOG("Failed to decode info json: [json_file]") return - access_identifier = cached_json["access_identifier"] - instance_name = text2num(cached_json["instance_name"]) - host_path = cached_json["host_path"] - if(cached_json["api_validate_only"]) + access_identifier = cached_json["accessIdentifier"] + instance_name = text2num(cached_json["instanceName"]) + host_path = cached_json["hostPath"] + if(cached_json["apiValidateOnly"]) Export(TGS4_COMM_VALIDATE) del(world) - chat_channels_json_path = cached_json["chat_channels_json"] - chat_commands_json_path = cached_json["chat_commands_json"] + chat_channels_json_path = cached_json["chatChannelsJson"] + chat_commands_json_path = cached_json["chatCommandsJson"] src.event_handler = event_handler - instance_name = cached_json["instance_name"] + instance_name = cached_json["instanceName"] cached_test_merges = list() - var/json = cached_json["test_merges"] + var/json = cached_json["testMerges"] for(var/I in json) var/datum/tgs_revision_information/test_merge/tm = new tm.number = text2num(I) var/list/entry = json[I] - tm.pull_request_commit = entry["pr_commit"] + tm.pull_request_commit = entry["prCommit"] tm.author = entry["author"] tm.title = entry["title"] tm.commit = entry["commit"] - tm.origin_commit = entry["origin_commit"] - tm.time_merged = text2num(entry["time_merged"]) + tm.origin_commit = entry["originCommit"] + tm.time_merged = text2num(entry["timeMerged"]) tm.comment = entry["comment"] tm.url = entry["url"] cached_revision = new cached_revision.commit = cached_json["commit"] - cached_revision.origin_commit = cached_json["origin_commit"] + cached_revision.origin_commit = cached_json["originCommit"] ListCustomCommands() @@ -208,10 +208,10 @@ /datum/tgs_api/v4/proc/DecodeChannel(channel_json) var/datum/tgs_chat_channel/channel = new channel.id = channel_json["id"] - channel.friendly_name = channel_json["friendly_name"] - channel.connection_name = channel_json["connection_name"] - channel.is_admin_channel = channel_json["is_admin_channel"] - channel.is_private_channel = channel_json["is_private_channel"] || FALSE + channel.friendly_name = channel_json["friendlyName"] + channel.connection_name = channel_json["connectionName"] + channel.is_admin_channel = channel_json["isAdminChannel"] + channel.is_private_channel = channel_json["isPrivateChannel"] || FALSE return channel #undef TGS4_TOPIC_COMMAND diff --git a/src/DMAPI/tgs/v4/commands.dm b/src/DMAPI/tgs/v4/commands.dm index 48ca6e98de..88ee61e882 100644 --- a/src/DMAPI/tgs/v4/commands.dm +++ b/src/DMAPI/tgs/v4/commands.dm @@ -28,7 +28,7 @@ var/datum/tgs_chat_user/u = new u.id = user["id"] - u.friendly_name = user["friendly_name"] + u.friendly_name = user["friendlyName"] u.mention = user["mention"] u.channel = DecodeChannel(user["channel"]) diff --git a/src/Tgstation.Server.Api/Models/CompilerStatus.cs b/src/Tgstation.Server.Api/Models/CompilerStatus.cs index d29df195e6..4509c9c6f5 100644 --- a/src/Tgstation.Server.Api/Models/CompilerStatus.cs +++ b/src/Tgstation.Server.Api/Models/CompilerStatus.cs @@ -11,6 +11,10 @@ /// The is idle /// Idle, + /// + /// Pre-compile scripts are running + /// + PreCompile, /// /// The is being copied /// @@ -36,6 +40,10 @@ /// Symlinking, /// + /// Post-compile scripts are running + /// + PostCompile, + /// /// A failed compile job is being erased /// Cleanup diff --git a/src/Tgstation.Server.Host/Components/Byond/ByondManager.cs b/src/Tgstation.Server.Host/Components/Byond/ByondManager.cs index f66005b42d..89f1991a3c 100644 --- a/src/Tgstation.Server.Host/Components/Byond/ByondManager.cs +++ b/src/Tgstation.Server.Host/Components/Byond/ByondManager.cs @@ -163,7 +163,7 @@ namespace Tgstation.Server.Host.Components.Byond var versionToUse = requiredVersion ?? ActiveVersion; if (versionToUse == null) throw new InvalidOperationException("No BYOND versions installed!"); - await InstallVersion(requiredVersion, cancellationToken).ConfigureAwait(false); + await InstallVersion(versionToUse, cancellationToken).ConfigureAwait(false); var versionKey = VersionKey(versionToUse); diff --git a/src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs b/src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs index e994fddfae..242f1c40ac 100644 --- a/src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs +++ b/src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs @@ -183,5 +183,35 @@ namespace Tgstation.Server.Host.Components.Compiler return new DmbProvider(compileJob, ioManager, () => CleanJob(compileJob)); } } + + /// + public async Task CleanUnusedCompileJobs(CompileJob exceptThisOne, CancellationToken cancellationToken) + { + List jobIdsToSkip; + //don't clean locked directories + lock (this) + jobIdsToSkip = jobLockCounts.Select(x => x.Key).ToList(); + + List jobUidsToNotErase = null; + + //find the uids of locked directories + await databaseContextFactory.UseContext(async db => + { + jobUidsToNotErase = await db.CompileJobs.Where(x => x.Job.Instance.Id == instance.Id && jobIdsToSkip.Contains(x.Id) && x.DirectoryName.HasValue).Select(x => x.DirectoryName.Value.ToString().ToUpperInvariant()).ToListAsync(cancellationToken).ConfigureAwait(false); + }).ConfigureAwait(false); + + //add the other exemption + if (exceptThisOne != null) + jobUidsToNotErase.Add(exceptThisOne.DirectoryName.Value.ToString().ToUpperInvariant()); + + //cleanup + var directories = await ioManager.GetDirectories(".", cancellationToken).ConfigureAwait(false); + await Task.WhenAll(directories.Select(x => + { + if (jobUidsToNotErase.Contains(x.ToUpperInvariant())) + return Task.CompletedTask; + return ioManager.DeleteDirectory(x, cancellationToken); + })).ConfigureAwait(false); + } } } diff --git a/src/Tgstation.Server.Host/Components/Compiler/DreamMaker.cs b/src/Tgstation.Server.Host/Components/Compiler/DreamMaker.cs index 33beec3f99..ce8125afc5 100644 --- a/src/Tgstation.Server.Host/Components/Compiler/DreamMaker.cs +++ b/src/Tgstation.Server.Host/Components/Compiler/DreamMaker.cs @@ -35,7 +35,7 @@ namespace Tgstation.Server.Host.Components.Compiler /// /// Extension for .dmes /// - const string DmeExtension = ".dme"; + const string DmeExtension = "dme"; /// public CompilerStatus Status { get; private set; } @@ -115,7 +115,7 @@ namespace Tgstation.Server.Host.Components.Compiler }; var dirA = ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName); - var provider = new TemporaryDmbProvider(ioManager.ResolvePath(ioManager.GetDirectoryName(dirA)), ioManager.ResolvePath(ioManager.ConcatPath(dirA, String.Concat(job.DmeName, DmbExtension)))); + var provider = new TemporaryDmbProvider(ioManager.ResolvePath(ioManager.GetDirectoryName(dirA)), ioManager.ResolvePath(ioManager.ConcatPath(dirA, String.Concat(job.DmeName, DmbExtension))), job); var timeoutAt = DateTimeOffset.Now.AddSeconds(timeout); using (var controller = await sessionControllerFactory.LaunchNew(launchParameters, provider, byondLock, true, true, true, cancellationToken).ConfigureAwait(false)) @@ -143,10 +143,11 @@ namespace Tgstation.Server.Host.Components.Compiler using (var dm = new Process()) { dm.StartInfo.FileName = dreamMakerPath; - dm.StartInfo.Arguments = String.Format(CultureInfo.InvariantCulture, "-clean {0}{1}", job.DmeName, DmeExtension); + dm.StartInfo.Arguments = String.Format(CultureInfo.InvariantCulture, "-clean {0}.{1}", job.DmeName, DmeExtension); dm.StartInfo.WorkingDirectory = ioManager.ResolvePath(ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName)); dm.StartInfo.RedirectStandardOutput = true; dm.StartInfo.RedirectStandardError = true; + dm.StartInfo.UseShellExecute = false; var OutputList = new StringBuilder(); var eventHandler = new DataReceivedEventHandler( delegate (object sender, DataReceivedEventArgs e) @@ -163,6 +164,8 @@ namespace Tgstation.Server.Host.Components.Compiler dm.Exited += (a, b) => dmTcs.SetResult(null); dm.Start(); + dm.BeginOutputReadLine(); + dm.BeginErrorReadLine(); try { using (cancellationToken.Register(() => dmTcs.SetCanceled())) @@ -191,7 +194,7 @@ namespace Tgstation.Server.Host.Components.Compiler async Task ModifyDme(Models.CompileJob job, CancellationToken cancellationToken) { var dirA = ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName); - var dmePath = ioManager.ConcatPath(dirA, String.Concat(job.DmeName, DmeExtension)); + var dmePath = ioManager.ConcatPath(dirA, String.Join('.', job.DmeName, DmeExtension)); var dmeReadTask = ioManager.ReadAllBytes(dmePath, cancellationToken); var dmeModificationsTask = configuration.CopyDMFilesTo(dmePath, ioManager.ResolvePath(dirA), cancellationToken); @@ -228,15 +231,27 @@ namespace Tgstation.Server.Host.Components.Compiler public async Task Compile(string projectName, uint apiValidateTimeout, IRepository repository, CancellationToken cancellationToken) { logger.LogTrace("Begin Compile"); + + var job = new Models.CompileJob + { + DirectoryName = Guid.NewGuid(), + DmeName = projectName + }; + + lock (this) + { + if(Status != CompilerStatus.Idle) + { + job.Output = "There is already a compile in progress!"; + return job; + } + + Status = CompilerStatus.PreCompile; + } await eventConsumer.HandleEvent(EventType.CompileStart, new List{ repository.Origin }, cancellationToken).ConfigureAwait(false); try { Status = CompilerStatus.Copying; - var job = new Models.CompileJob - { - DirectoryName = Guid.NewGuid(), - DmeName = projectName - }; await ioManager.CreateDirectory(job.DirectoryName.ToString(), cancellationToken).ConfigureAwait(false); var dirA = ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName); var dirB = ioManager.ConcatPath(job.DirectoryName.ToString(), BDirectoryName); @@ -248,7 +263,10 @@ namespace Tgstation.Server.Host.Components.Compiler { await ioManager.DeleteDirectory(job.DirectoryName.ToString(), CancellationToken.None).ConfigureAwait(false); } - catch { } + catch (Exception e) + { + logger.LogWarning("Error cleaning up compile directory {0}! Exception: {1}", ioManager.ResolvePath(job.DirectoryName.ToString()), e); + } }; try @@ -262,12 +280,14 @@ namespace Tgstation.Server.Host.Components.Compiler if (job.DmeName == null) { - job.DmeName = (await ioManager.GetFilesWithExtension(dirA, DmeExtension, cancellationToken).ConfigureAwait(false)).FirstOrDefault(); - if (job.DmeName == default) + var path = (await ioManager.GetFilesWithExtension(dirA, DmeExtension, cancellationToken).ConfigureAwait(false)).FirstOrDefault(); + if (path == default) { job.Output = "Unable to find any .dme!"; return job; } + var dmeWithExtension = ioManager.GetFileName(path); + job.DmeName = dmeWithExtension.Substring(0, dmeWithExtension.Length - DmeExtension.Length - 1); } await ModifyDme(job, cancellationToken).ConfigureAwait(false); @@ -309,6 +329,7 @@ namespace Tgstation.Server.Host.Components.Compiler var symBTask = configuration.SymlinkStaticFilesTo(ioManager.ResolvePath(dirB), cancellationToken); await Task.WhenAll(symATask, symBTask).ConfigureAwait(false); + Status = CompilerStatus.PostCompile; await eventConsumer.HandleEvent(EventType.CompileComplete, null, cancellationToken).ConfigureAwait(false); } await compileJobConsumer.LoadCompileJob(job, cancellationToken).ConfigureAwait(false); diff --git a/src/Tgstation.Server.Host/Components/Compiler/IDmbFactory.cs b/src/Tgstation.Server.Host/Components/Compiler/IDmbFactory.cs index e79f55b6f8..6efc0f69b9 100644 --- a/src/Tgstation.Server.Host/Components/Compiler/IDmbFactory.cs +++ b/src/Tgstation.Server.Host/Components/Compiler/IDmbFactory.cs @@ -29,5 +29,14 @@ namespace Tgstation.Server.Host.Components.Compiler /// The to make the for /// A new IDmbProvider FromCompileJob(CompileJob compileJob); + + /// + /// Deletes all compile jobs that are inactive in the Game folder + /// + /// An optional compile job to not delete + /// The for the operation + /// A representing the running operation + Task CleanUnusedCompileJobs(CompileJob exceptThisOne, CancellationToken cancellationToken); + } } diff --git a/src/Tgstation.Server.Host/Components/Compiler/TemporaryDmbProvider.cs b/src/Tgstation.Server.Host/Components/Compiler/TemporaryDmbProvider.cs index dcf5c0affe..3cd0962106 100644 --- a/src/Tgstation.Server.Host/Components/Compiler/TemporaryDmbProvider.cs +++ b/src/Tgstation.Server.Host/Components/Compiler/TemporaryDmbProvider.cs @@ -18,17 +18,19 @@ namespace Tgstation.Server.Host.Components.Compiler public string SecondaryDirectory => throw new NotSupportedException(); /// - public CompileJob CompileJob => null; + public CompileJob CompileJob { get; } /// /// Construct a /// /// The value of /// The value of - public TemporaryDmbProvider(string directory, string dmb) + /// The value of + public TemporaryDmbProvider(string directory, string dmb, CompileJob compileJob) { DmbName = dmb ?? throw new ArgumentNullException(nameof(dmb)); PrimaryDirectory = directory ?? throw new ArgumentNullException(nameof(directory)); + CompileJob = compileJob ?? throw new ArgumentNullException(nameof(compileJob)); } /// diff --git a/src/Tgstation.Server.Host/Components/Instance.cs b/src/Tgstation.Server.Host/Components/Instance.cs index fcd335a27a..07aa3575cb 100644 --- a/src/Tgstation.Server.Host/Components/Instance.cs +++ b/src/Tgstation.Server.Host/Components/Instance.cs @@ -194,7 +194,16 @@ namespace Tgstation.Server.Host.Components } /// - public Task StartAsync(CancellationToken cancellationToken) => Task.WhenAll(SetAutoUpdateInterval(metadata.AutoUpdateInterval), Configuration.StartAsync(cancellationToken), ByondManager.StartAsync(cancellationToken), Watchdog.StartAsync(cancellationToken), Chat.StartAsync(cancellationToken), compileJobConsumer.StartAsync(cancellationToken)); + public async Task StartAsync(CancellationToken cancellationToken) + { + await Task.WhenAll(SetAutoUpdateInterval(metadata.AutoUpdateInterval), Configuration.StartAsync(cancellationToken), ByondManager.StartAsync(cancellationToken), Watchdog.StartAsync(cancellationToken), Chat.StartAsync(cancellationToken), compileJobConsumer.StartAsync(cancellationToken)).ConfigureAwait(false); + CompileJob latestCompileJob = null; + await databaseContextFactory.UseContext(async db => + { + latestCompileJob = await db.CompileJobs.Where(x => x.Job.Instance.Id == metadata.Id && x.Job.ExceptionDetails == null).OrderByDescending(x => x.Job.StoppedAt).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + }).ConfigureAwait(false); + await dmbFactory.CleanUnusedCompileJobs(latestCompileJob, cancellationToken).ConfigureAwait(false); + } /// public Task StopAsync(CancellationToken cancellationToken) => Task.WhenAll(SetAutoUpdateInterval(null), Configuration.StopAsync(cancellationToken), ByondManager.StopAsync(cancellationToken), Watchdog.StopAsync(cancellationToken), Chat.StopAsync(cancellationToken), compileJobConsumer.StopAsync(cancellationToken)); diff --git a/src/Tgstation.Server.Host/IO/DefaultIOManager.cs b/src/Tgstation.Server.Host/IO/DefaultIOManager.cs index 65ae98fd1f..c33e46aca8 100644 --- a/src/Tgstation.Server.Host/IO/DefaultIOManager.cs +++ b/src/Tgstation.Server.Host/IO/DefaultIOManager.cs @@ -61,34 +61,40 @@ namespace Tgstation.Server.Host.IO /// The destination directory path /// Files and folders to ignore at the root level /// The for the operation - /// A representing the running operation - async Task CopyDirectoryImpl(string src, string dest, IEnumerable ignore, CancellationToken cancellationToken) + /// A of s representing the running operation + IEnumerable CopyDirectoryImpl(string src, string dest, IEnumerable ignore, CancellationToken cancellationToken) { - await CreateDirectory(dest, cancellationToken).ConfigureAwait(false); - var dir = new DirectoryInfo(src); - cancellationToken.ThrowIfCancellationRequested(); - - var dirs = dir.EnumerateDirectories(); - var files = dir.EnumerateFiles(); - - var fileCopyTasks = files.Select(x => + var atLeastOneSubDir = false; + foreach (var I in dir.EnumerateDirectories()) { - cancellationToken.ThrowIfCancellationRequested(); - if (ignore != null && ignore.Contains(x.Name)) - return Task.CompletedTask; - return CopyFile(x.FullName, Path.Combine(dest, x.Name), cancellationToken); - }); + if (ignore != null && ignore.Contains(I.Name)) + continue; + foreach (var J in CopyDirectoryImpl(I.FullName, Path.Combine(dest, I.Name), null, cancellationToken)) + { + atLeastOneSubDir = true; + yield return J; + } + } - var directoryCopyTasks = dirs.Select(x => + async Task CopyThisDirectory() { - cancellationToken.ThrowIfCancellationRequested(); - if (ignore != null && ignore.Contains(x.Name)) - return Task.CompletedTask; - return CopyDirectoryImpl(x.FullName, Path.Combine(dest, x.Name), null, cancellationToken); - }); + if (!atLeastOneSubDir) + await CreateDirectory(dest, cancellationToken).ConfigureAwait(false); //save on createdir calls - await Task.WhenAll(fileCopyTasks.Concat(directoryCopyTasks)).ConfigureAwait(false); + var tasks = new List(); + + await dir.EnumerateFiles().ToAsyncEnumerable().ForEachAsync(I => + { + if (ignore != null && ignore.Contains(I.Name)) + return; + tasks.Add(CopyFile(I.FullName, Path.Combine(dest, I.Name), cancellationToken)); + }).ConfigureAwait(false); + + await Task.WhenAll(tasks).ConfigureAwait(false); + }; + + yield return CopyThisDirectory(); } /// @@ -101,7 +107,8 @@ namespace Tgstation.Server.Host.IO src = ResolvePath(src); dest = ResolvePath(dest); - await CopyDirectoryImpl(src, dest, ignore, cancellationToken).ConfigureAwait(false); + foreach (var directoryCopy in CopyDirectoryImpl(src, dest, ignore, cancellationToken)) + await directoryCopy.ConfigureAwait(false); } /// From 06ce3fff086a7df7d55418632865215c3f9ae353 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Sun, 5 Aug 2018 14:27:39 -0400 Subject: [PATCH 02/36] Remove errant spaces from CompilerStatus.cs --- .../Models/CompilerStatus.cs | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/Tgstation.Server.Api/Models/CompilerStatus.cs b/src/Tgstation.Server.Api/Models/CompilerStatus.cs index 4509c9c6f5..d3ed992ab1 100644 --- a/src/Tgstation.Server.Api/Models/CompilerStatus.cs +++ b/src/Tgstation.Server.Api/Models/CompilerStatus.cs @@ -1,32 +1,32 @@ namespace Tgstation.Server.Api.Models { - /// - /// Status of the for an - /// + /// + /// Status of the for an + /// #pragma warning disable CA1717 // Only FlagsAttribute enums should have plural names public enum CompilerStatus #pragma warning restore CA1717 // Only FlagsAttribute enums should have plural names { - /// - /// The is idle - /// + /// + /// The is idle + /// Idle, /// /// Pre-compile scripts are running /// PreCompile, - /// - /// The is being copied - /// - Copying, + /// + /// The is being copied + /// + Copying, /// /// The .dme is having it's server side modifications applied /// Modifying, - /// - /// DreamMaker is running - /// - Compiling, + /// + /// DreamMaker is running + /// + Compiling, /// /// The DMAPI is being verified /// From 77e1633866c27b17d8b887c74a28b5140be80bc8 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Sun, 5 Aug 2018 22:31:37 -0400 Subject: [PATCH 03/36] The concept of commit early commit often doesn't seem to apply to me --- docs/API.dox | 55 ++++++++++++++++++- .../Models/CompilerStatus.cs | 8 +-- .../Components/Byond/WindowsByondInstaller.cs | 15 +++-- .../Components/Compiler/DmbFactory.cs | 49 ++++++++++------- .../Components/Compiler/DreamMaker.cs | 28 ++++++---- .../Compiler/ICompileJobConsumer.cs | 2 +- .../Components/EventType.cs | 3 +- .../Components/IInstance.cs | 5 ++ .../Components/Instance.cs | 20 ++++--- .../Components/InstanceFactory.cs | 2 +- .../Components/StaticFiles/Configuration.cs | 5 +- .../Components/StaticFiles/ScriptExecutor.cs | 4 +- .../Components/Watchdog/LaunchResult.cs | 7 +-- .../Components/Watchdog/Session.cs | 3 - .../Watchdog/SessionControllerFactory.cs | 30 +++++----- .../Controllers/DreamMakerController.cs | 27 +++++++-- .../Controllers/RepositoryController.cs | 24 ++++---- 17 files changed, 185 insertions(+), 102 deletions(-) diff --git a/docs/API.dox b/docs/API.dox index f8ddafc8a4..6783c24120 100644 --- a/docs/API.dox +++ b/docs/API.dox @@ -137,7 +137,7 @@ I DELETE "/InstanceUser/{UserId}" => OK Users with the permission to modify @ref Tgstation.Server.Api.Models.Instance objects can also gain user editing rights for any Instance. See @ref api_instance -@section api_ver Getting the server version +@section api_ver Version The version of TGS running can be retireved with this request @@ -196,6 +196,24 @@ Instances can be detached which will delete all meta knowledge of the instance ( DELETE "/Instance/{InstanceId}" => OK +@subsection api_job Jobs + +Some requests return @ref Tgstation.Server.Api.Models.Job objects. These are long running tasks the server will perform asyncronously and can be polled for status. + +To list all jobs in an Instance use the following request + +I GET "/Job" => Array of @ref Tgstation.Server.Api.Models.Job + +Note that the response for this request will only have the @ref Tgstation.Server.Api.Models.Job.Id field populated + +To get full details of a job use the following request: + +I GET "/Job/{JobId}" => @ref Tgstation.Server.Api.Models.Job + +To cancel a running job (If the job can be cancelled and you have sufficient rights) use the following request: + +I DELETE "/Job/{JobId}" => OK + @subsection api_chat Chat Bots Each chat bot is represented by a @ref Tgstation.Server.Api.Models.ChatSettings object @@ -219,4 +237,39 @@ I GET "/Chat/{ChatSettingsId}" => @ref Tgstation.Server.Api.Models.ChatSettings Also note that if the @ref Tgstation.Server.Api.Models.ChatSettings.Channels is present in a POST request, the list will fully replace any active channels +@subsection api_byond Byond Version Management + +To get the Byond version used for new compilations use the following request: + +I GET "/Byond" => @ref Tgstation.Server.Api.Models.Byond + +To set the active Byond version: + +I POST "/Byond" @ref Tgstation.Server.Api.Models.Byond => @ref Tgstation.Server.Api.Models.Byond + +To list all installed Byond versions use the following request: + +I GET "/Byond/List" => Array of @ref Tgstation.Server.Api.Models.Byond + +@subsection Git Repository Management + +To read the current repository state use the following request: + +I GET "/Repository" => @ref Tgstation.Server.Api.Models.Repository + +To clone the repository if it doesn't yet exist use the following request: + +I PUT "/Repository" => @ref Tgstation.Server.Api.Models.Repository + +The clone job will be represented by the @ref Tgstation.Server.Api.Models.Repository.ActiveJob field. Specify the @ref Tgstation.Server.Api.Models.Origin URL. Optionally specify the initial @ref Tgstation.Server.Api.Models.Repository.Reference as a git tag or branch. Be sure to specify the authentication fields if necessary to access your repository + +To delete an existing repository make the following request: + +I DELETE "/Repository" => OK + +Modifications to the repository are done with the following request: + +I POST "/Repository" => @ref Tgstation.Server.Api.Models.Repository => @ref Tgstation.Server.Api.Models.Repository + +Each update creates a job specified in the @ref Tgstation.Server.Api.Models.Repository.ActiveJob field jobs will be queued in succession. */ diff --git a/src/Tgstation.Server.Api/Models/CompilerStatus.cs b/src/Tgstation.Server.Api/Models/CompilerStatus.cs index d3ed992ab1..dc71d316f9 100644 --- a/src/Tgstation.Server.Api/Models/CompilerStatus.cs +++ b/src/Tgstation.Server.Api/Models/CompilerStatus.cs @@ -12,14 +12,14 @@ /// Idle, /// - /// Pre-compile scripts are running - /// - PreCompile, - /// /// The is being copied /// Copying, /// + /// Pre-compile scripts are running + /// + PreCompile, + /// /// The .dme is having it's server side modifications applied /// Modifying, diff --git a/src/Tgstation.Server.Host/Components/Byond/WindowsByondInstaller.cs b/src/Tgstation.Server.Host/Components/Byond/WindowsByondInstaller.cs index 3e5422c62d..d2a7329345 100644 --- a/src/Tgstation.Server.Host/Components/Byond/WindowsByondInstaller.cs +++ b/src/Tgstation.Server.Host/Components/Byond/WindowsByondInstaller.cs @@ -127,23 +127,22 @@ namespace Tgstation.Server.Host.Components.Byond p.StartInfo.WorkingDirectory = rbdx; p.EnableRaisingEvents = true; var tcs = new TaskCompletionSource(); - p.Exited += (a, b) => tcs.SetResult(null); + p.Exited += (a, b) => tcs.TrySetResult(null); try { p.Start(); - using (cancellationToken.Register(() => - { - p.Kill(); - tcs.SetCanceled(); - })) + using (cancellationToken.Register(() => tcs.TrySetCanceled())) await tcs.Task.ConfigureAwait(false); } finally { try { - p.Kill(); - p.WaitForExit(); + if (!p.HasExited) + { + p.Kill(); + p.WaitForExit(); + } } catch (InvalidOperationException) { } } diff --git a/src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs b/src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs index 242f1c40ac..2835269fe8 100644 --- a/src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs +++ b/src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs @@ -1,4 +1,5 @@ using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; using System; using System.Collections.Generic; using System.Linq; @@ -34,15 +35,19 @@ namespace Tgstation.Server.Host.Components.Compiler /// readonly IIOManager ioManager; /// - /// The for + /// The for the /// - readonly CancellationTokenSource cleanupCts; + readonly ILogger logger; /// /// The for the /// readonly Api.Models.Instance instance; - + /// + /// The for + /// + readonly CancellationTokenSource cleanupCts; + /// /// representing calls to /// @@ -63,11 +68,13 @@ namespace Tgstation.Server.Host.Components.Compiler /// /// The value of /// The value of + /// The value of /// The value of - public DmbFactory(IDatabaseContextFactory databaseContextFactory, IIOManager ioManager, Api.Models.Instance instance) + public DmbFactory(IDatabaseContextFactory databaseContextFactory, IIOManager ioManager, ILogger logger, Api.Models.Instance instance) { this.databaseContextFactory = databaseContextFactory ?? throw new ArgumentNullException(nameof(databaseContextFactory)); this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); this.instance = instance ?? throw new ArgumentNullException(nameof(instance)); cleanupTask = Task.CompletedTask; @@ -141,27 +148,19 @@ namespace Tgstation.Server.Host.Components.Compiler else task = newerDmbTcs.Task; - var result = await task.ConfigureAwait(false); - //so there's currently a race condition in DreamMakerController where the setting of CompileJob.RevisionInformation and thus IDmbProvider.RevisionInformation can be delayed to after this if someone tries to start the server instantly after compiling - //This is a terrible terrible hack to get around that - //I'm sorry future me, I can't think of any other way to fix this other than giving DreamMaker an IDatabaseContext or having the controller load the CompileJob - await Task.Delay(new TimeSpan(0, 0, 10), cancellationToken).ConfigureAwait(false); - return result; + return await task.ConfigureAwait(false); } /// public Task StartAsync(CancellationToken cancellationToken) => databaseContextFactory.UseContext(async (db) => { //where complete clause not necessary, only successful COMPILEjobs get in the db - var cj = await db.Instances.Where(x => x.Id == instance.Id).SelectMany(x => x.RevisionInformations).SelectMany(x => x.CompileJobs).OrderByDescending(x => x.Job.StoppedAt).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + var cj = await db.CompileJobs.Where(x => x.Job.Instance.Id == instance.Id).OrderByDescending(x => x.Job.StoppedAt).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); if (cj == default(CompileJob)) return; - var directoriesTask = ioManager.GetDirectories(".", cancellationToken); - var compileJobTask = LoadCompileJob(cj, false, cancellationToken); - //delete all other compile jobs - var directories = await directoriesTask.ConfigureAwait(false); - await Task.WhenAll(directories.Where(x => x != cj.Job.ToString()).Select(x => ioManager.DeleteDirectory(x, cancellationToken))).ConfigureAwait(false); - await compileJobTask.ConfigureAwait(false); + await LoadCompileJob(cj, false, cancellationToken).ConfigureAwait(false); + + //we dont do CleanUnusedCompileJobs here because the watchdog may have plans for them yet }); /// @@ -206,11 +205,19 @@ namespace Tgstation.Server.Host.Components.Compiler //cleanup var directories = await ioManager.GetDirectories(".", cancellationToken).ConfigureAwait(false); - await Task.WhenAll(directories.Select(x => + await Task.WhenAll(directories.Select(async x => { - if (jobUidsToNotErase.Contains(x.ToUpperInvariant())) - return Task.CompletedTask; - return ioManager.DeleteDirectory(x, cancellationToken); + var nameOnly = ioManager.GetFileName(x); + if (jobUidsToNotErase.Contains(nameOnly.ToUpperInvariant())) + return; + try + { + await ioManager.DeleteDirectory(x, cancellationToken).ConfigureAwait(false); + } + catch (Exception e) + { + logger.LogWarning("Error deleting directory {0}! Exception: {1}", x, e); + } })).ConfigureAwait(false); } } diff --git a/src/Tgstation.Server.Host/Components/Compiler/DreamMaker.cs b/src/Tgstation.Server.Host/Components/Compiler/DreamMaker.cs index ce8125afc5..858bc3792b 100644 --- a/src/Tgstation.Server.Host/Components/Compiler/DreamMaker.cs +++ b/src/Tgstation.Server.Host/Components/Compiler/DreamMaker.cs @@ -115,15 +115,19 @@ namespace Tgstation.Server.Host.Components.Compiler }; var dirA = ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName); - var provider = new TemporaryDmbProvider(ioManager.ResolvePath(ioManager.GetDirectoryName(dirA)), ioManager.ResolvePath(ioManager.ConcatPath(dirA, String.Concat(job.DmeName, DmbExtension))), job); + var provider = new TemporaryDmbProvider(ioManager.ResolvePath(dirA), String.Join('.', job.DmeName, DmbExtension), job); var timeoutAt = DateTimeOffset.Now.AddSeconds(timeout); using (var controller = await sessionControllerFactory.LaunchNew(launchParameters, provider, byondLock, true, true, true, cancellationToken).ConfigureAwait(false)) { - var timeoutTask = Task.Delay(timeoutAt - DateTimeOffset.Now, cancellationToken); + var now = DateTimeOffset.Now; + if (now < timeoutAt) + { + var timeoutTask = Task.Delay(timeoutAt - DateTimeOffset.Now, cancellationToken); - await Task.WhenAny(controller.Lifetime, timeoutTask).ConfigureAwait(false); - cancellationToken.ThrowIfCancellationRequested(); + await Task.WhenAny(controller.Lifetime, timeoutTask).ConfigureAwait(false); + cancellationToken.ThrowIfCancellationRequested(); + } if (!controller.Lifetime.IsCompleted) return false; @@ -161,14 +165,14 @@ namespace Tgstation.Server.Host.Components.Compiler dm.EnableRaisingEvents = true; var dmTcs = new TaskCompletionSource(); - dm.Exited += (a, b) => dmTcs.SetResult(null); + dm.Exited += (a, b) => dmTcs.TrySetResult(null); dm.Start(); dm.BeginOutputReadLine(); dm.BeginErrorReadLine(); try { - using (cancellationToken.Register(() => dmTcs.SetCanceled())) + using (cancellationToken.Register(() => dmTcs.TrySetCanceled())) await dmTcs.Task.ConfigureAwait(false); } finally @@ -246,12 +250,10 @@ namespace Tgstation.Server.Host.Components.Compiler return job; } - Status = CompilerStatus.PreCompile; + Status = CompilerStatus.Copying; } - await eventConsumer.HandleEvent(EventType.CompileStart, new List{ repository.Origin }, cancellationToken).ConfigureAwait(false); try { - Status = CompilerStatus.Copying; await ioManager.CreateDirectory(job.DirectoryName.ToString(), cancellationToken).ConfigureAwait(false); var dirA = ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName); var dirB = ioManager.ConcatPath(job.DirectoryName.ToString(), BDirectoryName); @@ -273,9 +275,14 @@ namespace Tgstation.Server.Host.Components.Compiler { //copy the repository var fullDirA = ioManager.ResolvePath(dirA); + var repoOrigin = repository.Origin; using (repository) await repository.CopyTo(fullDirA, cancellationToken).ConfigureAwait(false); + Status = CompilerStatus.PreCompile; + + await eventConsumer.HandleEvent(EventType.CompileStart, new List { ioManager.ResolvePath(ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName)), repoOrigin }, cancellationToken).ConfigureAwait(false); + Status = CompilerStatus.Modifying; if (job.DmeName == null) @@ -304,7 +311,7 @@ namespace Tgstation.Server.Host.Components.Compiler Status = CompilerStatus.Verifying; - ddVerified = job.ExitCode == 0 && await VerifyApi(apiValidateTimeout, job, byondLock, cancellationToken).ConfigureAwait(false); + ddVerified = job.ExitCode == 0 && await VerifyApi(apiValidateTimeout, job, byondLock, cancellationToken).ConfigureAwait(false); } if (!ddVerified) @@ -332,7 +339,6 @@ namespace Tgstation.Server.Host.Components.Compiler Status = CompilerStatus.PostCompile; await eventConsumer.HandleEvent(EventType.CompileComplete, null, cancellationToken).ConfigureAwait(false); } - await compileJobConsumer.LoadCompileJob(job, cancellationToken).ConfigureAwait(false); return job; } catch diff --git a/src/Tgstation.Server.Host/Components/Compiler/ICompileJobConsumer.cs b/src/Tgstation.Server.Host/Components/Compiler/ICompileJobConsumer.cs index 398d6d038c..8095e3321e 100644 --- a/src/Tgstation.Server.Host/Components/Compiler/ICompileJobConsumer.cs +++ b/src/Tgstation.Server.Host/Components/Compiler/ICompileJobConsumer.cs @@ -9,7 +9,7 @@ namespace Tgstation.Server.Host.Components.Compiler /// /// Sink for s /// - interface ICompileJobConsumer : IHostedService, IDisposable + public interface ICompileJobConsumer : IHostedService, IDisposable { /// /// Load a new into the diff --git a/src/Tgstation.Server.Host/Components/EventType.cs b/src/Tgstation.Server.Host/Components/EventType.cs index e9442b0d10..89e5076fd6 100644 --- a/src/Tgstation.Server.Host/Components/EventType.cs +++ b/src/Tgstation.Server.Host/Components/EventType.cs @@ -38,9 +38,8 @@ /// No parameters /// ByondChangeComplete = 7, - /// - /// Parameters: Origin commit sha + /// Parameters: Game directory path, origin commit sha /// CompileStart = 8, /// diff --git a/src/Tgstation.Server.Host/Components/IInstance.cs b/src/Tgstation.Server.Host/Components/IInstance.cs index 946a95b16a..917eaa3e66 100644 --- a/src/Tgstation.Server.Host/Components/IInstance.cs +++ b/src/Tgstation.Server.Host/Components/IInstance.cs @@ -40,6 +40,11 @@ namespace Tgstation.Server.Host.Components /// IChat Chat { get; } + /// + /// The for the + /// + ICompileJobConsumer CompileJobConsumer { get; } + /// /// The for the /// diff --git a/src/Tgstation.Server.Host/Components/Instance.cs b/src/Tgstation.Server.Host/Components/Instance.cs index 07aa3575cb..a47476351f 100644 --- a/src/Tgstation.Server.Host/Components/Instance.cs +++ b/src/Tgstation.Server.Host/Components/Instance.cs @@ -35,10 +35,8 @@ namespace Tgstation.Server.Host.Components /// public StaticFiles.IConfiguration Configuration { get; } - /// - /// The for the - /// - readonly ICompileJobConsumer compileJobConsumer; + /// + public ICompileJobConsumer CompileJobConsumer { get; } /// /// The for the @@ -79,7 +77,7 @@ namespace Tgstation.Server.Host.Components /// The value of /// The value of /// The value of - /// The value of + /// The value of /// The value of /// The value of /// The value of @@ -92,7 +90,7 @@ namespace Tgstation.Server.Host.Components Watchdog = watchdog ?? throw new ArgumentNullException(nameof(watchdog)); Chat = chat ?? throw new ArgumentNullException(nameof(chat)); Configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); - this.compileJobConsumer = compileJobConsumer ?? throw new ArgumentNullException(nameof(compileJobConsumer)); + CompileJobConsumer = compileJobConsumer ?? throw new ArgumentNullException(nameof(compileJobConsumer)); this.databaseContextFactory = databaseContextFactory ?? throw new ArgumentNullException(nameof(databaseContextFactory)); this.dmbFactory = dmbFactory ?? throw new ArgumentNullException(nameof(dmbFactory)); this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); @@ -102,7 +100,7 @@ namespace Tgstation.Server.Host.Components public void Dispose() { timerCts?.Dispose(); - compileJobConsumer.Dispose(); + CompileJobConsumer.Dispose(); Configuration.Dispose(); Chat.Dispose(); Watchdog.Dispose(); @@ -196,7 +194,11 @@ namespace Tgstation.Server.Host.Components /// public async Task StartAsync(CancellationToken cancellationToken) { - await Task.WhenAll(SetAutoUpdateInterval(metadata.AutoUpdateInterval), Configuration.StartAsync(cancellationToken), ByondManager.StartAsync(cancellationToken), Watchdog.StartAsync(cancellationToken), Chat.StartAsync(cancellationToken), compileJobConsumer.StartAsync(cancellationToken)).ConfigureAwait(false); + await Task.WhenAll(SetAutoUpdateInterval(metadata.AutoUpdateInterval), Configuration.StartAsync(cancellationToken), ByondManager.StartAsync(cancellationToken), Chat.StartAsync(cancellationToken), CompileJobConsumer.StartAsync(cancellationToken)).ConfigureAwait(false); + + //dependent on so many things, its just safer this way + await Watchdog.StartAsync(cancellationToken).ConfigureAwait(false); + CompileJob latestCompileJob = null; await databaseContextFactory.UseContext(async db => { @@ -206,7 +208,7 @@ namespace Tgstation.Server.Host.Components } /// - public Task StopAsync(CancellationToken cancellationToken) => Task.WhenAll(SetAutoUpdateInterval(null), Configuration.StopAsync(cancellationToken), ByondManager.StopAsync(cancellationToken), Watchdog.StopAsync(cancellationToken), Chat.StopAsync(cancellationToken), compileJobConsumer.StopAsync(cancellationToken)); + public Task StopAsync(CancellationToken cancellationToken) => Task.WhenAll(SetAutoUpdateInterval(null), Configuration.StopAsync(cancellationToken), ByondManager.StopAsync(cancellationToken), Watchdog.StopAsync(cancellationToken), Chat.StopAsync(cancellationToken), CompileJobConsumer.StopAsync(cancellationToken)); /// public async Task SetAutoUpdateInterval(int? newInterval) diff --git a/src/Tgstation.Server.Host/Components/InstanceFactory.cs b/src/Tgstation.Server.Host/Components/InstanceFactory.cs index 87f26407be..e5b48d68b2 100644 --- a/src/Tgstation.Server.Host/Components/InstanceFactory.cs +++ b/src/Tgstation.Server.Host/Components/InstanceFactory.cs @@ -130,7 +130,7 @@ namespace Tgstation.Server.Host.Components var configuration = new StaticFiles.Configuration(configurationIoManager, synchronousIOManager, symlinkFactory, scriptExecutor, loggerFactory.CreateLogger()); var eventConsumer = new EventConsumer(configuration); - var dmbFactory = new DmbFactory(databaseContextFactory, gameIoManager, metadata.CloneMetadata()); + var dmbFactory = new DmbFactory(databaseContextFactory, gameIoManager, loggerFactory.CreateLogger(), metadata.CloneMetadata()); try { var repoManager = new RepositoryManager(metadata.RepositorySettings, repoIoManager, eventConsumer); diff --git a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs index 6ff0183354..92595de192 100644 --- a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs +++ b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs @@ -27,9 +27,10 @@ namespace Tgstation.Server.Host.Components.StaticFiles static readonly IReadOnlyDictionary EventTypeScriptFileNameMap = new Dictionary { + { EventType.CompileStart, "PreCompile" } }; - static readonly string SystemScriptFileExtension = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? ".bat" : ".sh"; + static readonly string SystemScriptFileExtension = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "bat" : "sh"; /// /// The for @@ -298,7 +299,7 @@ namespace Tgstation.Server.Host.Components.StaticFiles var files = await ioManager.GetFilesWithExtension(EventScriptsSubdirectory, SystemScriptFileExtension, cancellationToken).ConfigureAwait(false); var resolvedScriptsDir = ioManager.ResolvePath(EventScriptsSubdirectory); - foreach (var I in files.Where(x => x.StartsWith(scriptName, StringComparison.Ordinal))) + foreach (var I in files.Select(x => ioManager.GetFileName(x)).Where(x => x.StartsWith(scriptName, StringComparison.Ordinal))) if ((await scriptExecutor.ExecuteScript(ioManager.ConcatPath(resolvedScriptsDir, I), parameters, cancellationToken).ConfigureAwait(false)) != 0) return false; } diff --git a/src/Tgstation.Server.Host/Components/StaticFiles/ScriptExecutor.cs b/src/Tgstation.Server.Host/Components/StaticFiles/ScriptExecutor.cs index 14a2a97805..ff2a90c625 100644 --- a/src/Tgstation.Server.Host/Components/StaticFiles/ScriptExecutor.cs +++ b/src/Tgstation.Server.Host/Components/StaticFiles/ScriptExecutor.cs @@ -47,11 +47,11 @@ namespace Tgstation.Server.Host.Components.StaticFiles process.EnableRaisingEvents = true; var tcs = new TaskCompletionSource(); - process.Exited += (a, b) => tcs.SetResult(null); + process.Exited += (a, b) => tcs.TrySetResult(null); try { process.Start(); - using (cancellationToken.Register(() => tcs.SetCanceled())) + using (cancellationToken.Register(() => tcs.TrySetCanceled())) await tcs.Task.ConfigureAwait(false); } catch (InvalidOperationException) diff --git a/src/Tgstation.Server.Host/Components/Watchdog/LaunchResult.cs b/src/Tgstation.Server.Host/Components/Watchdog/LaunchResult.cs index 3e8c3d51b4..81b2be9fb9 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/LaunchResult.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/LaunchResult.cs @@ -18,12 +18,7 @@ namespace Tgstation.Server.Host.Components.Watchdog /// public int? ExitCode { get; set; } - /// - /// The peak virtual memory usage in bytes - /// - public long PeakMemory { get; set; } - /// - public override string ToString() => String.Format(CultureInfo.InvariantCulture, "Exit Code: {0}, RAM: {1}, Time {2}ms", ExitCode, PeakMemory, StartupTime.TotalMilliseconds); + public override string ToString() => String.Format(CultureInfo.InvariantCulture, "Exit Code: {0}, Time {1}ms", ExitCode, StartupTime.TotalMilliseconds); } } \ No newline at end of file diff --git a/src/Tgstation.Server.Host/Components/Watchdog/Session.cs b/src/Tgstation.Server.Host/Components/Watchdog/Session.cs index 29b7eb8844..b585b32315 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/Session.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/Session.cs @@ -52,11 +52,8 @@ namespace Tgstation.Server.Host.Components.Watchdog var result = new LaunchResult { ExitCode = process.HasExited ? (int?)process.ExitCode : null, - PeakMemory = process.PeakWorkingSet64, StartupTime = DateTimeOffset.Now - startTime }; - if (result.PeakMemory == 0) //linux, best we can do honestly, test if this even works - result.PeakMemory = process.WorkingSet64; return result; }, default, TaskCreationOptions.LongRunning, TaskScheduler.Current); lifetimeTask = new TaskCompletionSource(); diff --git a/src/Tgstation.Server.Host/Components/Watchdog/SessionControllerFactory.cs b/src/Tgstation.Server.Host/Components/Watchdog/SessionControllerFactory.cs index 1afc9a1571..a0d6cb7c34 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/SessionControllerFactory.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/SessionControllerFactory.cs @@ -116,26 +116,28 @@ namespace Tgstation.Server.Host.Components.Watchdog InstanceName = instance.Name, Revision = dmbProvider.CompileJob.RevisionInformation }; - interopInfo.TestMerges.AddRange(dmbProvider.CompileJob.RevisionInformation.ActiveTestMerges.Select(x => x.TestMerge).Select(x => new TestMerge - { - Author = x.Author, - Body = x.BodyAtMerge, - Comment = x.Comment, - CommitSha = x.PrimaryRevisionInformation.CommitSha, - Number = x.Number, - OriginCommitSha = x.PrimaryRevisionInformation.OriginCommitSha, - PullRequestCommit = x.PullRequestRevision, - TimeMerged = x.MergedAt.Ticks, - Title = x.TitleAtMerge, - Url = x.Url - })); + + if (dmbProvider.CompileJob.RevisionInformation != null) //null while compiling + interopInfo.TestMerges.AddRange(dmbProvider.CompileJob.RevisionInformation.ActiveTestMerges.Select(x => x.TestMerge).Select(x => new TestMerge + { + Author = x.Author, + Body = x.BodyAtMerge, + Comment = x.Comment, + CommitSha = x.PrimaryRevisionInformation.CommitSha, + Number = x.Number, + OriginCommitSha = x.PrimaryRevisionInformation.OriginCommitSha, + PullRequestCommit = x.PullRequestRevision, + TimeMerged = x.MergedAt.Ticks, + Title = x.TitleAtMerge, + Url = x.Url + })); var interopJsonFile = GuidJsonFile(); var interopJson = JsonConvert.SerializeObject(interopInfo); var basePath = primaryDirectory ? dmbProvider.PrimaryDirectory : dmbProvider.SecondaryDirectory; - var localIoManager = new ResolvingIOManager(ioManager, ioManager.ConcatPath(basePath, dmbProvider.DmbName)); + var localIoManager = new ResolvingIOManager(ioManager, basePath); var chatJsonTrackingTask = chat.TrackJsons(basePath, interopInfo.ChatChannelsJson, interopInfo.ChatCommandsJson, cancellationToken); diff --git a/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs b/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs index 460a6c598c..8b666c2a8b 100644 --- a/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs +++ b/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs @@ -108,7 +108,8 @@ namespace Tgstation.Server.Host.Controllers var instance = instanceManager.GetInstance(instanceModel); CompileJob compileJob; - string repoSha = null; + Task revInfoTask; + string repoSha; using (var repo = await instance.RepositoryManager.LoadRepository(cancellationToken).ConfigureAwait(false)) { if (repo == null) @@ -117,11 +118,15 @@ namespace Tgstation.Server.Host.Controllers return; } repoSha = repo.Head; + revInfoTask = databaseContext.RevisionInformations.Where(x => x.CommitSha == repoSha).Select(x => new RevisionInformation { Id = x.Id }).FirstOrDefaultAsync(); compileJob = await instance.DreamMaker.Compile(projectName, timeout.Value, repo, cancellationToken).ConfigureAwait(false); } + if (compileJob.DMApiValidated != true) + return; + compileJob.Job = job; - compileJob.RevisionInformation = await databaseContext.RevisionInformations.Where(x => x.CommitSha == repoSha).Select(x => new RevisionInformation { Id = x.Id }).FirstOrDefaultAsync().ConfigureAwait(false); + compileJob.RevisionInformation = await revInfoTask.ConfigureAwait(false); if (compileJob.RevisionInformation == default) { @@ -134,12 +139,24 @@ namespace Tgstation.Server.Host.Controllers Id = Instance.Id } }; - DatabaseContext.Instances.Attach(compileJob.RevisionInformation.Instance); + databaseContext.Instances.Attach(compileJob.RevisionInformation.Instance); } databaseContext.CompileJobs.Add(compileJob); - //default ct because we don't want to give up after getting this far - await databaseContext.Save(default).ConfigureAwait(false); + await databaseContext.Save(cancellationToken).ConfigureAwait(false); + + //now load the entire compile job tree into the consumer + //default ct because we don't want to give up after getting this far since we already set this job as staged in the db + var finalCompileJob = await databaseContext.CompileJobs.Where(x => x.Id == compileJob.Id) + .Include(x => x.Job).ThenInclude(x => x.StartedBy) + .Include(x => x.RevisionInformation).ThenInclude(x => x.PrimaryTestMerge).ThenInclude(x => x.MergedBy) + .Include(x => x.RevisionInformation).ThenInclude(x => x.ActiveTestMerges).ThenInclude(x => x.TestMerge).ThenInclude(x => x.MergedBy) + .FirstOrDefaultAsync().ConfigureAwait(false); //can't wait to see that query + if (finalCompileJob == null) + //lol git fucked + return; + + await instance.CompileJobConsumer.LoadCompileJob(finalCompileJob, default).ConfigureAwait(false); } } } diff --git a/src/Tgstation.Server.Host/Controllers/RepositoryController.cs b/src/Tgstation.Server.Host/Controllers/RepositoryController.cs index 5b0209649f..116a71f2a1 100644 --- a/src/Tgstation.Server.Host/Controllers/RepositoryController.cs +++ b/src/Tgstation.Server.Host/Controllers/RepositoryController.cs @@ -300,7 +300,7 @@ namespace Tgstation.Server.Host.Controllers await jobManager.RegisterOperation(job, async (paramJob, serviceProvider, progressReporter, ct) => { - using (var repo = await instanceManager.GetInstance(Instance).RepositoryManager.LoadRepository(cancellationToken).ConfigureAwait(false)) + using (var repo = await instanceManager.GetInstance(Instance).RepositoryManager.LoadRepository(ct).ConfigureAwait(false)) { if (repo == null) throw new InvalidOperationException("Repository could not be loaded!"); @@ -330,13 +330,13 @@ namespace Tgstation.Server.Host.Controllers }; databaseContext.Instances.Attach(attachedInstance); - await LoadRevisionInformation(repo, databaseContext, attachedInstance, null, x => lastRevisionInfo = x, cancellationToken).ConfigureAwait(false); + await LoadRevisionInformation(repo, databaseContext, attachedInstance, null, x => lastRevisionInfo = x, ct).ConfigureAwait(false); //apply new rev info, tracking applied test merges async Task UpdateRevInfo() { var last = lastRevisionInfo; - await LoadRevisionInformation(repo, databaseContext, attachedInstance, last.OriginCommitSha, x => lastRevisionInfo = x, cancellationToken).ConfigureAwait(false); + await LoadRevisionInformation(repo, databaseContext, attachedInstance, last.OriginCommitSha, x => lastRevisionInfo = x, ct).ConfigureAwait(false); lastRevisionInfo.ActiveTestMerges.AddRange(last.ActiveTestMerges); }; @@ -347,11 +347,11 @@ namespace Tgstation.Server.Host.Controllers { if (!repo.Tracking && model.Reference == null) throw new InvalidOperationException("Not on an updatable reference!"); - await repo.FetchOrigin(currentModel.AccessUser, currentModel.AccessToken, x => progressReporter(x / numFetches), cancellationToken).ConfigureAwait(false); + await repo.FetchOrigin(currentModel.AccessUser, currentModel.AccessToken, x => progressReporter(x / numFetches), ct).ConfigureAwait(false); doneFetches = 1; if (!modelHasShaOrReference) { - var fastForward = await repo.MergeOrigin(committerName, currentModel.CommitterEmail, cancellationToken).ConfigureAwait(false); + var fastForward = await repo.MergeOrigin(committerName, currentModel.CommitterEmail, ct).ConfigureAwait(false); if (!fastForward.HasValue) throw new InvalidOperationException("Merge conflict occurred during origin update!"); await UpdateRevInfo().ConfigureAwait(false); @@ -366,17 +366,17 @@ namespace Tgstation.Server.Host.Controllers if ((model.CheckoutSha != null && repo.Head.ToUpperInvariant() != model.CheckoutSha.ToUpperInvariant()) || (model.Reference != null && repo.Reference != model.Reference)) { - await repo.CheckoutObject(model.CheckoutSha ?? model.Reference, cancellationToken).ConfigureAwait(false); - await LoadRevisionInformation(repo, databaseContext, attachedInstance, null, x => lastRevisionInfo = x, cancellationToken).ConfigureAwait(false); //we've either seen origin before or what we're checking out is on origin + await repo.CheckoutObject(model.CheckoutSha ?? model.Reference, ct).ConfigureAwait(false); + await LoadRevisionInformation(repo, databaseContext, attachedInstance, null, x => lastRevisionInfo = x, ct).ConfigureAwait(false); //we've either seen origin before or what we're checking out is on origin } if (model.UpdateFromOrigin == true && model.Reference != null) { if (!repo.Tracking) throw new InvalidOperationException("Checked out reference does not track a remote object!"); - await repo.ResetToOrigin(cancellationToken).ConfigureAwait(false); - await repo.Sychronize(currentModel.AccessUser, currentModel.AccessToken, true, cancellationToken).ConfigureAwait(false); - await LoadRevisionInformation(repo, databaseContext, attachedInstance, null, x => lastRevisionInfo = x, cancellationToken).ConfigureAwait(false); + await repo.ResetToOrigin(ct).ConfigureAwait(false); + await repo.Sychronize(currentModel.AccessUser, currentModel.AccessToken, true, ct).ConfigureAwait(false); + await LoadRevisionInformation(repo, databaseContext, attachedInstance, null, x => lastRevisionInfo = x, ct).ConfigureAwait(false); //repo head is on origin so force this //will update the db if necessary lastRevisionInfo.OriginCommitSha = repo.Head; @@ -414,7 +414,7 @@ namespace Tgstation.Server.Host.Controllers errorMessage = "P.R.E. NOT FOUND"; } - var mergeResult = await repo.AddTestMerge(I.Number, I.PullRequestRevision, committerName, currentModel.CommitterEmail, String.Format(CultureInfo.InvariantCulture, "Test merge of pull request #{0}{1}{2}", I.Number, I.Comment != null ? Environment.NewLine : null, I.Comment), currentModel.AccessUser, currentModel.AccessToken, x => progressReporter((x + 100 * doneFetches) / numFetches), cancellationToken).ConfigureAwait(false); + var mergeResult = await repo.AddTestMerge(I.Number, I.PullRequestRevision, committerName, currentModel.CommitterEmail, String.Format(CultureInfo.InvariantCulture, "Test merge of pull request #{0}{1}{2}", I.Number, I.Comment != null ? Environment.NewLine : null, I.Comment), currentModel.AccessUser, currentModel.AccessToken, x => progressReporter((x + 100 * doneFetches) / numFetches), ct).ConfigureAwait(false); if (!mergeResult.HasValue) //conflict, we don't care, dd already knows continue; @@ -451,7 +451,7 @@ namespace Tgstation.Server.Host.Controllers await repo.Sychronize(currentModel.AccessUser, currentModel.AccessToken, false, ct).ConfigureAwait(false); await UpdateRevInfo().ConfigureAwait(false); } - await databaseContext.Save(cancellationToken).ConfigureAwait(false); + await databaseContext.Save(ct).ConfigureAwait(false); } catch { From 98b19792d849b7e636d949f97a13d280c6d1277b Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Tue, 7 Aug 2018 11:25:45 -0400 Subject: [PATCH 04/36] Cleanup interop a bit --- src/DMAPI/tgs.dm | 10 +++++-- .../Components/InstanceManager.cs | 2 ++ .../Controllers/InteropController.cs | 27 ++++++++++++----- src/Tgstation.Server.Host/Core/Application.cs | 2 ++ src/Tgstation.Server.Host/Core/JobManager.cs | 30 ++++++++++++++----- src/Tgstation.Server.Host/appsettings.json | 2 +- 6 files changed, 55 insertions(+), 18 deletions(-) diff --git a/src/DMAPI/tgs.dm b/src/DMAPI/tgs.dm index 628b17cac6..03ad61a092 100644 --- a/src/DMAPI/tgs.dm +++ b/src/DMAPI/tgs.dm @@ -47,14 +47,14 @@ #define TGS_EVENT_PORT_SWAP -2 //before a port change is about to happen, extra parameter is new port #define TGS_EVENT_REBOOT_MODE_CHANGE -1 //before a reboot mode change, extras parameters are the current and new reboot mode enums +//TODO + //OTHER ENUMS #define TGS_REBOOT_MODE_NORMAL 0 #define TGS_REBOOT_MODE_SHUTDOWN 1 #define TGS_REBOOT_MODE_RESTART 2 -//TODO - //REQUIRED HOOKS //Call this somewhere in /world/New() that is always run @@ -74,6 +74,12 @@ /world/proc/TgsReboot() return +//OPTIONAL HOOKS + +//called with debug messages +/world/proc/TgsDebug(message) + return + //DATUM DEFINITIONS //unless otherwise specified all datums defined here should be considered read-only, warranty void if written diff --git a/src/Tgstation.Server.Host/Components/InstanceManager.cs b/src/Tgstation.Server.Host/Components/InstanceManager.cs index 8c395077c3..6259ad60e5 100644 --- a/src/Tgstation.Server.Host/Components/InstanceManager.cs +++ b/src/Tgstation.Server.Host/Components/InstanceManager.cs @@ -178,10 +178,12 @@ namespace Tgstation.Server.Host.Components var tasks = new List(); await dbInstances.ForEachAsync(metadata => tasks.Add(metadata.Online.Value ? OnlineInstance(metadata, cancellationToken) : Task.CompletedTask), cancellationToken).ConfigureAwait(false); await Task.WhenAll(tasks).ConfigureAwait(false); + logger.LogInformation("Instance manager ready!"); application.Ready(null); } catch (Exception e) { + logger.LogCritical("Instance manager startup error! Exception: {0}", e); application.Ready(e); } }); diff --git a/src/Tgstation.Server.Host/Controllers/InteropController.cs b/src/Tgstation.Server.Host/Controllers/InteropController.cs index f7b7b808e9..990f2f671f 100644 --- a/src/Tgstation.Server.Host/Controllers/InteropController.cs +++ b/src/Tgstation.Server.Host/Controllers/InteropController.cs @@ -1,11 +1,12 @@ using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; +using Microsoft.Net.Http.Headers; using System; +using System.Linq; +using System.Net.Http.Headers; using System.Threading; using System.Threading.Tasks; using Tgstation.Server.Host.Components; -using Tgstation.Server.Host.Models; -using Tgstation.Server.Host.Security; namespace Tgstation.Server.Host.Controllers { @@ -13,23 +14,27 @@ namespace Tgstation.Server.Host.Controllers /// Handles requests from DreamDaemon /// [Route("/Interop")] - public sealed class InteropController : ApiController + public sealed class InteropController : Controller { /// /// The for the /// readonly IInstanceManager instanceManager; + /// + /// The for the + /// + readonly ILogger logger; + /// /// Construct an /// - /// The for the - /// The for the /// The value of - /// The for the - public InteropController(IInstanceManager instanceManager, IDatabaseContext databaseContext, IAuthenticationContextFactory authenticationContextFactory, ILogger logger) : base(databaseContext, authenticationContextFactory, logger, false) + /// The value of + public InteropController(IInstanceManager instanceManager, ILogger logger) { this.instanceManager = instanceManager ?? throw new ArgumentNullException(nameof(instanceManager)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); } /// @@ -40,6 +45,14 @@ namespace Tgstation.Server.Host.Controllers [HttpGet] public async Task HandleInterop(CancellationToken cancellationToken) { + //since this is the only identifying factor of a TGS server we want to pretend we don't exist unless it at least has the correct BYOND headers + if (!Request.Headers.TryGetValue(HeaderNames.UserAgent, out var userAgentValues) + || !ProductInfoHeaderValue.TryParse(userAgentValues.FirstOrDefault(), out var clientUserAgent) + || clientUserAgent.Product.Name != "libbyond") + return Unauthorized(); + + logger.LogDebug("Request from BYOND: {0}", Request.QueryString); + var result = await instanceManager.HandleWorldExport(Request.Query, cancellationToken).ConfigureAwait(false); //explain things in very simple terms dream daemon can understand //EXCEPT DREAMDAENEN BUGS diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index 168e747e3f..028f60509b 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -243,6 +243,8 @@ namespace Tgstation.Server.Host.Core logger.LogInformation(VersionString); + logger.LogDebug("Configuring middleware..."); + serverAddresses = applicationBuilder.ServerFeatures.Get(); applicationBuilder.UseDeveloperExceptionPage(); //it is not worth it to limit this, you should only ever get it if you're an authorized user diff --git a/src/Tgstation.Server.Host/Core/JobManager.cs b/src/Tgstation.Server.Host/Core/JobManager.cs index b05e90757c..2bd6734e6e 100644 --- a/src/Tgstation.Server.Host/Core/JobManager.cs +++ b/src/Tgstation.Server.Host/Core/JobManager.cs @@ -1,6 +1,6 @@ using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; using System; using System.Collections.Generic; using System.Linq; @@ -17,6 +17,12 @@ namespace Tgstation.Server.Host.Core /// The for the /// readonly IServiceProvider serviceProvider; + + /// + /// The for the + /// + readonly ILogger logger; + /// /// of to running s /// @@ -26,9 +32,11 @@ namespace Tgstation.Server.Host.Core /// Construct a /// /// The value of - public JobManager(IServiceProvider serviceProvider) + /// The value of + public JobManager(IServiceProvider serviceProvider, ILogger logger) { this.serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); jobs = new Dictionary(); } @@ -144,20 +152,26 @@ namespace Tgstation.Server.Host.Core /// public async Task StartAsync(CancellationToken cancellationToken) { + logger.LogTrace("Starting job manager..."); using (var scope = serviceProvider.CreateScope()) { var databaseContext = scope.ServiceProvider.GetRequiredService(); //mark all jobs as cancelled - var enumerator = await databaseContext.Jobs.Where(y => !y.Cancelled.Value && !y.StoppedAt.HasValue).Select(y => y.Id).ToListAsync(cancellationToken).ConfigureAwait(false); - foreach(var I in enumerator) + var badJobs = await databaseContext.Jobs.Where(y => !y.Cancelled.Value && !y.StoppedAt.HasValue).Select(y => y.Id).ToListAsync(cancellationToken).ConfigureAwait(false); + if (badJobs.Count > 0) { - var job = new Job { Id = I }; - databaseContext.Jobs.Attach(job); - job.Cancelled = true; + logger.LogTrace("Cleaning {0} unfinished jobs...", badJobs.Count); + foreach (var I in enumerator) + { + var job = new Job { Id = I }; + databaseContext.Jobs.Attach(job); + job.Cancelled = true; + } + await databaseContext.Save(cancellationToken).ConfigureAwait(false); } - await databaseContext.Save(cancellationToken).ConfigureAwait(false); } + logger.LogDebug("Job manager started!"); } /// diff --git a/src/Tgstation.Server.Host/appsettings.json b/src/Tgstation.Server.Host/appsettings.json index 0f47b8e2ae..f97d9bf4e7 100644 --- a/src/Tgstation.Server.Host/appsettings.json +++ b/src/Tgstation.Server.Host/appsettings.json @@ -15,7 +15,7 @@ "Console": { "LogLevel": { "Default": "Trace", - "Microsoft": "Information" + "Microsoft": "Warning" } }, "LogLevel": { From 4c851168ab86fd300092dd3a58f7cfa37bffb5de Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Tue, 7 Aug 2018 11:35:58 -0400 Subject: [PATCH 05/36] Fixups --- src/DMAPI/tgs.dm | 6 ------ src/Tgstation.Server.Host/Core/JobManager.cs | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/src/DMAPI/tgs.dm b/src/DMAPI/tgs.dm index 03ad61a092..05a4b3e9a5 100644 --- a/src/DMAPI/tgs.dm +++ b/src/DMAPI/tgs.dm @@ -74,12 +74,6 @@ /world/proc/TgsReboot() return -//OPTIONAL HOOKS - -//called with debug messages -/world/proc/TgsDebug(message) - return - //DATUM DEFINITIONS //unless otherwise specified all datums defined here should be considered read-only, warranty void if written diff --git a/src/Tgstation.Server.Host/Core/JobManager.cs b/src/Tgstation.Server.Host/Core/JobManager.cs index 2bd6734e6e..8cf31a4801 100644 --- a/src/Tgstation.Server.Host/Core/JobManager.cs +++ b/src/Tgstation.Server.Host/Core/JobManager.cs @@ -162,7 +162,7 @@ namespace Tgstation.Server.Host.Core if (badJobs.Count > 0) { logger.LogTrace("Cleaning {0} unfinished jobs...", badJobs.Count); - foreach (var I in enumerator) + foreach (var I in badJobs) { var job = new Job { Id = I }; databaseContext.Jobs.Attach(job); From 499a61848861c886c1ec1cf7e6a0bc2209a28f20 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Tue, 7 Aug 2018 11:37:19 -0400 Subject: [PATCH 06/36] Helpful message --- src/DMAPI/tgs/v4/api.dm | 1 + 1 file changed, 1 insertion(+) diff --git a/src/DMAPI/tgs/v4/api.dm b/src/DMAPI/tgs/v4/api.dm index 5e499e6815..87fb1b80a1 100644 --- a/src/DMAPI/tgs/v4/api.dm +++ b/src/DMAPI/tgs/v4/api.dm @@ -62,6 +62,7 @@ instance_name = text2num(cached_json["instanceName"]) host_path = cached_json["hostPath"] if(cached_json["apiValidateOnly"]) + TGS_INFO_LOG("Validating API and exiting...") Export(TGS4_COMM_VALIDATE) del(world) From a8b75bffe4c820cbb1ead69c61d5ea485b28f5be Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Tue, 7 Aug 2018 11:54:22 -0400 Subject: [PATCH 07/36] Logging without microsoft BS in the way is so much cleaner --- .../Controllers/ApiController.cs | 2 +- .../Controllers/HomeController.cs | 3 +++ .../Controllers/InteropController.cs | 2 +- src/Tgstation.Server.Host/Core/JobManager.cs | 4 +++ .../Models/DatabaseContext.cs | 27 +++++++++++++------ .../MySqlDesignTimeDbContextFactory.cs | 3 ++- .../SqlServerDesignTimeDbContextFactory.cs | 3 ++- .../Models/MySqlDatabaseContext.cs | 4 ++- .../Models/SqlServerDatabaseContext.cs | 4 ++- .../Models/SqliteDatabaseContext.cs | 15 +++-------- 10 files changed, 42 insertions(+), 25 deletions(-) diff --git a/src/Tgstation.Server.Host/Controllers/ApiController.cs b/src/Tgstation.Server.Host/Controllers/ApiController.cs index b97a3eb2e3..e013a02d7e 100644 --- a/src/Tgstation.Server.Host/Controllers/ApiController.cs +++ b/src/Tgstation.Server.Host/Controllers/ApiController.cs @@ -188,7 +188,7 @@ namespace Tgstation.Server.Host.Controllers } } - Logger.LogInformation("Request made by User ID {0}. Api version: {1}. User-Agent: {2}", AuthenticationContext?.User.Id.ToString(CultureInfo.InvariantCulture) ?? "NULL", ApiHeaders.ApiVersion, ApiHeaders.UserAgent); + Logger.LogTrace("Request made by User ID {0}. Api version: {1}. User-Agent: {2}. Type: {3}. Route {4}", AuthenticationContext?.User.Id.ToString(CultureInfo.InvariantCulture) ?? "NULL", ApiHeaders.ApiVersion, ApiHeaders.UserAgent, Request.Method, Request.Path); await base.OnActionExecutionAsync(context, next).ConfigureAwait(false); } } diff --git a/src/Tgstation.Server.Host/Controllers/HomeController.cs b/src/Tgstation.Server.Host/Controllers/HomeController.cs index bc1b3f63fa..08cc614df5 100644 --- a/src/Tgstation.Server.Host/Controllers/HomeController.cs +++ b/src/Tgstation.Server.Host/Controllers/HomeController.cs @@ -127,6 +127,9 @@ namespace Tgstation.Server.Host.Controllers var token = tokenFactory.CreateToken(user, out var expiry); if (identity != null) identityCache.CacheSystemIdentity(user, identity, expiry.AddSeconds(10)); //expire the identity slightly after the auth token in case of lag + + Logger.LogDebug("Successfully logged in user {0} ({1})!", user.Id, user.CanonicalName); + return Json(token); } } diff --git a/src/Tgstation.Server.Host/Controllers/InteropController.cs b/src/Tgstation.Server.Host/Controllers/InteropController.cs index 990f2f671f..55bd53d731 100644 --- a/src/Tgstation.Server.Host/Controllers/InteropController.cs +++ b/src/Tgstation.Server.Host/Controllers/InteropController.cs @@ -14,7 +14,7 @@ namespace Tgstation.Server.Host.Controllers /// Handles requests from DreamDaemon /// [Route("/Interop")] - public sealed class InteropController : Controller + public sealed class InteropController : Controller //not an ApiController because "lol im byond and who is headers?" { /// /// The for the diff --git a/src/Tgstation.Server.Host/Core/JobManager.cs b/src/Tgstation.Server.Host/Core/JobManager.cs index 8cf31a4801..4dc1d28461 100644 --- a/src/Tgstation.Server.Host/Core/JobManager.cs +++ b/src/Tgstation.Server.Host/Core/JobManager.cs @@ -89,13 +89,16 @@ namespace Tgstation.Server.Host.Core databaseContext = scope.ServiceProvider.GetRequiredService(); databaseContext.Jobs.Attach(job); } + logger.LogDebug("Job {0} completed!", job.Id); } catch (OperationCanceledException) { + logger.LogDebug("Job {0} exited with cancelled!", job.Id); job.Cancelled = true; } catch (Exception e) { + logger.LogDebug("Job {0} exited with error! Exception: {1}", job.Id, e); job.ExceptionDetails = e.ToString(); } job.StoppedAt = DateTimeOffset.Now; @@ -136,6 +139,7 @@ namespace Tgstation.Server.Host.Core } databaseContext.Jobs.Add(job); await databaseContext.Save(cancellationToken).ConfigureAwait(false); + logger.LogDebug("Starting job {0}: {1}...", job.Id, job.Description); var jobHandler = JobHandler.Create(x => RunJob(job, (jobParam, serviceProvider, ct) => operation(jobParam, serviceProvider, y => { diff --git a/src/Tgstation.Server.Host/Models/DatabaseContext.cs b/src/Tgstation.Server.Host/Models/DatabaseContext.cs index 1a9e5bac6e..9165ab4f15 100644 --- a/src/Tgstation.Server.Host/Models/DatabaseContext.cs +++ b/src/Tgstation.Server.Host/Models/DatabaseContext.cs @@ -1,4 +1,5 @@ using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using System; #if !DEBUG @@ -62,6 +63,11 @@ namespace Tgstation.Server.Host.Models /// public DbSet WatchdogReattachInformations { get; set; } + /// + /// The for the + /// + protected ILogger Logger { get; } + /// /// The connection string for the /// @@ -82,15 +88,18 @@ namespace Tgstation.Server.Host.Models /// The for the /// The containing the value of /// The value of - public DatabaseContext(DbContextOptions dbContextOptions, IOptions databaseConfigurationOptions, IDatabaseSeeder databaseSeeder) : base(dbContextOptions) + /// The value of + public DatabaseContext(DbContextOptions dbContextOptions, IOptions databaseConfigurationOptions, IDatabaseSeeder databaseSeeder, ILogger logger) : base(dbContextOptions) { databaseConfiguration = databaseConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(databaseConfigurationOptions)); this.databaseSeeder = databaseSeeder ?? throw new ArgumentNullException(nameof(databaseSeeder)); + Logger = logger ?? throw new ArgumentNullException(nameof(logger)); } /// protected override void OnModelCreating(ModelBuilder modelBuilder) { + Logger.LogDebug("Building entity framework context..."); base.OnModelCreating(modelBuilder); var userModel = modelBuilder.Entity(); @@ -125,16 +134,12 @@ namespace Tgstation.Server.Host.Models instanceModel.HasMany(x => x.Jobs).WithOne(x => x.Instance).OnDelete(DeleteBehavior.Cascade); } - /// - protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) - { - base.OnConfiguring(optionsBuilder); - } - /// public async Task Initialize(CancellationToken cancellationToken) { + Logger.LogInformation("Migrating database..."); #if DEBUG + Logger.LogWarning("Running in debug mode. Using all or nothing strategy!"); await Database.EnsureCreatedAsync().ConfigureAwait(false); var wasEmpty = (await Users.CountAsync().ConfigureAwait(false)) == 0; #else @@ -143,9 +148,15 @@ namespace Tgstation.Server.Host.Models await Database.MigrateAsync(cancellationToken).ConfigureAwait(false); #endif if (wasEmpty) + { + Logger.LogInformation("Seeding database..."); await databaseSeeder.SeedDatabase(this, cancellationToken).ConfigureAwait(false); - else if(databaseConfiguration.ResetAdminPassword) + } + else if (databaseConfiguration.ResetAdminPassword) + { + Logger.LogWarning("Enabling and resetting admin password due to configuration!"); await databaseSeeder.ResetAdminPassword(this, cancellationToken).ConfigureAwait(false); + } } /// diff --git a/src/Tgstation.Server.Host/Models/Migrations/MySqlDesignTimeDbContextFactory.cs b/src/Tgstation.Server.Host/Models/Migrations/MySqlDesignTimeDbContextFactory.cs index fd55e611c1..14aafaca46 100644 --- a/src/Tgstation.Server.Host/Models/Migrations/MySqlDesignTimeDbContextFactory.cs +++ b/src/Tgstation.Server.Host/Models/Migrations/MySqlDesignTimeDbContextFactory.cs @@ -1,6 +1,7 @@ using Microsoft.AspNetCore.Identity; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Design; +using Microsoft.Extensions.Logging; using Tgstation.Server.Host.Security; namespace Tgstation.Server.Host.Models.Migrations @@ -9,6 +10,6 @@ namespace Tgstation.Server.Host.Models.Migrations sealed class MySqlDesignTimeDbContextFactory : IDesignTimeDbContextFactory { /// - public MySqlDatabaseContext CreateDbContext(string[] args) => new MySqlDatabaseContext(new DbContextOptions(), DesignTimeDbContextFactoryHelpers.GetDbContextOptions(), new DatabaseSeeder(new CryptographySuite(new PasswordHasher()))); + public MySqlDatabaseContext CreateDbContext(string[] args) => new MySqlDatabaseContext(new DbContextOptions(), DesignTimeDbContextFactoryHelpers.GetDbContextOptions(), new DatabaseSeeder(new CryptographySuite(new PasswordHasher())), new LoggerFactory().CreateLogger()); } } diff --git a/src/Tgstation.Server.Host/Models/Migrations/SqlServerDesignTimeDbContextFactory.cs b/src/Tgstation.Server.Host/Models/Migrations/SqlServerDesignTimeDbContextFactory.cs index 75a1b97053..bf903b09c8 100644 --- a/src/Tgstation.Server.Host/Models/Migrations/SqlServerDesignTimeDbContextFactory.cs +++ b/src/Tgstation.Server.Host/Models/Migrations/SqlServerDesignTimeDbContextFactory.cs @@ -1,6 +1,7 @@ using Microsoft.AspNetCore.Identity; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Design; +using Microsoft.Extensions.Logging; using Tgstation.Server.Host.Security; namespace Tgstation.Server.Host.Models.Migrations @@ -9,6 +10,6 @@ namespace Tgstation.Server.Host.Models.Migrations sealed class SqlServerDesignTimeDbContextFactory : IDesignTimeDbContextFactory { /// - public SqlServerDatabaseContext CreateDbContext(string[] args) => new SqlServerDatabaseContext(new DbContextOptions(), DesignTimeDbContextFactoryHelpers.GetDbContextOptions(), new DatabaseSeeder(new CryptographySuite(new PasswordHasher()))); + public SqlServerDatabaseContext CreateDbContext(string[] args) => new SqlServerDatabaseContext(new DbContextOptions(), DesignTimeDbContextFactoryHelpers.GetDbContextOptions(), new DatabaseSeeder(new CryptographySuite(new PasswordHasher())), new LoggerFactory().CreateLogger()); } } diff --git a/src/Tgstation.Server.Host/Models/MySqlDatabaseContext.cs b/src/Tgstation.Server.Host/Models/MySqlDatabaseContext.cs index 0f70b29fcd..eb6f0c81cd 100644 --- a/src/Tgstation.Server.Host/Models/MySqlDatabaseContext.cs +++ b/src/Tgstation.Server.Host/Models/MySqlDatabaseContext.cs @@ -1,4 +1,5 @@ using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Tgstation.Server.Host.Configuration; @@ -15,7 +16,8 @@ namespace Tgstation.Server.Host.Models /// The for the /// The of for the /// The for the - public MySqlDatabaseContext(DbContextOptions dbContextOptions, IOptions databaseConfiguration, IDatabaseSeeder databaseSeeder) : base(dbContextOptions, databaseConfiguration, databaseSeeder) + /// The for the + public MySqlDatabaseContext(DbContextOptions dbContextOptions, IOptions databaseConfiguration, IDatabaseSeeder databaseSeeder, ILogger logger) : base(dbContextOptions, databaseConfiguration, databaseSeeder, logger) { } /// diff --git a/src/Tgstation.Server.Host/Models/SqlServerDatabaseContext.cs b/src/Tgstation.Server.Host/Models/SqlServerDatabaseContext.cs index 19f6736cc9..b0f7ae1372 100644 --- a/src/Tgstation.Server.Host/Models/SqlServerDatabaseContext.cs +++ b/src/Tgstation.Server.Host/Models/SqlServerDatabaseContext.cs @@ -1,4 +1,5 @@ using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Tgstation.Server.Host.Configuration; @@ -15,7 +16,8 @@ namespace Tgstation.Server.Host.Models /// The for the /// The of for the /// The for the - public SqlServerDatabaseContext(DbContextOptions dbContextOptions, IOptions databaseConfiguration, IDatabaseSeeder databaseSeeder) : base(dbContextOptions, databaseConfiguration, databaseSeeder) + /// The for the + public SqlServerDatabaseContext(DbContextOptions dbContextOptions, IOptions databaseConfiguration, IDatabaseSeeder databaseSeeder, ILogger logger) : base(dbContextOptions, databaseConfiguration, databaseSeeder, logger) { } /// diff --git a/src/Tgstation.Server.Host/Models/SqliteDatabaseContext.cs b/src/Tgstation.Server.Host/Models/SqliteDatabaseContext.cs index 011893939c..9af5b1be9a 100644 --- a/src/Tgstation.Server.Host/Models/SqliteDatabaseContext.cs +++ b/src/Tgstation.Server.Host/Models/SqliteDatabaseContext.cs @@ -11,22 +11,15 @@ namespace Tgstation.Server.Host.Models /// sealed class SqliteDatabaseContext : DatabaseContext { - /// - /// The for the - /// - readonly ILogger logger; - /// /// Construct a /// /// The for the /// The of for the /// The for the - /// The value of - public SqliteDatabaseContext(DbContextOptions dbContextOptions, IOptions databaseConfiguration, IDatabaseSeeder databaseSeeder, ILogger logger) : base(dbContextOptions, databaseConfiguration, databaseSeeder) - { - this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } + /// The for the + public SqliteDatabaseContext(DbContextOptions dbContextOptions, IOptions databaseConfiguration, IDatabaseSeeder databaseSeeder, ILogger logger) : base(dbContextOptions, databaseConfiguration, databaseSeeder, logger) + { } /// protected override void OnConfiguring(DbContextOptionsBuilder options) @@ -35,7 +28,7 @@ namespace Tgstation.Server.Host.Models //on the off chance that connection string is null here we default to a db file next to the executable since this is the default database if (ConnectionString == null) { - logger.LogWarning("No database configured! Defaulting to SQLite in the working directory!"); + Logger.LogWarning("No database configured! Defaulting to SQLite in the working directory!"); options.UseSqlite("Data Source=TgsDatabase.db3"); } else From acf891498c22ea450e39ba0af404afd6e691d61d Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Tue, 7 Aug 2018 13:40:45 -0400 Subject: [PATCH 08/36] Fixes SessionController disposal --- .../Components/Watchdog/SessionController.cs | 44 ++++++++++++++++--- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Watchdog/SessionController.cs b/src/Tgstation.Server.Host/Components/Watchdog/SessionController.cs index b2ecc29db6..613b53d264 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/SessionController.cs @@ -145,6 +145,11 @@ namespace Tgstation.Server.Host.Components.Watchdog /// bool apiValidated; + /// + /// If should be kept alive instead + /// + bool released; + /// /// Construct a /// @@ -171,22 +176,50 @@ namespace Tgstation.Server.Host.Components.Watchdog portClosed = false; disposed = false; apiValidated = false; + released = false; rebootTcs = new TaskCompletionSource(); } + /// + /// Finalize the + /// + ~SessionController() => Dispose(false); + /// public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + /// + void Dispose(bool disposing) { lock (this) { if (disposed) return; - session.Dispose(); - interopContext.Dispose(); - Dmb?.Dispose(); //will be null when released - chatJsonTrackingContext.Dispose(); - disposed = true; + if (disposing) + { + if (!released) + session.Terminate(); + session.Dispose(); + interopContext.Dispose(); + Dmb?.Dispose(); //will be null when released + chatJsonTrackingContext.Dispose(); + disposed = true; + } + else + { + if (logger != null) + logger.LogError("Being disposed via finalizer!"); + if (!released) + if (session != null) + session.Terminate(); + else if (logger != null) + logger.LogCritical("Unable to terminate active DreamDaemon session due to finalizer ordering!"); + } } } @@ -256,6 +289,7 @@ namespace Tgstation.Server.Host.Components.Watchdog //we still don't want to dispose the dmb yet, even though we're keeping it alive var tmpProvider = reattachInformation.Dmb; reattachInformation.Dmb = null; + released = true; Dispose(); Dmb.KeepAlive(); reattachInformation.Dmb = tmpProvider; From a526ff0138109c2680e5a06a3335cae9d3a0866d Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Tue, 7 Aug 2018 13:41:08 -0400 Subject: [PATCH 09/36] Log cleaned game directories at init --- .../Components/Compiler/DmbFactory.cs | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs b/src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs index 2835269fe8..648a7a095a 100644 --- a/src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs +++ b/src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs @@ -205,20 +205,24 @@ namespace Tgstation.Server.Host.Components.Compiler //cleanup var directories = await ioManager.GetDirectories(".", cancellationToken).ConfigureAwait(false); - await Task.WhenAll(directories.Select(async x => + if (directories.Count > 0) { - var nameOnly = ioManager.GetFileName(x); - if (jobUidsToNotErase.Contains(nameOnly.ToUpperInvariant())) - return; - try + logger.LogDebug("Cleaning {0} unused game folders...", directories.Count); + await Task.WhenAll(directories.Select(async x => { - await ioManager.DeleteDirectory(x, cancellationToken).ConfigureAwait(false); - } - catch (Exception e) - { - logger.LogWarning("Error deleting directory {0}! Exception: {1}", x, e); - } - })).ConfigureAwait(false); + var nameOnly = ioManager.GetFileName(x); + if (jobUidsToNotErase.Contains(nameOnly.ToUpperInvariant())) + return; + try + { + await ioManager.DeleteDirectory(x, cancellationToken).ConfigureAwait(false); + } + catch (Exception e) + { + logger.LogWarning("Error deleting directory {0}! Exception: {1}", x, e); + } + })).ConfigureAwait(false); + } } } } From 1d62bb7e2114360e3094d60f7b3c91c637395754 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Tue, 7 Aug 2018 13:41:26 -0400 Subject: [PATCH 10/36] Remove spammy DisposeAndNullController log --- src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs index 315685ad42..75cea931bb 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs @@ -182,7 +182,6 @@ namespace Tgstation.Server.Host.Components.Watchdog /// void DisposeAndNullControllers() { - logger.LogTrace("DisposeAndNullControllers"); alphaServer?.Dispose(); alphaServer = null; bravoServer?.Dispose(); From 076b80595baab60bb3665ce4fccb30b369399d3c Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Tue, 7 Aug 2018 13:42:39 -0400 Subject: [PATCH 11/36] Database migration style now determined by config --- .gitignore | 1 - .../Configuration/DatabaseConfiguration.cs | 5 ++ .../Models/DatabaseContext.cs | 52 +++++++++++-------- .../Tgstation.Server.Host.csproj | 3 ++ .../appsettings.Development.json | 8 +++ src/Tgstation.Server.Host/appsettings.json | 1 + 6 files changed, 48 insertions(+), 22 deletions(-) create mode 100644 src/Tgstation.Server.Host/appsettings.Development.json diff --git a/.gitignore b/.gitignore index 5074f2b86e..a8f3e2c779 100644 --- a/.gitignore +++ b/.gitignore @@ -11,7 +11,6 @@ artifacts/ *DS_Store *.sln.ide /TestResults -/src/Tgstation.Server.Host/appsettings.Development.json /tests/DMAPI/travistester.lk /tests/DMAPI/travistester.int /tests/DMAPI/travistester.dmb diff --git a/src/Tgstation.Server.Host/Configuration/DatabaseConfiguration.cs b/src/Tgstation.Server.Host/Configuration/DatabaseConfiguration.cs index a75d463a48..07f8690601 100644 --- a/src/Tgstation.Server.Host/Configuration/DatabaseConfiguration.cs +++ b/src/Tgstation.Server.Host/Configuration/DatabaseConfiguration.cs @@ -24,5 +24,10 @@ /// The connection string for the database /// public string ConnectionString { get; set; } + + /// + /// If the database should use direct table creation instead of automatic migrations. Should not be used in production! + /// + public bool NoMigrations { get; set; } } } diff --git a/src/Tgstation.Server.Host/Models/DatabaseContext.cs b/src/Tgstation.Server.Host/Models/DatabaseContext.cs index 9165ab4f15..67d44688a1 100644 --- a/src/Tgstation.Server.Host/Models/DatabaseContext.cs +++ b/src/Tgstation.Server.Host/Models/DatabaseContext.cs @@ -2,9 +2,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using System; -#if !DEBUG using System.Linq; -#endif using System.Threading; using System.Threading.Tasks; using Tgstation.Server.Host.Configuration; @@ -47,22 +45,22 @@ namespace Tgstation.Server.Host.Models /// public DbSet Jobs { get; set; } + /// + public DbSet ReattachInformations { get; set; } + + /// + public DbSet WatchdogReattachInformations { get; set; } + /// /// The s in the /// public DbSet TestMerges { get; set; } - /// - public DbSet ReattachInformations { get; set; } - /// /// The s om the /// public DbSet RevInfoTestMerges { get; set; } - /// - public DbSet WatchdogReattachInformations { get; set; } - /// /// The for the /// @@ -77,6 +75,7 @@ namespace Tgstation.Server.Host.Models /// The for the /// readonly DatabaseConfiguration databaseConfiguration; + /// /// The for the /// @@ -99,7 +98,7 @@ namespace Tgstation.Server.Host.Models /// protected override void OnModelCreating(ModelBuilder modelBuilder) { - Logger.LogDebug("Building entity framework context..."); + Logger.LogTrace("Building entity framework context..."); base.OnModelCreating(modelBuilder); var userModel = modelBuilder.Entity(); @@ -138,24 +137,35 @@ namespace Tgstation.Server.Host.Models public async Task Initialize(CancellationToken cancellationToken) { Logger.LogInformation("Migrating database..."); -#if DEBUG - Logger.LogWarning("Running in debug mode. Using all or nothing strategy!"); - await Database.EnsureCreatedAsync().ConfigureAwait(false); - var wasEmpty = (await Users.CountAsync().ConfigureAwait(false)) == 0; -#else - var migrations = await Database.GetAppliedMigrationsAsync().ConfigureAwait(false); - var wasEmpty = !migrations.Any(); - await Database.MigrateAsync(cancellationToken).ConfigureAwait(false); -#endif + + var wasEmpty = false; + if (!databaseConfiguration.NoMigrations) + { + Logger.LogWarning("Running in debug mode. Using all or nothing migration strategy!"); + await Database.EnsureCreatedAsync(cancellationToken).ConfigureAwait(false); + } + else + { + var migrations = await Database.GetAppliedMigrationsAsync(cancellationToken).ConfigureAwait(false); + wasEmpty = !migrations.Any(); + await Database.MigrateAsync(cancellationToken).ConfigureAwait(false); + } + + wasEmpty |= (await Users.CountAsync(cancellationToken).ConfigureAwait(false)) == 0; + if (wasEmpty) { Logger.LogInformation("Seeding database..."); await databaseSeeder.SeedDatabase(this, cancellationToken).ConfigureAwait(false); } - else if (databaseConfiguration.ResetAdminPassword) + else { - Logger.LogWarning("Enabling and resetting admin password due to configuration!"); - await databaseSeeder.ResetAdminPassword(this, cancellationToken).ConfigureAwait(false); + Logger.LogDebug("No migrations applied!"); + if (databaseConfiguration.ResetAdminPassword) + { + Logger.LogWarning("Enabling and resetting admin password due to configuration!"); + await databaseSeeder.ResetAdminPassword(this, cancellationToken).ConfigureAwait(false); + } } } diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index 575d65fa34..a68eac4eb5 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -54,6 +54,9 @@ + + PreserveNewest + PreserveNewest diff --git a/src/Tgstation.Server.Host/appsettings.Development.json b/src/Tgstation.Server.Host/appsettings.Development.json new file mode 100644 index 0000000000..d40e70e835 --- /dev/null +++ b/src/Tgstation.Server.Host/appsettings.Development.json @@ -0,0 +1,8 @@ +{ + "General": { + "DisableFileLogging": true + }, + "Database": { + "NoMigrations": true + } +} diff --git a/src/Tgstation.Server.Host/appsettings.json b/src/Tgstation.Server.Host/appsettings.json index f97d9bf4e7..1c91417297 100644 --- a/src/Tgstation.Server.Host/appsettings.json +++ b/src/Tgstation.Server.Host/appsettings.json @@ -29,6 +29,7 @@ "UpdatePackageAssetName": "ServerUpdatePackage.zip" }, "Database": { + "NoMigrations": false, "DatabaseType": "SqlServer", "ResetAdminPassword": false, "ConnectionString": "Data Source=(local);Initial Catalog=TGS;Integrated Security=True" From 66bd039ba86c1362a6a1552b76acb58cc8f2d95d Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Tue, 7 Aug 2018 13:42:51 -0400 Subject: [PATCH 12/36] Fix grammar --- src/Tgstation.Server.Host/Core/JobManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Core/JobManager.cs b/src/Tgstation.Server.Host/Core/JobManager.cs index 4dc1d28461..bb2833d916 100644 --- a/src/Tgstation.Server.Host/Core/JobManager.cs +++ b/src/Tgstation.Server.Host/Core/JobManager.cs @@ -93,7 +93,7 @@ namespace Tgstation.Server.Host.Core } catch (OperationCanceledException) { - logger.LogDebug("Job {0} exited with cancelled!", job.Id); + logger.LogDebug("Job {0} cancelled!", job.Id); job.Cancelled = true; } catch (Exception e) From 98af3958680b3627f80805d7da78aa084b13977f Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Tue, 7 Aug 2018 13:43:12 -0400 Subject: [PATCH 13/36] Demote Configure log to Trace --- src/Tgstation.Server.Host/Core/Application.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index 028f60509b..79b103979b 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -243,7 +243,7 @@ namespace Tgstation.Server.Host.Core logger.LogInformation(VersionString); - logger.LogDebug("Configuring middleware..."); + logger.LogTrace("Configuring middleware..."); serverAddresses = applicationBuilder.ServerFeatures.Get(); From a4eb214d958532fa7c77dc7c3dc5a5bd55d5e0f6 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Tue, 7 Aug 2018 13:44:37 -0400 Subject: [PATCH 14/36] Remove support for Ultrasafe configurations due to the fucking file read prompts --- .../Models/CompilerStatus.cs | 8 +-- .../Models/DreamDaemonSecurity.cs | 2 +- .../Components/Compiler/DreamMaker.cs | 60 ++++++++++++++++--- .../Components/Compiler/IDreamMaker.cs | 5 +- .../Components/EventType.cs | 2 +- .../Components/Instance.cs | 12 ++-- .../Controllers/DreamDaemonController.cs | 3 + .../Controllers/DreamMakerController.cs | 9 ++- .../Controllers/InstanceController.cs | 2 +- 9 files changed, 78 insertions(+), 25 deletions(-) diff --git a/src/Tgstation.Server.Api/Models/CompilerStatus.cs b/src/Tgstation.Server.Api/Models/CompilerStatus.cs index dc71d316f9..9cb0a7a5ed 100644 --- a/src/Tgstation.Server.Api/Models/CompilerStatus.cs +++ b/src/Tgstation.Server.Api/Models/CompilerStatus.cs @@ -32,6 +32,10 @@ /// Verifying, /// + /// Post-compile scripts are running + /// + PostCompile, + /// /// The compile results are being duplicated /// Duplicating, @@ -40,10 +44,6 @@ /// Symlinking, /// - /// Post-compile scripts are running - /// - PostCompile, - /// /// A failed compile job is being erased /// Cleanup diff --git a/src/Tgstation.Server.Api/Models/DreamDaemonSecurity.cs b/src/Tgstation.Server.Api/Models/DreamDaemonSecurity.cs index 094332d1e4..68804af41b 100644 --- a/src/Tgstation.Server.Api/Models/DreamDaemonSecurity.cs +++ b/src/Tgstation.Server.Api/Models/DreamDaemonSecurity.cs @@ -14,7 +14,7 @@ /// Safe, /// - /// Server will not be able to run shell commands or access anything but temporary files + /// Server will not be able to run shell commands or access anything but temporary files. Currently unsupported! /// Ultrasafe } diff --git a/src/Tgstation.Server.Host/Components/Compiler/DreamMaker.cs b/src/Tgstation.Server.Host/Components/Compiler/DreamMaker.cs index 858bc3792b..4a62f8130e 100644 --- a/src/Tgstation.Server.Host/Components/Compiler/DreamMaker.cs +++ b/src/Tgstation.Server.Host/Components/Compiler/DreamMaker.cs @@ -100,22 +100,24 @@ namespace Tgstation.Server.Host.Components.Compiler /// Run a quick DD instance to test the DMAPI is installed on the target code /// /// The timeout in seconds for validation + /// The level to use to validate the API /// The for the operation /// The current /// The for the operation /// A resulting in if the DMAPI was successfully validated, otherwise - async Task VerifyApi(uint timeout, Models.CompileJob job, IByondExecutableLock byondLock, CancellationToken cancellationToken) + async Task VerifyApi(uint timeout, DreamDaemonSecurity securityLevel, Models.CompileJob job, IByondExecutableLock byondLock, CancellationToken cancellationToken) { + logger.LogTrace("Verifying DMAPI..."); var launchParameters = new DreamDaemonLaunchParameters { AllowWebClient = false, PrimaryPort = 0, //pick any port - SecurityLevel = DreamDaemonSecurity.Safe, //all it needs to read the file and exit + SecurityLevel = securityLevel, //all it needs to read the file and exit StartupTimeout = timeout }; var dirA = ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName); - var provider = new TemporaryDmbProvider(ioManager.ResolvePath(dirA), String.Join('.', job.DmeName, DmbExtension), job); + var provider = new TemporaryDmbProvider(ioManager.ResolvePath(dirA), String.Concat(job.DmeName, DmbExtension), job); var timeoutAt = DateTimeOffset.Now.AddSeconds(timeout); using (var controller = await sessionControllerFactory.LaunchNew(launchParameters, provider, byondLock, true, true, true, cancellationToken).ConfigureAwait(false)) @@ -129,9 +131,14 @@ namespace Tgstation.Server.Host.Components.Compiler cancellationToken.ThrowIfCancellationRequested(); } if (!controller.Lifetime.IsCompleted) + { + logger.LogDebug("API validation timed out!"); return false; + } - return controller.ApiValidated; + var validated = controller.ApiValidated; + logger.LogTrace("API valid: {0}", validated); + return validated; } } @@ -167,6 +174,7 @@ namespace Tgstation.Server.Host.Components.Compiler var dmTcs = new TaskCompletionSource(); dm.Exited += (a, b) => dmTcs.TrySetResult(null); + logger.LogTrace("Running DreamMaker..."); dm.Start(); dm.BeginOutputReadLine(); dm.BeginErrorReadLine(); @@ -184,8 +192,10 @@ namespace Tgstation.Server.Host.Components.Compiler } } - job.Output = OutputList.ToString(); job.ExitCode = dm.ExitCode; + logger.LogDebug("DreamMaker exit code: {0}", job.ExitCode); + job.Output = OutputList.ToString(); + logger.LogTrace("DreamMaker output: {0}", job.Output); } } @@ -209,7 +219,18 @@ namespace Tgstation.Server.Host.Components.Compiler var dmeModifications = await dmeModificationsTask.ConfigureAwait(false); if (dmeModifications == null || dmeModifications.TotalDmeOverwrite) + { + if (dmeModifications != null) + logger.LogDebug(".dme replacement configured!"); + else + logger.LogTrace("No .dme modifications required."); return; + } + + if (dmeModifications.HeadIncludeLine != null) + logger.LogDebug("Head .dme include line: {0}", dmeModifications.HeadIncludeLine); + if (dmeModifications.TailIncludeLine != null) + logger.LogDebug("Tail .dme include line: {0}", dmeModifications.TailIncludeLine); var dmeLines = new List(dme.Split(new[] { Environment.NewLine }, StringSplitOptions.None)); for (var I = 0; I < dmeLines.Count; ++I) @@ -232,8 +253,14 @@ namespace Tgstation.Server.Host.Components.Compiler } /// - public async Task Compile(string projectName, uint apiValidateTimeout, IRepository repository, CancellationToken cancellationToken) + public async Task Compile(string projectName, DreamDaemonSecurity securityLevel, uint apiValidateTimeout, IRepository repository, CancellationToken cancellationToken) { + if (repository == null) + throw new ArgumentNullException(nameof(repository)); + + if (securityLevel == DreamDaemonSecurity.Ultrasafe) + throw new ArgumentOutOfRangeException(nameof(securityLevel), securityLevel, "Cannot compile with ultrasafe security!"); + logger.LogTrace("Begin Compile"); var job = new Models.CompileJob @@ -242,11 +269,14 @@ namespace Tgstation.Server.Host.Components.Compiler DmeName = projectName }; + logger.LogTrace("Compile output GUID: {0}", job.DirectoryName); + lock (this) { if(Status != CompilerStatus.Idle) { job.Output = "There is already a compile in progress!"; + logger.LogInformation(job.Output); return job; } @@ -260,6 +290,7 @@ namespace Tgstation.Server.Host.Components.Compiler async Task CleanupFailedCompile() { + logger.LogTrace("Cleaning compile directory..."); Status = CompilerStatus.Cleanup; try { @@ -273,6 +304,7 @@ namespace Tgstation.Server.Host.Components.Compiler try { + logger.LogTrace("Copying repository to game directory..."); //copy the repository var fullDirA = ioManager.ResolvePath(dirA); var repoOrigin = repository.Origin; @@ -287,16 +319,20 @@ namespace Tgstation.Server.Host.Components.Compiler if (job.DmeName == null) { + logger.LogTrace("Searching for available .dmes..."); var path = (await ioManager.GetFilesWithExtension(dirA, DmeExtension, cancellationToken).ConfigureAwait(false)).FirstOrDefault(); if (path == default) { job.Output = "Unable to find any .dme!"; + logger.LogWarning(job.Output); return job; } var dmeWithExtension = ioManager.GetFileName(path); job.DmeName = dmeWithExtension.Substring(0, dmeWithExtension.Length - DmeExtension.Length - 1); } + logger.LogDebug("Selected {0}.dme for compilation!", job.DmeName); + await ModifyDme(job, cancellationToken).ConfigureAwait(false); Status = CompilerStatus.Compiling; @@ -311,7 +347,7 @@ namespace Tgstation.Server.Host.Components.Compiler Status = CompilerStatus.Verifying; - ddVerified = job.ExitCode == 0 && await VerifyApi(apiValidateTimeout, job, byondLock, cancellationToken).ConfigureAwait(false); + ddVerified = job.ExitCode == 0 && await VerifyApi(apiValidateTimeout, securityLevel, job, byondLock, cancellationToken).ConfigureAwait(false); } if (!ddVerified) @@ -324,11 +360,17 @@ namespace Tgstation.Server.Host.Components.Compiler { job.DMApiValidated = true; + logger.LogTrace("Running post compile event..."); + Status = CompilerStatus.PostCompile; + await eventConsumer.HandleEvent(EventType.CompileComplete, new List { ioManager.ResolvePath(ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName)) }, cancellationToken).ConfigureAwait(false); + + logger.LogTrace("Duplicating compiled game..."); Status = CompilerStatus.Duplicating; //duplicate the dmb et al await ioManager.CopyDirectory(dirA, dirB, null, cancellationToken).ConfigureAwait(false); + logger.LogTrace("Applying static game file symlinks..."); Status = CompilerStatus.Symlinking; //symlink in the static data @@ -336,8 +378,8 @@ namespace Tgstation.Server.Host.Components.Compiler var symBTask = configuration.SymlinkStaticFilesTo(ioManager.ResolvePath(dirB), cancellationToken); await Task.WhenAll(symATask, symBTask).ConfigureAwait(false); - Status = CompilerStatus.PostCompile; - await eventConsumer.HandleEvent(EventType.CompileComplete, null, cancellationToken).ConfigureAwait(false); + + logger.LogDebug("Compile complete!"); } return job; } diff --git a/src/Tgstation.Server.Host/Components/Compiler/IDreamMaker.cs b/src/Tgstation.Server.Host/Components/Compiler/IDreamMaker.cs index da6878c667..d1e31dad33 100644 --- a/src/Tgstation.Server.Host/Components/Compiler/IDreamMaker.cs +++ b/src/Tgstation.Server.Host/Components/Compiler/IDreamMaker.cs @@ -1,7 +1,7 @@ using System.Threading; using System.Threading.Tasks; +using Tgstation.Server.Api.Models; using Tgstation.Server.Host.Components.Repository; -using Tgstation.Server.Host.Models; namespace Tgstation.Server.Host.Components.Compiler { @@ -19,10 +19,11 @@ namespace Tgstation.Server.Host.Components.Compiler /// Starts a compile /// /// The optional name of the .dme to compile without the extension if not pre + /// The level allowed for API validation /// The time in seconds to wait while validating the API /// The to copy from /// The for the operation /// A resulting in the partially populated for the operation. In particular, note the field will only have it's field populated - Task Compile(string projectName, uint apiValidateTimeout, IRepository repository, CancellationToken cancellationToken); + Task Compile(string projectName, DreamDaemonSecurity securityLevel, uint apiValidateTimeout, IRepository repository, CancellationToken cancellationToken); } } \ No newline at end of file diff --git a/src/Tgstation.Server.Host/Components/EventType.cs b/src/Tgstation.Server.Host/Components/EventType.cs index 89e5076fd6..57de1c2bac 100644 --- a/src/Tgstation.Server.Host/Components/EventType.cs +++ b/src/Tgstation.Server.Host/Components/EventType.cs @@ -51,7 +51,7 @@ /// CompileFailure = 10, /// - /// No parameters + /// Parameters: Game directory path /// CompileComplete = 11, diff --git a/src/Tgstation.Server.Host/Components/Instance.cs b/src/Tgstation.Server.Host/Components/Instance.cs index a47476351f..f5baaa0b6d 100644 --- a/src/Tgstation.Server.Host/Components/Instance.cs +++ b/src/Tgstation.Server.Host/Components/Instance.cs @@ -122,15 +122,19 @@ namespace Tgstation.Server.Host.Components RepositorySettings repositorySettings = null; string projectName = null; - uint timeout = 0; + DreamDaemonSettings ddSettings = null; var dbTask = databaseContextFactory.UseContext(async (db) => { var instanceQuery = db.Instances.Where(x => x.Id == metadata.Id); - var timeoutTask = instanceQuery.Select(x => x.DreamDaemonSettings.StartupTimeout).FirstAsync(cancellationToken); + var ddSettingsTask = instanceQuery.Select(x => x.DreamDaemonSettings).Select(x => new DreamDaemonSettings + { + StartupTimeout = x.StartupTimeout, + SecurityLevel = x.SecurityLevel + }).FirstAsync(cancellationToken); var projectNameTask = instanceQuery.Select(x => x.DreamMakerSettings.ProjectName).FirstOrDefaultAsync(cancellationToken); repositorySettings = await instanceQuery.Select(x => x.RepositorySettings).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); projectName = await projectNameTask.ConfigureAwait(false); - timeout = (await timeoutTask.ConfigureAwait(false)).Value; + ddSettings = await ddSettingsTask.ConfigureAwait(false); }); using (var repo = await RepositoryManager.LoadRepository(cancellationToken).ConfigureAwait(false)) { @@ -170,7 +174,7 @@ namespace Tgstation.Server.Host.Components if (repositorySettings.AutoUpdatesSynchronize.Value && startSha != repo.Head) await repo.Sychronize(repositorySettings.AccessUser, repositorySettings.AccessToken, shouldSyncTracked, cancellationToken).ConfigureAwait(false); - var job = await DreamMaker.Compile(projectName, timeout, repo, cancellationToken).ConfigureAwait(false); + var job = await DreamMaker.Compile(projectName, ddSettings.SecurityLevel.Value, ddSettings.StartupTimeout.Value, repo, cancellationToken).ConfigureAwait(false); } } catch (OperationCanceledException) { } diff --git a/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs b/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs index 5e9cf98168..91b434a737 100644 --- a/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs +++ b/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs @@ -169,6 +169,9 @@ namespace Tgstation.Server.Host.Controllers || !CheckModified(x => x.SoftShutdown, DreamDaemonRights.SoftShutdown) || !CheckModified(x => x.StartupTimeout, DreamDaemonRights.SetStartupTimeout)) return Forbid(); + + if (current.SecurityLevel == DreamDaemonSecurity.Ultrasafe) + return BadRequest(new ErrorMessage { Message = "TGS does not support the ultrasafe DreamDaemon configuration!" }); var wd = instanceManager.GetInstance(Instance).Watchdog; diff --git a/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs b/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs index 8b666c2a8b..4c7502d025 100644 --- a/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs +++ b/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs @@ -101,9 +101,12 @@ namespace Tgstation.Server.Host.Controllers var instanceManager = serviceProvider.GetRequiredService(); var databaseContext = serviceProvider.GetRequiredService(); - var timeoutTask = databaseContext.DreamDaemonSettings.Where(x => x.InstanceId == instanceModel.Id).Select(x => x.StartupTimeout).FirstOrDefaultAsync(cancellationToken); + var ddSettingsTask = databaseContext.DreamDaemonSettings.Where(x => x.InstanceId == instanceModel.Id).Select(x => new DreamDaemonSettings{ + StartupTimeout = x.StartupTimeout, + SecurityLevel = x.SecurityLevel + }).FirstOrDefaultAsync(cancellationToken); var projectName = await databaseContext.DreamMakerSettings.Where(x => x.InstanceId == instanceModel.Id).Select(x => x.ProjectName).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); - var timeout = await timeoutTask.ConfigureAwait(false); + var ddSettings = await ddSettingsTask.ConfigureAwait(false); var instance = instanceManager.GetInstance(instanceModel); @@ -119,7 +122,7 @@ namespace Tgstation.Server.Host.Controllers } repoSha = repo.Head; revInfoTask = databaseContext.RevisionInformations.Where(x => x.CommitSha == repoSha).Select(x => new RevisionInformation { Id = x.Id }).FirstOrDefaultAsync(); - compileJob = await instance.DreamMaker.Compile(projectName, timeout.Value, repo, cancellationToken).ConfigureAwait(false); + compileJob = await instance.DreamMaker.Compile(projectName, ddSettings.SecurityLevel.Value, ddSettings.StartupTimeout.Value, repo, cancellationToken).ConfigureAwait(false); } if (compileJob.DMApiValidated != true) diff --git a/src/Tgstation.Server.Host/Controllers/InstanceController.cs b/src/Tgstation.Server.Host/Controllers/InstanceController.cs index b1430a4fd1..a25768c465 100644 --- a/src/Tgstation.Server.Host/Controllers/InstanceController.cs +++ b/src/Tgstation.Server.Host/Controllers/InstanceController.cs @@ -117,7 +117,7 @@ namespace Tgstation.Server.Host.Controllers AutoStart = false, PrimaryPort = 1337, SecondaryPort = 1338, - SecurityLevel = DreamDaemonSecurity.Ultrasafe, + SecurityLevel = DreamDaemonSecurity.Safe, SoftRestart = false, SoftShutdown = false, StartupTimeout = 20 From 757ceb7154719952ce9ee36b49adc124d445e84d Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Tue, 7 Aug 2018 13:44:55 -0400 Subject: [PATCH 15/36] Use system null for logging user ID's --- src/Tgstation.Server.Host/Controllers/ApiController.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Controllers/ApiController.cs b/src/Tgstation.Server.Host/Controllers/ApiController.cs index e013a02d7e..3574f228cd 100644 --- a/src/Tgstation.Server.Host/Controllers/ApiController.cs +++ b/src/Tgstation.Server.Host/Controllers/ApiController.cs @@ -188,7 +188,7 @@ namespace Tgstation.Server.Host.Controllers } } - Logger.LogTrace("Request made by User ID {0}. Api version: {1}. User-Agent: {2}. Type: {3}. Route {4}", AuthenticationContext?.User.Id.ToString(CultureInfo.InvariantCulture) ?? "NULL", ApiHeaders.ApiVersion, ApiHeaders.UserAgent, Request.Method, Request.Path); + Logger.LogTrace("Request made by User ID {0}. Api version: {1}. User-Agent: {2}. Type: {3}. Route {4}", AuthenticationContext?.User.Id.ToString(CultureInfo.InvariantCulture), ApiHeaders.ApiVersion, ApiHeaders.UserAgent, Request.Method, Request.Path); await base.OnActionExecutionAsync(context, next).ConfigureAwait(false); } } From 57ac907c3a236ea5edbea528a0891f012e225148 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Tue, 7 Aug 2018 13:45:24 -0400 Subject: [PATCH 16/36] Fixes login logging. Buffs system identity expiration time --- src/Tgstation.Server.Host/Controllers/HomeController.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Tgstation.Server.Host/Controllers/HomeController.cs b/src/Tgstation.Server.Host/Controllers/HomeController.cs index 08cc614df5..0718e569a4 100644 --- a/src/Tgstation.Server.Host/Controllers/HomeController.cs +++ b/src/Tgstation.Server.Host/Controllers/HomeController.cs @@ -126,9 +126,9 @@ namespace Tgstation.Server.Host.Controllers var token = tokenFactory.CreateToken(user, out var expiry); if (identity != null) - identityCache.CacheSystemIdentity(user, identity, expiry.AddSeconds(10)); //expire the identity slightly after the auth token in case of lag + identityCache.CacheSystemIdentity(user, identity, expiry.AddMinutes(1)); //expire the identity slightly after the auth token in case of lag - Logger.LogDebug("Successfully logged in user {0} ({1})!", user.Id, user.CanonicalName); + Logger.LogDebug("Successfully logged in user {0}!", user.Id); return Json(token); } From fbed911ef357c0d07f0182fda4d5634b52c84328 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Tue, 7 Aug 2018 13:45:36 -0400 Subject: [PATCH 17/36] Adds executor logging --- .../Components/Watchdog/Executor.cs | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Components/Watchdog/Executor.cs b/src/Tgstation.Server.Host/Components/Watchdog/Executor.cs index 5ff46f692c..85ec573591 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/Executor.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/Executor.cs @@ -1,4 +1,5 @@ -using System; +using Microsoft.Extensions.Logging; +using System; using System.Diagnostics; using System.Globalization; using Tgstation.Server.Api.Models; @@ -10,6 +11,11 @@ namespace Tgstation.Server.Host.Components.Watchdog /// sealed class Executor : IExecutor { + /// + /// The for the + /// + readonly ILogger logger; + /// /// Change a given into the appropriate DreamDaemon command line word /// @@ -30,6 +36,15 @@ namespace Tgstation.Server.Host.Components.Watchdog } } + /// + /// Construct an + /// + /// The value of + public Executor(ILogger logger) + { + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + /// public ISession AttachToDreamDaemon(int processId, IByondExecutableLock byondLock) => new Session(Process.GetProcessById(processId), byondLock); @@ -57,6 +72,8 @@ namespace Tgstation.Server.Host.Components.Watchdog launchParameters.AllowWebClient.Value ? "-webclient " : String.Empty, SecurityWord(launchParameters.SecurityLevel.Value), parameters); + + logger.LogTrace("Running DreamDaemon in {0}: {1} {2}", proc.StartInfo.WorkingDirectory, proc.StartInfo.FileName, proc.StartInfo.Arguments); proc.Start(); From 4166ecccae598754492b807e216f45e43d4fc74d Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Tue, 7 Aug 2018 13:45:54 -0400 Subject: [PATCH 18/36] No more critical logs if application startup is cancelled --- src/Tgstation.Server.Host/Components/InstanceManager.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Tgstation.Server.Host/Components/InstanceManager.cs b/src/Tgstation.Server.Host/Components/InstanceManager.cs index 6259ad60e5..6c61afd878 100644 --- a/src/Tgstation.Server.Host/Components/InstanceManager.cs +++ b/src/Tgstation.Server.Host/Components/InstanceManager.cs @@ -181,6 +181,10 @@ namespace Tgstation.Server.Host.Components logger.LogInformation("Instance manager ready!"); application.Ready(null); } + catch (OperationCanceledException) + { + logger.LogInformation("Cancelled instance manager initialization!"); + } catch (Exception e) { logger.LogCritical("Instance manager startup error! Exception: {0}", e); From 8900a2cb80463d4aeb83661b222e55a85ee57f89 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Tue, 7 Aug 2018 20:14:13 -0400 Subject: [PATCH 19/36] Fix V4 API not activating --- src/DMAPI/tgs/core/core.dm | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/DMAPI/tgs/core/core.dm b/src/DMAPI/tgs/core/core.dm index 198ddc7d07..bd44452413 100644 --- a/src/DMAPI/tgs/core/core.dm +++ b/src/DMAPI/tgs/core/core.dm @@ -33,6 +33,10 @@ switch(major) if(2) return /datum/tgs_api/v3210 + if(4) + switch(major) + if(0) + return /datum/tgs_api/v4 if(super != null && major != null && minor != null && patch != null && tgs_version > TgsMaximumAPIVersion()) TGS_ERROR_LOG("Detected unknown API version! Defaulting to latest. Update the DMAPI to fix this problem.") From 694755bf0f13c3951e90fc9a4eada04af677f885 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Tue, 7 Aug 2018 20:15:14 -0400 Subject: [PATCH 20/36] Fix validate command --- src/DMAPI/tgs/v4/api.dm | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/DMAPI/tgs/v4/api.dm b/src/DMAPI/tgs/v4/api.dm index 87fb1b80a1..a4263c2446 100644 --- a/src/DMAPI/tgs/v4/api.dm +++ b/src/DMAPI/tgs/v4/api.dm @@ -11,7 +11,7 @@ #define TGS4_COMM_ONLINE "tgs_on" #define TGS4_COMM_IDENTIFY "tgs_ident" -#define TGS4_COMM_VALIDATE "tgs_vali" +#define TGS4_COMM_VALIDATE "tgs_validate" #define TGS4_COMM_SERVER_PRIMED "tgs_prime" #define TGS4_COMM_WORLD_REBOOT "tgs_reboot" #define TGS4_COMM_END_PROCESS "tgs_kill" @@ -92,6 +92,8 @@ ListCustomCommands() + return TRUE + /datum/tgs_api/v4/OnInitializationComplete() Export(TGS4_COMM_SERVER_PRIMED) var/tgs4_secret_sleep_offline_sauce = 24051994 From 615c199cb036c50c3e6f77ad4d52460c61a29a76 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Tue, 7 Aug 2018 20:15:43 -0400 Subject: [PATCH 21/36] Fix interop json serialization --- .../Components/Chat/JsonTrackingContext.cs | 9 ++++++++- .../Components/Watchdog/SessionControllerFactory.cs | 9 ++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Chat/JsonTrackingContext.cs b/src/Tgstation.Server.Host/Components/Chat/JsonTrackingContext.cs index dd552f87ff..c7482f38ed 100644 --- a/src/Tgstation.Server.Host/Components/Chat/JsonTrackingContext.cs +++ b/src/Tgstation.Server.Host/Components/Chat/JsonTrackingContext.cs @@ -1,4 +1,5 @@ using Newtonsoft.Json; +using Newtonsoft.Json.Serialization; using System; using System.Collections.Generic; using System.Text; @@ -58,7 +59,13 @@ namespace Tgstation.Server.Host.Components.Chat public async Task SetChannels(IEnumerable channels, CancellationToken cancellationToken) { using (await SemaphoreSlimContext.Lock(channelsSemaphore, cancellationToken).ConfigureAwait(false)) - await ioManager.WriteAllBytes(channelsPath, Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(channels)), cancellationToken).ConfigureAwait(false); + await ioManager.WriteAllBytes(channelsPath, Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(channels, Formatting.Indented, new JsonSerializerSettings + { + ContractResolver = new DefaultContractResolver + { + NamingStrategy = new CamelCaseNamingStrategy() + } + })), cancellationToken).ConfigureAwait(false); } } } diff --git a/src/Tgstation.Server.Host/Components/Watchdog/SessionControllerFactory.cs b/src/Tgstation.Server.Host/Components/Watchdog/SessionControllerFactory.cs index a0d6cb7c34..45d028d0dc 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/SessionControllerFactory.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/SessionControllerFactory.cs @@ -1,6 +1,7 @@ using Byond.TopicSender; using Microsoft.Extensions.Logging; using Newtonsoft.Json; +using Newtonsoft.Json.Serialization; using System; using System.Globalization; using System.Linq; @@ -134,7 +135,13 @@ namespace Tgstation.Server.Host.Components.Watchdog var interopJsonFile = GuidJsonFile(); - var interopJson = JsonConvert.SerializeObject(interopInfo); + var interopJson = JsonConvert.SerializeObject(interopInfo, Formatting.Indented, new JsonSerializerSettings + { + ContractResolver = new DefaultContractResolver + { + NamingStrategy = new CamelCaseNamingStrategy() + } + }); var basePath = primaryDirectory ? dmbProvider.PrimaryDirectory : dmbProvider.SecondaryDirectory; var localIoManager = new ResolvingIOManager(ioManager, basePath); From 7023baaecd9105cedf8d94d8b5a3304698820e9d Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Tue, 7 Aug 2018 20:16:56 -0400 Subject: [PATCH 22/36] Fix symlinking invocations when the path already exists --- .../Components/StaticFiles/Configuration.cs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs index 92595de192..c62ee59bc5 100644 --- a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs +++ b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs @@ -224,7 +224,19 @@ namespace Tgstation.Server.Host.Components.StaticFiles else task = ioManager.GetDirectories(GameStaticFilesSubdirectory, cancellationToken); var entries = await task.ConfigureAwait(false); - await Task.WhenAll(entries.Select(x => symlinkFactory.CreateSymbolicLink(ioManager.ResolvePath(x), ioManager.ConcatPath(destination, x), cancellationToken))).ConfigureAwait(false); + + await Task.WhenAll(task.Result.Select(async x => + { + var destPath = ioManager.ConcatPath(destination, ioManager.GetFileName(x)); + logger.LogTrace("Symlinking {0} to {1}...", x, destPath); + var fileExistsTask = ioManager.FileExists(destPath, cancellationToken); + if (await ioManager.DirectoryExists(destPath, cancellationToken).ConfigureAwait(false)) + await ioManager.DeleteDirectory(destPath, cancellationToken).ConfigureAwait(false); + var fileExists = await fileExistsTask.ConfigureAwait(false); + if (fileExists) + await ioManager.DeleteFile(destPath, cancellationToken).ConfigureAwait(false); + await symlinkFactory.CreateSymbolicLink(ioManager.ResolvePath(x), ioManager.ResolvePath(destPath), cancellationToken).ConfigureAwait(false); + })).ConfigureAwait(false); } await Task.WhenAll(SymlinkBase(true), SymlinkBase(false)).ConfigureAwait(false); From 62054c049964e796b0e70ced1b669b7d52477992 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Tue, 7 Aug 2018 20:17:09 -0400 Subject: [PATCH 23/36] Remove unused usings --- .../Components/Watchdog/InteropConstants.cs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Watchdog/InteropConstants.cs b/src/Tgstation.Server.Host/Components/Watchdog/InteropConstants.cs index 9b08efcdf5..23ad40c9af 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/InteropConstants.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/InteropConstants.cs @@ -1,8 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace Tgstation.Server.Host.Components.Watchdog +namespace Tgstation.Server.Host.Components.Watchdog { static class InteropConstants { From 00dd5d29f214b4eb8041eabb3e733554fb4da1b6 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Tue, 7 Aug 2018 20:17:46 -0400 Subject: [PATCH 24/36] Catch occasional InvalidOperationException in Session lifetimeTask --- .../Components/Watchdog/Session.cs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Watchdog/Session.cs b/src/Tgstation.Server.Host/Components/Watchdog/Session.cs index b585b32315..a26f14e130 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/Session.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/Session.cs @@ -57,8 +57,16 @@ namespace Tgstation.Server.Host.Components.Watchdog return result; }, default, TaskCreationOptions.LongRunning, TaskScheduler.Current); lifetimeTask = new TaskCompletionSource(); - process.EnableRaisingEvents = true; - process.Exited += (a, b) => lifetimeTask.SetResult(process.ExitCode); + try + { + process.EnableRaisingEvents = true; + process.Exited += (a, b) => lifetimeTask.TrySetResult(process.ExitCode); + } + catch (InvalidOperationException) + { + //dead proccess + lifetimeTask.TrySetResult(process.ExitCode); + } } /// From ce35010c5d8d4caa0cd75c39e015963a3fe02485 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Tue, 7 Aug 2018 20:18:14 -0400 Subject: [PATCH 25/36] Fix selected revision information not being attached to context in compile flow --- src/Tgstation.Server.Host/Controllers/DreamMakerController.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs b/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs index 4c7502d025..856b041626 100644 --- a/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs +++ b/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs @@ -144,6 +144,8 @@ namespace Tgstation.Server.Host.Controllers }; databaseContext.Instances.Attach(compileJob.RevisionInformation.Instance); } + else + databaseContext.RevisionInformations.Attach(compileJob.RevisionInformation); databaseContext.CompileJobs.Add(compileJob); await databaseContext.Save(cancellationToken).ConfigureAwait(false); From 604d80c9c2e12c7a2b1281bf7fcd05be2a4db6c5 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Tue, 7 Aug 2018 20:18:39 -0400 Subject: [PATCH 26/36] Adds method to read latest Job --- src/Tgstation.Server.Host/Controllers/JobController.cs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/Tgstation.Server.Host/Controllers/JobController.cs b/src/Tgstation.Server.Host/Controllers/JobController.cs index 16f628cba4..b946cbe94c 100644 --- a/src/Tgstation.Server.Host/Controllers/JobController.cs +++ b/src/Tgstation.Server.Host/Controllers/JobController.cs @@ -35,6 +35,16 @@ namespace Tgstation.Server.Host.Controllers this.jobManager = jobManager ?? throw new ArgumentNullException(nameof(jobManager)); } + /// + [TgsAuthorize] + public override async Task Read(CancellationToken cancellationToken) + { + var result = await DatabaseContext.Jobs.Where(x => x.Instance.Id == Instance.Id).OrderByDescending(x => x.StartedAt).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + if (result == null) + return StatusCode((int)HttpStatusCode.Gone); + return Json(result); + } + /// [TgsAuthorize] public override async Task List(CancellationToken cancellationToken) From 243cdec07e745b0b6cae6537580ccdb4985c3c52 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Tue, 7 Aug 2018 20:19:04 -0400 Subject: [PATCH 27/36] Adds symlink detection to DeleteDirectory --- src/Tgstation.Server.Host/IO/DefaultIOManager.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/IO/DefaultIOManager.cs b/src/Tgstation.Server.Host/IO/DefaultIOManager.cs index c33e46aca8..95a512d017 100644 --- a/src/Tgstation.Server.Host/IO/DefaultIOManager.cs +++ b/src/Tgstation.Server.Host/IO/DefaultIOManager.cs @@ -34,7 +34,11 @@ namespace Tgstation.Server.Host.IO foreach (var subDir in dir.EnumerateDirectories()) { cancellationToken.ThrowIfCancellationRequested(); - tasks.Add(NormalizeAndDelete(subDir, cancellationToken)); + if (!subDir.Attributes.HasFlag(FileAttributes.Directory) || subDir.Attributes.HasFlag(FileAttributes.ReparsePoint)) + //this is probably a symlink + subDir.Delete(); + else + tasks.Add(NormalizeAndDelete(subDir, cancellationToken)); } foreach (var file in dir.EnumerateFiles()) { From 2440f9cc37ab7ca63c2482b6a7254d4dbd8614a8 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Tue, 7 Aug 2018 20:19:27 -0400 Subject: [PATCH 28/36] JobManager attaches model BEFORE invoking --- src/Tgstation.Server.Host/Core/JobManager.cs | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/Tgstation.Server.Host/Core/JobManager.cs b/src/Tgstation.Server.Host/Core/JobManager.cs index bb2833d916..e4e13aa4d9 100644 --- a/src/Tgstation.Server.Host/Core/JobManager.cs +++ b/src/Tgstation.Server.Host/Core/JobManager.cs @@ -80,15 +80,11 @@ namespace Tgstation.Server.Host.Core { var oldJob = job; job = new Job { Id = oldJob.Id }; - try - { - await operation(job, scope.ServiceProvider, cancellationToken).ConfigureAwait(false); - } - finally - { - databaseContext = scope.ServiceProvider.GetRequiredService(); - databaseContext.Jobs.Attach(job); - } + databaseContext = scope.ServiceProvider.GetRequiredService(); + databaseContext.Jobs.Attach(job); + + await operation(job, scope.ServiceProvider, cancellationToken).ConfigureAwait(false); + logger.LogDebug("Job {0} completed!", job.Id); } catch (OperationCanceledException) From e50a1ab44d8e8df55d991a3af252212b47e8f2ef Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Tue, 7 Aug 2018 20:19:49 -0400 Subject: [PATCH 29/36] Fix interop not using InteropConstants --- src/Tgstation.Server.Host/Components/InstanceManager.cs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/InstanceManager.cs b/src/Tgstation.Server.Host/Components/InstanceManager.cs index 6c61afd878..46b39b8e8c 100644 --- a/src/Tgstation.Server.Host/Components/InstanceManager.cs +++ b/src/Tgstation.Server.Host/Components/InstanceManager.cs @@ -8,6 +8,7 @@ using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; +using Tgstation.Server.Host.Components.Watchdog; using Tgstation.Server.Host.Core; using Tgstation.Server.Host.IO; @@ -16,11 +17,6 @@ namespace Tgstation.Server.Host.Components /// sealed class InstanceManager : IInstanceManager, IHostedService, IInteropRegistrar, IDisposable { - /// - /// HTTP GET query key for interop access identifiers - /// - const string AccessIdentifierQueryKey = "access"; - /// /// The for the /// @@ -222,7 +218,7 @@ namespace Tgstation.Server.Host.Components if (query == null) throw new ArgumentNullException(nameof(query)); - if (!query.TryGetValue(AccessIdentifierQueryKey, out StringValues values)) + if (!query.TryGetValue(InteropConstants.DMInteropAccessIdentifier, out StringValues values)) return null; var accessIdentifier = values.FirstOrDefault(); if (accessIdentifier == default) From 568d169731e121d26b838142f83d9afd3edcf6c1 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Tue, 7 Aug 2018 20:20:03 -0400 Subject: [PATCH 30/36] Spaces => tab --- src/Tgstation.Server.Host/Security/IdentityCache.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Security/IdentityCache.cs b/src/Tgstation.Server.Host/Security/IdentityCache.cs index 81850ab4f2..1d1f19a681 100644 --- a/src/Tgstation.Server.Host/Security/IdentityCache.cs +++ b/src/Tgstation.Server.Host/Security/IdentityCache.cs @@ -31,7 +31,7 @@ namespace Tgstation.Server.Host.Security lock (cachedIdentities) { if (cachedIdentities.TryGetValue(user.Id, out var identCache)) - identCache.Dispose(); //also clears it out + identCache.Dispose(); //also clears it out identCache = new IdentityCacheObject(systemIdentity.Clone(), () => { lock (cachedIdentities) From 1ddce44310b9f451d1e17353a084ff67cabd4313 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Tue, 7 Aug 2018 21:33:34 -0400 Subject: [PATCH 31/36] Remove Job requirement from DmbProviders --- .../Components/Compiler/DmbFactory.cs | 109 +++++++++++------- .../Components/Compiler/IDmbFactory.cs | 5 +- .../Components/ReattachInfoHandler.cs | 3 +- .../Watchdog/ReattachInformation.cs | 8 +- .../Components/Watchdog/Watchdog.cs | 9 +- .../Watchdog/WatchdogReattachInformation.cs | 12 +- 6 files changed, 88 insertions(+), 58 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs b/src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs index 648a7a095a..b2b319473b 100644 --- a/src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs +++ b/src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs @@ -55,7 +55,7 @@ namespace Tgstation.Server.Host.Components.Compiler /// /// resulting in the latest yet to exist /// - TaskCompletionSource newerDmbTcs; + TaskCompletionSource newerDmbTcs; /// /// The latest /// @@ -78,12 +78,13 @@ namespace Tgstation.Server.Host.Components.Compiler this.instance = instance ?? throw new ArgumentNullException(nameof(instance)); cleanupTask = Task.CompletedTask; + newerDmbTcs = new TaskCompletionSource(); cleanupCts = new CancellationTokenSource(); jobLockCounts = new Dictionary(); } /// - public void Dispose() => cleanupCts.Dispose(); + public void Dispose() => cleanupCts.Dispose(); //we don't dispose nextDmbProvider here, since it might be the only thing we have /// /// Delete the of @@ -91,75 +92,71 @@ namespace Tgstation.Server.Host.Components.Compiler /// The to clean void CleanJob(CompileJob job) { + logger.LogTrace("Cleaning compile job {0} => {1}", job.Id, job.DirectoryName); async Task HandleCleanup() { var deleteJob = ioManager.DeleteDirectory(job.DirectoryName.ToString(), cleanupCts.Token); Task otherTask; - lock (this) - otherTask = cleanupTask; + //lock (this) //already locked below + otherTask = cleanupTask; await Task.WhenAll(otherTask, deleteJob).ConfigureAwait(false); } lock (this) { - var currentVal = jobLockCounts[job.Id]; - if (--jobLockCounts[job.Id] == 0) + if (!jobLockCounts.TryGetValue(job.Id, out var currentVal) || --jobLockCounts[job.Id] == 0) + { + jobLockCounts.Remove(job.Id); cleanupTask = HandleCleanup(); + } } } /// - public Task LoadCompileJob(CompileJob job, CancellationToken cancellationToken) => LoadCompileJob(job, true, cancellationToken); - - async Task LoadCompileJob(CompileJob job, bool setAsStagedInDb, CancellationToken cancellationToken) + public async Task LoadCompileJob(CompileJob job, CancellationToken cancellationToken) { if (job == null) throw new ArgumentNullException(nameof(job)); - if (job.DMApiValidated != true || job.Job.Cancelled.Value || job.Job.ExceptionDetails != null || job.Job.StoppedAt == null) + if (job.DMApiValidated != true || job.Job.Cancelled == true || job.Job.ExceptionDetails != null) throw new InvalidOperationException("Cannot load incomplete compile job!"); - if (setAsStagedInDb) - await databaseContextFactory.UseContext(async db => - { - var ddsettings = new DreamDaemonSettings - { - InstanceId = instance.Id - }; - db.DreamDaemonSettings.Attach(ddsettings); - ddsettings.StagedCompileJob = job; - await db.Save(cancellationToken).ConfigureAwait(false); - }).ConfigureAwait(false); + var newProvider = await FromCompileJob(job, cancellationToken).ConfigureAwait(false); + if (newProvider == null) + return; lock (this) { - var oldDmbProvider = nextDmbProvider; - if (oldDmbProvider != null && oldDmbProvider.CompileJob.Job.StoppedAt < oldDmbProvider.CompileJob.Job.StoppedAt) - throw new InvalidOperationException("Loaded compile job older than current job!"); - nextDmbProvider = FromCompileJob(job); + nextDmbProvider?.Dispose(); + nextDmbProvider = newProvider; newerDmbTcs.SetResult(nextDmbProvider); - newerDmbTcs = new TaskCompletionSource(); + newerDmbTcs = new TaskCompletionSource(); } } /// public async Task LockNextDmb(CancellationToken cancellationToken) { - Task task; - lock (this) - if (nextDmbProvider != null) - return nextDmbProvider; - else + if (nextDmbProvider == null) + { + Task task; + lock (this) task = newerDmbTcs.Task; - - return await task.ConfigureAwait(false); + await task.ConfigureAwait(false); + } + lock (this) + { + ++jobLockCounts[nextDmbProvider.CompileJob.Id]; + return nextDmbProvider; + } } /// public Task StartAsync(CancellationToken cancellationToken) => databaseContextFactory.UseContext(async (db) => { //where complete clause not necessary, only successful COMPILEjobs get in the db - var cj = await db.CompileJobs.Where(x => x.Job.Instance.Id == instance.Id).OrderByDescending(x => x.Job.StoppedAt).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + var cj = await db.CompileJobs.Where(x => x.Job.Instance.Id == instance.Id && !x.Job.Cancelled.Value && x.Job.ExceptionDetails == null && x.Job.StoppedAt != null) + .Include(x => x.Job) + .OrderByDescending(x => x.Job.StoppedAt).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); if (cj == default(CompileJob)) return; - await LoadCompileJob(cj, false, cancellationToken).ConfigureAwait(false); - + await LoadCompileJob(cj, cancellationToken).ConfigureAwait(false); //we dont do CleanUnusedCompileJobs here because the watchdog may have plans for them yet }); @@ -171,15 +168,41 @@ namespace Tgstation.Server.Host.Components.Compiler } /// - public IDmbProvider FromCompileJob(CompileJob compileJob) + public async Task FromCompileJob(CompileJob compileJob, CancellationToken cancellationToken) { - lock (this) + logger.LogTrace("Loading compile job {0}...", compileJob.Id); + var providerSubmitted = false; + var newProvider = new DmbProvider(compileJob, ioManager, () => { - if (!jobLockCounts.TryGetValue(compileJob.Id, out int value)) - jobLockCounts.Add(compileJob.Id, 1); - else - jobLockCounts[compileJob.Id] = ++value; - return new DmbProvider(compileJob, ioManager, () => CleanJob(compileJob)); + if (providerSubmitted) + CleanJob(compileJob); + }); + + try + { + var primaryCheckTask = ioManager.FileExists(ioManager.ConcatPath(newProvider.PrimaryDirectory, newProvider.DmbName), cancellationToken); + var secondaryCheckTask = ioManager.FileExists(ioManager.ConcatPath(newProvider.PrimaryDirectory, newProvider.DmbName), cancellationToken); + + if (!(await primaryCheckTask.ConfigureAwait(false) && await secondaryCheckTask.ConfigureAwait(false))) + { + logger.LogWarning("Error loading compile job, .dmb missing!"); + return null; //omae wa mou shinderu + } + + lock (this) + { + if (!jobLockCounts.TryGetValue(compileJob.Id, out int value)) + jobLockCounts.Add(compileJob.Id, 1); + else + jobLockCounts[compileJob.Id] = ++value; + providerSubmitted = true; + return newProvider; + } + } + finally + { + if (!providerSubmitted) + newProvider.Dispose(); } } diff --git a/src/Tgstation.Server.Host/Components/Compiler/IDmbFactory.cs b/src/Tgstation.Server.Host/Components/Compiler/IDmbFactory.cs index 6efc0f69b9..a76611577c 100644 --- a/src/Tgstation.Server.Host/Components/Compiler/IDmbFactory.cs +++ b/src/Tgstation.Server.Host/Components/Compiler/IDmbFactory.cs @@ -27,8 +27,9 @@ namespace Tgstation.Server.Host.Components.Compiler /// Gets a for a given /// /// The to make the for - /// A new - IDmbProvider FromCompileJob(CompileJob compileJob); + /// The for the operation + /// A resulting in a new representing the on success, on failure + Task FromCompileJob(CompileJob compileJob, CancellationToken cancellationToken); /// /// Deletes all compile jobs that are inactive in the Game folder diff --git a/src/Tgstation.Server.Host/Components/ReattachInfoHandler.cs b/src/Tgstation.Server.Host/Components/ReattachInfoHandler.cs index a4135a29fa..5e831d0651 100644 --- a/src/Tgstation.Server.Host/Components/ReattachInfoHandler.cs +++ b/src/Tgstation.Server.Host/Components/ReattachInfoHandler.cs @@ -80,7 +80,8 @@ namespace Tgstation.Server.Host.Components await databaseContextFactory.UseContext(async (db) => result = await db.Instances.Where(x => x.Id == metadata.Id).Select(x => x.WatchdogReattachInformation).FirstAsync(cancellationToken).ConfigureAwait(false) ).ConfigureAwait(false); - return new WatchdogReattachInformation(result, dmbFactory); + var bravoDmbTask = dmbFactory.FromCompileJob(result.Bravo.CompileJob, cancellationToken); + return new WatchdogReattachInformation(result, await dmbFactory.FromCompileJob(result.Alpha.CompileJob, cancellationToken).ConfigureAwait(false), await bravoDmbTask.ConfigureAwait(false)); } } } diff --git a/src/Tgstation.Server.Host/Components/Watchdog/ReattachInformation.cs b/src/Tgstation.Server.Host/Components/Watchdog/ReattachInformation.cs index 271d840692..c403d7cae6 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/ReattachInformation.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/ReattachInformation.cs @@ -1,4 +1,4 @@ -using Tgstation.Server.Host.Components.Compiler; +using System; using Tgstation.Server.Host.Models; namespace Tgstation.Server.Host.Components.Watchdog @@ -22,10 +22,10 @@ namespace Tgstation.Server.Host.Components.Watchdog /// Construct a from a given and /// /// The to copy values from - /// The used to assign - public ReattachInformation(Models.ReattachInformation copy, IDmbFactory dmbFactory) : base(copy) + /// The value of + public ReattachInformation(Models.ReattachInformation copy, IDmbProvider dmb) : base(copy) { - Dmb = dmbFactory.FromCompileJob(copy.CompileJob); + Dmb = dmb ?? throw new ArgumentNullException(nameof(dmb)); } } } \ No newline at end of file diff --git a/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs index 75cea931bb..3248a31598 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs @@ -314,7 +314,12 @@ namespace Tgstation.Server.Host.Components.Watchdog //either way try to start it using the active server's dmb as a backup try { - var dmbBackup = dmbFactory.FromCompileJob(monitorState.ActiveServer.Dmb.CompileJob); + var dmbBackup = await dmbFactory.FromCompileJob(monitorState.ActiveServer.Dmb.CompileJob, cancellationToken).ConfigureAwait(false); + + if (dmbBackup == null) //NANI!? + //just give up, if THAT compile job is failing then the ActiveServer is gonna crash soon too or already has + throw new Exception("Creating backup DMB provider failed!"); + monitorState.InactiveServer = await sessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbBackup, null, false, !monitorState.ActiveServer.IsPrimary, false, cancellationToken).ConfigureAwait(false); usedMostRecentDmb = false; await chat.SendWatchdogMessage("Staging newest DMB on inactive server failed: {0} Falling back to previous dmb...", cancellationToken).ConfigureAwait(false); @@ -326,7 +331,7 @@ namespace Tgstation.Server.Host.Components.Watchdog catch (Exception e2) { //fuuuuucckkk - logger.LogError("Backup strategy failed! Monitor will restart when active server reboots! This Exception: {0}", e2.ToString()); + logger.LogError("Backup strategy failed! Monitor will restart when active server reboots! Exception: {0}", e2.ToString()); monitorState.InactiveServerCritFail = true; await chat.SendWatchdogMessage("Attempted reboot of inactive server failed. Watchdog will reset when active server fails or exits", cancellationToken).ConfigureAwait(false); return true; //we didn't use the old dmb diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogReattachInformation.cs b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogReattachInformation.cs index efe645c044..0ed75fed6f 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogReattachInformation.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogReattachInformation.cs @@ -1,5 +1,4 @@ -using Tgstation.Server.Host.Components.Compiler; -using Tgstation.Server.Host.Models; +using Tgstation.Server.Host.Models; namespace Tgstation.Server.Host.Components.Watchdog { @@ -27,13 +26,14 @@ namespace Tgstation.Server.Host.Components.Watchdog /// Construct a from a given with a given /// /// The to copy information from - /// The used to build the s - public WatchdogReattachInformation(Models.WatchdogReattachInformation copy, IDmbFactory dmbFactory): base(copy) + /// The used to build + /// The used to build + public WatchdogReattachInformation(Models.WatchdogReattachInformation copy, IDmbProvider dmbAlpha, IDmbProvider dmbBravo): base(copy) { if (copy.Alpha != null) - Alpha = new ReattachInformation(copy.Alpha, dmbFactory); + Alpha = new ReattachInformation(copy.Alpha, dmbAlpha); if (copy.Bravo != null) - Bravo = new ReattachInformation(copy.Bravo, dmbFactory); + Bravo = new ReattachInformation(copy.Bravo, dmbBravo); } } } From 5c9aae969b6a591f08c2b2d1c238b6c5b11e7b44 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Tue, 7 Aug 2018 21:40:43 -0400 Subject: [PATCH 32/36] Fix doc comments --- .../Components/Compiler/DmbProvider.cs | 9 ++------- .../Components/Watchdog/ReattachInformation.cs | 2 +- .../Components/Watchdog/SessionController.cs | 2 +- .../Components/Watchdog/WatchdogReattachInformation.cs | 2 +- src/Tgstation.Server.Host/Models/DatabaseContext.cs | 2 +- .../Models/SqliteDatabaseContext.cs | 2 +- 6 files changed, 7 insertions(+), 12 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Compiler/DmbProvider.cs b/src/Tgstation.Server.Host/Components/Compiler/DmbProvider.cs index 3650c69ddc..bf042d1912 100644 --- a/src/Tgstation.Server.Host/Components/Compiler/DmbProvider.cs +++ b/src/Tgstation.Server.Host/Components/Compiler/DmbProvider.cs @@ -46,15 +46,10 @@ namespace Tgstation.Server.Host.Components.Compiler this.onDispose = onDispose ?? throw new ArgumentNullException(nameof(onDispose)); } - ~DmbProvider() => Dispose(); + /// + public void Dispose() => onDispose?.Invoke(); /// - public void Dispose() - { - onDispose?.Invoke(); - GC.SuppressFinalize(this); - } - public void KeepAlive() => onDispose = null; } } diff --git a/src/Tgstation.Server.Host/Components/Watchdog/ReattachInformation.cs b/src/Tgstation.Server.Host/Components/Watchdog/ReattachInformation.cs index c403d7cae6..9570ce7673 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/ReattachInformation.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/ReattachInformation.cs @@ -19,7 +19,7 @@ namespace Tgstation.Server.Host.Components.Watchdog public ReattachInformation() { } /// - /// Construct a from a given and + /// Construct a from a given and /// /// The to copy values from /// The value of diff --git a/src/Tgstation.Server.Host/Components/Watchdog/SessionController.cs b/src/Tgstation.Server.Host/Components/Watchdog/SessionController.cs index 613b53d264..ea131270e6 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/SessionController.cs @@ -274,7 +274,7 @@ namespace Tgstation.Server.Host.Components.Watchdog } /// - /// Throws an if has been called + /// Throws an if has been called /// void CheckDisposed() { diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogReattachInformation.cs b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogReattachInformation.cs index 0ed75fed6f..5f7542e68d 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogReattachInformation.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogReattachInformation.cs @@ -23,7 +23,7 @@ namespace Tgstation.Server.Host.Components.Watchdog public WatchdogReattachInformation() { } /// - /// Construct a from a given with a given + /// Construct a from a given with a given and /// /// The to copy information from /// The used to build diff --git a/src/Tgstation.Server.Host/Models/DatabaseContext.cs b/src/Tgstation.Server.Host/Models/DatabaseContext.cs index 67d44688a1..411836458d 100644 --- a/src/Tgstation.Server.Host/Models/DatabaseContext.cs +++ b/src/Tgstation.Server.Host/Models/DatabaseContext.cs @@ -87,7 +87,7 @@ namespace Tgstation.Server.Host.Models /// The for the /// The containing the value of /// The value of - /// The value of + /// The value of public DatabaseContext(DbContextOptions dbContextOptions, IOptions databaseConfigurationOptions, IDatabaseSeeder databaseSeeder, ILogger logger) : base(dbContextOptions) { databaseConfiguration = databaseConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(databaseConfigurationOptions)); diff --git a/src/Tgstation.Server.Host/Models/SqliteDatabaseContext.cs b/src/Tgstation.Server.Host/Models/SqliteDatabaseContext.cs index 9af5b1be9a..224f08ff0b 100644 --- a/src/Tgstation.Server.Host/Models/SqliteDatabaseContext.cs +++ b/src/Tgstation.Server.Host/Models/SqliteDatabaseContext.cs @@ -17,7 +17,7 @@ namespace Tgstation.Server.Host.Models /// The for the /// The of for the /// The for the - /// The for the + /// The for the public SqliteDatabaseContext(DbContextOptions dbContextOptions, IOptions databaseConfiguration, IDatabaseSeeder databaseSeeder, ILogger logger) : base(dbContextOptions, databaseConfiguration, databaseSeeder, logger) { } From c38f8302a4e2711fc1bfd830010defcb9d2dd435 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Tue, 7 Aug 2018 21:57:05 -0400 Subject: [PATCH 33/36] Test merging optimization --- .../Controllers/RepositoryController.cs | 125 +++++++++++------- 1 file changed, 74 insertions(+), 51 deletions(-) diff --git a/src/Tgstation.Server.Host/Controllers/RepositoryController.cs b/src/Tgstation.Server.Host/Controllers/RepositoryController.cs index 116a71f2a1..a0e8d65633 100644 --- a/src/Tgstation.Server.Host/Controllers/RepositoryController.cs +++ b/src/Tgstation.Server.Host/Controllers/RepositoryController.cs @@ -382,67 +382,90 @@ namespace Tgstation.Server.Host.Controllers lastRevisionInfo.OriginCommitSha = repo.Head; } } - + //test merging if (newTestMerges) { - var gitHubClient = currentModel.AccessToken != null ? gitHubClientFactory.CreateClient(currentModel.AccessToken) : gitHubClientFactory.CreateClient(); - var contextUser = new Models.User + //optimization: if we've already merged these exact same commits in this fashion before, just find the rev info for it and check it out + Models.RevisionInformation revInfoWereLookingFor = null; + if(lastRevisionInfo.OriginCommitSha == lastRevisionInfo.CommitSha) { - Id = AuthenticationContext.User.Id - }; - databaseContext.Users.Attach(contextUser); + foreach (var I in model.NewTestMerges) + //normalize the shas to lowercase ala libgit2 +#pragma warning disable CA1308 // Normalize strings to uppercase + I.PullRequestRevision = I.PullRequestRevision?.ToLowerInvariant(); +#pragma warning restore CA1308 // Normalize strings to uppercase - var repoOwner = repo.GitHubOwner; - var repoName = repo.GitHubRepoName; - foreach (var I in model.NewTestMerges) + revInfoWereLookingFor = await databaseContext.RevisionInformations.Where( + x => x.OriginCommitSha == lastRevisionInfo.OriginCommitSha + && x.ActiveTestMerges.Count == model.NewTestMerges.Count) + //split here cause this bit probably has to be done locally + .Where(x => x.ActiveTestMerges.Select(y => y.TestMerge).All(y => model.NewTestMerges.Any(z => y.Number == z.Number && y.PullRequestRevision.StartsWith(z.PullRequestRevision, StringComparison.Ordinal)))).FirstOrDefaultAsync(ct).ConfigureAwait(false); + } + + + if (revInfoWereLookingFor != null) + await repo.ResetToSha(revInfoWereLookingFor.CommitSha, cancellationToken).ConfigureAwait(false); + else { - Octokit.PullRequest pr = null; - string errorMessage = null; - try + var gitHubClient = currentModel.AccessToken != null ? gitHubClientFactory.CreateClient(currentModel.AccessToken) : gitHubClientFactory.CreateClient(); + var contextUser = new Models.User { - pr = await gitHubClient.PullRequest.Get(repoOwner, repoName, I.Number).ConfigureAwait(false); - } - catch (Octokit.RateLimitExceededException) - { - //you look at your anonymous access and sigh - errorMessage = "P.R.E. RATE LIMITED"; - } - catch (Octokit.NotFoundException) - { - //you look at your shithub access and sigh - errorMessage = "P.R.E. NOT FOUND"; - } - - var mergeResult = await repo.AddTestMerge(I.Number, I.PullRequestRevision, committerName, currentModel.CommitterEmail, String.Format(CultureInfo.InvariantCulture, "Test merge of pull request #{0}{1}{2}", I.Number, I.Comment != null ? Environment.NewLine : null, I.Comment), currentModel.AccessUser, currentModel.AccessToken, x => progressReporter((x + 100 * doneFetches) / numFetches), ct).ConfigureAwait(false); - - if (!mergeResult.HasValue) //conflict, we don't care, dd already knows - continue; - - ++doneFetches; - - var revInfoUpdateTask = UpdateRevInfo(); - - var tm = new Models.TestMerge - { - Author = pr?.User.Login ?? errorMessage, - BodyAtMerge = pr?.Body ?? errorMessage ?? String.Empty, - MergedAt = DateTimeOffset.Now, - TitleAtMerge = pr?.Title ?? errorMessage ?? String.Empty, - Comment = I.Comment, - Number = I.Number, - MergedBy = contextUser, - PullRequestRevision = I.PullRequestRevision, - Url = pr?.HtmlUrl ?? errorMessage + Id = AuthenticationContext.User.Id }; + databaseContext.Users.Attach(contextUser); - await revInfoUpdateTask.ConfigureAwait(false); - - lastRevisionInfo.PrimaryTestMerge = tm; - lastRevisionInfo.ActiveTestMerges.Add(new RevInfoTestMerge + var repoOwner = repo.GitHubOwner; + var repoName = repo.GitHubRepoName; + foreach (var I in model.NewTestMerges) { - TestMerge = tm - }); + Octokit.PullRequest pr = null; + string errorMessage = null; + try + { + pr = await gitHubClient.PullRequest.Get(repoOwner, repoName, I.Number).ConfigureAwait(false); + } + catch (Octokit.RateLimitExceededException) + { + //you look at your anonymous access and sigh + errorMessage = "P.R.E. RATE LIMITED"; + } + catch (Octokit.NotFoundException) + { + //you look at your shithub access and sigh + errorMessage = "P.R.E. NOT FOUND"; + } + + var mergeResult = await repo.AddTestMerge(I.Number, I.PullRequestRevision, committerName, currentModel.CommitterEmail, String.Format(CultureInfo.InvariantCulture, "Test merge of pull request #{0}{1}{2}", I.Number, I.Comment != null ? Environment.NewLine : null, I.Comment), currentModel.AccessUser, currentModel.AccessToken, x => progressReporter((x + 100 * doneFetches) / numFetches), ct).ConfigureAwait(false); + + if (!mergeResult.HasValue) //conflict, we don't care, dd already knows + continue; + + ++doneFetches; + + var revInfoUpdateTask = UpdateRevInfo(); + + var tm = new Models.TestMerge + { + Author = pr?.User.Login ?? errorMessage, + BodyAtMerge = pr?.Body ?? errorMessage ?? String.Empty, + MergedAt = DateTimeOffset.Now, + TitleAtMerge = pr?.Title ?? errorMessage ?? String.Empty, + Comment = I.Comment, + Number = I.Number, + MergedBy = contextUser, + PullRequestRevision = I.PullRequestRevision, + Url = pr?.HtmlUrl ?? errorMessage + }; + + await revInfoUpdateTask.ConfigureAwait(false); + + lastRevisionInfo.PrimaryTestMerge = tm; + lastRevisionInfo.ActiveTestMerges.Add(new RevInfoTestMerge + { + TestMerge = tm + }); + } } } From a2b2c850f6d045f61c9e8af4a370ca6a32a28ef1 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Tue, 7 Aug 2018 22:01:12 -0400 Subject: [PATCH 34/36] Fix cleaning unused game folders message --- .../Components/Compiler/DmbFactory.cs | 33 ++++++++++--------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs b/src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs index b2b319473b..7141a80077 100644 --- a/src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs +++ b/src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs @@ -228,23 +228,26 @@ namespace Tgstation.Server.Host.Components.Compiler //cleanup var directories = await ioManager.GetDirectories(".", cancellationToken).ConfigureAwait(false); - if (directories.Count > 0) + int deleting = 0; + var tasks = directories.Select(async x => { - logger.LogDebug("Cleaning {0} unused game folders...", directories.Count); - await Task.WhenAll(directories.Select(async x => + var nameOnly = ioManager.GetFileName(x); + if (jobUidsToNotErase.Contains(nameOnly.ToUpperInvariant())) + return; + try { - var nameOnly = ioManager.GetFileName(x); - if (jobUidsToNotErase.Contains(nameOnly.ToUpperInvariant())) - return; - try - { - await ioManager.DeleteDirectory(x, cancellationToken).ConfigureAwait(false); - } - catch (Exception e) - { - logger.LogWarning("Error deleting directory {0}! Exception: {1}", x, e); - } - })).ConfigureAwait(false); + ++deleting; + await ioManager.DeleteDirectory(x, cancellationToken).ConfigureAwait(false); + } + catch (Exception e) + { + logger.LogWarning("Error deleting directory {0}! Exception: {1}", x, e); + } + }).ToList(); + if (deleting > 0) + { + logger.LogDebug("Cleaning {0} unused game folders...", deleting); + await Task.WhenAll().ConfigureAwait(false); } } } From e6d85d8827f04876da72901fe1d41f18756e6897 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Tue, 7 Aug 2018 22:03:23 -0400 Subject: [PATCH 35/36] Add missing info to startup CompileJob select --- src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs b/src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs index 7141a80077..945fc898f1 100644 --- a/src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs +++ b/src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs @@ -152,7 +152,9 @@ namespace Tgstation.Server.Host.Components.Compiler { //where complete clause not necessary, only successful COMPILEjobs get in the db var cj = await db.CompileJobs.Where(x => x.Job.Instance.Id == instance.Id && !x.Job.Cancelled.Value && x.Job.ExceptionDetails == null && x.Job.StoppedAt != null) - .Include(x => x.Job) + .Include(x => x.Job).ThenInclude(x => x.StartedBy) + .Include(x => x.RevisionInformation).ThenInclude(x => x.PrimaryTestMerge).ThenInclude(x => x.MergedBy) + .Include(x => x.RevisionInformation).ThenInclude(x => x.ActiveTestMerges).ThenInclude(x => x.TestMerge).ThenInclude(x => x.MergedBy) .OrderByDescending(x => x.Job.StoppedAt).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); if (cj == default(CompileJob)) return; From c27ea103a7fc05c9dda85167975095c932131d74 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Tue, 7 Aug 2018 22:06:22 -0400 Subject: [PATCH 36/36] Remove dumb stuff from the prototype --- v4_prototype_TODO.txt | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/v4_prototype_TODO.txt b/v4_prototype_TODO.txt index afe9140947..7333b2c540 100644 --- a/v4_prototype_TODO.txt +++ b/v4_prototype_TODO.txt @@ -1,15 +1,5 @@ -Watchdog performance counters - - Remember to catch PlatformNotSupportedException() - -Install as network service instead of LocalSystem if win10 >= Creators update to allow for unprivileged symlinks - Verify the byond cache folder location on linux -Don't throw arg null exceptions with null [FromBody]'s, apparently that's allowed, return bad request instead - -Test repo more -Test byond -Test compile Test watchdog test configuration