diff --git a/.gitignore b/.gitignore index 5074f2b86e..a8f3e2c779 100644 --- a/.gitignore +++ b/.gitignore @@ -11,7 +11,6 @@ artifacts/ *DS_Store *.sln.ide /TestResults -/src/Tgstation.Server.Host/appsettings.Development.json /tests/DMAPI/travistester.lk /tests/DMAPI/travistester.int /tests/DMAPI/travistester.dmb diff --git a/docs/API.dox b/docs/API.dox index f8ddafc8a4..6783c24120 100644 --- a/docs/API.dox +++ b/docs/API.dox @@ -137,7 +137,7 @@ I DELETE "/InstanceUser/{UserId}" => OK Users with the permission to modify @ref Tgstation.Server.Api.Models.Instance objects can also gain user editing rights for any Instance. See @ref api_instance -@section api_ver Getting the server version +@section api_ver Version The version of TGS running can be retireved with this request @@ -196,6 +196,24 @@ Instances can be detached which will delete all meta knowledge of the instance ( DELETE "/Instance/{InstanceId}" => OK +@subsection api_job Jobs + +Some requests return @ref Tgstation.Server.Api.Models.Job objects. These are long running tasks the server will perform asyncronously and can be polled for status. + +To list all jobs in an Instance use the following request + +I GET "/Job" => Array of @ref Tgstation.Server.Api.Models.Job + +Note that the response for this request will only have the @ref Tgstation.Server.Api.Models.Job.Id field populated + +To get full details of a job use the following request: + +I GET "/Job/{JobId}" => @ref Tgstation.Server.Api.Models.Job + +To cancel a running job (If the job can be cancelled and you have sufficient rights) use the following request: + +I DELETE "/Job/{JobId}" => OK + @subsection api_chat Chat Bots Each chat bot is represented by a @ref Tgstation.Server.Api.Models.ChatSettings object @@ -219,4 +237,39 @@ I GET "/Chat/{ChatSettingsId}" => @ref Tgstation.Server.Api.Models.ChatSettings Also note that if the @ref Tgstation.Server.Api.Models.ChatSettings.Channels is present in a POST request, the list will fully replace any active channels +@subsection api_byond Byond Version Management + +To get the Byond version used for new compilations use the following request: + +I GET "/Byond" => @ref Tgstation.Server.Api.Models.Byond + +To set the active Byond version: + +I POST "/Byond" @ref Tgstation.Server.Api.Models.Byond => @ref Tgstation.Server.Api.Models.Byond + +To list all installed Byond versions use the following request: + +I GET "/Byond/List" => Array of @ref Tgstation.Server.Api.Models.Byond + +@subsection Git Repository Management + +To read the current repository state use the following request: + +I GET "/Repository" => @ref Tgstation.Server.Api.Models.Repository + +To clone the repository if it doesn't yet exist use the following request: + +I PUT "/Repository" => @ref Tgstation.Server.Api.Models.Repository + +The clone job will be represented by the @ref Tgstation.Server.Api.Models.Repository.ActiveJob field. Specify the @ref Tgstation.Server.Api.Models.Origin URL. Optionally specify the initial @ref Tgstation.Server.Api.Models.Repository.Reference as a git tag or branch. Be sure to specify the authentication fields if necessary to access your repository + +To delete an existing repository make the following request: + +I DELETE "/Repository" => OK + +Modifications to the repository are done with the following request: + +I POST "/Repository" => @ref Tgstation.Server.Api.Models.Repository => @ref Tgstation.Server.Api.Models.Repository + +Each update creates a job specified in the @ref Tgstation.Server.Api.Models.Repository.ActiveJob field jobs will be queued in succession. */ diff --git a/src/DMAPI/tgs.dm b/src/DMAPI/tgs.dm index 628b17cac6..05a4b3e9a5 100644 --- a/src/DMAPI/tgs.dm +++ b/src/DMAPI/tgs.dm @@ -47,14 +47,14 @@ #define TGS_EVENT_PORT_SWAP -2 //before a port change is about to happen, extra parameter is new port #define TGS_EVENT_REBOOT_MODE_CHANGE -1 //before a reboot mode change, extras parameters are the current and new reboot mode enums +//TODO + //OTHER ENUMS #define TGS_REBOOT_MODE_NORMAL 0 #define TGS_REBOOT_MODE_SHUTDOWN 1 #define TGS_REBOOT_MODE_RESTART 2 -//TODO - //REQUIRED HOOKS //Call this somewhere in /world/New() that is always run diff --git a/src/DMAPI/tgs/core/core.dm b/src/DMAPI/tgs/core/core.dm index 198ddc7d07..bd44452413 100644 --- a/src/DMAPI/tgs/core/core.dm +++ b/src/DMAPI/tgs/core/core.dm @@ -33,6 +33,10 @@ switch(major) if(2) return /datum/tgs_api/v3210 + if(4) + switch(major) + if(0) + return /datum/tgs_api/v4 if(super != null && major != null && minor != null && patch != null && tgs_version > TgsMaximumAPIVersion()) TGS_ERROR_LOG("Detected unknown API version! Defaulting to latest. Update the DMAPI to fix this problem.") diff --git a/src/DMAPI/tgs/v4/api.dm b/src/DMAPI/tgs/v4/api.dm index e62de1c25e..a4263c2446 100644 --- a/src/DMAPI/tgs/v4/api.dm +++ b/src/DMAPI/tgs/v4/api.dm @@ -11,7 +11,7 @@ #define TGS4_COMM_ONLINE "tgs_on" #define TGS4_COMM_IDENTIFY "tgs_ident" -#define TGS4_COMM_VALIDATE "tgs_vali" +#define TGS4_COMM_VALIDATE "tgs_validate" #define TGS4_COMM_SERVER_PRIMED "tgs_prime" #define TGS4_COMM_WORLD_REBOOT "tgs_reboot" #define TGS4_COMM_END_PROCESS "tgs_kill" @@ -58,39 +58,42 @@ TGS_ERROR_LOG("Failed to decode info json: [json_file]") return - access_identifier = cached_json["access_identifier"] - instance_name = text2num(cached_json["instance_name"]) - host_path = cached_json["host_path"] - if(cached_json["api_validate_only"]) + access_identifier = cached_json["accessIdentifier"] + instance_name = text2num(cached_json["instanceName"]) + host_path = cached_json["hostPath"] + if(cached_json["apiValidateOnly"]) + TGS_INFO_LOG("Validating API and exiting...") Export(TGS4_COMM_VALIDATE) del(world) - chat_channels_json_path = cached_json["chat_channels_json"] - chat_commands_json_path = cached_json["chat_commands_json"] + chat_channels_json_path = cached_json["chatChannelsJson"] + chat_commands_json_path = cached_json["chatCommandsJson"] src.event_handler = event_handler - instance_name = cached_json["instance_name"] + instance_name = cached_json["instanceName"] cached_test_merges = list() - var/json = cached_json["test_merges"] + var/json = cached_json["testMerges"] for(var/I in json) var/datum/tgs_revision_information/test_merge/tm = new tm.number = text2num(I) var/list/entry = json[I] - tm.pull_request_commit = entry["pr_commit"] + tm.pull_request_commit = entry["prCommit"] tm.author = entry["author"] tm.title = entry["title"] tm.commit = entry["commit"] - tm.origin_commit = entry["origin_commit"] - tm.time_merged = text2num(entry["time_merged"]) + tm.origin_commit = entry["originCommit"] + tm.time_merged = text2num(entry["timeMerged"]) tm.comment = entry["comment"] tm.url = entry["url"] cached_revision = new cached_revision.commit = cached_json["commit"] - cached_revision.origin_commit = cached_json["origin_commit"] + cached_revision.origin_commit = cached_json["originCommit"] ListCustomCommands() + return TRUE + /datum/tgs_api/v4/OnInitializationComplete() Export(TGS4_COMM_SERVER_PRIMED) var/tgs4_secret_sleep_offline_sauce = 24051994 @@ -208,10 +211,10 @@ /datum/tgs_api/v4/proc/DecodeChannel(channel_json) var/datum/tgs_chat_channel/channel = new channel.id = channel_json["id"] - channel.friendly_name = channel_json["friendly_name"] - channel.connection_name = channel_json["connection_name"] - channel.is_admin_channel = channel_json["is_admin_channel"] - channel.is_private_channel = channel_json["is_private_channel"] || FALSE + channel.friendly_name = channel_json["friendlyName"] + channel.connection_name = channel_json["connectionName"] + channel.is_admin_channel = channel_json["isAdminChannel"] + channel.is_private_channel = channel_json["isPrivateChannel"] || FALSE return channel #undef TGS4_TOPIC_COMMAND diff --git a/src/DMAPI/tgs/v4/commands.dm b/src/DMAPI/tgs/v4/commands.dm index 48ca6e98de..88ee61e882 100644 --- a/src/DMAPI/tgs/v4/commands.dm +++ b/src/DMAPI/tgs/v4/commands.dm @@ -28,7 +28,7 @@ var/datum/tgs_chat_user/u = new u.id = user["id"] - u.friendly_name = user["friendly_name"] + u.friendly_name = user["friendlyName"] u.mention = user["mention"] u.channel = DecodeChannel(user["channel"]) diff --git a/src/Tgstation.Server.Api/Models/CompilerStatus.cs b/src/Tgstation.Server.Api/Models/CompilerStatus.cs index d29df195e6..9cb0a7a5ed 100644 --- a/src/Tgstation.Server.Api/Models/CompilerStatus.cs +++ b/src/Tgstation.Server.Api/Models/CompilerStatus.cs @@ -1,33 +1,41 @@ namespace Tgstation.Server.Api.Models { - /// - /// Status of the for an - /// + /// + /// Status of the for an + /// #pragma warning disable CA1717 // Only FlagsAttribute enums should have plural names public enum CompilerStatus #pragma warning restore CA1717 // Only FlagsAttribute enums should have plural names { - /// - /// The is idle - /// + /// + /// The is idle + /// Idle, - /// - /// The is being copied - /// - Copying, + /// + /// The is being copied + /// + Copying, + /// + /// Pre-compile scripts are running + /// + PreCompile, /// /// The .dme is having it's server side modifications applied /// Modifying, - /// - /// DreamMaker is running - /// - Compiling, + /// + /// DreamMaker is running + /// + Compiling, /// /// The DMAPI is being verified /// Verifying, /// + /// Post-compile scripts are running + /// + PostCompile, + /// /// The compile results are being duplicated /// Duplicating, diff --git a/src/Tgstation.Server.Api/Models/DreamDaemonSecurity.cs b/src/Tgstation.Server.Api/Models/DreamDaemonSecurity.cs index 094332d1e4..68804af41b 100644 --- a/src/Tgstation.Server.Api/Models/DreamDaemonSecurity.cs +++ b/src/Tgstation.Server.Api/Models/DreamDaemonSecurity.cs @@ -14,7 +14,7 @@ /// Safe, /// - /// Server will not be able to run shell commands or access anything but temporary files + /// Server will not be able to run shell commands or access anything but temporary files. Currently unsupported! /// Ultrasafe } diff --git a/src/Tgstation.Server.Host/Components/Byond/ByondManager.cs b/src/Tgstation.Server.Host/Components/Byond/ByondManager.cs index f66005b42d..89f1991a3c 100644 --- a/src/Tgstation.Server.Host/Components/Byond/ByondManager.cs +++ b/src/Tgstation.Server.Host/Components/Byond/ByondManager.cs @@ -163,7 +163,7 @@ namespace Tgstation.Server.Host.Components.Byond var versionToUse = requiredVersion ?? ActiveVersion; if (versionToUse == null) throw new InvalidOperationException("No BYOND versions installed!"); - await InstallVersion(requiredVersion, cancellationToken).ConfigureAwait(false); + await InstallVersion(versionToUse, cancellationToken).ConfigureAwait(false); var versionKey = VersionKey(versionToUse); diff --git a/src/Tgstation.Server.Host/Components/Byond/WindowsByondInstaller.cs b/src/Tgstation.Server.Host/Components/Byond/WindowsByondInstaller.cs index 3e5422c62d..d2a7329345 100644 --- a/src/Tgstation.Server.Host/Components/Byond/WindowsByondInstaller.cs +++ b/src/Tgstation.Server.Host/Components/Byond/WindowsByondInstaller.cs @@ -127,23 +127,22 @@ namespace Tgstation.Server.Host.Components.Byond p.StartInfo.WorkingDirectory = rbdx; p.EnableRaisingEvents = true; var tcs = new TaskCompletionSource(); - p.Exited += (a, b) => tcs.SetResult(null); + p.Exited += (a, b) => tcs.TrySetResult(null); try { p.Start(); - using (cancellationToken.Register(() => - { - p.Kill(); - tcs.SetCanceled(); - })) + using (cancellationToken.Register(() => tcs.TrySetCanceled())) await tcs.Task.ConfigureAwait(false); } finally { try { - p.Kill(); - p.WaitForExit(); + if (!p.HasExited) + { + p.Kill(); + p.WaitForExit(); + } } catch (InvalidOperationException) { } } diff --git a/src/Tgstation.Server.Host/Components/Chat/JsonTrackingContext.cs b/src/Tgstation.Server.Host/Components/Chat/JsonTrackingContext.cs index dd552f87ff..c7482f38ed 100644 --- a/src/Tgstation.Server.Host/Components/Chat/JsonTrackingContext.cs +++ b/src/Tgstation.Server.Host/Components/Chat/JsonTrackingContext.cs @@ -1,4 +1,5 @@ using Newtonsoft.Json; +using Newtonsoft.Json.Serialization; using System; using System.Collections.Generic; using System.Text; @@ -58,7 +59,13 @@ namespace Tgstation.Server.Host.Components.Chat public async Task SetChannels(IEnumerable channels, CancellationToken cancellationToken) { using (await SemaphoreSlimContext.Lock(channelsSemaphore, cancellationToken).ConfigureAwait(false)) - await ioManager.WriteAllBytes(channelsPath, Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(channels)), cancellationToken).ConfigureAwait(false); + await ioManager.WriteAllBytes(channelsPath, Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(channels, Formatting.Indented, new JsonSerializerSettings + { + ContractResolver = new DefaultContractResolver + { + NamingStrategy = new CamelCaseNamingStrategy() + } + })), cancellationToken).ConfigureAwait(false); } } } diff --git a/src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs b/src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs index e994fddfae..945fc898f1 100644 --- a/src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs +++ b/src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs @@ -1,4 +1,5 @@ using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; using System; using System.Collections.Generic; using System.Linq; @@ -34,15 +35,19 @@ namespace Tgstation.Server.Host.Components.Compiler /// readonly IIOManager ioManager; /// - /// The for + /// The for the /// - readonly CancellationTokenSource cleanupCts; + readonly ILogger logger; /// /// The for the /// readonly Api.Models.Instance instance; - + /// + /// The for + /// + readonly CancellationTokenSource cleanupCts; + /// /// representing calls to /// @@ -50,7 +55,7 @@ namespace Tgstation.Server.Host.Components.Compiler /// /// resulting in the latest yet to exist /// - TaskCompletionSource newerDmbTcs; + TaskCompletionSource newerDmbTcs; /// /// The latest /// @@ -63,20 +68,23 @@ namespace Tgstation.Server.Host.Components.Compiler /// /// The value of /// The value of + /// The value of /// The value of - public DmbFactory(IDatabaseContextFactory databaseContextFactory, IIOManager ioManager, Api.Models.Instance instance) + public DmbFactory(IDatabaseContextFactory databaseContextFactory, IIOManager ioManager, ILogger logger, Api.Models.Instance instance) { this.databaseContextFactory = databaseContextFactory ?? throw new ArgumentNullException(nameof(databaseContextFactory)); this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); this.instance = instance ?? throw new ArgumentNullException(nameof(instance)); cleanupTask = Task.CompletedTask; + newerDmbTcs = new TaskCompletionSource(); cleanupCts = new CancellationTokenSource(); jobLockCounts = new Dictionary(); } /// - public void Dispose() => cleanupCts.Dispose(); + public void Dispose() => cleanupCts.Dispose(); //we don't dispose nextDmbProvider here, since it might be the only thing we have /// /// Delete the of @@ -84,84 +92,74 @@ namespace Tgstation.Server.Host.Components.Compiler /// The to clean void CleanJob(CompileJob job) { + logger.LogTrace("Cleaning compile job {0} => {1}", job.Id, job.DirectoryName); async Task HandleCleanup() { var deleteJob = ioManager.DeleteDirectory(job.DirectoryName.ToString(), cleanupCts.Token); Task otherTask; - lock (this) - otherTask = cleanupTask; + //lock (this) //already locked below + otherTask = cleanupTask; await Task.WhenAll(otherTask, deleteJob).ConfigureAwait(false); } lock (this) { - var currentVal = jobLockCounts[job.Id]; - if (--jobLockCounts[job.Id] == 0) + if (!jobLockCounts.TryGetValue(job.Id, out var currentVal) || --jobLockCounts[job.Id] == 0) + { + jobLockCounts.Remove(job.Id); cleanupTask = HandleCleanup(); + } } } /// - public Task LoadCompileJob(CompileJob job, CancellationToken cancellationToken) => LoadCompileJob(job, true, cancellationToken); - - async Task LoadCompileJob(CompileJob job, bool setAsStagedInDb, CancellationToken cancellationToken) + public async Task LoadCompileJob(CompileJob job, CancellationToken cancellationToken) { if (job == null) throw new ArgumentNullException(nameof(job)); - if (job.DMApiValidated != true || job.Job.Cancelled.Value || job.Job.ExceptionDetails != null || job.Job.StoppedAt == null) + if (job.DMApiValidated != true || job.Job.Cancelled == true || job.Job.ExceptionDetails != null) throw new InvalidOperationException("Cannot load incomplete compile job!"); - if (setAsStagedInDb) - await databaseContextFactory.UseContext(async db => - { - var ddsettings = new DreamDaemonSettings - { - InstanceId = instance.Id - }; - db.DreamDaemonSettings.Attach(ddsettings); - ddsettings.StagedCompileJob = job; - await db.Save(cancellationToken).ConfigureAwait(false); - }).ConfigureAwait(false); + var newProvider = await FromCompileJob(job, cancellationToken).ConfigureAwait(false); + if (newProvider == null) + return; lock (this) { - var oldDmbProvider = nextDmbProvider; - if (oldDmbProvider != null && oldDmbProvider.CompileJob.Job.StoppedAt < oldDmbProvider.CompileJob.Job.StoppedAt) - throw new InvalidOperationException("Loaded compile job older than current job!"); - nextDmbProvider = FromCompileJob(job); + nextDmbProvider?.Dispose(); + nextDmbProvider = newProvider; newerDmbTcs.SetResult(nextDmbProvider); - newerDmbTcs = new TaskCompletionSource(); + newerDmbTcs = new TaskCompletionSource(); } } /// public async Task LockNextDmb(CancellationToken cancellationToken) { - Task task; - lock (this) - if (nextDmbProvider != null) - return nextDmbProvider; - else + if (nextDmbProvider == null) + { + Task task; + lock (this) task = newerDmbTcs.Task; - - var result = await task.ConfigureAwait(false); - //so there's currently a race condition in DreamMakerController where the setting of CompileJob.RevisionInformation and thus IDmbProvider.RevisionInformation can be delayed to after this if someone tries to start the server instantly after compiling - //This is a terrible terrible hack to get around that - //I'm sorry future me, I can't think of any other way to fix this other than giving DreamMaker an IDatabaseContext or having the controller load the CompileJob - await Task.Delay(new TimeSpan(0, 0, 10), cancellationToken).ConfigureAwait(false); - return result; + await task.ConfigureAwait(false); + } + lock (this) + { + ++jobLockCounts[nextDmbProvider.CompileJob.Id]; + return nextDmbProvider; + } } /// public Task StartAsync(CancellationToken cancellationToken) => databaseContextFactory.UseContext(async (db) => { //where complete clause not necessary, only successful COMPILEjobs get in the db - var cj = await db.Instances.Where(x => x.Id == instance.Id).SelectMany(x => x.RevisionInformations).SelectMany(x => x.CompileJobs).OrderByDescending(x => x.Job.StoppedAt).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + var cj = await db.CompileJobs.Where(x => x.Job.Instance.Id == instance.Id && !x.Job.Cancelled.Value && x.Job.ExceptionDetails == null && x.Job.StoppedAt != null) + .Include(x => x.Job).ThenInclude(x => x.StartedBy) + .Include(x => x.RevisionInformation).ThenInclude(x => x.PrimaryTestMerge).ThenInclude(x => x.MergedBy) + .Include(x => x.RevisionInformation).ThenInclude(x => x.ActiveTestMerges).ThenInclude(x => x.TestMerge).ThenInclude(x => x.MergedBy) + .OrderByDescending(x => x.Job.StoppedAt).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); if (cj == default(CompileJob)) return; - var directoriesTask = ioManager.GetDirectories(".", cancellationToken); - var compileJobTask = LoadCompileJob(cj, false, cancellationToken); - //delete all other compile jobs - var directories = await directoriesTask.ConfigureAwait(false); - await Task.WhenAll(directories.Where(x => x != cj.Job.ToString()).Select(x => ioManager.DeleteDirectory(x, cancellationToken))).ConfigureAwait(false); - await compileJobTask.ConfigureAwait(false); + await LoadCompileJob(cj, cancellationToken).ConfigureAwait(false); + //we dont do CleanUnusedCompileJobs here because the watchdog may have plans for them yet }); /// @@ -172,15 +170,86 @@ namespace Tgstation.Server.Host.Components.Compiler } /// - public IDmbProvider FromCompileJob(CompileJob compileJob) + public async Task FromCompileJob(CompileJob compileJob, CancellationToken cancellationToken) { - lock (this) + logger.LogTrace("Loading compile job {0}...", compileJob.Id); + var providerSubmitted = false; + var newProvider = new DmbProvider(compileJob, ioManager, () => { - if (!jobLockCounts.TryGetValue(compileJob.Id, out int value)) - jobLockCounts.Add(compileJob.Id, 1); - else - jobLockCounts[compileJob.Id] = ++value; - return new DmbProvider(compileJob, ioManager, () => CleanJob(compileJob)); + if (providerSubmitted) + CleanJob(compileJob); + }); + + try + { + var primaryCheckTask = ioManager.FileExists(ioManager.ConcatPath(newProvider.PrimaryDirectory, newProvider.DmbName), cancellationToken); + var secondaryCheckTask = ioManager.FileExists(ioManager.ConcatPath(newProvider.PrimaryDirectory, newProvider.DmbName), cancellationToken); + + if (!(await primaryCheckTask.ConfigureAwait(false) && await secondaryCheckTask.ConfigureAwait(false))) + { + logger.LogWarning("Error loading compile job, .dmb missing!"); + return null; //omae wa mou shinderu + } + + lock (this) + { + if (!jobLockCounts.TryGetValue(compileJob.Id, out int value)) + jobLockCounts.Add(compileJob.Id, 1); + else + jobLockCounts[compileJob.Id] = ++value; + providerSubmitted = true; + return newProvider; + } + } + finally + { + if (!providerSubmitted) + newProvider.Dispose(); + } + } + + /// + public async Task CleanUnusedCompileJobs(CompileJob exceptThisOne, CancellationToken cancellationToken) + { + List jobIdsToSkip; + //don't clean locked directories + lock (this) + jobIdsToSkip = jobLockCounts.Select(x => x.Key).ToList(); + + List jobUidsToNotErase = null; + + //find the uids of locked directories + await databaseContextFactory.UseContext(async db => + { + jobUidsToNotErase = await db.CompileJobs.Where(x => x.Job.Instance.Id == instance.Id && jobIdsToSkip.Contains(x.Id) && x.DirectoryName.HasValue).Select(x => x.DirectoryName.Value.ToString().ToUpperInvariant()).ToListAsync(cancellationToken).ConfigureAwait(false); + }).ConfigureAwait(false); + + //add the other exemption + if (exceptThisOne != null) + jobUidsToNotErase.Add(exceptThisOne.DirectoryName.Value.ToString().ToUpperInvariant()); + + //cleanup + var directories = await ioManager.GetDirectories(".", cancellationToken).ConfigureAwait(false); + int deleting = 0; + var tasks = directories.Select(async x => + { + var nameOnly = ioManager.GetFileName(x); + if (jobUidsToNotErase.Contains(nameOnly.ToUpperInvariant())) + return; + try + { + ++deleting; + await ioManager.DeleteDirectory(x, cancellationToken).ConfigureAwait(false); + } + catch (Exception e) + { + logger.LogWarning("Error deleting directory {0}! Exception: {1}", x, e); + } + }).ToList(); + if (deleting > 0) + { + logger.LogDebug("Cleaning {0} unused game folders...", deleting); + await Task.WhenAll().ConfigureAwait(false); } } } diff --git a/src/Tgstation.Server.Host/Components/Compiler/DmbProvider.cs b/src/Tgstation.Server.Host/Components/Compiler/DmbProvider.cs index 3650c69ddc..bf042d1912 100644 --- a/src/Tgstation.Server.Host/Components/Compiler/DmbProvider.cs +++ b/src/Tgstation.Server.Host/Components/Compiler/DmbProvider.cs @@ -46,15 +46,10 @@ namespace Tgstation.Server.Host.Components.Compiler this.onDispose = onDispose ?? throw new ArgumentNullException(nameof(onDispose)); } - ~DmbProvider() => Dispose(); + /// + public void Dispose() => onDispose?.Invoke(); /// - public void Dispose() - { - onDispose?.Invoke(); - GC.SuppressFinalize(this); - } - public void KeepAlive() => onDispose = null; } } diff --git a/src/Tgstation.Server.Host/Components/Compiler/DreamMaker.cs b/src/Tgstation.Server.Host/Components/Compiler/DreamMaker.cs index 33beec3f99..4a62f8130e 100644 --- a/src/Tgstation.Server.Host/Components/Compiler/DreamMaker.cs +++ b/src/Tgstation.Server.Host/Components/Compiler/DreamMaker.cs @@ -35,7 +35,7 @@ namespace Tgstation.Server.Host.Components.Compiler /// /// Extension for .dmes /// - const string DmeExtension = ".dme"; + const string DmeExtension = "dme"; /// public CompilerStatus Status { get; private set; } @@ -100,34 +100,45 @@ namespace Tgstation.Server.Host.Components.Compiler /// Run a quick DD instance to test the DMAPI is installed on the target code /// /// The timeout in seconds for validation + /// The level to use to validate the API /// The for the operation /// The current /// The for the operation /// A resulting in if the DMAPI was successfully validated, otherwise - async Task VerifyApi(uint timeout, Models.CompileJob job, IByondExecutableLock byondLock, CancellationToken cancellationToken) + async Task VerifyApi(uint timeout, DreamDaemonSecurity securityLevel, Models.CompileJob job, IByondExecutableLock byondLock, CancellationToken cancellationToken) { + logger.LogTrace("Verifying DMAPI..."); var launchParameters = new DreamDaemonLaunchParameters { AllowWebClient = false, PrimaryPort = 0, //pick any port - SecurityLevel = DreamDaemonSecurity.Safe, //all it needs to read the file and exit + SecurityLevel = securityLevel, //all it needs to read the file and exit StartupTimeout = timeout }; var dirA = ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName); - var provider = new TemporaryDmbProvider(ioManager.ResolvePath(ioManager.GetDirectoryName(dirA)), ioManager.ResolvePath(ioManager.ConcatPath(dirA, String.Concat(job.DmeName, DmbExtension)))); + var provider = new TemporaryDmbProvider(ioManager.ResolvePath(dirA), String.Concat(job.DmeName, DmbExtension), job); var timeoutAt = DateTimeOffset.Now.AddSeconds(timeout); using (var controller = await sessionControllerFactory.LaunchNew(launchParameters, provider, byondLock, true, true, true, cancellationToken).ConfigureAwait(false)) { - var timeoutTask = Task.Delay(timeoutAt - DateTimeOffset.Now, cancellationToken); + var now = DateTimeOffset.Now; + if (now < timeoutAt) + { + var timeoutTask = Task.Delay(timeoutAt - DateTimeOffset.Now, cancellationToken); - await Task.WhenAny(controller.Lifetime, timeoutTask).ConfigureAwait(false); - cancellationToken.ThrowIfCancellationRequested(); + await Task.WhenAny(controller.Lifetime, timeoutTask).ConfigureAwait(false); + cancellationToken.ThrowIfCancellationRequested(); + } if (!controller.Lifetime.IsCompleted) + { + logger.LogDebug("API validation timed out!"); return false; + } - return controller.ApiValidated; + var validated = controller.ApiValidated; + logger.LogTrace("API valid: {0}", validated); + return validated; } } @@ -143,10 +154,11 @@ namespace Tgstation.Server.Host.Components.Compiler using (var dm = new Process()) { dm.StartInfo.FileName = dreamMakerPath; - dm.StartInfo.Arguments = String.Format(CultureInfo.InvariantCulture, "-clean {0}{1}", job.DmeName, DmeExtension); + dm.StartInfo.Arguments = String.Format(CultureInfo.InvariantCulture, "-clean {0}.{1}", job.DmeName, DmeExtension); dm.StartInfo.WorkingDirectory = ioManager.ResolvePath(ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName)); dm.StartInfo.RedirectStandardOutput = true; dm.StartInfo.RedirectStandardError = true; + dm.StartInfo.UseShellExecute = false; var OutputList = new StringBuilder(); var eventHandler = new DataReceivedEventHandler( delegate (object sender, DataReceivedEventArgs e) @@ -160,12 +172,15 @@ namespace Tgstation.Server.Host.Components.Compiler dm.EnableRaisingEvents = true; var dmTcs = new TaskCompletionSource(); - dm.Exited += (a, b) => dmTcs.SetResult(null); + dm.Exited += (a, b) => dmTcs.TrySetResult(null); + logger.LogTrace("Running DreamMaker..."); dm.Start(); + dm.BeginOutputReadLine(); + dm.BeginErrorReadLine(); try { - using (cancellationToken.Register(() => dmTcs.SetCanceled())) + using (cancellationToken.Register(() => dmTcs.TrySetCanceled())) await dmTcs.Task.ConfigureAwait(false); } finally @@ -177,8 +192,10 @@ namespace Tgstation.Server.Host.Components.Compiler } } - job.Output = OutputList.ToString(); job.ExitCode = dm.ExitCode; + logger.LogDebug("DreamMaker exit code: {0}", job.ExitCode); + job.Output = OutputList.ToString(); + logger.LogTrace("DreamMaker output: {0}", job.Output); } } @@ -191,7 +208,7 @@ namespace Tgstation.Server.Host.Components.Compiler async Task ModifyDme(Models.CompileJob job, CancellationToken cancellationToken) { var dirA = ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName); - var dmePath = ioManager.ConcatPath(dirA, String.Concat(job.DmeName, DmeExtension)); + var dmePath = ioManager.ConcatPath(dirA, String.Join('.', job.DmeName, DmeExtension)); var dmeReadTask = ioManager.ReadAllBytes(dmePath, cancellationToken); var dmeModificationsTask = configuration.CopyDMFilesTo(dmePath, ioManager.ResolvePath(dirA), cancellationToken); @@ -202,7 +219,18 @@ namespace Tgstation.Server.Host.Components.Compiler var dmeModifications = await dmeModificationsTask.ConfigureAwait(false); if (dmeModifications == null || dmeModifications.TotalDmeOverwrite) + { + if (dmeModifications != null) + logger.LogDebug(".dme replacement configured!"); + else + logger.LogTrace("No .dme modifications required."); return; + } + + if (dmeModifications.HeadIncludeLine != null) + logger.LogDebug("Head .dme include line: {0}", dmeModifications.HeadIncludeLine); + if (dmeModifications.TailIncludeLine != null) + logger.LogDebug("Tail .dme include line: {0}", dmeModifications.TailIncludeLine); var dmeLines = new List(dme.Split(new[] { Environment.NewLine }, StringSplitOptions.None)); for (var I = 0; I < dmeLines.Count; ++I) @@ -225,51 +253,86 @@ namespace Tgstation.Server.Host.Components.Compiler } /// - public async Task Compile(string projectName, uint apiValidateTimeout, IRepository repository, CancellationToken cancellationToken) + public async Task Compile(string projectName, DreamDaemonSecurity securityLevel, uint apiValidateTimeout, IRepository repository, CancellationToken cancellationToken) { + if (repository == null) + throw new ArgumentNullException(nameof(repository)); + + if (securityLevel == DreamDaemonSecurity.Ultrasafe) + throw new ArgumentOutOfRangeException(nameof(securityLevel), securityLevel, "Cannot compile with ultrasafe security!"); + logger.LogTrace("Begin Compile"); - await eventConsumer.HandleEvent(EventType.CompileStart, new List{ repository.Origin }, cancellationToken).ConfigureAwait(false); + + var job = new Models.CompileJob + { + DirectoryName = Guid.NewGuid(), + DmeName = projectName + }; + + logger.LogTrace("Compile output GUID: {0}", job.DirectoryName); + + lock (this) + { + if(Status != CompilerStatus.Idle) + { + job.Output = "There is already a compile in progress!"; + logger.LogInformation(job.Output); + return job; + } + + Status = CompilerStatus.Copying; + } try { - Status = CompilerStatus.Copying; - var job = new Models.CompileJob - { - DirectoryName = Guid.NewGuid(), - DmeName = projectName - }; await ioManager.CreateDirectory(job.DirectoryName.ToString(), cancellationToken).ConfigureAwait(false); var dirA = ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName); var dirB = ioManager.ConcatPath(job.DirectoryName.ToString(), BDirectoryName); async Task CleanupFailedCompile() { + logger.LogTrace("Cleaning compile directory..."); Status = CompilerStatus.Cleanup; try { await ioManager.DeleteDirectory(job.DirectoryName.ToString(), CancellationToken.None).ConfigureAwait(false); } - catch { } + catch (Exception e) + { + logger.LogWarning("Error cleaning up compile directory {0}! Exception: {1}", ioManager.ResolvePath(job.DirectoryName.ToString()), e); + } }; try { + logger.LogTrace("Copying repository to game directory..."); //copy the repository var fullDirA = ioManager.ResolvePath(dirA); + var repoOrigin = repository.Origin; using (repository) await repository.CopyTo(fullDirA, cancellationToken).ConfigureAwait(false); + Status = CompilerStatus.PreCompile; + + await eventConsumer.HandleEvent(EventType.CompileStart, new List { ioManager.ResolvePath(ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName)), repoOrigin }, cancellationToken).ConfigureAwait(false); + Status = CompilerStatus.Modifying; if (job.DmeName == null) { - job.DmeName = (await ioManager.GetFilesWithExtension(dirA, DmeExtension, cancellationToken).ConfigureAwait(false)).FirstOrDefault(); - if (job.DmeName == default) + logger.LogTrace("Searching for available .dmes..."); + var path = (await ioManager.GetFilesWithExtension(dirA, DmeExtension, cancellationToken).ConfigureAwait(false)).FirstOrDefault(); + if (path == default) { job.Output = "Unable to find any .dme!"; + logger.LogWarning(job.Output); return job; } + var dmeWithExtension = ioManager.GetFileName(path); + job.DmeName = dmeWithExtension.Substring(0, dmeWithExtension.Length - DmeExtension.Length - 1); } + logger.LogDebug("Selected {0}.dme for compilation!", job.DmeName); + await ModifyDme(job, cancellationToken).ConfigureAwait(false); Status = CompilerStatus.Compiling; @@ -284,7 +347,7 @@ namespace Tgstation.Server.Host.Components.Compiler Status = CompilerStatus.Verifying; - ddVerified = job.ExitCode == 0 && await VerifyApi(apiValidateTimeout, job, byondLock, cancellationToken).ConfigureAwait(false); + ddVerified = job.ExitCode == 0 && await VerifyApi(apiValidateTimeout, securityLevel, job, byondLock, cancellationToken).ConfigureAwait(false); } if (!ddVerified) @@ -297,11 +360,17 @@ namespace Tgstation.Server.Host.Components.Compiler { job.DMApiValidated = true; + logger.LogTrace("Running post compile event..."); + Status = CompilerStatus.PostCompile; + await eventConsumer.HandleEvent(EventType.CompileComplete, new List { ioManager.ResolvePath(ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName)) }, cancellationToken).ConfigureAwait(false); + + logger.LogTrace("Duplicating compiled game..."); Status = CompilerStatus.Duplicating; //duplicate the dmb et al await ioManager.CopyDirectory(dirA, dirB, null, cancellationToken).ConfigureAwait(false); + logger.LogTrace("Applying static game file symlinks..."); Status = CompilerStatus.Symlinking; //symlink in the static data @@ -309,9 +378,9 @@ namespace Tgstation.Server.Host.Components.Compiler var symBTask = configuration.SymlinkStaticFilesTo(ioManager.ResolvePath(dirB), cancellationToken); await Task.WhenAll(symATask, symBTask).ConfigureAwait(false); - await eventConsumer.HandleEvent(EventType.CompileComplete, null, cancellationToken).ConfigureAwait(false); + + logger.LogDebug("Compile complete!"); } - await compileJobConsumer.LoadCompileJob(job, cancellationToken).ConfigureAwait(false); return job; } catch diff --git a/src/Tgstation.Server.Host/Components/Compiler/ICompileJobConsumer.cs b/src/Tgstation.Server.Host/Components/Compiler/ICompileJobConsumer.cs index 398d6d038c..8095e3321e 100644 --- a/src/Tgstation.Server.Host/Components/Compiler/ICompileJobConsumer.cs +++ b/src/Tgstation.Server.Host/Components/Compiler/ICompileJobConsumer.cs @@ -9,7 +9,7 @@ namespace Tgstation.Server.Host.Components.Compiler /// /// Sink for s /// - interface ICompileJobConsumer : IHostedService, IDisposable + public interface ICompileJobConsumer : IHostedService, IDisposable { /// /// Load a new into the diff --git a/src/Tgstation.Server.Host/Components/Compiler/IDmbFactory.cs b/src/Tgstation.Server.Host/Components/Compiler/IDmbFactory.cs index e79f55b6f8..a76611577c 100644 --- a/src/Tgstation.Server.Host/Components/Compiler/IDmbFactory.cs +++ b/src/Tgstation.Server.Host/Components/Compiler/IDmbFactory.cs @@ -27,7 +27,17 @@ namespace Tgstation.Server.Host.Components.Compiler /// Gets a for a given /// /// The to make the for - /// A new - IDmbProvider FromCompileJob(CompileJob compileJob); + /// The for the operation + /// A resulting in a new representing the on success, on failure + Task FromCompileJob(CompileJob compileJob, CancellationToken cancellationToken); + + /// + /// Deletes all compile jobs that are inactive in the Game folder + /// + /// An optional compile job to not delete + /// The for the operation + /// A representing the running operation + Task CleanUnusedCompileJobs(CompileJob exceptThisOne, CancellationToken cancellationToken); + } } diff --git a/src/Tgstation.Server.Host/Components/Compiler/IDreamMaker.cs b/src/Tgstation.Server.Host/Components/Compiler/IDreamMaker.cs index da6878c667..d1e31dad33 100644 --- a/src/Tgstation.Server.Host/Components/Compiler/IDreamMaker.cs +++ b/src/Tgstation.Server.Host/Components/Compiler/IDreamMaker.cs @@ -1,7 +1,7 @@ using System.Threading; using System.Threading.Tasks; +using Tgstation.Server.Api.Models; using Tgstation.Server.Host.Components.Repository; -using Tgstation.Server.Host.Models; namespace Tgstation.Server.Host.Components.Compiler { @@ -19,10 +19,11 @@ namespace Tgstation.Server.Host.Components.Compiler /// Starts a compile /// /// The optional name of the .dme to compile without the extension if not pre + /// The level allowed for API validation /// The time in seconds to wait while validating the API /// The to copy from /// The for the operation /// A resulting in the partially populated for the operation. In particular, note the field will only have it's field populated - Task Compile(string projectName, uint apiValidateTimeout, IRepository repository, CancellationToken cancellationToken); + Task Compile(string projectName, DreamDaemonSecurity securityLevel, uint apiValidateTimeout, IRepository repository, CancellationToken cancellationToken); } } \ No newline at end of file diff --git a/src/Tgstation.Server.Host/Components/Compiler/TemporaryDmbProvider.cs b/src/Tgstation.Server.Host/Components/Compiler/TemporaryDmbProvider.cs index dcf5c0affe..3cd0962106 100644 --- a/src/Tgstation.Server.Host/Components/Compiler/TemporaryDmbProvider.cs +++ b/src/Tgstation.Server.Host/Components/Compiler/TemporaryDmbProvider.cs @@ -18,17 +18,19 @@ namespace Tgstation.Server.Host.Components.Compiler public string SecondaryDirectory => throw new NotSupportedException(); /// - public CompileJob CompileJob => null; + public CompileJob CompileJob { get; } /// /// Construct a /// /// The value of /// The value of - public TemporaryDmbProvider(string directory, string dmb) + /// The value of + public TemporaryDmbProvider(string directory, string dmb, CompileJob compileJob) { DmbName = dmb ?? throw new ArgumentNullException(nameof(dmb)); PrimaryDirectory = directory ?? throw new ArgumentNullException(nameof(directory)); + CompileJob = compileJob ?? throw new ArgumentNullException(nameof(compileJob)); } /// diff --git a/src/Tgstation.Server.Host/Components/EventType.cs b/src/Tgstation.Server.Host/Components/EventType.cs index e9442b0d10..57de1c2bac 100644 --- a/src/Tgstation.Server.Host/Components/EventType.cs +++ b/src/Tgstation.Server.Host/Components/EventType.cs @@ -38,9 +38,8 @@ /// No parameters /// ByondChangeComplete = 7, - /// - /// Parameters: Origin commit sha + /// Parameters: Game directory path, origin commit sha /// CompileStart = 8, /// @@ -52,7 +51,7 @@ /// CompileFailure = 10, /// - /// No parameters + /// Parameters: Game directory path /// CompileComplete = 11, diff --git a/src/Tgstation.Server.Host/Components/IInstance.cs b/src/Tgstation.Server.Host/Components/IInstance.cs index 946a95b16a..917eaa3e66 100644 --- a/src/Tgstation.Server.Host/Components/IInstance.cs +++ b/src/Tgstation.Server.Host/Components/IInstance.cs @@ -40,6 +40,11 @@ namespace Tgstation.Server.Host.Components /// IChat Chat { get; } + /// + /// The for the + /// + ICompileJobConsumer CompileJobConsumer { get; } + /// /// The for the /// diff --git a/src/Tgstation.Server.Host/Components/Instance.cs b/src/Tgstation.Server.Host/Components/Instance.cs index fcd335a27a..f5baaa0b6d 100644 --- a/src/Tgstation.Server.Host/Components/Instance.cs +++ b/src/Tgstation.Server.Host/Components/Instance.cs @@ -35,10 +35,8 @@ namespace Tgstation.Server.Host.Components /// public StaticFiles.IConfiguration Configuration { get; } - /// - /// The for the - /// - readonly ICompileJobConsumer compileJobConsumer; + /// + public ICompileJobConsumer CompileJobConsumer { get; } /// /// The for the @@ -79,7 +77,7 @@ namespace Tgstation.Server.Host.Components /// The value of /// The value of /// The value of - /// The value of + /// The value of /// The value of /// The value of /// The value of @@ -92,7 +90,7 @@ namespace Tgstation.Server.Host.Components Watchdog = watchdog ?? throw new ArgumentNullException(nameof(watchdog)); Chat = chat ?? throw new ArgumentNullException(nameof(chat)); Configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); - this.compileJobConsumer = compileJobConsumer ?? throw new ArgumentNullException(nameof(compileJobConsumer)); + CompileJobConsumer = compileJobConsumer ?? throw new ArgumentNullException(nameof(compileJobConsumer)); this.databaseContextFactory = databaseContextFactory ?? throw new ArgumentNullException(nameof(databaseContextFactory)); this.dmbFactory = dmbFactory ?? throw new ArgumentNullException(nameof(dmbFactory)); this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); @@ -102,7 +100,7 @@ namespace Tgstation.Server.Host.Components public void Dispose() { timerCts?.Dispose(); - compileJobConsumer.Dispose(); + CompileJobConsumer.Dispose(); Configuration.Dispose(); Chat.Dispose(); Watchdog.Dispose(); @@ -124,15 +122,19 @@ namespace Tgstation.Server.Host.Components RepositorySettings repositorySettings = null; string projectName = null; - uint timeout = 0; + DreamDaemonSettings ddSettings = null; var dbTask = databaseContextFactory.UseContext(async (db) => { var instanceQuery = db.Instances.Where(x => x.Id == metadata.Id); - var timeoutTask = instanceQuery.Select(x => x.DreamDaemonSettings.StartupTimeout).FirstAsync(cancellationToken); + var ddSettingsTask = instanceQuery.Select(x => x.DreamDaemonSettings).Select(x => new DreamDaemonSettings + { + StartupTimeout = x.StartupTimeout, + SecurityLevel = x.SecurityLevel + }).FirstAsync(cancellationToken); var projectNameTask = instanceQuery.Select(x => x.DreamMakerSettings.ProjectName).FirstOrDefaultAsync(cancellationToken); repositorySettings = await instanceQuery.Select(x => x.RepositorySettings).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); projectName = await projectNameTask.ConfigureAwait(false); - timeout = (await timeoutTask.ConfigureAwait(false)).Value; + ddSettings = await ddSettingsTask.ConfigureAwait(false); }); using (var repo = await RepositoryManager.LoadRepository(cancellationToken).ConfigureAwait(false)) { @@ -172,7 +174,7 @@ namespace Tgstation.Server.Host.Components if (repositorySettings.AutoUpdatesSynchronize.Value && startSha != repo.Head) await repo.Sychronize(repositorySettings.AccessUser, repositorySettings.AccessToken, shouldSyncTracked, cancellationToken).ConfigureAwait(false); - var job = await DreamMaker.Compile(projectName, timeout, repo, cancellationToken).ConfigureAwait(false); + var job = await DreamMaker.Compile(projectName, ddSettings.SecurityLevel.Value, ddSettings.StartupTimeout.Value, repo, cancellationToken).ConfigureAwait(false); } } catch (OperationCanceledException) { } @@ -194,10 +196,23 @@ namespace Tgstation.Server.Host.Components } /// - public Task StartAsync(CancellationToken cancellationToken) => Task.WhenAll(SetAutoUpdateInterval(metadata.AutoUpdateInterval), Configuration.StartAsync(cancellationToken), ByondManager.StartAsync(cancellationToken), Watchdog.StartAsync(cancellationToken), Chat.StartAsync(cancellationToken), compileJobConsumer.StartAsync(cancellationToken)); + public async Task StartAsync(CancellationToken cancellationToken) + { + await Task.WhenAll(SetAutoUpdateInterval(metadata.AutoUpdateInterval), Configuration.StartAsync(cancellationToken), ByondManager.StartAsync(cancellationToken), Chat.StartAsync(cancellationToken), CompileJobConsumer.StartAsync(cancellationToken)).ConfigureAwait(false); + + //dependent on so many things, its just safer this way + await Watchdog.StartAsync(cancellationToken).ConfigureAwait(false); + + CompileJob latestCompileJob = null; + await databaseContextFactory.UseContext(async db => + { + latestCompileJob = await db.CompileJobs.Where(x => x.Job.Instance.Id == metadata.Id && x.Job.ExceptionDetails == null).OrderByDescending(x => x.Job.StoppedAt).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + }).ConfigureAwait(false); + await dmbFactory.CleanUnusedCompileJobs(latestCompileJob, cancellationToken).ConfigureAwait(false); + } /// - public Task StopAsync(CancellationToken cancellationToken) => Task.WhenAll(SetAutoUpdateInterval(null), Configuration.StopAsync(cancellationToken), ByondManager.StopAsync(cancellationToken), Watchdog.StopAsync(cancellationToken), Chat.StopAsync(cancellationToken), compileJobConsumer.StopAsync(cancellationToken)); + public Task StopAsync(CancellationToken cancellationToken) => Task.WhenAll(SetAutoUpdateInterval(null), Configuration.StopAsync(cancellationToken), ByondManager.StopAsync(cancellationToken), Watchdog.StopAsync(cancellationToken), Chat.StopAsync(cancellationToken), CompileJobConsumer.StopAsync(cancellationToken)); /// public async Task SetAutoUpdateInterval(int? newInterval) diff --git a/src/Tgstation.Server.Host/Components/InstanceFactory.cs b/src/Tgstation.Server.Host/Components/InstanceFactory.cs index 87f26407be..e5b48d68b2 100644 --- a/src/Tgstation.Server.Host/Components/InstanceFactory.cs +++ b/src/Tgstation.Server.Host/Components/InstanceFactory.cs @@ -130,7 +130,7 @@ namespace Tgstation.Server.Host.Components var configuration = new StaticFiles.Configuration(configurationIoManager, synchronousIOManager, symlinkFactory, scriptExecutor, loggerFactory.CreateLogger()); var eventConsumer = new EventConsumer(configuration); - var dmbFactory = new DmbFactory(databaseContextFactory, gameIoManager, metadata.CloneMetadata()); + var dmbFactory = new DmbFactory(databaseContextFactory, gameIoManager, loggerFactory.CreateLogger(), metadata.CloneMetadata()); try { var repoManager = new RepositoryManager(metadata.RepositorySettings, repoIoManager, eventConsumer); diff --git a/src/Tgstation.Server.Host/Components/InstanceManager.cs b/src/Tgstation.Server.Host/Components/InstanceManager.cs index 8c395077c3..46b39b8e8c 100644 --- a/src/Tgstation.Server.Host/Components/InstanceManager.cs +++ b/src/Tgstation.Server.Host/Components/InstanceManager.cs @@ -8,6 +8,7 @@ using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; +using Tgstation.Server.Host.Components.Watchdog; using Tgstation.Server.Host.Core; using Tgstation.Server.Host.IO; @@ -16,11 +17,6 @@ namespace Tgstation.Server.Host.Components /// sealed class InstanceManager : IInstanceManager, IHostedService, IInteropRegistrar, IDisposable { - /// - /// HTTP GET query key for interop access identifiers - /// - const string AccessIdentifierQueryKey = "access"; - /// /// The for the /// @@ -178,10 +174,16 @@ namespace Tgstation.Server.Host.Components var tasks = new List(); await dbInstances.ForEachAsync(metadata => tasks.Add(metadata.Online.Value ? OnlineInstance(metadata, cancellationToken) : Task.CompletedTask), cancellationToken).ConfigureAwait(false); await Task.WhenAll(tasks).ConfigureAwait(false); + logger.LogInformation("Instance manager ready!"); application.Ready(null); } + catch (OperationCanceledException) + { + logger.LogInformation("Cancelled instance manager initialization!"); + } catch (Exception e) { + logger.LogCritical("Instance manager startup error! Exception: {0}", e); application.Ready(e); } }); @@ -216,7 +218,7 @@ namespace Tgstation.Server.Host.Components if (query == null) throw new ArgumentNullException(nameof(query)); - if (!query.TryGetValue(AccessIdentifierQueryKey, out StringValues values)) + if (!query.TryGetValue(InteropConstants.DMInteropAccessIdentifier, out StringValues values)) return null; var accessIdentifier = values.FirstOrDefault(); if (accessIdentifier == default) diff --git a/src/Tgstation.Server.Host/Components/ReattachInfoHandler.cs b/src/Tgstation.Server.Host/Components/ReattachInfoHandler.cs index a4135a29fa..5e831d0651 100644 --- a/src/Tgstation.Server.Host/Components/ReattachInfoHandler.cs +++ b/src/Tgstation.Server.Host/Components/ReattachInfoHandler.cs @@ -80,7 +80,8 @@ namespace Tgstation.Server.Host.Components await databaseContextFactory.UseContext(async (db) => result = await db.Instances.Where(x => x.Id == metadata.Id).Select(x => x.WatchdogReattachInformation).FirstAsync(cancellationToken).ConfigureAwait(false) ).ConfigureAwait(false); - return new WatchdogReattachInformation(result, dmbFactory); + var bravoDmbTask = dmbFactory.FromCompileJob(result.Bravo.CompileJob, cancellationToken); + return new WatchdogReattachInformation(result, await dmbFactory.FromCompileJob(result.Alpha.CompileJob, cancellationToken).ConfigureAwait(false), await bravoDmbTask.ConfigureAwait(false)); } } } diff --git a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs index 6ff0183354..c62ee59bc5 100644 --- a/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs +++ b/src/Tgstation.Server.Host/Components/StaticFiles/Configuration.cs @@ -27,9 +27,10 @@ namespace Tgstation.Server.Host.Components.StaticFiles static readonly IReadOnlyDictionary EventTypeScriptFileNameMap = new Dictionary { + { EventType.CompileStart, "PreCompile" } }; - static readonly string SystemScriptFileExtension = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? ".bat" : ".sh"; + static readonly string SystemScriptFileExtension = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "bat" : "sh"; /// /// The for @@ -223,7 +224,19 @@ namespace Tgstation.Server.Host.Components.StaticFiles else task = ioManager.GetDirectories(GameStaticFilesSubdirectory, cancellationToken); var entries = await task.ConfigureAwait(false); - await Task.WhenAll(entries.Select(x => symlinkFactory.CreateSymbolicLink(ioManager.ResolvePath(x), ioManager.ConcatPath(destination, x), cancellationToken))).ConfigureAwait(false); + + await Task.WhenAll(task.Result.Select(async x => + { + var destPath = ioManager.ConcatPath(destination, ioManager.GetFileName(x)); + logger.LogTrace("Symlinking {0} to {1}...", x, destPath); + var fileExistsTask = ioManager.FileExists(destPath, cancellationToken); + if (await ioManager.DirectoryExists(destPath, cancellationToken).ConfigureAwait(false)) + await ioManager.DeleteDirectory(destPath, cancellationToken).ConfigureAwait(false); + var fileExists = await fileExistsTask.ConfigureAwait(false); + if (fileExists) + await ioManager.DeleteFile(destPath, cancellationToken).ConfigureAwait(false); + await symlinkFactory.CreateSymbolicLink(ioManager.ResolvePath(x), ioManager.ResolvePath(destPath), cancellationToken).ConfigureAwait(false); + })).ConfigureAwait(false); } await Task.WhenAll(SymlinkBase(true), SymlinkBase(false)).ConfigureAwait(false); @@ -298,7 +311,7 @@ namespace Tgstation.Server.Host.Components.StaticFiles var files = await ioManager.GetFilesWithExtension(EventScriptsSubdirectory, SystemScriptFileExtension, cancellationToken).ConfigureAwait(false); var resolvedScriptsDir = ioManager.ResolvePath(EventScriptsSubdirectory); - foreach (var I in files.Where(x => x.StartsWith(scriptName, StringComparison.Ordinal))) + foreach (var I in files.Select(x => ioManager.GetFileName(x)).Where(x => x.StartsWith(scriptName, StringComparison.Ordinal))) if ((await scriptExecutor.ExecuteScript(ioManager.ConcatPath(resolvedScriptsDir, I), parameters, cancellationToken).ConfigureAwait(false)) != 0) return false; } diff --git a/src/Tgstation.Server.Host/Components/StaticFiles/ScriptExecutor.cs b/src/Tgstation.Server.Host/Components/StaticFiles/ScriptExecutor.cs index 14a2a97805..ff2a90c625 100644 --- a/src/Tgstation.Server.Host/Components/StaticFiles/ScriptExecutor.cs +++ b/src/Tgstation.Server.Host/Components/StaticFiles/ScriptExecutor.cs @@ -47,11 +47,11 @@ namespace Tgstation.Server.Host.Components.StaticFiles process.EnableRaisingEvents = true; var tcs = new TaskCompletionSource(); - process.Exited += (a, b) => tcs.SetResult(null); + process.Exited += (a, b) => tcs.TrySetResult(null); try { process.Start(); - using (cancellationToken.Register(() => tcs.SetCanceled())) + using (cancellationToken.Register(() => tcs.TrySetCanceled())) await tcs.Task.ConfigureAwait(false); } catch (InvalidOperationException) diff --git a/src/Tgstation.Server.Host/Components/Watchdog/Executor.cs b/src/Tgstation.Server.Host/Components/Watchdog/Executor.cs index 5ff46f692c..85ec573591 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/Executor.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/Executor.cs @@ -1,4 +1,5 @@ -using System; +using Microsoft.Extensions.Logging; +using System; using System.Diagnostics; using System.Globalization; using Tgstation.Server.Api.Models; @@ -10,6 +11,11 @@ namespace Tgstation.Server.Host.Components.Watchdog /// sealed class Executor : IExecutor { + /// + /// The for the + /// + readonly ILogger logger; + /// /// Change a given into the appropriate DreamDaemon command line word /// @@ -30,6 +36,15 @@ namespace Tgstation.Server.Host.Components.Watchdog } } + /// + /// Construct an + /// + /// The value of + public Executor(ILogger logger) + { + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + /// public ISession AttachToDreamDaemon(int processId, IByondExecutableLock byondLock) => new Session(Process.GetProcessById(processId), byondLock); @@ -57,6 +72,8 @@ namespace Tgstation.Server.Host.Components.Watchdog launchParameters.AllowWebClient.Value ? "-webclient " : String.Empty, SecurityWord(launchParameters.SecurityLevel.Value), parameters); + + logger.LogTrace("Running DreamDaemon in {0}: {1} {2}", proc.StartInfo.WorkingDirectory, proc.StartInfo.FileName, proc.StartInfo.Arguments); proc.Start(); diff --git a/src/Tgstation.Server.Host/Components/Watchdog/InteropConstants.cs b/src/Tgstation.Server.Host/Components/Watchdog/InteropConstants.cs index 9b08efcdf5..23ad40c9af 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/InteropConstants.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/InteropConstants.cs @@ -1,8 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace Tgstation.Server.Host.Components.Watchdog +namespace Tgstation.Server.Host.Components.Watchdog { static class InteropConstants { diff --git a/src/Tgstation.Server.Host/Components/Watchdog/LaunchResult.cs b/src/Tgstation.Server.Host/Components/Watchdog/LaunchResult.cs index 3e8c3d51b4..81b2be9fb9 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/LaunchResult.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/LaunchResult.cs @@ -18,12 +18,7 @@ namespace Tgstation.Server.Host.Components.Watchdog /// public int? ExitCode { get; set; } - /// - /// The peak virtual memory usage in bytes - /// - public long PeakMemory { get; set; } - /// - public override string ToString() => String.Format(CultureInfo.InvariantCulture, "Exit Code: {0}, RAM: {1}, Time {2}ms", ExitCode, PeakMemory, StartupTime.TotalMilliseconds); + public override string ToString() => String.Format(CultureInfo.InvariantCulture, "Exit Code: {0}, Time {1}ms", ExitCode, StartupTime.TotalMilliseconds); } } \ No newline at end of file diff --git a/src/Tgstation.Server.Host/Components/Watchdog/ReattachInformation.cs b/src/Tgstation.Server.Host/Components/Watchdog/ReattachInformation.cs index 271d840692..9570ce7673 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/ReattachInformation.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/ReattachInformation.cs @@ -1,4 +1,4 @@ -using Tgstation.Server.Host.Components.Compiler; +using System; using Tgstation.Server.Host.Models; namespace Tgstation.Server.Host.Components.Watchdog @@ -19,13 +19,13 @@ namespace Tgstation.Server.Host.Components.Watchdog public ReattachInformation() { } /// - /// Construct a from a given and + /// Construct a from a given and /// /// The to copy values from - /// The used to assign - public ReattachInformation(Models.ReattachInformation copy, IDmbFactory dmbFactory) : base(copy) + /// The value of + public ReattachInformation(Models.ReattachInformation copy, IDmbProvider dmb) : base(copy) { - Dmb = dmbFactory.FromCompileJob(copy.CompileJob); + Dmb = dmb ?? throw new ArgumentNullException(nameof(dmb)); } } } \ No newline at end of file diff --git a/src/Tgstation.Server.Host/Components/Watchdog/Session.cs b/src/Tgstation.Server.Host/Components/Watchdog/Session.cs index 29b7eb8844..a26f14e130 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/Session.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/Session.cs @@ -52,16 +52,21 @@ namespace Tgstation.Server.Host.Components.Watchdog var result = new LaunchResult { ExitCode = process.HasExited ? (int?)process.ExitCode : null, - PeakMemory = process.PeakWorkingSet64, StartupTime = DateTimeOffset.Now - startTime }; - if (result.PeakMemory == 0) //linux, best we can do honestly, test if this even works - result.PeakMemory = process.WorkingSet64; return result; }, default, TaskCreationOptions.LongRunning, TaskScheduler.Current); lifetimeTask = new TaskCompletionSource(); - process.EnableRaisingEvents = true; - process.Exited += (a, b) => lifetimeTask.SetResult(process.ExitCode); + try + { + process.EnableRaisingEvents = true; + process.Exited += (a, b) => lifetimeTask.TrySetResult(process.ExitCode); + } + catch (InvalidOperationException) + { + //dead proccess + lifetimeTask.TrySetResult(process.ExitCode); + } } /// diff --git a/src/Tgstation.Server.Host/Components/Watchdog/SessionController.cs b/src/Tgstation.Server.Host/Components/Watchdog/SessionController.cs index b2ecc29db6..ea131270e6 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/SessionController.cs @@ -145,6 +145,11 @@ namespace Tgstation.Server.Host.Components.Watchdog /// bool apiValidated; + /// + /// If should be kept alive instead + /// + bool released; + /// /// Construct a /// @@ -171,22 +176,50 @@ namespace Tgstation.Server.Host.Components.Watchdog portClosed = false; disposed = false; apiValidated = false; + released = false; rebootTcs = new TaskCompletionSource(); } + /// + /// Finalize the + /// + ~SessionController() => Dispose(false); + /// public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + /// + void Dispose(bool disposing) { lock (this) { if (disposed) return; - session.Dispose(); - interopContext.Dispose(); - Dmb?.Dispose(); //will be null when released - chatJsonTrackingContext.Dispose(); - disposed = true; + if (disposing) + { + if (!released) + session.Terminate(); + session.Dispose(); + interopContext.Dispose(); + Dmb?.Dispose(); //will be null when released + chatJsonTrackingContext.Dispose(); + disposed = true; + } + else + { + if (logger != null) + logger.LogError("Being disposed via finalizer!"); + if (!released) + if (session != null) + session.Terminate(); + else if (logger != null) + logger.LogCritical("Unable to terminate active DreamDaemon session due to finalizer ordering!"); + } } } @@ -241,7 +274,7 @@ namespace Tgstation.Server.Host.Components.Watchdog } /// - /// Throws an if has been called + /// Throws an if has been called /// void CheckDisposed() { @@ -256,6 +289,7 @@ namespace Tgstation.Server.Host.Components.Watchdog //we still don't want to dispose the dmb yet, even though we're keeping it alive var tmpProvider = reattachInformation.Dmb; reattachInformation.Dmb = null; + released = true; Dispose(); Dmb.KeepAlive(); reattachInformation.Dmb = tmpProvider; diff --git a/src/Tgstation.Server.Host/Components/Watchdog/SessionControllerFactory.cs b/src/Tgstation.Server.Host/Components/Watchdog/SessionControllerFactory.cs index 1afc9a1571..45d028d0dc 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/SessionControllerFactory.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/SessionControllerFactory.cs @@ -1,6 +1,7 @@ using Byond.TopicSender; using Microsoft.Extensions.Logging; using Newtonsoft.Json; +using Newtonsoft.Json.Serialization; using System; using System.Globalization; using System.Linq; @@ -116,26 +117,34 @@ namespace Tgstation.Server.Host.Components.Watchdog InstanceName = instance.Name, Revision = dmbProvider.CompileJob.RevisionInformation }; - interopInfo.TestMerges.AddRange(dmbProvider.CompileJob.RevisionInformation.ActiveTestMerges.Select(x => x.TestMerge).Select(x => new TestMerge - { - Author = x.Author, - Body = x.BodyAtMerge, - Comment = x.Comment, - CommitSha = x.PrimaryRevisionInformation.CommitSha, - Number = x.Number, - OriginCommitSha = x.PrimaryRevisionInformation.OriginCommitSha, - PullRequestCommit = x.PullRequestRevision, - TimeMerged = x.MergedAt.Ticks, - Title = x.TitleAtMerge, - Url = x.Url - })); + + if (dmbProvider.CompileJob.RevisionInformation != null) //null while compiling + interopInfo.TestMerges.AddRange(dmbProvider.CompileJob.RevisionInformation.ActiveTestMerges.Select(x => x.TestMerge).Select(x => new TestMerge + { + Author = x.Author, + Body = x.BodyAtMerge, + Comment = x.Comment, + CommitSha = x.PrimaryRevisionInformation.CommitSha, + Number = x.Number, + OriginCommitSha = x.PrimaryRevisionInformation.OriginCommitSha, + PullRequestCommit = x.PullRequestRevision, + TimeMerged = x.MergedAt.Ticks, + Title = x.TitleAtMerge, + Url = x.Url + })); var interopJsonFile = GuidJsonFile(); - var interopJson = JsonConvert.SerializeObject(interopInfo); + var interopJson = JsonConvert.SerializeObject(interopInfo, Formatting.Indented, new JsonSerializerSettings + { + ContractResolver = new DefaultContractResolver + { + NamingStrategy = new CamelCaseNamingStrategy() + } + }); var basePath = primaryDirectory ? dmbProvider.PrimaryDirectory : dmbProvider.SecondaryDirectory; - var localIoManager = new ResolvingIOManager(ioManager, ioManager.ConcatPath(basePath, dmbProvider.DmbName)); + var localIoManager = new ResolvingIOManager(ioManager, basePath); var chatJsonTrackingTask = chat.TrackJsons(basePath, interopInfo.ChatChannelsJson, interopInfo.ChatCommandsJson, cancellationToken); diff --git a/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs index 315685ad42..3248a31598 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs @@ -182,7 +182,6 @@ namespace Tgstation.Server.Host.Components.Watchdog /// void DisposeAndNullControllers() { - logger.LogTrace("DisposeAndNullControllers"); alphaServer?.Dispose(); alphaServer = null; bravoServer?.Dispose(); @@ -315,7 +314,12 @@ namespace Tgstation.Server.Host.Components.Watchdog //either way try to start it using the active server's dmb as a backup try { - var dmbBackup = dmbFactory.FromCompileJob(monitorState.ActiveServer.Dmb.CompileJob); + var dmbBackup = await dmbFactory.FromCompileJob(monitorState.ActiveServer.Dmb.CompileJob, cancellationToken).ConfigureAwait(false); + + if (dmbBackup == null) //NANI!? + //just give up, if THAT compile job is failing then the ActiveServer is gonna crash soon too or already has + throw new Exception("Creating backup DMB provider failed!"); + monitorState.InactiveServer = await sessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbBackup, null, false, !monitorState.ActiveServer.IsPrimary, false, cancellationToken).ConfigureAwait(false); usedMostRecentDmb = false; await chat.SendWatchdogMessage("Staging newest DMB on inactive server failed: {0} Falling back to previous dmb...", cancellationToken).ConfigureAwait(false); @@ -327,7 +331,7 @@ namespace Tgstation.Server.Host.Components.Watchdog catch (Exception e2) { //fuuuuucckkk - logger.LogError("Backup strategy failed! Monitor will restart when active server reboots! This Exception: {0}", e2.ToString()); + logger.LogError("Backup strategy failed! Monitor will restart when active server reboots! Exception: {0}", e2.ToString()); monitorState.InactiveServerCritFail = true; await chat.SendWatchdogMessage("Attempted reboot of inactive server failed. Watchdog will reset when active server fails or exits", cancellationToken).ConfigureAwait(false); return true; //we didn't use the old dmb diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogReattachInformation.cs b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogReattachInformation.cs index efe645c044..5f7542e68d 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogReattachInformation.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogReattachInformation.cs @@ -1,5 +1,4 @@ -using Tgstation.Server.Host.Components.Compiler; -using Tgstation.Server.Host.Models; +using Tgstation.Server.Host.Models; namespace Tgstation.Server.Host.Components.Watchdog { @@ -24,16 +23,17 @@ namespace Tgstation.Server.Host.Components.Watchdog public WatchdogReattachInformation() { } /// - /// Construct a from a given with a given + /// Construct a from a given with a given and /// /// The to copy information from - /// The used to build the s - public WatchdogReattachInformation(Models.WatchdogReattachInformation copy, IDmbFactory dmbFactory): base(copy) + /// The used to build + /// The used to build + public WatchdogReattachInformation(Models.WatchdogReattachInformation copy, IDmbProvider dmbAlpha, IDmbProvider dmbBravo): base(copy) { if (copy.Alpha != null) - Alpha = new ReattachInformation(copy.Alpha, dmbFactory); + Alpha = new ReattachInformation(copy.Alpha, dmbAlpha); if (copy.Bravo != null) - Bravo = new ReattachInformation(copy.Bravo, dmbFactory); + Bravo = new ReattachInformation(copy.Bravo, dmbBravo); } } } diff --git a/src/Tgstation.Server.Host/Configuration/DatabaseConfiguration.cs b/src/Tgstation.Server.Host/Configuration/DatabaseConfiguration.cs index a75d463a48..07f8690601 100644 --- a/src/Tgstation.Server.Host/Configuration/DatabaseConfiguration.cs +++ b/src/Tgstation.Server.Host/Configuration/DatabaseConfiguration.cs @@ -24,5 +24,10 @@ /// The connection string for the database /// public string ConnectionString { get; set; } + + /// + /// If the database should use direct table creation instead of automatic migrations. Should not be used in production! + /// + public bool NoMigrations { get; set; } } } diff --git a/src/Tgstation.Server.Host/Controllers/ApiController.cs b/src/Tgstation.Server.Host/Controllers/ApiController.cs index b97a3eb2e3..3574f228cd 100644 --- a/src/Tgstation.Server.Host/Controllers/ApiController.cs +++ b/src/Tgstation.Server.Host/Controllers/ApiController.cs @@ -188,7 +188,7 @@ namespace Tgstation.Server.Host.Controllers } } - Logger.LogInformation("Request made by User ID {0}. Api version: {1}. User-Agent: {2}", AuthenticationContext?.User.Id.ToString(CultureInfo.InvariantCulture) ?? "NULL", ApiHeaders.ApiVersion, ApiHeaders.UserAgent); + Logger.LogTrace("Request made by User ID {0}. Api version: {1}. User-Agent: {2}. Type: {3}. Route {4}", AuthenticationContext?.User.Id.ToString(CultureInfo.InvariantCulture), ApiHeaders.ApiVersion, ApiHeaders.UserAgent, Request.Method, Request.Path); await base.OnActionExecutionAsync(context, next).ConfigureAwait(false); } } diff --git a/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs b/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs index 5e9cf98168..91b434a737 100644 --- a/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs +++ b/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs @@ -169,6 +169,9 @@ namespace Tgstation.Server.Host.Controllers || !CheckModified(x => x.SoftShutdown, DreamDaemonRights.SoftShutdown) || !CheckModified(x => x.StartupTimeout, DreamDaemonRights.SetStartupTimeout)) return Forbid(); + + if (current.SecurityLevel == DreamDaemonSecurity.Ultrasafe) + return BadRequest(new ErrorMessage { Message = "TGS does not support the ultrasafe DreamDaemon configuration!" }); var wd = instanceManager.GetInstance(Instance).Watchdog; diff --git a/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs b/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs index 460a6c598c..856b041626 100644 --- a/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs +++ b/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs @@ -101,14 +101,18 @@ namespace Tgstation.Server.Host.Controllers var instanceManager = serviceProvider.GetRequiredService(); var databaseContext = serviceProvider.GetRequiredService(); - var timeoutTask = databaseContext.DreamDaemonSettings.Where(x => x.InstanceId == instanceModel.Id).Select(x => x.StartupTimeout).FirstOrDefaultAsync(cancellationToken); + var ddSettingsTask = databaseContext.DreamDaemonSettings.Where(x => x.InstanceId == instanceModel.Id).Select(x => new DreamDaemonSettings{ + StartupTimeout = x.StartupTimeout, + SecurityLevel = x.SecurityLevel + }).FirstOrDefaultAsync(cancellationToken); var projectName = await databaseContext.DreamMakerSettings.Where(x => x.InstanceId == instanceModel.Id).Select(x => x.ProjectName).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); - var timeout = await timeoutTask.ConfigureAwait(false); + var ddSettings = await ddSettingsTask.ConfigureAwait(false); var instance = instanceManager.GetInstance(instanceModel); CompileJob compileJob; - string repoSha = null; + Task revInfoTask; + string repoSha; using (var repo = await instance.RepositoryManager.LoadRepository(cancellationToken).ConfigureAwait(false)) { if (repo == null) @@ -117,11 +121,15 @@ namespace Tgstation.Server.Host.Controllers return; } repoSha = repo.Head; - compileJob = await instance.DreamMaker.Compile(projectName, timeout.Value, repo, cancellationToken).ConfigureAwait(false); + revInfoTask = databaseContext.RevisionInformations.Where(x => x.CommitSha == repoSha).Select(x => new RevisionInformation { Id = x.Id }).FirstOrDefaultAsync(); + compileJob = await instance.DreamMaker.Compile(projectName, ddSettings.SecurityLevel.Value, ddSettings.StartupTimeout.Value, repo, cancellationToken).ConfigureAwait(false); } + if (compileJob.DMApiValidated != true) + return; + compileJob.Job = job; - compileJob.RevisionInformation = await databaseContext.RevisionInformations.Where(x => x.CommitSha == repoSha).Select(x => new RevisionInformation { Id = x.Id }).FirstOrDefaultAsync().ConfigureAwait(false); + compileJob.RevisionInformation = await revInfoTask.ConfigureAwait(false); if (compileJob.RevisionInformation == default) { @@ -134,12 +142,26 @@ namespace Tgstation.Server.Host.Controllers Id = Instance.Id } }; - DatabaseContext.Instances.Attach(compileJob.RevisionInformation.Instance); + databaseContext.Instances.Attach(compileJob.RevisionInformation.Instance); } + else + databaseContext.RevisionInformations.Attach(compileJob.RevisionInformation); databaseContext.CompileJobs.Add(compileJob); - //default ct because we don't want to give up after getting this far - await databaseContext.Save(default).ConfigureAwait(false); + await databaseContext.Save(cancellationToken).ConfigureAwait(false); + + //now load the entire compile job tree into the consumer + //default ct because we don't want to give up after getting this far since we already set this job as staged in the db + var finalCompileJob = await databaseContext.CompileJobs.Where(x => x.Id == compileJob.Id) + .Include(x => x.Job).ThenInclude(x => x.StartedBy) + .Include(x => x.RevisionInformation).ThenInclude(x => x.PrimaryTestMerge).ThenInclude(x => x.MergedBy) + .Include(x => x.RevisionInformation).ThenInclude(x => x.ActiveTestMerges).ThenInclude(x => x.TestMerge).ThenInclude(x => x.MergedBy) + .FirstOrDefaultAsync().ConfigureAwait(false); //can't wait to see that query + if (finalCompileJob == null) + //lol git fucked + return; + + await instance.CompileJobConsumer.LoadCompileJob(finalCompileJob, default).ConfigureAwait(false); } } } diff --git a/src/Tgstation.Server.Host/Controllers/HomeController.cs b/src/Tgstation.Server.Host/Controllers/HomeController.cs index bc1b3f63fa..0718e569a4 100644 --- a/src/Tgstation.Server.Host/Controllers/HomeController.cs +++ b/src/Tgstation.Server.Host/Controllers/HomeController.cs @@ -126,7 +126,10 @@ namespace Tgstation.Server.Host.Controllers var token = tokenFactory.CreateToken(user, out var expiry); if (identity != null) - identityCache.CacheSystemIdentity(user, identity, expiry.AddSeconds(10)); //expire the identity slightly after the auth token in case of lag + identityCache.CacheSystemIdentity(user, identity, expiry.AddMinutes(1)); //expire the identity slightly after the auth token in case of lag + + Logger.LogDebug("Successfully logged in user {0}!", user.Id); + return Json(token); } } diff --git a/src/Tgstation.Server.Host/Controllers/InstanceController.cs b/src/Tgstation.Server.Host/Controllers/InstanceController.cs index b1430a4fd1..a25768c465 100644 --- a/src/Tgstation.Server.Host/Controllers/InstanceController.cs +++ b/src/Tgstation.Server.Host/Controllers/InstanceController.cs @@ -117,7 +117,7 @@ namespace Tgstation.Server.Host.Controllers AutoStart = false, PrimaryPort = 1337, SecondaryPort = 1338, - SecurityLevel = DreamDaemonSecurity.Ultrasafe, + SecurityLevel = DreamDaemonSecurity.Safe, SoftRestart = false, SoftShutdown = false, StartupTimeout = 20 diff --git a/src/Tgstation.Server.Host/Controllers/InteropController.cs b/src/Tgstation.Server.Host/Controllers/InteropController.cs index f7b7b808e9..55bd53d731 100644 --- a/src/Tgstation.Server.Host/Controllers/InteropController.cs +++ b/src/Tgstation.Server.Host/Controllers/InteropController.cs @@ -1,11 +1,12 @@ using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; +using Microsoft.Net.Http.Headers; using System; +using System.Linq; +using System.Net.Http.Headers; using System.Threading; using System.Threading.Tasks; using Tgstation.Server.Host.Components; -using Tgstation.Server.Host.Models; -using Tgstation.Server.Host.Security; namespace Tgstation.Server.Host.Controllers { @@ -13,23 +14,27 @@ namespace Tgstation.Server.Host.Controllers /// Handles requests from DreamDaemon /// [Route("/Interop")] - public sealed class InteropController : ApiController + public sealed class InteropController : Controller //not an ApiController because "lol im byond and who is headers?" { /// /// The for the /// readonly IInstanceManager instanceManager; + /// + /// The for the + /// + readonly ILogger logger; + /// /// Construct an /// - /// The for the - /// The for the /// The value of - /// The for the - public InteropController(IInstanceManager instanceManager, IDatabaseContext databaseContext, IAuthenticationContextFactory authenticationContextFactory, ILogger logger) : base(databaseContext, authenticationContextFactory, logger, false) + /// The value of + public InteropController(IInstanceManager instanceManager, ILogger logger) { this.instanceManager = instanceManager ?? throw new ArgumentNullException(nameof(instanceManager)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); } /// @@ -40,6 +45,14 @@ namespace Tgstation.Server.Host.Controllers [HttpGet] public async Task HandleInterop(CancellationToken cancellationToken) { + //since this is the only identifying factor of a TGS server we want to pretend we don't exist unless it at least has the correct BYOND headers + if (!Request.Headers.TryGetValue(HeaderNames.UserAgent, out var userAgentValues) + || !ProductInfoHeaderValue.TryParse(userAgentValues.FirstOrDefault(), out var clientUserAgent) + || clientUserAgent.Product.Name != "libbyond") + return Unauthorized(); + + logger.LogDebug("Request from BYOND: {0}", Request.QueryString); + var result = await instanceManager.HandleWorldExport(Request.Query, cancellationToken).ConfigureAwait(false); //explain things in very simple terms dream daemon can understand //EXCEPT DREAMDAENEN BUGS diff --git a/src/Tgstation.Server.Host/Controllers/JobController.cs b/src/Tgstation.Server.Host/Controllers/JobController.cs index 16f628cba4..b946cbe94c 100644 --- a/src/Tgstation.Server.Host/Controllers/JobController.cs +++ b/src/Tgstation.Server.Host/Controllers/JobController.cs @@ -35,6 +35,16 @@ namespace Tgstation.Server.Host.Controllers this.jobManager = jobManager ?? throw new ArgumentNullException(nameof(jobManager)); } + /// + [TgsAuthorize] + public override async Task Read(CancellationToken cancellationToken) + { + var result = await DatabaseContext.Jobs.Where(x => x.Instance.Id == Instance.Id).OrderByDescending(x => x.StartedAt).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + if (result == null) + return StatusCode((int)HttpStatusCode.Gone); + return Json(result); + } + /// [TgsAuthorize] public override async Task List(CancellationToken cancellationToken) diff --git a/src/Tgstation.Server.Host/Controllers/RepositoryController.cs b/src/Tgstation.Server.Host/Controllers/RepositoryController.cs index 5b0209649f..a0e8d65633 100644 --- a/src/Tgstation.Server.Host/Controllers/RepositoryController.cs +++ b/src/Tgstation.Server.Host/Controllers/RepositoryController.cs @@ -300,7 +300,7 @@ namespace Tgstation.Server.Host.Controllers await jobManager.RegisterOperation(job, async (paramJob, serviceProvider, progressReporter, ct) => { - using (var repo = await instanceManager.GetInstance(Instance).RepositoryManager.LoadRepository(cancellationToken).ConfigureAwait(false)) + using (var repo = await instanceManager.GetInstance(Instance).RepositoryManager.LoadRepository(ct).ConfigureAwait(false)) { if (repo == null) throw new InvalidOperationException("Repository could not be loaded!"); @@ -330,13 +330,13 @@ namespace Tgstation.Server.Host.Controllers }; databaseContext.Instances.Attach(attachedInstance); - await LoadRevisionInformation(repo, databaseContext, attachedInstance, null, x => lastRevisionInfo = x, cancellationToken).ConfigureAwait(false); + await LoadRevisionInformation(repo, databaseContext, attachedInstance, null, x => lastRevisionInfo = x, ct).ConfigureAwait(false); //apply new rev info, tracking applied test merges async Task UpdateRevInfo() { var last = lastRevisionInfo; - await LoadRevisionInformation(repo, databaseContext, attachedInstance, last.OriginCommitSha, x => lastRevisionInfo = x, cancellationToken).ConfigureAwait(false); + await LoadRevisionInformation(repo, databaseContext, attachedInstance, last.OriginCommitSha, x => lastRevisionInfo = x, ct).ConfigureAwait(false); lastRevisionInfo.ActiveTestMerges.AddRange(last.ActiveTestMerges); }; @@ -347,11 +347,11 @@ namespace Tgstation.Server.Host.Controllers { if (!repo.Tracking && model.Reference == null) throw new InvalidOperationException("Not on an updatable reference!"); - await repo.FetchOrigin(currentModel.AccessUser, currentModel.AccessToken, x => progressReporter(x / numFetches), cancellationToken).ConfigureAwait(false); + await repo.FetchOrigin(currentModel.AccessUser, currentModel.AccessToken, x => progressReporter(x / numFetches), ct).ConfigureAwait(false); doneFetches = 1; if (!modelHasShaOrReference) { - var fastForward = await repo.MergeOrigin(committerName, currentModel.CommitterEmail, cancellationToken).ConfigureAwait(false); + var fastForward = await repo.MergeOrigin(committerName, currentModel.CommitterEmail, ct).ConfigureAwait(false); if (!fastForward.HasValue) throw new InvalidOperationException("Merge conflict occurred during origin update!"); await UpdateRevInfo().ConfigureAwait(false); @@ -366,83 +366,106 @@ namespace Tgstation.Server.Host.Controllers if ((model.CheckoutSha != null && repo.Head.ToUpperInvariant() != model.CheckoutSha.ToUpperInvariant()) || (model.Reference != null && repo.Reference != model.Reference)) { - await repo.CheckoutObject(model.CheckoutSha ?? model.Reference, cancellationToken).ConfigureAwait(false); - await LoadRevisionInformation(repo, databaseContext, attachedInstance, null, x => lastRevisionInfo = x, cancellationToken).ConfigureAwait(false); //we've either seen origin before or what we're checking out is on origin + await repo.CheckoutObject(model.CheckoutSha ?? model.Reference, ct).ConfigureAwait(false); + await LoadRevisionInformation(repo, databaseContext, attachedInstance, null, x => lastRevisionInfo = x, ct).ConfigureAwait(false); //we've either seen origin before or what we're checking out is on origin } if (model.UpdateFromOrigin == true && model.Reference != null) { if (!repo.Tracking) throw new InvalidOperationException("Checked out reference does not track a remote object!"); - await repo.ResetToOrigin(cancellationToken).ConfigureAwait(false); - await repo.Sychronize(currentModel.AccessUser, currentModel.AccessToken, true, cancellationToken).ConfigureAwait(false); - await LoadRevisionInformation(repo, databaseContext, attachedInstance, null, x => lastRevisionInfo = x, cancellationToken).ConfigureAwait(false); + await repo.ResetToOrigin(ct).ConfigureAwait(false); + await repo.Sychronize(currentModel.AccessUser, currentModel.AccessToken, true, ct).ConfigureAwait(false); + await LoadRevisionInformation(repo, databaseContext, attachedInstance, null, x => lastRevisionInfo = x, ct).ConfigureAwait(false); //repo head is on origin so force this //will update the db if necessary lastRevisionInfo.OriginCommitSha = repo.Head; } } - + //test merging if (newTestMerges) { - var gitHubClient = currentModel.AccessToken != null ? gitHubClientFactory.CreateClient(currentModel.AccessToken) : gitHubClientFactory.CreateClient(); - var contextUser = new Models.User + //optimization: if we've already merged these exact same commits in this fashion before, just find the rev info for it and check it out + Models.RevisionInformation revInfoWereLookingFor = null; + if(lastRevisionInfo.OriginCommitSha == lastRevisionInfo.CommitSha) { - Id = AuthenticationContext.User.Id - }; - databaseContext.Users.Attach(contextUser); + foreach (var I in model.NewTestMerges) + //normalize the shas to lowercase ala libgit2 +#pragma warning disable CA1308 // Normalize strings to uppercase + I.PullRequestRevision = I.PullRequestRevision?.ToLowerInvariant(); +#pragma warning restore CA1308 // Normalize strings to uppercase - var repoOwner = repo.GitHubOwner; - var repoName = repo.GitHubRepoName; - foreach (var I in model.NewTestMerges) + revInfoWereLookingFor = await databaseContext.RevisionInformations.Where( + x => x.OriginCommitSha == lastRevisionInfo.OriginCommitSha + && x.ActiveTestMerges.Count == model.NewTestMerges.Count) + //split here cause this bit probably has to be done locally + .Where(x => x.ActiveTestMerges.Select(y => y.TestMerge).All(y => model.NewTestMerges.Any(z => y.Number == z.Number && y.PullRequestRevision.StartsWith(z.PullRequestRevision, StringComparison.Ordinal)))).FirstOrDefaultAsync(ct).ConfigureAwait(false); + } + + + if (revInfoWereLookingFor != null) + await repo.ResetToSha(revInfoWereLookingFor.CommitSha, cancellationToken).ConfigureAwait(false); + else { - Octokit.PullRequest pr = null; - string errorMessage = null; - try + var gitHubClient = currentModel.AccessToken != null ? gitHubClientFactory.CreateClient(currentModel.AccessToken) : gitHubClientFactory.CreateClient(); + var contextUser = new Models.User { - pr = await gitHubClient.PullRequest.Get(repoOwner, repoName, I.Number).ConfigureAwait(false); - } - catch (Octokit.RateLimitExceededException) - { - //you look at your anonymous access and sigh - errorMessage = "P.R.E. RATE LIMITED"; - } - catch (Octokit.NotFoundException) - { - //you look at your shithub access and sigh - errorMessage = "P.R.E. NOT FOUND"; - } - - var mergeResult = await repo.AddTestMerge(I.Number, I.PullRequestRevision, committerName, currentModel.CommitterEmail, String.Format(CultureInfo.InvariantCulture, "Test merge of pull request #{0}{1}{2}", I.Number, I.Comment != null ? Environment.NewLine : null, I.Comment), currentModel.AccessUser, currentModel.AccessToken, x => progressReporter((x + 100 * doneFetches) / numFetches), cancellationToken).ConfigureAwait(false); - - if (!mergeResult.HasValue) //conflict, we don't care, dd already knows - continue; - - ++doneFetches; - - var revInfoUpdateTask = UpdateRevInfo(); - - var tm = new Models.TestMerge - { - Author = pr?.User.Login ?? errorMessage, - BodyAtMerge = pr?.Body ?? errorMessage ?? String.Empty, - MergedAt = DateTimeOffset.Now, - TitleAtMerge = pr?.Title ?? errorMessage ?? String.Empty, - Comment = I.Comment, - Number = I.Number, - MergedBy = contextUser, - PullRequestRevision = I.PullRequestRevision, - Url = pr?.HtmlUrl ?? errorMessage + Id = AuthenticationContext.User.Id }; + databaseContext.Users.Attach(contextUser); - await revInfoUpdateTask.ConfigureAwait(false); - - lastRevisionInfo.PrimaryTestMerge = tm; - lastRevisionInfo.ActiveTestMerges.Add(new RevInfoTestMerge + var repoOwner = repo.GitHubOwner; + var repoName = repo.GitHubRepoName; + foreach (var I in model.NewTestMerges) { - TestMerge = tm - }); + Octokit.PullRequest pr = null; + string errorMessage = null; + try + { + pr = await gitHubClient.PullRequest.Get(repoOwner, repoName, I.Number).ConfigureAwait(false); + } + catch (Octokit.RateLimitExceededException) + { + //you look at your anonymous access and sigh + errorMessage = "P.R.E. RATE LIMITED"; + } + catch (Octokit.NotFoundException) + { + //you look at your shithub access and sigh + errorMessage = "P.R.E. NOT FOUND"; + } + + var mergeResult = await repo.AddTestMerge(I.Number, I.PullRequestRevision, committerName, currentModel.CommitterEmail, String.Format(CultureInfo.InvariantCulture, "Test merge of pull request #{0}{1}{2}", I.Number, I.Comment != null ? Environment.NewLine : null, I.Comment), currentModel.AccessUser, currentModel.AccessToken, x => progressReporter((x + 100 * doneFetches) / numFetches), ct).ConfigureAwait(false); + + if (!mergeResult.HasValue) //conflict, we don't care, dd already knows + continue; + + ++doneFetches; + + var revInfoUpdateTask = UpdateRevInfo(); + + var tm = new Models.TestMerge + { + Author = pr?.User.Login ?? errorMessage, + BodyAtMerge = pr?.Body ?? errorMessage ?? String.Empty, + MergedAt = DateTimeOffset.Now, + TitleAtMerge = pr?.Title ?? errorMessage ?? String.Empty, + Comment = I.Comment, + Number = I.Number, + MergedBy = contextUser, + PullRequestRevision = I.PullRequestRevision, + Url = pr?.HtmlUrl ?? errorMessage + }; + + await revInfoUpdateTask.ConfigureAwait(false); + + lastRevisionInfo.PrimaryTestMerge = tm; + lastRevisionInfo.ActiveTestMerges.Add(new RevInfoTestMerge + { + TestMerge = tm + }); + } } } @@ -451,7 +474,7 @@ namespace Tgstation.Server.Host.Controllers await repo.Sychronize(currentModel.AccessUser, currentModel.AccessToken, false, ct).ConfigureAwait(false); await UpdateRevInfo().ConfigureAwait(false); } - await databaseContext.Save(cancellationToken).ConfigureAwait(false); + await databaseContext.Save(ct).ConfigureAwait(false); } catch { diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index 168e747e3f..79b103979b 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -243,6 +243,8 @@ namespace Tgstation.Server.Host.Core logger.LogInformation(VersionString); + logger.LogTrace("Configuring middleware..."); + serverAddresses = applicationBuilder.ServerFeatures.Get(); applicationBuilder.UseDeveloperExceptionPage(); //it is not worth it to limit this, you should only ever get it if you're an authorized user diff --git a/src/Tgstation.Server.Host/Core/JobManager.cs b/src/Tgstation.Server.Host/Core/JobManager.cs index b05e90757c..e4e13aa4d9 100644 --- a/src/Tgstation.Server.Host/Core/JobManager.cs +++ b/src/Tgstation.Server.Host/Core/JobManager.cs @@ -1,6 +1,6 @@ using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; using System; using System.Collections.Generic; using System.Linq; @@ -17,6 +17,12 @@ namespace Tgstation.Server.Host.Core /// The for the /// readonly IServiceProvider serviceProvider; + + /// + /// The for the + /// + readonly ILogger logger; + /// /// of to running s /// @@ -26,9 +32,11 @@ namespace Tgstation.Server.Host.Core /// Construct a /// /// The value of - public JobManager(IServiceProvider serviceProvider) + /// The value of + public JobManager(IServiceProvider serviceProvider, ILogger logger) { this.serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); jobs = new Dictionary(); } @@ -72,22 +80,21 @@ namespace Tgstation.Server.Host.Core { var oldJob = job; job = new Job { Id = oldJob.Id }; - try - { - await operation(job, scope.ServiceProvider, cancellationToken).ConfigureAwait(false); - } - finally - { - databaseContext = scope.ServiceProvider.GetRequiredService(); - databaseContext.Jobs.Attach(job); - } + databaseContext = scope.ServiceProvider.GetRequiredService(); + databaseContext.Jobs.Attach(job); + + await operation(job, scope.ServiceProvider, cancellationToken).ConfigureAwait(false); + + logger.LogDebug("Job {0} completed!", job.Id); } catch (OperationCanceledException) { + logger.LogDebug("Job {0} cancelled!", job.Id); job.Cancelled = true; } catch (Exception e) { + logger.LogDebug("Job {0} exited with error! Exception: {1}", job.Id, e); job.ExceptionDetails = e.ToString(); } job.StoppedAt = DateTimeOffset.Now; @@ -128,6 +135,7 @@ namespace Tgstation.Server.Host.Core } databaseContext.Jobs.Add(job); await databaseContext.Save(cancellationToken).ConfigureAwait(false); + logger.LogDebug("Starting job {0}: {1}...", job.Id, job.Description); var jobHandler = JobHandler.Create(x => RunJob(job, (jobParam, serviceProvider, ct) => operation(jobParam, serviceProvider, y => { @@ -144,20 +152,26 @@ namespace Tgstation.Server.Host.Core /// public async Task StartAsync(CancellationToken cancellationToken) { + logger.LogTrace("Starting job manager..."); using (var scope = serviceProvider.CreateScope()) { var databaseContext = scope.ServiceProvider.GetRequiredService(); //mark all jobs as cancelled - var enumerator = await databaseContext.Jobs.Where(y => !y.Cancelled.Value && !y.StoppedAt.HasValue).Select(y => y.Id).ToListAsync(cancellationToken).ConfigureAwait(false); - foreach(var I in enumerator) + var badJobs = await databaseContext.Jobs.Where(y => !y.Cancelled.Value && !y.StoppedAt.HasValue).Select(y => y.Id).ToListAsync(cancellationToken).ConfigureAwait(false); + if (badJobs.Count > 0) { - var job = new Job { Id = I }; - databaseContext.Jobs.Attach(job); - job.Cancelled = true; + logger.LogTrace("Cleaning {0} unfinished jobs...", badJobs.Count); + foreach (var I in badJobs) + { + var job = new Job { Id = I }; + databaseContext.Jobs.Attach(job); + job.Cancelled = true; + } + await databaseContext.Save(cancellationToken).ConfigureAwait(false); } - await databaseContext.Save(cancellationToken).ConfigureAwait(false); } + logger.LogDebug("Job manager started!"); } /// diff --git a/src/Tgstation.Server.Host/IO/DefaultIOManager.cs b/src/Tgstation.Server.Host/IO/DefaultIOManager.cs index 65ae98fd1f..95a512d017 100644 --- a/src/Tgstation.Server.Host/IO/DefaultIOManager.cs +++ b/src/Tgstation.Server.Host/IO/DefaultIOManager.cs @@ -34,7 +34,11 @@ namespace Tgstation.Server.Host.IO foreach (var subDir in dir.EnumerateDirectories()) { cancellationToken.ThrowIfCancellationRequested(); - tasks.Add(NormalizeAndDelete(subDir, cancellationToken)); + if (!subDir.Attributes.HasFlag(FileAttributes.Directory) || subDir.Attributes.HasFlag(FileAttributes.ReparsePoint)) + //this is probably a symlink + subDir.Delete(); + else + tasks.Add(NormalizeAndDelete(subDir, cancellationToken)); } foreach (var file in dir.EnumerateFiles()) { @@ -61,34 +65,40 @@ namespace Tgstation.Server.Host.IO /// The destination directory path /// Files and folders to ignore at the root level /// The for the operation - /// A representing the running operation - async Task CopyDirectoryImpl(string src, string dest, IEnumerable ignore, CancellationToken cancellationToken) + /// A of s representing the running operation + IEnumerable CopyDirectoryImpl(string src, string dest, IEnumerable ignore, CancellationToken cancellationToken) { - await CreateDirectory(dest, cancellationToken).ConfigureAwait(false); - var dir = new DirectoryInfo(src); - cancellationToken.ThrowIfCancellationRequested(); - - var dirs = dir.EnumerateDirectories(); - var files = dir.EnumerateFiles(); - - var fileCopyTasks = files.Select(x => + var atLeastOneSubDir = false; + foreach (var I in dir.EnumerateDirectories()) { - cancellationToken.ThrowIfCancellationRequested(); - if (ignore != null && ignore.Contains(x.Name)) - return Task.CompletedTask; - return CopyFile(x.FullName, Path.Combine(dest, x.Name), cancellationToken); - }); + if (ignore != null && ignore.Contains(I.Name)) + continue; + foreach (var J in CopyDirectoryImpl(I.FullName, Path.Combine(dest, I.Name), null, cancellationToken)) + { + atLeastOneSubDir = true; + yield return J; + } + } - var directoryCopyTasks = dirs.Select(x => + async Task CopyThisDirectory() { - cancellationToken.ThrowIfCancellationRequested(); - if (ignore != null && ignore.Contains(x.Name)) - return Task.CompletedTask; - return CopyDirectoryImpl(x.FullName, Path.Combine(dest, x.Name), null, cancellationToken); - }); + if (!atLeastOneSubDir) + await CreateDirectory(dest, cancellationToken).ConfigureAwait(false); //save on createdir calls - await Task.WhenAll(fileCopyTasks.Concat(directoryCopyTasks)).ConfigureAwait(false); + var tasks = new List(); + + await dir.EnumerateFiles().ToAsyncEnumerable().ForEachAsync(I => + { + if (ignore != null && ignore.Contains(I.Name)) + return; + tasks.Add(CopyFile(I.FullName, Path.Combine(dest, I.Name), cancellationToken)); + }).ConfigureAwait(false); + + await Task.WhenAll(tasks).ConfigureAwait(false); + }; + + yield return CopyThisDirectory(); } /// @@ -101,7 +111,8 @@ namespace Tgstation.Server.Host.IO src = ResolvePath(src); dest = ResolvePath(dest); - await CopyDirectoryImpl(src, dest, ignore, cancellationToken).ConfigureAwait(false); + foreach (var directoryCopy in CopyDirectoryImpl(src, dest, ignore, cancellationToken)) + await directoryCopy.ConfigureAwait(false); } /// diff --git a/src/Tgstation.Server.Host/Models/DatabaseContext.cs b/src/Tgstation.Server.Host/Models/DatabaseContext.cs index 1a9e5bac6e..411836458d 100644 --- a/src/Tgstation.Server.Host/Models/DatabaseContext.cs +++ b/src/Tgstation.Server.Host/Models/DatabaseContext.cs @@ -1,9 +1,8 @@ using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using System; -#if !DEBUG using System.Linq; -#endif using System.Threading; using System.Threading.Tasks; using Tgstation.Server.Host.Configuration; @@ -46,21 +45,26 @@ namespace Tgstation.Server.Host.Models /// public DbSet Jobs { get; set; } + /// + public DbSet ReattachInformations { get; set; } + + /// + public DbSet WatchdogReattachInformations { get; set; } + /// /// The s in the /// public DbSet TestMerges { get; set; } - /// - public DbSet ReattachInformations { get; set; } - /// /// The s om the /// public DbSet RevInfoTestMerges { get; set; } - /// - public DbSet WatchdogReattachInformations { get; set; } + /// + /// The for the + /// + protected ILogger Logger { get; } /// /// The connection string for the @@ -71,6 +75,7 @@ namespace Tgstation.Server.Host.Models /// The for the /// readonly DatabaseConfiguration databaseConfiguration; + /// /// The for the /// @@ -82,15 +87,18 @@ namespace Tgstation.Server.Host.Models /// The for the /// The containing the value of /// The value of - public DatabaseContext(DbContextOptions dbContextOptions, IOptions databaseConfigurationOptions, IDatabaseSeeder databaseSeeder) : base(dbContextOptions) + /// The value of + public DatabaseContext(DbContextOptions dbContextOptions, IOptions databaseConfigurationOptions, IDatabaseSeeder databaseSeeder, ILogger logger) : base(dbContextOptions) { databaseConfiguration = databaseConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(databaseConfigurationOptions)); this.databaseSeeder = databaseSeeder ?? throw new ArgumentNullException(nameof(databaseSeeder)); + Logger = logger ?? throw new ArgumentNullException(nameof(logger)); } /// protected override void OnModelCreating(ModelBuilder modelBuilder) { + Logger.LogTrace("Building entity framework context..."); base.OnModelCreating(modelBuilder); var userModel = modelBuilder.Entity(); @@ -125,27 +133,40 @@ namespace Tgstation.Server.Host.Models instanceModel.HasMany(x => x.Jobs).WithOne(x => x.Instance).OnDelete(DeleteBehavior.Cascade); } - /// - protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) - { - base.OnConfiguring(optionsBuilder); - } - /// public async Task Initialize(CancellationToken cancellationToken) { -#if DEBUG - await Database.EnsureCreatedAsync().ConfigureAwait(false); - var wasEmpty = (await Users.CountAsync().ConfigureAwait(false)) == 0; -#else - var migrations = await Database.GetAppliedMigrationsAsync().ConfigureAwait(false); - var wasEmpty = !migrations.Any(); - await Database.MigrateAsync(cancellationToken).ConfigureAwait(false); -#endif + Logger.LogInformation("Migrating database..."); + + var wasEmpty = false; + if (!databaseConfiguration.NoMigrations) + { + Logger.LogWarning("Running in debug mode. Using all or nothing migration strategy!"); + await Database.EnsureCreatedAsync(cancellationToken).ConfigureAwait(false); + } + else + { + var migrations = await Database.GetAppliedMigrationsAsync(cancellationToken).ConfigureAwait(false); + wasEmpty = !migrations.Any(); + await Database.MigrateAsync(cancellationToken).ConfigureAwait(false); + } + + wasEmpty |= (await Users.CountAsync(cancellationToken).ConfigureAwait(false)) == 0; + if (wasEmpty) + { + Logger.LogInformation("Seeding database..."); await databaseSeeder.SeedDatabase(this, cancellationToken).ConfigureAwait(false); - else if(databaseConfiguration.ResetAdminPassword) - await databaseSeeder.ResetAdminPassword(this, cancellationToken).ConfigureAwait(false); + } + else + { + Logger.LogDebug("No migrations applied!"); + if (databaseConfiguration.ResetAdminPassword) + { + Logger.LogWarning("Enabling and resetting admin password due to configuration!"); + await databaseSeeder.ResetAdminPassword(this, cancellationToken).ConfigureAwait(false); + } + } } /// diff --git a/src/Tgstation.Server.Host/Models/Migrations/MySqlDesignTimeDbContextFactory.cs b/src/Tgstation.Server.Host/Models/Migrations/MySqlDesignTimeDbContextFactory.cs index fd55e611c1..14aafaca46 100644 --- a/src/Tgstation.Server.Host/Models/Migrations/MySqlDesignTimeDbContextFactory.cs +++ b/src/Tgstation.Server.Host/Models/Migrations/MySqlDesignTimeDbContextFactory.cs @@ -1,6 +1,7 @@ using Microsoft.AspNetCore.Identity; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Design; +using Microsoft.Extensions.Logging; using Tgstation.Server.Host.Security; namespace Tgstation.Server.Host.Models.Migrations @@ -9,6 +10,6 @@ namespace Tgstation.Server.Host.Models.Migrations sealed class MySqlDesignTimeDbContextFactory : IDesignTimeDbContextFactory { /// - public MySqlDatabaseContext CreateDbContext(string[] args) => new MySqlDatabaseContext(new DbContextOptions(), DesignTimeDbContextFactoryHelpers.GetDbContextOptions(), new DatabaseSeeder(new CryptographySuite(new PasswordHasher()))); + public MySqlDatabaseContext CreateDbContext(string[] args) => new MySqlDatabaseContext(new DbContextOptions(), DesignTimeDbContextFactoryHelpers.GetDbContextOptions(), new DatabaseSeeder(new CryptographySuite(new PasswordHasher())), new LoggerFactory().CreateLogger()); } } diff --git a/src/Tgstation.Server.Host/Models/Migrations/SqlServerDesignTimeDbContextFactory.cs b/src/Tgstation.Server.Host/Models/Migrations/SqlServerDesignTimeDbContextFactory.cs index 75a1b97053..bf903b09c8 100644 --- a/src/Tgstation.Server.Host/Models/Migrations/SqlServerDesignTimeDbContextFactory.cs +++ b/src/Tgstation.Server.Host/Models/Migrations/SqlServerDesignTimeDbContextFactory.cs @@ -1,6 +1,7 @@ using Microsoft.AspNetCore.Identity; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Design; +using Microsoft.Extensions.Logging; using Tgstation.Server.Host.Security; namespace Tgstation.Server.Host.Models.Migrations @@ -9,6 +10,6 @@ namespace Tgstation.Server.Host.Models.Migrations sealed class SqlServerDesignTimeDbContextFactory : IDesignTimeDbContextFactory { /// - public SqlServerDatabaseContext CreateDbContext(string[] args) => new SqlServerDatabaseContext(new DbContextOptions(), DesignTimeDbContextFactoryHelpers.GetDbContextOptions(), new DatabaseSeeder(new CryptographySuite(new PasswordHasher()))); + public SqlServerDatabaseContext CreateDbContext(string[] args) => new SqlServerDatabaseContext(new DbContextOptions(), DesignTimeDbContextFactoryHelpers.GetDbContextOptions(), new DatabaseSeeder(new CryptographySuite(new PasswordHasher())), new LoggerFactory().CreateLogger()); } } diff --git a/src/Tgstation.Server.Host/Models/MySqlDatabaseContext.cs b/src/Tgstation.Server.Host/Models/MySqlDatabaseContext.cs index 0f70b29fcd..eb6f0c81cd 100644 --- a/src/Tgstation.Server.Host/Models/MySqlDatabaseContext.cs +++ b/src/Tgstation.Server.Host/Models/MySqlDatabaseContext.cs @@ -1,4 +1,5 @@ using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Tgstation.Server.Host.Configuration; @@ -15,7 +16,8 @@ namespace Tgstation.Server.Host.Models /// The for the /// The of for the /// The for the - public MySqlDatabaseContext(DbContextOptions dbContextOptions, IOptions databaseConfiguration, IDatabaseSeeder databaseSeeder) : base(dbContextOptions, databaseConfiguration, databaseSeeder) + /// The for the + public MySqlDatabaseContext(DbContextOptions dbContextOptions, IOptions databaseConfiguration, IDatabaseSeeder databaseSeeder, ILogger logger) : base(dbContextOptions, databaseConfiguration, databaseSeeder, logger) { } /// diff --git a/src/Tgstation.Server.Host/Models/SqlServerDatabaseContext.cs b/src/Tgstation.Server.Host/Models/SqlServerDatabaseContext.cs index 19f6736cc9..b0f7ae1372 100644 --- a/src/Tgstation.Server.Host/Models/SqlServerDatabaseContext.cs +++ b/src/Tgstation.Server.Host/Models/SqlServerDatabaseContext.cs @@ -1,4 +1,5 @@ using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Tgstation.Server.Host.Configuration; @@ -15,7 +16,8 @@ namespace Tgstation.Server.Host.Models /// The for the /// The of for the /// The for the - public SqlServerDatabaseContext(DbContextOptions dbContextOptions, IOptions databaseConfiguration, IDatabaseSeeder databaseSeeder) : base(dbContextOptions, databaseConfiguration, databaseSeeder) + /// The for the + public SqlServerDatabaseContext(DbContextOptions dbContextOptions, IOptions databaseConfiguration, IDatabaseSeeder databaseSeeder, ILogger logger) : base(dbContextOptions, databaseConfiguration, databaseSeeder, logger) { } /// diff --git a/src/Tgstation.Server.Host/Models/SqliteDatabaseContext.cs b/src/Tgstation.Server.Host/Models/SqliteDatabaseContext.cs index 011893939c..224f08ff0b 100644 --- a/src/Tgstation.Server.Host/Models/SqliteDatabaseContext.cs +++ b/src/Tgstation.Server.Host/Models/SqliteDatabaseContext.cs @@ -11,22 +11,15 @@ namespace Tgstation.Server.Host.Models /// sealed class SqliteDatabaseContext : DatabaseContext { - /// - /// The for the - /// - readonly ILogger logger; - /// /// Construct a /// /// The for the /// The of for the /// The for the - /// The value of - public SqliteDatabaseContext(DbContextOptions dbContextOptions, IOptions databaseConfiguration, IDatabaseSeeder databaseSeeder, ILogger logger) : base(dbContextOptions, databaseConfiguration, databaseSeeder) - { - this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } + /// The for the + public SqliteDatabaseContext(DbContextOptions dbContextOptions, IOptions databaseConfiguration, IDatabaseSeeder databaseSeeder, ILogger logger) : base(dbContextOptions, databaseConfiguration, databaseSeeder, logger) + { } /// protected override void OnConfiguring(DbContextOptionsBuilder options) @@ -35,7 +28,7 @@ namespace Tgstation.Server.Host.Models //on the off chance that connection string is null here we default to a db file next to the executable since this is the default database if (ConnectionString == null) { - logger.LogWarning("No database configured! Defaulting to SQLite in the working directory!"); + Logger.LogWarning("No database configured! Defaulting to SQLite in the working directory!"); options.UseSqlite("Data Source=TgsDatabase.db3"); } else diff --git a/src/Tgstation.Server.Host/Security/IdentityCache.cs b/src/Tgstation.Server.Host/Security/IdentityCache.cs index 81850ab4f2..1d1f19a681 100644 --- a/src/Tgstation.Server.Host/Security/IdentityCache.cs +++ b/src/Tgstation.Server.Host/Security/IdentityCache.cs @@ -31,7 +31,7 @@ namespace Tgstation.Server.Host.Security lock (cachedIdentities) { if (cachedIdentities.TryGetValue(user.Id, out var identCache)) - identCache.Dispose(); //also clears it out + identCache.Dispose(); //also clears it out identCache = new IdentityCacheObject(systemIdentity.Clone(), () => { lock (cachedIdentities) diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index 575d65fa34..a68eac4eb5 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -54,6 +54,9 @@ + + PreserveNewest + PreserveNewest diff --git a/src/Tgstation.Server.Host/appsettings.Development.json b/src/Tgstation.Server.Host/appsettings.Development.json new file mode 100644 index 0000000000..d40e70e835 --- /dev/null +++ b/src/Tgstation.Server.Host/appsettings.Development.json @@ -0,0 +1,8 @@ +{ + "General": { + "DisableFileLogging": true + }, + "Database": { + "NoMigrations": true + } +} diff --git a/src/Tgstation.Server.Host/appsettings.json b/src/Tgstation.Server.Host/appsettings.json index 0f47b8e2ae..1c91417297 100644 --- a/src/Tgstation.Server.Host/appsettings.json +++ b/src/Tgstation.Server.Host/appsettings.json @@ -15,7 +15,7 @@ "Console": { "LogLevel": { "Default": "Trace", - "Microsoft": "Information" + "Microsoft": "Warning" } }, "LogLevel": { @@ -29,6 +29,7 @@ "UpdatePackageAssetName": "ServerUpdatePackage.zip" }, "Database": { + "NoMigrations": false, "DatabaseType": "SqlServer", "ResetAdminPassword": false, "ConnectionString": "Data Source=(local);Initial Catalog=TGS;Integrated Security=True" diff --git a/v4_prototype_TODO.txt b/v4_prototype_TODO.txt index afe9140947..7333b2c540 100644 --- a/v4_prototype_TODO.txt +++ b/v4_prototype_TODO.txt @@ -1,15 +1,5 @@ -Watchdog performance counters - - Remember to catch PlatformNotSupportedException() - -Install as network service instead of LocalSystem if win10 >= Creators update to allow for unprivileged symlinks - Verify the byond cache folder location on linux -Don't throw arg null exceptions with null [FromBody]'s, apparently that's allowed, return bad request instead - -Test repo more -Test byond -Test compile Test watchdog test configuration