From 86b59072737b897a58a54d7fd477fd47c5188b34 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Sat, 15 Sep 2018 13:05:53 -0400 Subject: [PATCH 01/11] Adds the PostComplete Job step --- src/Tgstation.Server.Host/Core/JobManager.cs | 52 ++++++++++++++------ src/Tgstation.Server.Host/Models/Job.cs | 13 ++++- 2 files changed, 50 insertions(+), 15 deletions(-) diff --git a/src/Tgstation.Server.Host/Core/JobManager.cs b/src/Tgstation.Server.Host/Core/JobManager.cs index 47ac244289..1876ecda4e 100644 --- a/src/Tgstation.Server.Host/Core/JobManager.cs +++ b/src/Tgstation.Server.Host/Core/JobManager.cs @@ -70,13 +70,35 @@ namespace Tgstation.Server.Host.Core /// The for the operation /// A representing the running operation async Task RunJob(Job job, Func operation, CancellationToken cancellationToken) - { + { try { using (var scope = serviceProvider.CreateScope()) { + async Task HandleExceptions(Task task) + { + try + { + await task.ConfigureAwait(false); + } + catch (OperationCanceledException) + { + logger.LogDebug("Job {0} cancelled!", job.Id); + job.Cancelled = true; + } + catch (Exception e) + { + job.ExceptionDetails = e is JobException ? e.Message : e.ToString(); + logger.LogDebug("Job {0} exited with error! Exception: {1}", job.Id, job.ExceptionDetails); + } + finally + { + job.StoppedAt = DateTimeOffset.Now; + } + } + IDatabaseContext databaseContext = null; - try + async Task RunJobInternal() { var oldJob = job; job = new Job { Id = oldJob.Id }; @@ -86,19 +108,21 @@ namespace Tgstation.Server.Host.Core await operation(job, scope.ServiceProvider, cancellationToken).ConfigureAwait(false); logger.LogDebug("Job {0} completed!", job.Id); - } - catch (OperationCanceledException) - { - logger.LogDebug("Job {0} cancelled!", job.Id); - job.Cancelled = true; - } - catch (Exception e) - { - job.ExceptionDetails = e is JobException ? e.Message : e.ToString(); - logger.LogDebug("Job {0} exited with error! Exception: {1}", job.Id, job.ExceptionDetails); - } - job.StoppedAt = DateTimeOffset.Now; + }; + + await HandleExceptions(RunJobInternal()).ConfigureAwait(false); + await databaseContext.Save(default).ConfigureAwait(false); + + bool JobErroredOrCancelled() => job.ExceptionDetails != null || job.Cancelled.Value; + + //ok so, now it's time for the post commit step if it exists + if (!JobErroredOrCancelled() && job.PostComplete != null) + { + await HandleExceptions(job.PostComplete(cancellationToken)).ConfigureAwait(false); + if (JobErroredOrCancelled()) + await databaseContext.Save(default).ConfigureAwait(false); + } } } finally diff --git a/src/Tgstation.Server.Host/Models/Job.cs b/src/Tgstation.Server.Host/Models/Job.cs index aeb31c0d0e..b601ecc46b 100644 --- a/src/Tgstation.Server.Host/Models/Job.cs +++ b/src/Tgstation.Server.Host/Models/Job.cs @@ -1,4 +1,8 @@ -using System.ComponentModel.DataAnnotations; +using System; +using System.ComponentModel.DataAnnotations; +using System.ComponentModel.DataAnnotations.Schema; +using System.Threading; +using System.Threading.Tasks; namespace Tgstation.Server.Host.Models { @@ -22,6 +26,13 @@ namespace Tgstation.Server.Host.Models [Required] public Instance Instance { get; set; } + /// + /// A to run after the job completes. This will not affect the time, unless it is cancelled or errors + /// + /// This should only be used where there are database dependencies that also rely on the Job itself completing A.K.A. manually initiated s + [NotMapped] + public Func PostComplete { get; set; } + /// /// Convert the to it's API form /// From a956ac239aef71de06f8eeb24bb6f201127a03f6 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Sat, 15 Sep 2018 13:15:45 -0400 Subject: [PATCH 02/11] Make DreamMakerController use the PostCompile system --- .../Controllers/DreamMakerController.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs b/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs index 823a3700da..746f04157b 100644 --- a/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs +++ b/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs @@ -189,10 +189,9 @@ namespace Tgstation.Server.Host.Controllers compileJob.Job = job; - databaseContext.CompileJobs.Add(compileJob); - await databaseContext.Save(cancellationToken).ConfigureAwait(false); + databaseContext.CompileJobs.Add(compileJob); //will be saved by job context - await instance.CompileJobConsumer.LoadCompileJob(compileJob, cancellationToken).ConfigureAwait(false); + job.PostComplete = ct => instance.CompileJobConsumer.LoadCompileJob(compileJob, ct); } } } From 3ab97ef6e1f78cc229307a100657863d3cc8e61c Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Sat, 15 Sep 2018 13:20:05 -0400 Subject: [PATCH 03/11] Inject IJobManager into the Instance component --- src/Tgstation.Server.Host/Components/Instance.cs | 9 ++++++++- .../Components/InstanceFactory.cs | 12 +++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Instance.cs b/src/Tgstation.Server.Host/Components/Instance.cs index 21e7edc06d..46b5dc3541 100644 --- a/src/Tgstation.Server.Host/Components/Instance.cs +++ b/src/Tgstation.Server.Host/Components/Instance.cs @@ -48,6 +48,11 @@ namespace Tgstation.Server.Host.Components /// readonly IDmbFactory dmbFactory; + /// + /// The for the + /// + readonly IJobManager jobManager; + /// /// The for the /// @@ -80,8 +85,9 @@ namespace Tgstation.Server.Host.Components /// The value of /// The value of /// The value of + /// The value of /// The value of - public Instance(Api.Models.Instance metadata, IRepositoryManager repositoryManager, IByondManager byondManager, IDreamMaker dreamMaker, IWatchdog watchdog, IChat chat, StaticFiles.IConfiguration configuration, ICompileJobConsumer compileJobConsumer, IDatabaseContextFactory databaseContextFactory, IDmbFactory dmbFactory, ILogger logger) + public Instance(Api.Models.Instance metadata, IRepositoryManager repositoryManager, IByondManager byondManager, IDreamMaker dreamMaker, IWatchdog watchdog, IChat chat, StaticFiles.IConfiguration configuration, ICompileJobConsumer compileJobConsumer, IDatabaseContextFactory databaseContextFactory, IDmbFactory dmbFactory, IJobManager jobManager, ILogger logger) { this.metadata = metadata ?? throw new ArgumentNullException(nameof(metadata)); RepositoryManager = repositoryManager ?? throw new ArgumentNullException(nameof(repositoryManager)); @@ -93,6 +99,7 @@ namespace Tgstation.Server.Host.Components CompileJobConsumer = compileJobConsumer ?? throw new ArgumentNullException(nameof(compileJobConsumer)); this.databaseContextFactory = databaseContextFactory ?? throw new ArgumentNullException(nameof(databaseContextFactory)); this.dmbFactory = dmbFactory ?? throw new ArgumentNullException(nameof(dmbFactory)); + this.jobManager = jobManager ?? throw new ArgumentNullException(nameof(jobManager)); this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); } diff --git a/src/Tgstation.Server.Host/Components/InstanceFactory.cs b/src/Tgstation.Server.Host/Components/InstanceFactory.cs index 11c9b9a197..35c0fd23df 100644 --- a/src/Tgstation.Server.Host/Components/InstanceFactory.cs +++ b/src/Tgstation.Server.Host/Components/InstanceFactory.cs @@ -8,7 +8,6 @@ using Tgstation.Server.Host.Components.Chat; using Tgstation.Server.Host.Components.Chat.Commands; using Tgstation.Server.Host.Components.Compiler; using Tgstation.Server.Host.Components.Repository; -using Tgstation.Server.Host.Components.StaticFiles; using Tgstation.Server.Host.Components.Watchdog; using Tgstation.Server.Host.Core; using Tgstation.Server.Host.IO; @@ -89,6 +88,11 @@ namespace Tgstation.Server.Host.Components /// readonly IWatchdogFactory watchdogFactory; + /// + /// The for the + /// + readonly IJobManager jobManager; + /// /// Construct an /// @@ -106,7 +110,8 @@ namespace Tgstation.Server.Host.Components /// The value of /// The value of /// The value of - public InstanceFactory(IIOManager ioManager, IDatabaseContextFactory databaseContextFactory, IApplication application, ILoggerFactory loggerFactory, IByondTopicSender byondTopicSender, IServerControl serverUpdater, ICryptographySuite cryptographySuite, ISynchronousIOManager synchronousIOManager, ISymlinkFactory symlinkFactory, IByondInstaller byondInstaller, IProviderFactory providerFactory, IProcessExecutor processExecutor, IPostWriteHandler postWriteHandler, IWatchdogFactory watchdogFactory) + /// The value of + public InstanceFactory(IIOManager ioManager, IDatabaseContextFactory databaseContextFactory, IApplication application, ILoggerFactory loggerFactory, IByondTopicSender byondTopicSender, IServerControl serverUpdater, ICryptographySuite cryptographySuite, ISynchronousIOManager synchronousIOManager, ISymlinkFactory symlinkFactory, IByondInstaller byondInstaller, IProviderFactory providerFactory, IProcessExecutor processExecutor, IPostWriteHandler postWriteHandler, IWatchdogFactory watchdogFactory, IJobManager jobManager) { this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); this.databaseContextFactory = databaseContextFactory ?? throw new ArgumentNullException(nameof(databaseContextFactory)); @@ -122,6 +127,7 @@ namespace Tgstation.Server.Host.Components this.processExecutor = processExecutor ?? throw new ArgumentNullException(nameof(processExecutor)); this.postWriteHandler = postWriteHandler ?? throw new ArgumentNullException(nameof(postWriteHandler)); this.watchdogFactory = watchdogFactory ?? throw new ArgumentNullException(nameof(watchdogFactory)); + this.jobManager = jobManager ?? throw new ArgumentNullException(nameof(jobManager)); } /// @@ -162,7 +168,7 @@ namespace Tgstation.Server.Host.Components { var dreamMaker = new DreamMaker(byond, gameIoManager, configuration, sessionControllerFactory, dmbFactory, application, eventConsumer, chat, processExecutor, watchdog, loggerFactory.CreateLogger()); - return new Instance(metadata.CloneMetadata(), repoManager, byond, dreamMaker, watchdog, chat, configuration, dmbFactory, databaseContextFactory, dmbFactory, loggerFactory.CreateLogger()); + return new Instance(metadata.CloneMetadata(), repoManager, byond, dreamMaker, watchdog, chat, configuration, dmbFactory, databaseContextFactory, dmbFactory, jobManager, loggerFactory.CreateLogger()); } catch { From a374651e38afc11552058ac47a9a329875919572 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Sat, 15 Sep 2018 13:56:29 -0400 Subject: [PATCH 04/11] Add IJobManager.WaitForCompletion --- src/Tgstation.Server.Host/Core/IJobManager.cs | 12 ++++++++++++ src/Tgstation.Server.Host/Core/JobManager.cs | 17 +++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/src/Tgstation.Server.Host/Core/IJobManager.cs b/src/Tgstation.Server.Host/Core/IJobManager.cs index 0d5a16842e..7389ce78d5 100644 --- a/src/Tgstation.Server.Host/Core/IJobManager.cs +++ b/src/Tgstation.Server.Host/Core/IJobManager.cs @@ -25,6 +25,18 @@ namespace Tgstation.Server.Host.Core /// A representing a running operation Task RegisterOperation(Job job, Func, CancellationToken, Task> operation, CancellationToken cancellationToken); + /// + /// Wait for a given to complete + /// + /// The to wait for + /// The to cancel the + /// A that will cancel the + /// The for the operation + /// A representing the +#pragma warning disable CA1068 // CancellationToken parameters must come last https://github.com/dotnet/roslyn-analyzers/issues/1816 + Task WaitForJobCompletion(Job job, User canceller, CancellationToken jobCancellationToken, CancellationToken cancellationToken); +#pragma warning restore CA1068 // CancellationToken parameters must come last + /// /// Cancels a give /// diff --git a/src/Tgstation.Server.Host/Core/JobManager.cs b/src/Tgstation.Server.Host/Core/JobManager.cs index 1876ecda4e..b9b8ba4b68 100644 --- a/src/Tgstation.Server.Host/Core/JobManager.cs +++ b/src/Tgstation.Server.Host/Core/JobManager.cs @@ -254,5 +254,22 @@ namespace Tgstation.Server.Host.Core return handler.Progress; } } + + /// + public async Task WaitForJobCompletion(Job job, User canceller, CancellationToken jobCancellationToken, CancellationToken cancellationToken) + { + JobHandler handler; + lock (this) + { + if (!jobs.TryGetValue(job.Id, out handler)) + return; + } + Task cancelTask = null; + using (jobCancellationToken.Register(() => cancelTask = CancelJob(job, canceller, true, cancellationToken))) + await handler.Wait(cancellationToken).ConfigureAwait(false); + + if (cancelTask != null) + await cancelTask.ConfigureAwait(false); + } } } From cad7ddc10f58e667e81dfc47139716ee8a8b7720 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Sat, 15 Sep 2018 14:03:44 -0400 Subject: [PATCH 05/11] Merge DreamMakerController and Instance auto update compile functions. Instance auto update now properly specifies it as a job --- .../Components/IInstance.cs | 11 ++ .../Components/Instance.cs | 130 ++++++++++++------ .../Controllers/DreamMakerController.cs | 68 +-------- 3 files changed, 99 insertions(+), 110 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/IInstance.cs b/src/Tgstation.Server.Host/Components/IInstance.cs index 8e5b2edf80..cf2c527a8a 100644 --- a/src/Tgstation.Server.Host/Components/IInstance.cs +++ b/src/Tgstation.Server.Host/Components/IInstance.cs @@ -1,5 +1,6 @@ using Microsoft.Extensions.Hosting; using System; +using System.Threading; using System.Threading.Tasks; using Tgstation.Server.Host.Components.Byond; using Tgstation.Server.Host.Components.Chat; @@ -75,5 +76,15 @@ namespace Tgstation.Server.Host.Components /// The new auto update inteval /// A representing the running operation Task SetAutoUpdateInterval(uint newInterval); + + /// + /// Run the compile job and insert it into the database. Meant to be called by a + /// + /// The running + /// The for the operation + /// The to report compilation progress + /// The for the operation + /// A representing the running operation + Task CompileProcess(Job job, IServiceProvider serviceProvider, Action progressReporter, CancellationToken cancellationToken); } } \ No newline at end of file diff --git a/src/Tgstation.Server.Host/Components/Instance.cs b/src/Tgstation.Server.Host/Components/Instance.cs index 46b5dc3541..1e99e0b49a 100644 --- a/src/Tgstation.Server.Host/Components/Instance.cs +++ b/src/Tgstation.Server.Host/Components/Instance.cs @@ -1,9 +1,11 @@ using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using System; using System.Linq; using System.Threading; using System.Threading.Tasks; +using Tgstation.Server.Api.Rights; using Tgstation.Server.Host.Components.Byond; using Tgstation.Server.Host.Components.Chat; using Tgstation.Server.Host.Components.Compiler; @@ -114,6 +116,59 @@ namespace Tgstation.Server.Host.Components RepositoryManager.Dispose(); } + /// + public async Task CompileProcess(Job job, IServiceProvider serviceProvider, Action progressReporter, CancellationToken cancellationToken) + { + var databaseContext = serviceProvider.GetRequiredService(); + + var ddSettingsTask = databaseContext.DreamDaemonSettings.Where(x => x.InstanceId == metadata.Id).Select(x => new DreamDaemonSettings + { + StartupTimeout = x.StartupTimeout, + SecurityLevel = x.SecurityLevel + }).FirstOrDefaultAsync(cancellationToken); + + var dreamMakerSettings = await databaseContext.DreamMakerSettings.Where(x => x.InstanceId == metadata.Id).FirstAsync(cancellationToken).ConfigureAwait(false); + if (dreamMakerSettings == default) + throw new JobException("Missing DreamMakerSettings in DB!"); + var ddSettings = await ddSettingsTask.ConfigureAwait(false); + if (ddSettings == default) + throw new JobException("Missing DreamDaemonSettings in DB!"); + + CompileJob compileJob; + RevisionInformation revInfo; + using (var repo = await RepositoryManager.LoadRepository(cancellationToken).ConfigureAwait(false)) + { + if (repo == null) + throw new JobException("Missing Repository!"); + + var repoSha = repo.Head; + revInfo = await databaseContext.RevisionInformations.Where(x => x.CommitSha == repoSha).Include(x => x.ActiveTestMerges).ThenInclude(x => x.TestMerge).FirstOrDefaultAsync().ConfigureAwait(false); + + if (revInfo == default) + { + revInfo = new RevisionInformation + { + CommitSha = repoSha, + OriginCommitSha = repoSha, + Instance = new Models.Instance + { + Id = metadata.Id + } + }; + logger.LogWarning(Repository.Repository.OriginTrackingErrorTemplate, repoSha); + databaseContext.Instances.Attach(revInfo.Instance); + } + + compileJob = await DreamMaker.Compile(revInfo, dreamMakerSettings, ddSettings.SecurityLevel.Value, ddSettings.StartupTimeout.Value, repo, cancellationToken).ConfigureAwait(false); + } + + compileJob.Job = job; + + databaseContext.CompileJobs.Add(compileJob); //will be saved by job context + + job.PostComplete = ct => CompileJobConsumer.LoadCompileJob(compileJob, ct); + } + /// /// Pull the repository and compile for every set of given /// @@ -129,33 +184,32 @@ namespace Tgstation.Server.Host.Components try { - CompileJob job = null; + Job job = null; + Models.User user = null; //need this the whole time + var noRepo = false; await databaseContextFactory.UseContext(async (db) => { - //start up queries we'll need in the future - var instanceQuery = db.Instances.Where(x => x.Id == metadata.Id); - var ddSettingsTask = instanceQuery.Select(x => x.DreamDaemonSettings).Select(x => new DreamDaemonSettings - { - StartupTimeout = x.StartupTimeout, - SecurityLevel = x.SecurityLevel - }).FirstAsync(cancellationToken); - var dmSettingsTask = instanceQuery.Select(x => x.DreamMakerSettings).FirstAsync(cancellationToken); - var repositorySettingsTask = instanceQuery.Select(x => x.RepositorySettings).FirstAsync(cancellationToken); + var userTask = db.Users.FirstAsync(cancellationToken); + + var repositorySettingsTask = db.RepositorySettings.Where(x => x.InstanceId == metadata.Id).FirstAsync(cancellationToken); using (var repo = await RepositoryManager.LoadRepository(cancellationToken).ConfigureAwait(false)) { if (repo == null) + { + //no repo, no auto updates + noRepo = true; return; + } - //start the rev info query - var startSha = repo.Head; - var revInfoTask = instanceQuery.SelectMany(x => x.RevisionInformations).Where(x => x.CommitSha == startSha).FirstOrDefaultAsync(cancellationToken); - - //need repo setting to fetch var repositorySettings = await repositorySettingsTask.ConfigureAwait(false); + + //the main point of auto update is to pull the remote await repo.FetchOrigin(repositorySettings.AccessUser, repositorySettings.AccessToken, null, cancellationToken).ConfigureAwait(false); + var startSha = repo.Head; + //take appropriate auto update actions bool shouldSyncTracked; if (repositorySettings.AutoUpdatesKeepTestMerges.Value) @@ -174,38 +228,28 @@ namespace Tgstation.Server.Host.Components //synch if necessary if (repositorySettings.AutoUpdatesSynchronize.Value && startSha != repo.Head) await repo.Sychronize(repositorySettings.AccessUser, repositorySettings.AccessToken, shouldSyncTracked, cancellationToken).ConfigureAwait(false); - - //finish other queries - var dmSettings = await dmSettingsTask.ConfigureAwait(false); - var ddSettings = await ddSettingsTask.ConfigureAwait(false); - var revInfo = await revInfoTask.ConfigureAwait(false); - - //null rev info handling - if (revInfo == default) - { - var currentSha = repo.Head; - revInfo = new RevisionInformation - { - CommitSha = currentSha, - OriginCommitSha = currentSha, - Instance = new Models.Instance - { - Id = metadata.Id - } - }; - logger.LogWarning(Repository.Repository.OriginTrackingErrorTemplate, currentSha); - db.Instances.Attach(revInfo.Instance); - } - - //finally start compile - job = await DreamMaker.Compile(revInfo, dmSettings, ddSettings.SecurityLevel.Value, ddSettings.StartupTimeout.Value, repo, cancellationToken).ConfigureAwait(false); } - db.CompileJobs.Add(job); - await db.Save(cancellationToken).ConfigureAwait(false); + user = await userTask.ConfigureAwait(false); + + //finally set up the job + job = new Job + { + StartedBy = user, + Instance = new Models.Instance + { + Id = metadata.Id + }, + Description = "Scheduled code deployment", + CancelRightsType = RightsType.DreamMaker, + CancelRight = (ulong)DreamMakerRights.CancelCompile + }; + + await jobManager.RegisterOperation(job, CompileProcess, cancellationToken).ConfigureAwait(false); }).ConfigureAwait(false); - await CompileJobConsumer.LoadCompileJob(job, cancellationToken).ConfigureAwait(false); + if(!noRepo) + await jobManager.WaitForJobCompletion(job, user, cancellationToken, default).ConfigureAwait(false); } catch (OperationCanceledException) { diff --git a/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs b/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs index 746f04157b..3c12cdc2bf 100644 --- a/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs +++ b/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs @@ -1,6 +1,5 @@ using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using System; using System.Linq; @@ -10,7 +9,6 @@ using System.Threading.Tasks; using Tgstation.Server.Api; using Tgstation.Server.Api.Rights; using Tgstation.Server.Host.Components; -using Tgstation.Server.Host.Components.Repository; using Tgstation.Server.Host.Core; using Tgstation.Server.Host.Models; using Tgstation.Server.Host.Security; @@ -97,7 +95,7 @@ namespace Tgstation.Server.Host.Controllers CancelRight = (ulong)DreamMakerRights.CancelCompile, Instance = Instance }; - await jobManager.RegisterOperation(job, (paramJob, serviceProvider, progressReporter, ct) => RunCompile(paramJob, serviceProvider, Instance, ct), cancellationToken).ConfigureAwait(false); + await jobManager.RegisterOperation(job, instanceManager.GetInstance(Instance).CompileProcess, cancellationToken).ConfigureAwait(false); return Accepted(job.ToApi()); } @@ -129,69 +127,5 @@ namespace Tgstation.Server.Host.Controllers await DatabaseContext.Save(cancellationToken).ConfigureAwait(false); return await Read(cancellationToken).ConfigureAwait(false); } - - /// - /// Run the compile job and insert it into the database - /// - /// The running - /// The for the operation - /// The for the operation - /// The for the operation - /// A representing the running operation - async Task RunCompile(Job job, IServiceProvider serviceProvider, Models.Instance instanceModel, CancellationToken cancellationToken) - { - var instanceManager = serviceProvider.GetRequiredService(); - var databaseContext = serviceProvider.GetRequiredService(); - - var ddSettingsTask = databaseContext.DreamDaemonSettings.Where(x => x.InstanceId == instanceModel.Id).Select(x => new DreamDaemonSettings - { - StartupTimeout = x.StartupTimeout, - SecurityLevel = x.SecurityLevel - }).FirstOrDefaultAsync(cancellationToken); - - - var dreamMakerSettings = await databaseContext.DreamMakerSettings.Where(x => x.InstanceId == instanceModel.Id).FirstAsync(cancellationToken).ConfigureAwait(false); - if (dreamMakerSettings == default) - throw new JobException("Missing DreamMakerSettings in DB!"); - var ddSettings = await ddSettingsTask.ConfigureAwait(false); - if (ddSettings == default) - throw new JobException("Missing DreamDaemonSettings in DB!"); - - var instance = instanceManager.GetInstance(instanceModel); - - CompileJob compileJob; - RevisionInformation revInfo; - using (var repo = await instance.RepositoryManager.LoadRepository(cancellationToken).ConfigureAwait(false)) - { - if (repo == null) - throw new JobException("Missing Repository!"); - - var repoSha = repo.Head; - revInfo = await databaseContext.RevisionInformations.Where(x => x.CommitSha == repoSha).Include(x => x.ActiveTestMerges).ThenInclude(x => x.TestMerge).FirstOrDefaultAsync().ConfigureAwait(false); - - if (revInfo == default) - { - revInfo = new RevisionInformation - { - CommitSha = repoSha, - OriginCommitSha = repoSha, - Instance = new Models.Instance - { - Id = Instance.Id - } - }; - Logger.LogWarning(Repository.OriginTrackingErrorTemplate, repoSha); - databaseContext.Instances.Attach(revInfo.Instance); - } - - compileJob = await instance.DreamMaker.Compile(revInfo, dreamMakerSettings, ddSettings.SecurityLevel.Value, ddSettings.StartupTimeout.Value, repo, cancellationToken).ConfigureAwait(false); - } - - compileJob.Job = job; - - databaseContext.CompileJobs.Add(compileJob); //will be saved by job context - - job.PostComplete = ct => instance.CompileJobConsumer.LoadCompileJob(compileJob, ct); - } } } From b6b8d8e06b2b697c6896863d36f0902c18b5a8b3 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Sat, 15 Sep 2018 14:05:26 -0400 Subject: [PATCH 06/11] Remove unused IInstance properties --- .../Components/IInstance.cs | 18 +----------------- .../Components/Instance.cs | 5 +---- 2 files changed, 2 insertions(+), 21 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/IInstance.cs b/src/Tgstation.Server.Host/Components/IInstance.cs index cf2c527a8a..921a408266 100644 --- a/src/Tgstation.Server.Host/Components/IInstance.cs +++ b/src/Tgstation.Server.Host/Components/IInstance.cs @@ -26,12 +26,7 @@ namespace Tgstation.Server.Host.Components /// The for the /// IByondManager ByondManager { get; } - - /// - /// The for the - /// - IDreamMaker DreamMaker { get; } - + /// /// The for the /// @@ -42,11 +37,6 @@ namespace Tgstation.Server.Host.Components /// IChat Chat { get; } - /// - /// The for the - /// - ICompileJobConsumer CompileJobConsumer { get; } - /// /// The for the /// @@ -58,12 +48,6 @@ namespace Tgstation.Server.Host.Components /// The latest if it exists CompileJob LatestCompileJob(); - /// - /// Get the associated with the - /// - /// The associated with the - Api.Models.Instance GetMetadata(); - /// /// Rename the /// diff --git a/src/Tgstation.Server.Host/Components/Instance.cs b/src/Tgstation.Server.Host/Components/Instance.cs index 1e99e0b49a..d904aadfa4 100644 --- a/src/Tgstation.Server.Host/Components/Instance.cs +++ b/src/Tgstation.Server.Host/Components/Instance.cs @@ -266,10 +266,7 @@ namespace Tgstation.Server.Host.Components break; } } - - /// - public Api.Models.Instance GetMetadata() => metadata.CloneMetadata(); - + /// public void Rename(string newName) { From 9222031ba116c77973d872a0624684315c2bbf7a Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Sat, 15 Sep 2018 14:07:20 -0400 Subject: [PATCH 07/11] Make use of IInstance.Rename --- src/Tgstation.Server.Host/Controllers/InstanceController.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Tgstation.Server.Host/Controllers/InstanceController.cs b/src/Tgstation.Server.Host/Controllers/InstanceController.cs index 277677d774..3545f1ce8a 100644 --- a/src/Tgstation.Server.Host/Controllers/InstanceController.cs +++ b/src/Tgstation.Server.Host/Controllers/InstanceController.cs @@ -295,6 +295,7 @@ namespace Tgstation.Server.Host.Controllers } var originalOnline = originalModel.Online.Value; + var renamed = model.Name != null && originalModel.Name != model.Name; if (CheckModified(x => x.AutoUpdateInterval, InstanceManagerRights.SetAutoUpdate) || CheckModified(x => x.ConfigurationType, InstanceManagerRights.SetConfiguration) @@ -311,6 +312,9 @@ namespace Tgstation.Server.Host.Controllers await DatabaseContext.Save(cancellationToken).ConfigureAwait(false); + if (renamed) + instanceManager.GetInstance(originalModel).Rename(originalModel.Name); + var oldAutoStart = originalModel.DreamDaemonSettings.AutoStart; try { From fee1e4a6070c46d593b455a35164b9b296537e3e Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Sat, 15 Sep 2018 14:11:43 -0400 Subject: [PATCH 08/11] Clean up queries involving CompileJobs Repeat after me: CompileJobs that didn't succeed will not be in the database CompileJobs that didn't succeed will not be in the database CompileJobs that didn't succeed will not be in the database CompileJobs that didn't succeed will not be in the database CompileJobs that didn't succeed will not be in the database CompileJobs that didn't succeed will not be in the database CompileJobs that didn't succeed will not be in the database CompileJobs that didn't succeed will not be in the database CompileJobs that didn't succeed will not be in the database CompileJobs that didn't succeed will not be in the database --- src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs | 4 ++-- src/Tgstation.Server.Host/Components/Instance.cs | 2 +- src/Tgstation.Server.Host/Controllers/DreamMakerController.cs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs b/src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs index 045e495fa0..c31ab7e326 100644 --- a/src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs +++ b/src/Tgstation.Server.Host/Components/Compiler/DmbFactory.cs @@ -169,7 +169,7 @@ namespace Tgstation.Server.Host.Components.Compiler public Task StartAsync(CancellationToken cancellationToken) => databaseContextFactory.UseContext(async (db) => { //where complete clause not necessary, only successful COMPILEjobs get in the db - var cj = await db.CompileJobs.Where(x => x.Job.Instance.Id == instance.Id && !x.Job.Cancelled.Value && x.Job.ExceptionDetails == null && x.Job.StoppedAt != null) + var cj = await db.CompileJobs.Where(x => x.Job.Instance.Id == instance.Id) .Include(x => x.Job).ThenInclude(x => x.StartedBy) .Include(x => x.RevisionInformation).ThenInclude(x => x.PrimaryTestMerge).ThenInclude(x => x.MergedBy) .Include(x => x.RevisionInformation).ThenInclude(x => x.ActiveTestMerges).ThenInclude(x => x.TestMerge).ThenInclude(x => x.MergedBy) @@ -245,7 +245,7 @@ namespace Tgstation.Server.Host.Components.Compiler //find the uids of locked directories await databaseContextFactory.UseContext(async db => { - jobUidsToNotErase = await db.CompileJobs.Where(x => x.Job.Instance.Id == instance.Id && jobIdsToSkip.Contains(x.Id) && x.DirectoryName.HasValue).Select(x => x.DirectoryName.Value.ToString().ToUpperInvariant()).ToListAsync(cancellationToken).ConfigureAwait(false); + jobUidsToNotErase = await db.CompileJobs.Where(x => x.Job.Instance.Id == instance.Id && jobIdsToSkip.Contains(x.Id)).Select(x => x.DirectoryName.Value.ToString().ToUpperInvariant()).ToListAsync(cancellationToken).ConfigureAwait(false); }).ConfigureAwait(false); //add the other exemption diff --git a/src/Tgstation.Server.Host/Components/Instance.cs b/src/Tgstation.Server.Host/Components/Instance.cs index d904aadfa4..fc9c3234cf 100644 --- a/src/Tgstation.Server.Host/Components/Instance.cs +++ b/src/Tgstation.Server.Host/Components/Instance.cs @@ -286,7 +286,7 @@ namespace Tgstation.Server.Host.Components CompileJob latestCompileJob = null; await databaseContextFactory.UseContext(async db => { - latestCompileJob = await db.CompileJobs.Where(x => x.Job.Instance.Id == metadata.Id && x.Job.ExceptionDetails == null).OrderByDescending(x => x.Job.StoppedAt).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + latestCompileJob = await db.CompileJobs.Where(x => x.Job.Instance.Id == metadata.Id).OrderByDescending(x => x.Job.StoppedAt).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); }).ConfigureAwait(false); await dmbFactory.CleanUnusedCompileJobs(latestCompileJob, cancellationToken).ConfigureAwait(false); } diff --git a/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs b/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs index 3c12cdc2bf..b847fef9e2 100644 --- a/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs +++ b/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs @@ -76,7 +76,7 @@ namespace Tgstation.Server.Host.Controllers [TgsAuthorize(DreamMakerRights.CompileJobs)] public override async Task List(CancellationToken cancellationToken) { - var compileJobs = await DatabaseContext.CompileJobs.Where(x => x.Job.Instance.Id == Instance.Id).OrderByDescending(x => x.Job.StartedAt).Select(x => new Api.Models.CompileJob + var compileJobs = await DatabaseContext.CompileJobs.Where(x => x.Job.Instance.Id == Instance.Id).OrderByDescending(x => x.Job.StoppedAt).Select(x => new Api.Models.CompileJob { Id = x.Id }).ToListAsync(cancellationToken).ConfigureAwait(false); From 1c0007398a93de11d2edb1c9d46670897cec3f37 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Sat, 15 Sep 2018 14:31:03 -0400 Subject: [PATCH 09/11] Move auto update repository changes into it's own job. Add some logging --- .../Components/Instance.cs | 82 ++++++++++++------- 1 file changed, 53 insertions(+), 29 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Instance.cs b/src/Tgstation.Server.Host/Components/Instance.cs index fc9c3234cf..f178039f9e 100644 --- a/src/Tgstation.Server.Host/Components/Instance.cs +++ b/src/Tgstation.Server.Host/Components/Instance.cs @@ -184,17 +184,31 @@ namespace Tgstation.Server.Host.Components try { - Job job = null; Models.User user = null; - //need this the whole time - var noRepo = false; - await databaseContextFactory.UseContext(async (db) => + await databaseContextFactory.UseContext(async (db) => user = await db.Users.FirstAsync(cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); + var repositoryUpdateJob = new Job { - var userTask = db.Users.FirstAsync(cancellationToken); + Instance = new Models.Instance + { + Id = metadata.Id + }, + Description = "Scheduled repository update", + CancelRightsType = RightsType.Repository, + CancelRight = (ulong)RepositoryRights.CancelPendingChanges + }; - var repositorySettingsTask = db.RepositorySettings.Where(x => x.InstanceId == metadata.Id).FirstAsync(cancellationToken); + var noRepo = false; + await jobManager.RegisterOperation(repositoryUpdateJob, async (paramJob, serviceProvider, progressReporter, jobCancellationToken) => + { + var db = serviceProvider.GetRequiredService(); + var repositorySettingsTask = db.RepositorySettings.Where(x => x.InstanceId == metadata.Id).FirstAsync(jobCancellationToken); - using (var repo = await RepositoryManager.LoadRepository(cancellationToken).ConfigureAwait(false)) + //assume 5 steps with synchronize + const int ProgressSections = 5; + const int ProgressStep = 100 / ProgressSections; + progressReporter(0 * ProgressStep); + + using (var repo = await RepositoryManager.LoadRepository(jobCancellationToken).ConfigureAwait(false)) { if (repo == null) { @@ -202,11 +216,17 @@ namespace Tgstation.Server.Host.Components noRepo = true; return; } + progressReporter(1 * ProgressStep); var repositorySettings = await repositorySettingsTask.ConfigureAwait(false); - + + const int SecondStepProgress = 2 * ProgressStep; + progressReporter(SecondStepProgress); + //the main point of auto update is to pull the remote - await repo.FetchOrigin(repositorySettings.AccessUser, repositorySettings.AccessToken, null, cancellationToken).ConfigureAwait(false); + await repo.FetchOrigin(repositorySettings.AccessUser, repositorySettings.AccessToken, x => progressReporter(SecondStepProgress + (x / ProgressSections)), jobCancellationToken).ConfigureAwait(false); + + progressReporter(3 * ProgressStep); var startSha = repo.Head; @@ -214,45 +234,48 @@ namespace Tgstation.Server.Host.Components bool shouldSyncTracked; if (repositorySettings.AutoUpdatesKeepTestMerges.Value) { - var result = await repo.MergeOrigin(repositorySettings.CommitterName, repositorySettings.CommitterEmail, cancellationToken).ConfigureAwait(false); + var result = await repo.MergeOrigin(repositorySettings.CommitterName, repositorySettings.CommitterEmail, jobCancellationToken).ConfigureAwait(false); if (!result.HasValue) return; shouldSyncTracked = result.Value; } else { - await repo.ResetToOrigin(cancellationToken).ConfigureAwait(false); + await repo.ResetToOrigin(jobCancellationToken).ConfigureAwait(false); shouldSyncTracked = true; } + progressReporter(4 * ProgressStep); //synch if necessary if (repositorySettings.AutoUpdatesSynchronize.Value && startSha != repo.Head) - await repo.Sychronize(repositorySettings.AccessUser, repositorySettings.AccessToken, shouldSyncTracked, cancellationToken).ConfigureAwait(false); + await repo.Sychronize(repositorySettings.AccessUser, repositorySettings.AccessToken, shouldSyncTracked, jobCancellationToken).ConfigureAwait(false); + + progressReporter(5 * ProgressStep); } + }, cancellationToken).ConfigureAwait(false); - user = await userTask.ConfigureAwait(false); + await jobManager.WaitForJobCompletion(repositoryUpdateJob, user, cancellationToken, default).ConfigureAwait(false); - //finally set up the job - job = new Job - { - StartedBy = user, - Instance = new Models.Instance - { - Id = metadata.Id - }, - Description = "Scheduled code deployment", - CancelRightsType = RightsType.DreamMaker, - CancelRight = (ulong)DreamMakerRights.CancelCompile - }; + if (noRepo) + continue; - await jobManager.RegisterOperation(job, CompileProcess, cancellationToken).ConfigureAwait(false); - }).ConfigureAwait(false); + //finally set up the job + var compileProcessJob = new Job + { + StartedBy = user, + Instance = repositoryUpdateJob.Instance, + Description = "Scheduled code deployment", + CancelRightsType = RightsType.DreamMaker, + CancelRight = (ulong)DreamMakerRights.CancelCompile + }; - if(!noRepo) - await jobManager.WaitForJobCompletion(job, user, cancellationToken, default).ConfigureAwait(false); + await jobManager.RegisterOperation(compileProcessJob, CompileProcess, cancellationToken).ConfigureAwait(false); + + await jobManager.WaitForJobCompletion(compileProcessJob, user, cancellationToken, default).ConfigureAwait(false); } catch (OperationCanceledException) { + logger.LogDebug("Cancelled auto update job!"); throw; } catch (Exception e) @@ -265,6 +288,7 @@ namespace Tgstation.Server.Host.Components { break; } + logger.LogTrace("Leaving auto update loop..."); } /// From 4eb0b89b0836303b42ab64c1b388fc334d11c5b2 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Sat, 15 Sep 2018 14:33:58 -0400 Subject: [PATCH 10/11] Stops assuming the admin user's ID --- src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs index 939661e74d..e51e28e9a2 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs @@ -1,4 +1,5 @@ using Byond.TopicSender; +using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; using Newtonsoft.Json; using Newtonsoft.Json.Serialization; @@ -807,11 +808,14 @@ namespace Tgstation.Server.Host.Components.Watchdog if (!autoStart) return; + long? adminUserId = null; + + await databaseContextFactory.UseContext(async db => adminUserId = await db.Users.Select(x => x.Id).FirstAsync(cancellationToken).ConfigureAwait(false)).ConfigureAwait(false); var job = new Models.Job { StartedBy = new Models.User { - Id = 1 //just use admin for this cause whatever + Id = adminUserId.Value }, Instance = new Models.Instance { From a8ae7efcafd1c63c997c0abac57088aba80bfa4c Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Sat, 15 Sep 2018 15:16:38 -0400 Subject: [PATCH 11/11] Sanitize your public API functions --- src/Tgstation.Server.Host/Components/Instance.cs | 8 ++++++++ src/Tgstation.Server.Host/Core/JobManager.cs | 6 ++++++ 2 files changed, 14 insertions(+) diff --git a/src/Tgstation.Server.Host/Components/Instance.cs b/src/Tgstation.Server.Host/Components/Instance.cs index f178039f9e..8f7b0e7700 100644 --- a/src/Tgstation.Server.Host/Components/Instance.cs +++ b/src/Tgstation.Server.Host/Components/Instance.cs @@ -119,6 +119,14 @@ namespace Tgstation.Server.Host.Components /// public async Task CompileProcess(Job job, IServiceProvider serviceProvider, Action progressReporter, CancellationToken cancellationToken) { + //DO NOT FOLLOW THE SUGGESTION FOR A THROW EXPRESSION HERE + if (job == null) + throw new ArgumentNullException(nameof(job)); + if (serviceProvider == null) + throw new ArgumentNullException(nameof(serviceProvider)); + if (progressReporter == null) + throw new ArgumentNullException(nameof(progressReporter)); + var databaseContext = serviceProvider.GetRequiredService(); var ddSettingsTask = databaseContext.DreamDaemonSettings.Where(x => x.InstanceId == metadata.Id).Select(x => new DreamDaemonSettings diff --git a/src/Tgstation.Server.Host/Core/JobManager.cs b/src/Tgstation.Server.Host/Core/JobManager.cs index b9b8ba4b68..cb034f445a 100644 --- a/src/Tgstation.Server.Host/Core/JobManager.cs +++ b/src/Tgstation.Server.Host/Core/JobManager.cs @@ -247,6 +247,8 @@ namespace Tgstation.Server.Host.Core /// public int? JobProgress(Job job) { + if (job == null) + throw new ArgumentNullException(nameof(job)); lock (this) { if (!jobs.TryGetValue(job.Id, out var handler)) @@ -258,6 +260,10 @@ namespace Tgstation.Server.Host.Core /// public async Task WaitForJobCompletion(Job job, User canceller, CancellationToken jobCancellationToken, CancellationToken cancellationToken) { + if (job == null) + throw new ArgumentNullException(nameof(job)); + if (canceller == null) + throw new ArgumentNullException(nameof(canceller)); JobHandler handler; lock (this) {