diff --git a/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs b/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs index 62a03990dd..cea2a3f7c7 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs @@ -15,7 +15,7 @@ namespace Tgstation.Server.Host.Components.Deployment /// /// Standard /// - sealed class DmbFactory : IDmbFactory, ICompileJobConsumer + sealed class DmbFactory : IDmbFactory, ICompileJobSink { /// public Task OnNewerDmb @@ -302,5 +302,13 @@ namespace Tgstation.Server.Host.Components.Deployment await Task.WhenAll(tasks).ConfigureAwait(false); } #pragma warning restore CA1506 + + /// + public CompileJob LatestCompileJob() + { + if (!DmbAvailable) + return null; + return LockNextDmb(0)?.CompileJob; + } } } diff --git a/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs b/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs index 3c2b21bb43..268a5fc202 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs @@ -1,4 +1,6 @@ -using Microsoft.Extensions.Logging; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using Octokit; using System; using System.Collections.Generic; using System.Globalization; @@ -13,8 +15,11 @@ using Tgstation.Server.Host.Components.Chat; using Tgstation.Server.Host.Components.Repository; using Tgstation.Server.Host.Components.Session; using Tgstation.Server.Host.Components.Watchdog; +using Tgstation.Server.Host.Core; +using Tgstation.Server.Host.Database; using Tgstation.Server.Host.IO; using Tgstation.Server.Host.Jobs; +using Tgstation.Server.Host.Models; using Tgstation.Server.Host.System; namespace Tgstation.Server.Host.Components.Deployment @@ -82,11 +87,31 @@ namespace Tgstation.Server.Host.Components.Deployment /// readonly IWatchdog watchdog; + /// + /// The for . + /// + readonly IRepositoryManager repositoryManager; + + /// + /// The for . + /// + readonly IGitHubClientFactory gitHubClientFactory; + + /// + /// The for . + /// + readonly ICompileJobSink compileJobConsumer; + /// /// The for /// readonly ILogger logger; + /// + /// The belongs to. + /// + readonly Api.Models.Instance metadata; + /// /// for . /// @@ -108,7 +133,11 @@ namespace Tgstation.Server.Host.Components.Deployment /// The value of /// The value of /// The value of + /// The value of . + /// The value of . + /// The value of . /// The value of + /// The value of . public DreamMaker( IByondManager byond, IIOManager ioManager, @@ -118,7 +147,11 @@ namespace Tgstation.Server.Host.Components.Deployment IChatManager chatManager, IProcessExecutor processExecutor, IWatchdog watchdog, - ILogger logger) + IGitHubClientFactory gitHubClientFactory, + ICompileJobSink compileJobConsumer, + IRepositoryManager repositoryManager, + ILogger logger, + Api.Models.Instance metadata) { this.byond = byond ?? throw new ArgumentNullException(nameof(byond)); this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); @@ -128,7 +161,11 @@ namespace Tgstation.Server.Host.Components.Deployment this.chatManager = chatManager ?? throw new ArgumentNullException(nameof(chatManager)); this.processExecutor = processExecutor ?? throw new ArgumentNullException(nameof(processExecutor)); this.watchdog = watchdog ?? throw new ArgumentNullException(nameof(watchdog)); + this.gitHubClientFactory = gitHubClientFactory ?? throw new ArgumentNullException(nameof(gitHubClientFactory)); + this.compileJobConsumer = compileJobConsumer ?? throw new ArgumentNullException(nameof(compileJobConsumer)); + this.repositoryManager = repositoryManager ?? throw new ArgumentNullException(nameof(repositoryManager)); this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + this.metadata = metadata ?? throw new ArgumentNullException(nameof(metadata)); compilingLock = new object(); } @@ -162,7 +199,7 @@ namespace Tgstation.Server.Host.Components.Deployment /// /// The timeout in seconds for validation /// The level to use to validate the API - /// The for the operation + /// The for the operation /// The current /// The port to use for API validation /// The for the operation @@ -231,7 +268,7 @@ namespace Tgstation.Server.Host.Components.Deployment /// Compiles a .dme with DreamMaker /// /// The path to the DreamMaker executable - /// The for the operation + /// The for the operation /// The for the operation /// A representing the running operation async Task RunDreamMaker(string dreamMakerPath, Models.CompileJob job, CancellationToken cancellationToken) @@ -260,7 +297,7 @@ namespace Tgstation.Server.Host.Components.Deployment /// /// Adds server side includes to the .dme being compiled /// - /// The for the operation + /// The for the operation /// The for the operation /// A representing the running operation async Task ModifyDme(Models.CompileJob job, CancellationToken cancellationToken) @@ -314,7 +351,7 @@ namespace Tgstation.Server.Host.Components.Deployment /// /// Cleans up a failed compile /// - /// The running + /// The running /// If the was cancelled /// The for the operation /// A representing the running operation @@ -338,7 +375,7 @@ namespace Tgstation.Server.Host.Components.Deployment /// /// Send a message to about a deployment /// - /// The for the deployment + /// The for the deployment /// The for the deployment /// The for the operation /// A representing the running operation @@ -396,7 +433,7 @@ namespace Tgstation.Server.Host.Components.Deployment /// /// Executes and populate a given /// - /// The to run and populate + /// The to run and populate /// The settings to use /// The to use /// The to use @@ -484,14 +521,6 @@ namespace Tgstation.Server.Host.Components.Deployment await Task.WhenAll(symATask, symBTask).ConfigureAwait(false); - await chatManager.SendUpdateMessage( - String.Format( - CultureInfo.InvariantCulture, - "Deployment complete! Changes will be applied when DreamDaemon {0}.", - watchdog.Running ? "reboots" : "is launched"), - cancellationToken) - .ConfigureAwait(false); - logger.LogDebug("Compile complete!"); } catch (Exception e) @@ -502,20 +531,180 @@ namespace Tgstation.Server.Host.Components.Deployment } /// - public async Task Compile(Models.RevisionInformation revisionInformation, Api.Models.DreamMaker dreamMakerSettings, uint apiValidateTimeout, IRepository repository, Action progressReporter, TimeSpan? estimatedDuration, CancellationToken cancellationToken) + public async Task DeploymentProcess( + Models.Job job, + IDatabaseContext databaseContext, + Action progressReporter, + CancellationToken cancellationToken) { - if (revisionInformation == null) - throw new ArgumentNullException(nameof(revisionInformation)); - - if (dreamMakerSettings == null) - throw new ArgumentNullException(nameof(dreamMakerSettings)); - - if (repository == null) - throw new ArgumentNullException(nameof(repository)); - +#pragma warning disable IDE0016 // Use 'throw' expression + if (job == null) + throw new ArgumentNullException(nameof(job)); +#pragma warning restore IDE0016 // Use 'throw' expression + if (databaseContext == null) + throw new ArgumentNullException(nameof(databaseContext)); if (progressReporter == null) throw new ArgumentNullException(nameof(progressReporter)); + var averageSpan = await CalculateExpectedDeploymentTime(databaseContext, cancellationToken).ConfigureAwait(false); + + var ddSettings = await databaseContext + .DreamDaemonSettings + .Where(x => x.InstanceId == metadata.Id) + .Select(x => new Models.DreamDaemonSettings + { + StartupTimeout = x.StartupTimeout, + }) + .FirstOrDefaultAsync(cancellationToken) + .ConfigureAwait(false); + if (ddSettings == default) + throw new JobException(ErrorCode.InstanceMissingDreamDaemonSettings); + + var dreamMakerSettings = await databaseContext.DreamMakerSettings.Where(x => x.InstanceId == metadata.Id).FirstAsync(cancellationToken).ConfigureAwait(false); + if (dreamMakerSettings == default) + throw new JobException(ErrorCode.InstanceMissingDreamMakerSettings); + + Models.RepositorySettings repositorySettings = null; + string repoOwner = null; + string repoName = null; + Models.CompileJob compileJob; + Models.RevisionInformation revInfo; + + using (var repo = await repositoryManager.LoadRepository(cancellationToken).ConfigureAwait(false)) + { + if (repo == null) + throw new JobException(ErrorCode.RepoMissing); + + if (repo.IsGitHubRepository) + { + repoOwner = repo.GitHubOwner; + repoName = repo.GitHubRepoName; + repositorySettings = await databaseContext + .RepositorySettings + .Where(x => x.InstanceId == metadata.Id) + .Select(x => new Models.RepositorySettings + { + AccessToken = x.AccessToken, + ShowTestMergeCommitters = x.ShowTestMergeCommitters, + PushTestMergeCommits = x.PushTestMergeCommits, + PostTestMergeComment = x.PostTestMergeComment + }) + .FirstOrDefaultAsync(cancellationToken) + .ConfigureAwait(false); + if (repositorySettings == default) + throw new JobException(ErrorCode.InstanceMissingRepositorySettings); + } + + var repoSha = repo.Head; + revInfo = await databaseContext + .RevisionInformations + .Where(x => x.CommitSha == repoSha && x.Instance.Id == metadata.Id) + .Include(x => x.ActiveTestMerges) + .ThenInclude(x => x.TestMerge) + .ThenInclude(x => x.MergedBy) + .FirstOrDefaultAsync(cancellationToken) + .ConfigureAwait(false); + + if (revInfo == default) + { + revInfo = new Models.RevisionInformation + { + CommitSha = repoSha, + OriginCommitSha = repoSha, + Instance = new Models.Instance + { + Id = metadata.Id + } + }; + + logger.LogWarning(Repository.Repository.OriginTrackingErrorTemplate, repoSha); + databaseContext.Instances.Attach(revInfo.Instance); + } + + compileJob = await Compile( + revInfo, + dreamMakerSettings, + ddSettings.StartupTimeout.Value, + repo, + progressReporter, + averageSpan, + cancellationToken) + .ConfigureAwait(false); + + try + { + compileJob.Job = job; + + databaseContext.CompileJobs.Add(compileJob); + + await PostDeploymentComments(compileJob, repositorySettings, repoOwner, repoName).ConfigureAwait(false); + + // The difficulty with compile jobs is they have a two part commit + await databaseContext.Save(cancellationToken).ConfigureAwait(false); + try + { + await compileJobConsumer.LoadCompileJob(compileJob, cancellationToken).ConfigureAwait(false); + } + catch + { + // So we need to un-commit the compile job if the above throws + databaseContext.CompileJobs.Remove(compileJob); + await databaseContext.Save(default).ConfigureAwait(false); + throw; + } + } + catch (Exception ex) + { + await CleanupFailedCompile(compileJob, ex is OperationCanceledException, default).ConfigureAwait(false); + throw; + } + } + + await eventConsumer.HandleEvent(EventType.DeploymentComplete, null, cancellationToken).ConfigureAwait(false); + + await chatManager.SendUpdateMessage( + String.Format( + CultureInfo.InvariantCulture, + "Deployment complete! Changes will be applied when DreamDaemon {0}.", + watchdog.Running ? "reboots" : "is launched"), + cancellationToken) + .ConfigureAwait(false); + } + + /// + /// Calculate the average length of a deployment using a given . + /// + /// The to retrieve previous deployment s from. + /// The for the operation. + /// A resulting in the average of the 10 previous deployments or if there are none. + async Task CalculateExpectedDeploymentTime(IDatabaseContext databaseContext, CancellationToken cancellationToken) + { + var previousCompileJobs = await databaseContext.CompileJobs + .Where(x => x.Job.Instance.Id == metadata.Id) + .OrderByDescending(x => x.Job.StoppedAt) + .Take(10) + .Select(x => new Models.Job + { + StoppedAt = x.Job.StoppedAt, + StartedAt = x.Job.StartedAt + }) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + TimeSpan? averageSpan = null; + if (previousCompileJobs.Count != 0) + { + var totalSpan = TimeSpan.Zero; + foreach (var I in previousCompileJobs) + totalSpan += I.StoppedAt.Value - I.StartedAt.Value; + averageSpan = totalSpan / previousCompileJobs.Count; + } + + return averageSpan; + } + + async Task Compile(Models.RevisionInformation revisionInformation, Api.Models.DreamMaker dreamMakerSettings, uint apiValidateTimeout, IRepository repository, Action progressReporter, TimeSpan? estimatedDuration, CancellationToken cancellationToken) + { logger.LogTrace("Begin Compile"); lock (compilingLock) @@ -556,5 +745,99 @@ namespace Tgstation.Server.Host.Components.Deployment await progressTask.ConfigureAwait(false); } } + + /// + /// Post deployment GitHub comments. + /// + /// The deployed . + /// The . + /// The GitHub repostiory owner. + /// The GitHub repostiory name. + /// A representing the running operation. + async Task PostDeploymentComments( + Models.CompileJob compileJob, + Models.RepositorySettings repositorySettings, + string repoOwner, + string repoName) + { + if (repositorySettings?.AccessToken == null) + return; + + // potential for commenting on a test merge change + var outgoingCompileJob = compileJobConsumer.LatestCompileJob(); + + if ((outgoingCompileJob != null && outgoingCompileJob.RevisionInformation.CommitSha == compileJob.RevisionInformation.CommitSha) || !repositorySettings.PostTestMergeComment.Value) + return; + + outgoingCompileJob ??= new Models.CompileJob + { + RevisionInformation = new Models.RevisionInformation + { + ActiveTestMerges = new List() + } + }; + + var gitHubClient = gitHubClientFactory.CreateClient(repositorySettings.AccessToken); + + async Task CommentOnPR(int prNumber, string comment) + { + try + { + await gitHubClient.Issue.Comment.Create(repoOwner, repoName, prNumber, comment).ConfigureAwait(false); + } + catch (ApiException e) + { + logger.LogWarning("Error posting GitHub comment! Exception: {0}", e); + } + } + + var tasks = new List(); + + string FormatTestMerge(Models.TestMerge testMerge, bool updated) => String.Format(CultureInfo.InvariantCulture, "#### Test Merge {4}{0}{0}##### Server Instance{0}{5}{1}{0}{0}##### Revision{0}Origin: {6}{0}Pull Request: {2}{0}Server: {7}{3}", + Environment.NewLine, + repositorySettings.ShowTestMergeCommitters.Value ? String.Format(CultureInfo.InvariantCulture, "{0}{0}##### Merged By{0}{1}", Environment.NewLine, testMerge.MergedBy.Name) : String.Empty, + testMerge.PullRequestRevision, + testMerge.Comment != null ? String.Format(CultureInfo.InvariantCulture, "{0}{0}##### Comment{0}{1}", Environment.NewLine, testMerge.Comment) : String.Empty, + updated ? "Updated" : "Deployed", + metadata.Name, + compileJob.RevisionInformation.OriginCommitSha, + compileJob.RevisionInformation.CommitSha); + + // added prs + foreach (var I in compileJob + .RevisionInformation + .ActiveTestMerges + .Select(x => x.TestMerge) + .Where(x => !outgoingCompileJob + .RevisionInformation + .ActiveTestMerges + .Any(y => y.TestMerge.Number == x.Number))) + tasks.Add(CommentOnPR(I.Number, FormatTestMerge(I, false))); + + // removed prs + foreach (var I in outgoingCompileJob + .RevisionInformation + .ActiveTestMerges + .Select(x => x.TestMerge) + .Where(x => !compileJob + .RevisionInformation + .ActiveTestMerges + .Any(y => y.TestMerge.Number == x.Number))) + tasks.Add(CommentOnPR(I.Number, "#### Test Merge Removed")); + + // updated prs + foreach (var I in compileJob + .RevisionInformation + .ActiveTestMerges + .Select(x => x.TestMerge) + .Where(x => outgoingCompileJob + .RevisionInformation + .ActiveTestMerges + .Any(y => y.TestMerge.Number == x.Number))) + tasks.Add(CommentOnPR(I.Number, FormatTestMerge(I, true))); + + if (tasks.Any()) + await Task.WhenAll(tasks).ConfigureAwait(false); + } } } diff --git a/src/Tgstation.Server.Host/Components/Deployment/ICompileJobConsumer.cs b/src/Tgstation.Server.Host/Components/Deployment/ICompileJobSink.cs similarity index 79% rename from src/Tgstation.Server.Host/Components/Deployment/ICompileJobConsumer.cs rename to src/Tgstation.Server.Host/Components/Deployment/ICompileJobSink.cs index a6af72fd99..fb2d80b88f 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/ICompileJobConsumer.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/ICompileJobSink.cs @@ -1,6 +1,4 @@ -using Microsoft.Extensions.Hosting; -using System; -using System.Threading; +using System.Threading; using System.Threading.Tasks; using Tgstation.Server.Host.Models; @@ -9,10 +7,10 @@ namespace Tgstation.Server.Host.Components.Deployment /// /// Sink for s /// - public interface ICompileJobConsumer : IHostedService, IDisposable + public interface ICompileJobSink : ILatestCompileJobProvider { /// - /// Load a new into the + /// Load a new into the /// /// The to load /// The for the operation diff --git a/src/Tgstation.Server.Host/Components/Deployment/IDmbFactory.cs b/src/Tgstation.Server.Host/Components/Deployment/IDmbFactory.cs index 4f5059c2be..16e4cef3fb 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/IDmbFactory.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/IDmbFactory.cs @@ -1,4 +1,5 @@ -using System; +using Microsoft.Extensions.Hosting; +using System; using System.Threading; using System.Threading.Tasks; using Tgstation.Server.Host.Models; @@ -8,7 +9,7 @@ namespace Tgstation.Server.Host.Components.Deployment /// /// Factory for s /// - public interface IDmbFactory + public interface IDmbFactory : ILatestCompileJobProvider, IHostedService, IDisposable { /// /// Get a that completes when the result of a call to will be different than the previous call if any diff --git a/src/Tgstation.Server.Host/Components/Deployment/IDreamMaker.cs b/src/Tgstation.Server.Host/Components/Deployment/IDreamMaker.cs index 81f6a6d881..e23a5c224f 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/IDreamMaker.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/IDreamMaker.cs @@ -1,7 +1,8 @@ using System; using System.Threading; using System.Threading.Tasks; -using Tgstation.Server.Host.Components.Repository; +using Tgstation.Server.Host.Database; +using Tgstation.Server.Host.Models; namespace Tgstation.Server.Host.Components.Deployment { @@ -11,16 +12,17 @@ namespace Tgstation.Server.Host.Components.Deployment public interface IDreamMaker { /// - /// Starts a compile + /// Create and a compile job and insert it into the database. Meant to be called by a . /// - /// The being compiled from the - /// The for the compile - /// The time in seconds to wait while validating the API - /// The to copy from - /// The to report compilation progress - /// The estimated amount of time the compile will take - /// The for the operation - /// A resulting in the partially populated for the operation. In particular, note the field will only have it's field populated - Task Compile(Models.RevisionInformation revisionInformation, Api.Models.DreamMaker dreamMakerSettings, uint apiValidateTimeout, IRepository repository, Action progressReporter, TimeSpan? estimatedDuration, CancellationToken cancellationToken); + /// The running . + /// The for the operation. + /// The to report compilation progress. + /// The for the operation. + /// A representing the running operation. + Task DeploymentProcess( + Job job, + IDatabaseContext databaseContext, + Action progressReporter, + CancellationToken cancellationToken); } } \ No newline at end of file diff --git a/src/Tgstation.Server.Host/Components/Deployment/ILatestCompileJobProvider.cs b/src/Tgstation.Server.Host/Components/Deployment/ILatestCompileJobProvider.cs new file mode 100644 index 0000000000..38a90c8dc6 --- /dev/null +++ b/src/Tgstation.Server.Host/Components/Deployment/ILatestCompileJobProvider.cs @@ -0,0 +1,16 @@ +using Tgstation.Server.Host.Models; + +namespace Tgstation.Server.Host.Components.Deployment +{ + /// + /// Provides the most recently deployed . + /// + public interface ILatestCompileJobProvider + { + /// + /// Gets the latest . + /// + /// The latest . + CompileJob LatestCompileJob(); + } +} diff --git a/src/Tgstation.Server.Host/Components/IInstance.cs b/src/Tgstation.Server.Host/Components/IInstance.cs index 8f9afece75..ee975d13a8 100644 --- a/src/Tgstation.Server.Host/Components/IInstance.cs +++ b/src/Tgstation.Server.Host/Components/IInstance.cs @@ -1,21 +1,19 @@ using Microsoft.Extensions.Hosting; using System; -using System.Threading; using System.Threading.Tasks; using Tgstation.Server.Host.Components.Byond; using Tgstation.Server.Host.Components.Chat; +using Tgstation.Server.Host.Components.Deployment; using Tgstation.Server.Host.Components.Repository; using Tgstation.Server.Host.Components.StaticFiles; using Tgstation.Server.Host.Components.Watchdog; -using Tgstation.Server.Host.Database; -using Tgstation.Server.Host.Models; namespace Tgstation.Server.Host.Components { /// /// For interacting with the instance services /// - public interface IInstance : IHostedService, IDisposable + public interface IInstance : ILatestCompileJobProvider, IHostedService, IDisposable { /// /// The for the @@ -27,6 +25,11 @@ namespace Tgstation.Server.Host.Components /// IByondManager ByondManager { get; } + /// + /// The for the . + /// + IDreamMaker DreamMaker { get; } + /// /// The for the /// @@ -42,12 +45,6 @@ namespace Tgstation.Server.Host.Components /// IConfiguration Configuration { get; } - /// - /// The latest staged - /// - /// The latest if it exists - CompileJob LatestCompileJob(); - /// /// Rename the /// @@ -60,15 +57,5 @@ namespace Tgstation.Server.Host.Components /// The new auto update inteval /// A representing the running operation Task SetAutoUpdateInterval(uint newInterval); - - /// - /// Run the compile job and insert it into the database. Meant to be called by a - /// - /// The running - /// The for the operation - /// The to report compilation progress - /// The for the operation - /// A representing the running operation - Task CompileProcess(Job job, IDatabaseContext databaseContext, Action progressReporter, CancellationToken cancellationToken); } } \ No newline at end of file diff --git a/src/Tgstation.Server.Host/Components/Instance.cs b/src/Tgstation.Server.Host/Components/Instance.cs index 87814d36c6..38bc7fbbc2 100644 --- a/src/Tgstation.Server.Host/Components/Instance.cs +++ b/src/Tgstation.Server.Host/Components/Instance.cs @@ -1,9 +1,7 @@ using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; -using Octokit; using System; using System.Collections.Generic; -using System.Globalization; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -13,7 +11,6 @@ using Tgstation.Server.Host.Components.Chat; using Tgstation.Server.Host.Components.Deployment; using Tgstation.Server.Host.Components.Repository; using Tgstation.Server.Host.Components.Watchdog; -using Tgstation.Server.Host.Core; using Tgstation.Server.Host.Database; using Tgstation.Server.Host.Jobs; using Tgstation.Server.Host.Models; @@ -39,15 +36,8 @@ namespace Tgstation.Server.Host.Components /// public StaticFiles.IConfiguration Configuration { get; } - /// - /// The for the - /// - readonly IDreamMaker dreamMaker; - - /// - /// The for the - /// - readonly ICompileJobConsumer compileJobConsumer; + /// + public IDreamMaker DreamMaker { get; } /// /// The for the @@ -69,11 +59,6 @@ namespace Tgstation.Server.Host.Components /// readonly IEventConsumer eventConsumer; - /// - /// The for the - /// - readonly IGitHubClientFactory gitHubClientFactory; - /// /// The for the /// @@ -105,16 +90,14 @@ namespace Tgstation.Server.Host.Components /// The value of /// The value of /// The value of - /// The value of + /// The value of /// The value of /// The value of /// The value of - /// The value of /// The value of /// The value of /// The value of /// The value of - /// The value of /// The value of public Instance( Api.Models.Instance metadata, @@ -125,27 +108,23 @@ namespace Tgstation.Server.Host.Components IChatManager chat, StaticFiles.IConfiguration configuration, - ICompileJobConsumer compileJobConsumer, IDatabaseContextFactory databaseContextFactory, IDmbFactory dmbFactory, IJobManager jobManager, IEventConsumer eventConsumer, - IGitHubClientFactory gitHubClientFactory, ILogger logger) { this.metadata = metadata ?? throw new ArgumentNullException(nameof(metadata)); RepositoryManager = repositoryManager ?? throw new ArgumentNullException(nameof(repositoryManager)); ByondManager = byondManager ?? throw new ArgumentNullException(nameof(byondManager)); - this.dreamMaker = dreamMaker ?? throw new ArgumentNullException(nameof(dreamMaker)); + DreamMaker = dreamMaker ?? throw new ArgumentNullException(nameof(dreamMaker)); Watchdog = watchdog ?? throw new ArgumentNullException(nameof(watchdog)); Chat = chat ?? throw new ArgumentNullException(nameof(chat)); Configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); - this.compileJobConsumer = compileJobConsumer ?? throw new ArgumentNullException(nameof(compileJobConsumer)); this.databaseContextFactory = databaseContextFactory ?? throw new ArgumentNullException(nameof(databaseContextFactory)); this.dmbFactory = dmbFactory ?? throw new ArgumentNullException(nameof(dmbFactory)); this.jobManager = jobManager ?? throw new ArgumentNullException(nameof(jobManager)); this.eventConsumer = eventConsumer ?? throw new ArgumentNullException(nameof(eventConsumer)); - this.gitHubClientFactory = gitHubClientFactory ?? throw new ArgumentNullException(nameof(gitHubClientFactory)); this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); timerLock = new object(); @@ -155,227 +134,13 @@ namespace Tgstation.Server.Host.Components public void Dispose() { timerCts?.Dispose(); - compileJobConsumer.Dispose(); Configuration.Dispose(); Chat.Dispose(); Watchdog.Dispose(); + dmbFactory.Dispose(); RepositoryManager.Dispose(); } - /// - public async Task CompileProcess(Job job, IDatabaseContext databaseContext, Action progressReporter, CancellationToken cancellationToken) - { -#pragma warning disable IDE0016 // Use 'throw' expression - if (job == null) - throw new ArgumentNullException(nameof(job)); -#pragma warning restore IDE0016 // Use 'throw' expression - if (databaseContext == null) - throw new ArgumentNullException(nameof(databaseContext)); - if (progressReporter == null) - throw new ArgumentNullException(nameof(progressReporter)); - - var ddSettings = await databaseContext.DreamDaemonSettings.Where(x => x.InstanceId == metadata.Id).Select(x => new DreamDaemonSettings - { - StartupTimeout = x.StartupTimeout, - }) - .FirstOrDefaultAsync(cancellationToken) - .ConfigureAwait(false); - if (ddSettings == default) - throw new JobException(Api.Models.ErrorCode.InstanceMissingDreamDaemonSettings); - - var previousCompileJobs = await databaseContext.CompileJobs - .Where(x => x.Job.Instance.Id == metadata.Id) - .OrderByDescending(x => x.Job.StoppedAt) - .Select(x => new Job - { - StoppedAt = x.Job.StoppedAt, - StartedAt = x.Job.StartedAt - }) - .Take(10) - .ToListAsync(cancellationToken) - .ConfigureAwait(false); - - var dreamMakerSettings = await databaseContext.DreamMakerSettings.Where(x => x.InstanceId == metadata.Id).FirstAsync(cancellationToken).ConfigureAwait(false); - if (dreamMakerSettings == default) - throw new JobException(Api.Models.ErrorCode.InstanceMissingDreamMakerSettings); - - RepositorySettings repositorySettings = null; - string repoOwner = null; - string repoName = null; - CompileJob compileJob; - RevisionInformation revInfo; - using (var repo = await RepositoryManager.LoadRepository(cancellationToken).ConfigureAwait(false)) - { - if (repo == null) - throw new JobException(Api.Models.ErrorCode.RepoMissing); - - if (repo.IsGitHubRepository) - { - repoOwner = repo.GitHubOwner; - repoName = repo.GitHubRepoName; - repositorySettings = await databaseContext - .RepositorySettings - .Where(x => x.InstanceId == metadata.Id) - .Select(x => new RepositorySettings - { - AccessToken = x.AccessToken, - ShowTestMergeCommitters = x.ShowTestMergeCommitters, - PushTestMergeCommits = x.PushTestMergeCommits, - PostTestMergeComment = x.PostTestMergeComment - }) - .FirstOrDefaultAsync(cancellationToken) - .ConfigureAwait(false); - if (repositorySettings == default) - throw new JobException(Api.Models.ErrorCode.InstanceMissingRepositorySettings); - } - - var repoSha = repo.Head; - revInfo = await databaseContext.RevisionInformations.Where(x => x.CommitSha == repoSha && x.Instance.Id == metadata.Id).Include(x => x.ActiveTestMerges).ThenInclude(x => x.TestMerge).ThenInclude(x => x.MergedBy).FirstOrDefaultAsync().ConfigureAwait(false); - - if (revInfo == default) - { - revInfo = new RevisionInformation - { - CommitSha = repoSha, - OriginCommitSha = repoSha, - Instance = new Models.Instance - { - Id = metadata.Id - } - }; - logger.LogWarning(Repository.Repository.OriginTrackingErrorTemplate, repoSha); - databaseContext.Instances.Attach(revInfo.Instance); - } - - TimeSpan? averageSpan = null; - if (previousCompileJobs.Count != 0) - { - var totalSpan = TimeSpan.Zero; - foreach (var I in previousCompileJobs) - totalSpan += I.StoppedAt.Value - I.StartedAt.Value; - averageSpan = totalSpan / previousCompileJobs.Count; - } - - compileJob = await dreamMaker.Compile(revInfo, dreamMakerSettings, ddSettings.StartupTimeout.Value, repo, progressReporter, averageSpan, cancellationToken).ConfigureAwait(false); - } - - compileJob.Job = job; - - databaseContext.CompileJobs.Add(compileJob); - - await PostDeploymentComments(compileJob, repositorySettings, repoOwner, repoName).ConfigureAwait(false); - - // The difficulty with compile jobs is they have a two part commit - await databaseContext.Save(cancellationToken).ConfigureAwait(false); - try - { - await compileJobConsumer.LoadCompileJob(compileJob, cancellationToken).ConfigureAwait(false); - } - catch - { - // So we need to un-commit the compile job if the above throws - databaseContext.CompileJobs.Remove(compileJob); - await databaseContext.Save(default).ConfigureAwait(false); - throw; - } - - await eventConsumer.HandleEvent(EventType.DeploymentComplete, null, cancellationToken).ConfigureAwait(false); - } - - /// - /// Post deployment GitHub comments. - /// - /// The deployed . - /// The . - /// The GitHub repostiory owner. - /// The GitHub repostiory name. - /// A representing the running operation. - async Task PostDeploymentComments( - CompileJob compileJob, - RepositorySettings repositorySettings, - string repoOwner, - string repoName) - { - if (repositorySettings?.AccessToken == null) - return; - - // potential for commenting on a test merge change - var outgoingCompileJob = LatestCompileJob(); - - if ((outgoingCompileJob != null && outgoingCompileJob.RevisionInformation.CommitSha == compileJob.RevisionInformation.CommitSha) || !repositorySettings.PostTestMergeComment.Value) - return; - - outgoingCompileJob ??= new CompileJob - { - RevisionInformation = new RevisionInformation - { - ActiveTestMerges = new List() - } - }; - - var gitHubClient = gitHubClientFactory.CreateClient(repositorySettings.AccessToken); - - async Task CommentOnPR(int prNumber, string comment) - { - try - { - await gitHubClient.Issue.Comment.Create(repoOwner, repoName, prNumber, comment).ConfigureAwait(false); - } - catch (ApiException e) - { - logger.LogWarning("Error posting GitHub comment! Exception: {0}", e); - } - } - - var tasks = new List(); - - string FormatTestMerge(TestMerge testMerge, bool updated) => String.Format(CultureInfo.InvariantCulture, "#### Test Merge {4}{0}{0}##### Server Instance{0}{5}{1}{0}{0}##### Revision{0}Origin: {6}{0}Pull Request: {2}{0}Server: {7}{3}", - Environment.NewLine, - repositorySettings.ShowTestMergeCommitters.Value ? String.Format(CultureInfo.InvariantCulture, "{0}{0}##### Merged By{0}{1}", Environment.NewLine, testMerge.MergedBy.Name) : String.Empty, - testMerge.PullRequestRevision, - testMerge.Comment != null ? String.Format(CultureInfo.InvariantCulture, "{0}{0}##### Comment{0}{1}", Environment.NewLine, testMerge.Comment) : String.Empty, - updated ? "Updated" : "Deployed", - metadata.Name, - compileJob.RevisionInformation.OriginCommitSha, - compileJob.RevisionInformation.CommitSha); - - // added prs - foreach (var I in compileJob - .RevisionInformation - .ActiveTestMerges - .Select(x => x.TestMerge) - .Where(x => !outgoingCompileJob - .RevisionInformation - .ActiveTestMerges - .Any(y => y.TestMerge.Number == x.Number))) - tasks.Add(CommentOnPR(I.Number, FormatTestMerge(I, false))); - - // removed prs - foreach (var I in outgoingCompileJob - .RevisionInformation - .ActiveTestMerges - .Select(x => x.TestMerge) - .Where(x => !compileJob - .RevisionInformation - .ActiveTestMerges - .Any(y => y.TestMerge.Number == x.Number))) - tasks.Add(CommentOnPR(I.Number, "#### Test Merge Removed")); - - // updated prs - foreach (var I in compileJob - .RevisionInformation - .ActiveTestMerges - .Select(x => x.TestMerge) - .Where(x => outgoingCompileJob - .RevisionInformation - .ActiveTestMerges - .Any(y => y.TestMerge.Number == x.Number))) - tasks.Add(CommentOnPR(I.Number, FormatTestMerge(I, true))); - - if (tasks.Any()) - await Task.WhenAll(tasks).ConfigureAwait(false); - } - /// /// Pull the repository and compile for every set of given /// @@ -393,11 +158,11 @@ namespace Tgstation.Server.Host.Components await eventConsumer.HandleEvent(EventType.InstanceAutoUpdateStart, new List(), cancellationToken).ConfigureAwait(false); try { - Models.User user = null; + User user = null; await databaseContextFactory.UseContext( async (db) => user = await db .Users - .Where(x => x.CanonicalName == Models.User.CanonicalizeName(Api.Models.User.AdminName)) + .Where(x => x.CanonicalName == User.CanonicalizeName(Api.Models.User.AdminName)) .FirstAsync(cancellationToken) .ConfigureAwait(false)) .ConfigureAwait(false); @@ -582,7 +347,10 @@ namespace Tgstation.Server.Host.Components CancelRight = (ulong)DreamMakerRights.CancelCompile }; - await jobManager.RegisterOperation(compileProcessJob, CompileProcess, cancellationToken).ConfigureAwait(false); + await jobManager.RegisterOperation( + compileProcessJob, + DreamMaker.DeploymentProcess, + cancellationToken).ConfigureAwait(false); await jobManager.WaitForJobCompletion(compileProcessJob, user, cancellationToken, default).ConfigureAwait(false); } @@ -617,7 +385,13 @@ namespace Tgstation.Server.Host.Components /// public async Task StartAsync(CancellationToken cancellationToken) { - await Task.WhenAll(SetAutoUpdateInterval(metadata.AutoUpdateInterval.Value), Configuration.StartAsync(cancellationToken), ByondManager.StartAsync(cancellationToken), Chat.StartAsync(cancellationToken), compileJobConsumer.StartAsync(cancellationToken)).ConfigureAwait(false); + await Task.WhenAll( + SetAutoUpdateInterval(metadata.AutoUpdateInterval.Value), + Configuration.StartAsync(cancellationToken), + ByondManager.StartAsync(cancellationToken), + Chat.StartAsync(cancellationToken), + dmbFactory.StartAsync(cancellationToken)) + .ConfigureAwait(false); // dependent on so many things, its just safer this way await Watchdog.StartAsync(cancellationToken).ConfigureAwait(false); @@ -634,7 +408,7 @@ namespace Tgstation.Server.Host.Components Configuration.StopAsync(cancellationToken), ByondManager.StopAsync(cancellationToken), Chat.StopAsync(cancellationToken), - compileJobConsumer.StopAsync(cancellationToken)) + dmbFactory.StopAsync(cancellationToken)) .ConfigureAwait(false); } @@ -668,11 +442,6 @@ namespace Tgstation.Server.Host.Components } /// - public CompileJob LatestCompileJob() - { - if (!dmbFactory.DmbAvailable) - return null; - return dmbFactory.LockNextDmb(0)?.CompileJob; - } + public CompileJob LatestCompileJob() => dmbFactory.LatestCompileJob(); } } diff --git a/src/Tgstation.Server.Host/Components/InstanceFactory.cs b/src/Tgstation.Server.Host/Components/InstanceFactory.cs index 7b841e1270..c1ec4fa817 100644 --- a/src/Tgstation.Server.Host/Components/InstanceFactory.cs +++ b/src/Tgstation.Server.Host/Components/InstanceFactory.cs @@ -254,9 +254,37 @@ namespace Tgstation.Server.Host.Components commandFactory.SetWatchdog(watchdog); try { - var dreamMaker = new DreamMaker(byond, gameIoManager, configuration, sessionControllerFactory, eventConsumer, chatManager, processExecutor, watchdog, loggerFactory.CreateLogger()); + Instance instance = null; + var dreamMaker = new DreamMaker( + byond, + gameIoManager, + configuration, + sessionControllerFactory, + eventConsumer, + chatManager, + processExecutor, + watchdog, + gitHubClientFactory, + dmbFactory, + repoManager, + loggerFactory.CreateLogger(), + metadata.CloneMetadata()); - return new Instance(metadata.CloneMetadata(), repoManager, byond, dreamMaker, watchdog, chatManager, configuration, dmbFactory, databaseContextFactory, dmbFactory, jobManager, eventConsumer, gitHubClientFactory, loggerFactory.CreateLogger()); + instance = new Instance( + metadata.CloneMetadata(), + repoManager, + byond, + dreamMaker, + watchdog, + chatManager, + configuration, + databaseContextFactory, + dmbFactory, + jobManager, + eventConsumer, + loggerFactory.CreateLogger()); + + return instance; } catch { diff --git a/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs b/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs index 2c4b619b56..3f011a9cea 100644 --- a/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs +++ b/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs @@ -127,7 +127,13 @@ namespace Tgstation.Server.Host.Controllers CancelRight = (ulong)DreamMakerRights.CancelCompile, Instance = Instance }; - await jobManager.RegisterOperation(job, instanceManager.GetInstance(Instance).CompileProcess, cancellationToken).ConfigureAwait(false); + + IInstance instance = instanceManager.GetInstance(Instance); + await jobManager.RegisterOperation( + job, + instance.DreamMaker.DeploymentProcess, + cancellationToken) + .ConfigureAwait(false); return Accepted(job.ToApi()); } diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index 7cee5f48af..9177220c50 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -115,9 +115,6 @@ - - PreserveNewest - PreserveNewest