From b9f5a58006fa31abd1cc5e8d1ee2b22620cc7c02 Mon Sep 17 00:00:00 2001 From: Cyberboss Date: Thu, 27 Sep 2018 16:55:11 -0400 Subject: [PATCH] Implement compiler progress heuristics --- .../Components/Compiler/DreamMaker.cs | 251 ++++++++++-------- .../Components/Compiler/IDreamMaker.cs | 8 +- .../Components/Instance.cs | 20 +- 3 files changed, 165 insertions(+), 114 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Compiler/DreamMaker.cs b/src/Tgstation.Server.Host/Components/Compiler/DreamMaker.cs index ee15c5d0e0..b4f885d4d1 100644 --- a/src/Tgstation.Server.Host/Components/Compiler/DreamMaker.cs +++ b/src/Tgstation.Server.Host/Components/Compiler/DreamMaker.cs @@ -267,7 +267,7 @@ namespace Tgstation.Server.Host.Components.Compiler } /// - public async Task Compile(Models.RevisionInformation revisionInformation, Api.Models.DreamMaker dreamMakerSettings, uint apiValidateTimeout, IRepository repository, CancellationToken cancellationToken) + public async Task Compile(Models.RevisionInformation revisionInformation, Api.Models.DreamMaker dreamMakerSettings, uint apiValidateTimeout, IRepository repository, Action progressReporter, TimeSpan? estimatedDuration, CancellationToken cancellationToken) { if (revisionInformation == null) throw new ArgumentNullException(nameof(revisionInformation)); @@ -278,6 +278,9 @@ namespace Tgstation.Server.Host.Components.Compiler if (repository == null) throw new ArgumentNullException(nameof(repository)); + if (progressReporter == null) + throw new ArgumentNullException(nameof(progressReporter)); + if (dreamMakerSettings.ApiValidationSecurityLevel == DreamDaemonSecurity.Ultrasafe) throw new ArgumentOutOfRangeException(nameof(dreamMakerSettings), dreamMakerSettings, "Cannot compile with ultrasafe security!"); @@ -299,137 +302,165 @@ namespace Tgstation.Server.Host.Components.Compiler compiling = true; } - try + using (var progressCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken)) { - var commitInsert = revisionInformation.CommitSha.Substring(0, 7); - string remoteCommitInsert; - if (revisionInformation.CommitSha == revisionInformation.OriginCommitSha) + async Task ProgressTask() { - commitInsert = String.Format(CultureInfo.InvariantCulture, "^{0}", commitInsert); - remoteCommitInsert = String.Empty; - } - else - remoteCommitInsert = String.Format(CultureInfo.InvariantCulture, ". Remote commit: ^{0}", revisionInformation.OriginCommitSha.Substring(0, 7)); + if (!estimatedDuration.HasValue) + return; - var testmergeInsert = revisionInformation.ActiveTestMerges.Count == 0 ? String.Empty : String.Format(CultureInfo.InvariantCulture, " (Test Merges: {0})", - String.Join(", ", revisionInformation.ActiveTestMerges.Select(x => x.TestMerge).Select(x => - { - var result = String.Format(CultureInfo.InvariantCulture, "#{0} at {1}", x.Number, x.PullRequestRevision.Substring(0, 7)); - if (x.Comment != null) - result += String.Format(CultureInfo.InvariantCulture, " ({0})", x.Comment); - return result; - }))); - - using (var byondLock = await byond.UseExecutables(null, cancellationToken).ConfigureAwait(false)) - { - await chat.SendUpdateMessage(String.Format(CultureInfo.InvariantCulture, "Deploying revision: {0}{1}{2} BYOND Version: {3}", commitInsert, testmergeInsert, remoteCommitInsert, byondLock.Version), cancellationToken).ConfigureAwait(false); - - async Task CleanupFailedCompile(bool cancelled) - { - logger.LogTrace("Cleaning compile directory..."); - var chatTask = chat.SendUpdateMessage(cancelled ? "Deploy cancelled!" : "Deploy failed!", cancellationToken); - try - { - await ioManager.DeleteDirectory(job.DirectoryName.ToString(), CancellationToken.None).ConfigureAwait(false); - } - catch (Exception e) - { - logger.LogWarning("Error cleaning up compile directory {0}! Exception: {1}", ioManager.ResolvePath(job.DirectoryName.ToString()), e); - } - - await chatTask.ConfigureAwait(false); - }; + progressReporter(0); + var ct = progressCts.Token; + var sleepInterval = estimatedDuration.Value / 100; try { - await ioManager.CreateDirectory(job.DirectoryName.ToString(), cancellationToken).ConfigureAwait(false); - - var dirA = ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName); - var dirB = ioManager.ConcatPath(job.DirectoryName.ToString(), BDirectoryName); - - logger.LogTrace("Copying repository to game directory..."); - //copy the repository - var fullDirA = ioManager.ResolvePath(dirA); - var repoOrigin = repository.Origin; - using (repository) - await repository.CopyTo(fullDirA, cancellationToken).ConfigureAwait(false); - - //run precompile scripts - var resolvedGameDirectory = ioManager.ResolvePath(ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName)); - await eventConsumer.HandleEvent(EventType.CompileStart, new List { resolvedGameDirectory, repoOrigin }, cancellationToken).ConfigureAwait(false); - - //determine the dme - if (job.DmeName == null) + for (var I = 0; I < 99; ++I) { - logger.LogTrace("Searching for available .dmes..."); - var path = (await ioManager.GetFilesWithExtension(dirA, DmeExtension, cancellationToken).ConfigureAwait(false)).FirstOrDefault(); - if (path == default) - throw new JobException("Unable to find any .dme!"); - var dmeWithExtension = ioManager.GetFileName(path); - job.DmeName = dmeWithExtension.Substring(0, dmeWithExtension.Length - DmeExtension.Length - 1); + await Task.Delay(sleepInterval, cancellationToken).ConfigureAwait(false); + progressReporter(I + 1); } - else if (!await ioManager.FileExists(ioManager.ConcatPath(dirA, String.Join('.', job.DmeName, DmeExtension)), cancellationToken).ConfigureAwait(false)) - throw new JobException("Unable to locate specified .dme!"); + } + catch (OperationCanceledException) { } + } - logger.LogDebug("Selected {0}.dme for compilation!", job.DmeName); + var progressTask = ProgressTask(); + try + { - await ModifyDme(job, cancellationToken).ConfigureAwait(false); - //run compiler, verify api - job.ByondVersion = byondLock.Version.ToString(); + var commitInsert = revisionInformation.CommitSha.Substring(0, 7); + string remoteCommitInsert; + if (revisionInformation.CommitSha == revisionInformation.OriginCommitSha) + { + commitInsert = String.Format(CultureInfo.InvariantCulture, "^{0}", commitInsert); + remoteCommitInsert = String.Empty; + } + else + remoteCommitInsert = String.Format(CultureInfo.InvariantCulture, ". Remote commit: ^{0}", revisionInformation.OriginCommitSha.Substring(0, 7)); - var exitCode = await RunDreamMaker(byondLock.DreamMakerPath, job, cancellationToken).ConfigureAwait(false); + var testmergeInsert = revisionInformation.ActiveTestMerges.Count == 0 ? String.Empty : String.Format(CultureInfo.InvariantCulture, " (Test Merges: {0})", + String.Join(", ", revisionInformation.ActiveTestMerges.Select(x => x.TestMerge).Select(x => + { + var result = String.Format(CultureInfo.InvariantCulture, "#{0} at {1}", x.Number, x.PullRequestRevision.Substring(0, 7)); + if (x.Comment != null) + result += String.Format(CultureInfo.InvariantCulture, " ({0})", x.Comment); + return result; + }))); + + using (var byondLock = await byond.UseExecutables(null, cancellationToken).ConfigureAwait(false)) + { + await chat.SendUpdateMessage(String.Format(CultureInfo.InvariantCulture, "Deploying revision: {0}{1}{2} BYOND Version: {3}", commitInsert, testmergeInsert, remoteCommitInsert, byondLock.Version), cancellationToken).ConfigureAwait(false); + + async Task CleanupFailedCompile(bool cancelled) + { + logger.LogTrace("Cleaning compile directory..."); + var chatTask = chat.SendUpdateMessage(cancelled ? "Deploy cancelled!" : "Deploy failed!", cancellationToken); + try + { + await ioManager.DeleteDirectory(job.DirectoryName.ToString(), CancellationToken.None).ConfigureAwait(false); + } + catch (Exception e) + { + logger.LogWarning("Error cleaning up compile directory {0}! Exception: {1}", ioManager.ResolvePath(job.DirectoryName.ToString()), e); + } + + await chatTask.ConfigureAwait(false); + }; try { - if (exitCode != 0) - throw new JobException(String.Format(CultureInfo.InvariantCulture, "DM exited with a non-zero code: {0}{1}{2}", exitCode, Environment.NewLine, job.Output)); + await ioManager.CreateDirectory(job.DirectoryName.ToString(), cancellationToken).ConfigureAwait(false); - await VerifyApi(apiValidateTimeout, dreamMakerSettings.ApiValidationSecurityLevel.Value, job, byondLock, dreamMakerSettings.ApiValidationPort.Value, cancellationToken).ConfigureAwait(false); + var dirA = ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName); + var dirB = ioManager.ConcatPath(job.DirectoryName.ToString(), BDirectoryName); + + logger.LogTrace("Copying repository to game directory..."); + //copy the repository + var fullDirA = ioManager.ResolvePath(dirA); + var repoOrigin = repository.Origin; + using (repository) + await repository.CopyTo(fullDirA, cancellationToken).ConfigureAwait(false); + + //run precompile scripts + var resolvedGameDirectory = ioManager.ResolvePath(ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName)); + await eventConsumer.HandleEvent(EventType.CompileStart, new List { resolvedGameDirectory, repoOrigin }, cancellationToken).ConfigureAwait(false); + + //determine the dme + if (job.DmeName == null) + { + logger.LogTrace("Searching for available .dmes..."); + var path = (await ioManager.GetFilesWithExtension(dirA, DmeExtension, cancellationToken).ConfigureAwait(false)).FirstOrDefault(); + if (path == default) + throw new JobException("Unable to find any .dme!"); + var dmeWithExtension = ioManager.GetFileName(path); + job.DmeName = dmeWithExtension.Substring(0, dmeWithExtension.Length - DmeExtension.Length - 1); + } + else if (!await ioManager.FileExists(ioManager.ConcatPath(dirA, String.Join('.', job.DmeName, DmeExtension)), cancellationToken).ConfigureAwait(false)) + throw new JobException("Unable to locate specified .dme!"); + + logger.LogDebug("Selected {0}.dme for compilation!", job.DmeName); + + await ModifyDme(job, cancellationToken).ConfigureAwait(false); + + //run compiler, verify api + job.ByondVersion = byondLock.Version.ToString(); + + var exitCode = await RunDreamMaker(byondLock.DreamMakerPath, job, cancellationToken).ConfigureAwait(false); + + try + { + if (exitCode != 0) + throw new JobException(String.Format(CultureInfo.InvariantCulture, "DM exited with a non-zero code: {0}{1}{2}", exitCode, Environment.NewLine, job.Output)); + + await VerifyApi(apiValidateTimeout, dreamMakerSettings.ApiValidationSecurityLevel.Value, job, byondLock, dreamMakerSettings.ApiValidationPort.Value, cancellationToken).ConfigureAwait(false); + } + catch (JobException) + { + //server never validated or compile failed + await eventConsumer.HandleEvent(EventType.CompileFailure, new List { resolvedGameDirectory, exitCode == 0 ? "1" : "0" }, cancellationToken).ConfigureAwait(false); + throw; + } + + logger.LogTrace("Running post compile event..."); + await eventConsumer.HandleEvent(EventType.CompileComplete, new List { ioManager.ResolvePath(ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName)) }, cancellationToken).ConfigureAwait(false); + + logger.LogTrace("Duplicating compiled game..."); + + //duplicate the dmb et al + await ioManager.CopyDirectory(dirA, dirB, null, cancellationToken).ConfigureAwait(false); + + logger.LogTrace("Applying static game file symlinks..."); + + //symlink in the static data + var symATask = configuration.SymlinkStaticFilesTo(fullDirA, cancellationToken); + var symBTask = configuration.SymlinkStaticFilesTo(ioManager.ResolvePath(dirB), cancellationToken); + + await Task.WhenAll(symATask, symBTask).ConfigureAwait(false); + + await chat.SendUpdateMessage(String.Format(CultureInfo.InvariantCulture, "Deployment complete!{0}", watchdog.Running ? " Changes will be applied on next server reboot." : String.Empty), cancellationToken).ConfigureAwait(false); + + logger.LogDebug("Compile complete!"); + return job; } - catch (JobException) + catch (Exception e) { - //server never validated or compile failed - await eventConsumer.HandleEvent(EventType.CompileFailure, new List { resolvedGameDirectory, exitCode == 0 ? "1" : "0" }, cancellationToken).ConfigureAwait(false); + await CleanupFailedCompile(e is OperationCanceledException).ConfigureAwait(false); throw; } - - logger.LogTrace("Running post compile event..."); - await eventConsumer.HandleEvent(EventType.CompileComplete, new List { ioManager.ResolvePath(ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName)) }, cancellationToken).ConfigureAwait(false); - - logger.LogTrace("Duplicating compiled game..."); - - //duplicate the dmb et al - await ioManager.CopyDirectory(dirA, dirB, null, cancellationToken).ConfigureAwait(false); - - logger.LogTrace("Applying static game file symlinks..."); - - //symlink in the static data - var symATask = configuration.SymlinkStaticFilesTo(fullDirA, cancellationToken); - var symBTask = configuration.SymlinkStaticFilesTo(ioManager.ResolvePath(dirB), cancellationToken); - - await Task.WhenAll(symATask, symBTask).ConfigureAwait(false); - - await chat.SendUpdateMessage(String.Format(CultureInfo.InvariantCulture, "Deployment complete!{0}", watchdog.Running ? " Changes will be applied on next server reboot." : String.Empty), cancellationToken).ConfigureAwait(false); - - logger.LogDebug("Compile complete!"); - return job; - } - catch (Exception e) - { - await CleanupFailedCompile(e is OperationCanceledException).ConfigureAwait(false); - throw; } } - } - catch (OperationCanceledException) - { - await eventConsumer.HandleEvent(EventType.CompileCancelled, null, default).ConfigureAwait(false); - throw; - } - finally - { - compiling = false; + catch (OperationCanceledException) + { + await eventConsumer.HandleEvent(EventType.CompileCancelled, null, default).ConfigureAwait(false); + throw; + } + finally + { + compiling = false; + progressCts.Cancel(); + await progressTask.ConfigureAwait(false); + } } } } diff --git a/src/Tgstation.Server.Host/Components/Compiler/IDreamMaker.cs b/src/Tgstation.Server.Host/Components/Compiler/IDreamMaker.cs index 83f83849fa..b0a489ae84 100644 --- a/src/Tgstation.Server.Host/Components/Compiler/IDreamMaker.cs +++ b/src/Tgstation.Server.Host/Components/Compiler/IDreamMaker.cs @@ -1,6 +1,6 @@ -using System.Threading; +using System; +using System.Threading; using System.Threading.Tasks; -using Tgstation.Server.Api.Models; using Tgstation.Server.Host.Components.Repository; namespace Tgstation.Server.Host.Components.Compiler @@ -17,8 +17,10 @@ namespace Tgstation.Server.Host.Components.Compiler /// The for the compile /// The time in seconds to wait while validating the API /// The to copy from + /// The to report compilation progress + /// The estimated amount of time the compile will take /// The for the operation /// A resulting in the partially populated for the operation. In particular, note the field will only have it's field populated - Task Compile(Models.RevisionInformation revisionInformation, Api.Models.DreamMaker dreamMakerSettings, uint apiValidateTimeout, IRepository repository, CancellationToken cancellationToken); + Task Compile(Models.RevisionInformation revisionInformation, Api.Models.DreamMaker dreamMakerSettings, uint apiValidateTimeout, IRepository repository, Action progressReporter, TimeSpan? estimatedDuration, CancellationToken cancellationToken); } } \ No newline at end of file diff --git a/src/Tgstation.Server.Host/Components/Instance.cs b/src/Tgstation.Server.Host/Components/Instance.cs index 50077a86db..b7d20f9566 100644 --- a/src/Tgstation.Server.Host/Components/Instance.cs +++ b/src/Tgstation.Server.Host/Components/Instance.cs @@ -132,6 +132,14 @@ namespace Tgstation.Server.Host.Components StartupTimeout = x.StartupTimeout, }).FirstOrDefaultAsync(cancellationToken); + var compileJobsTask = databaseContext.CompileJobs + .Where(x => x.Job.Instance.Id == metadata.Id) + .OrderByDescending(x => x.Job.StoppedAt) + .Include(x => x.Job) + .Select(x => x.Job.StoppedAt.Value - x.Job.StartedAt.Value) + .Take(10) + .ToListAsync(cancellationToken); + var dreamMakerSettings = await databaseContext.DreamMakerSettings.Where(x => x.InstanceId == metadata.Id).FirstAsync(cancellationToken).ConfigureAwait(false); if (dreamMakerSettings == default) throw new JobException("Missing DreamMakerSettings in DB!"); @@ -163,8 +171,18 @@ namespace Tgstation.Server.Host.Components logger.LogWarning(Repository.Repository.OriginTrackingErrorTemplate, repoSha); databaseContext.Instances.Attach(revInfo.Instance); } + + TimeSpan? averageSpan = null; + var previousCompileJobs = await compileJobsTask.ConfigureAwait(false); + if(previousCompileJobs.Count != 0) + { + var totalSpan = TimeSpan.Zero; + foreach (var I in previousCompileJobs) + totalSpan += I; + averageSpan = totalSpan / previousCompileJobs.Count; + } - compileJob = await DreamMaker.Compile(revInfo, dreamMakerSettings, ddSettings.StartupTimeout.Value, repo, cancellationToken).ConfigureAwait(false); + compileJob = await DreamMaker.Compile(revInfo, dreamMakerSettings, ddSettings.StartupTimeout.Value, repo, progressReporter, averageSpan, cancellationToken).ConfigureAwait(false); } compileJob.Job = job;