diff --git a/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs b/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs index 94e01c8af5..dd4caa7835 100644 --- a/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs +++ b/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs @@ -620,13 +620,61 @@ namespace Tgstation.Server.Host.Components.Chat } /// - public Task SendUpdateMessage(string message, CancellationToken cancellationToken) + public async Task> SendDeploymentMessage( + Models.RevisionInformation revisionInformation, + Version byondVersion, + DateTimeOffset? estimatedCompletionTime, + string gitHubOwner, + string gitHubRepo, + bool localCommitPushed, + CancellationToken cancellationToken) { List wdChannels; - message = String.Format(CultureInfo.InvariantCulture, "DM: {0}", message); lock (mappedChannels) // so it doesn't change while we're using it wdChannels = mappedChannels.Where(x => x.Value.IsUpdatesChannel).Select(x => x.Key).ToList(); - return SendMessage(message, wdChannels, cancellationToken); + + logger.LogTrace("Sending deployment message for RevisionInformation: {0}", revisionInformation.Id); + + var callbacks = new List>(); + + await Task.WhenAll( + wdChannels.Select( + async x => + { + ChannelMapping channelMapping; + lock (mappedChannels) + if (!mappedChannels.TryGetValue(x, out channelMapping)) + return; + IProvider provider; + lock (providers) + if (!providers.TryGetValue(channelMapping.ProviderId, out provider)) + return; + try + { + var callback = await provider.SendUpdateMessage( + revisionInformation, + byondVersion, + estimatedCompletionTime, + gitHubOwner, + gitHubRepo, + channelMapping.ProviderChannelId, + localCommitPushed, + cancellationToken) + .ConfigureAwait(false); + + callbacks.Add(callback); + } + catch (Exception ex) + { + logger.LogWarning( + "Error sending deploy message to provider {0}! Exception: {1}", + channelMapping.ProviderId, + ex); + } + })) + .ConfigureAwait(false); + + return (errorMessage, dreamMakerOutput) => Task.WhenAll(callbacks.Select(x => x(errorMessage, dreamMakerOutput))); } /// diff --git a/src/Tgstation.Server.Host/Components/Chat/IChatManager.cs b/src/Tgstation.Server.Host/Components/Chat/IChatManager.cs index 2c9b286c30..062e8fe1bd 100644 --- a/src/Tgstation.Server.Host/Components/Chat/IChatManager.cs +++ b/src/Tgstation.Server.Host/Components/Chat/IChatManager.cs @@ -62,12 +62,24 @@ namespace Tgstation.Server.Host.Components.Chat Task SendWatchdogMessage(string message, bool adminOnly, CancellationToken cancellationToken); /// - /// Send a chat to configured update channels + /// Send the message for a deployment to configured deployment channels. /// - /// The message being sent - /// The for the operation - /// A representing the running operation - Task SendUpdateMessage(string message, CancellationToken cancellationToken); + /// The of the deployment. + /// The BYOND of the deployment. + /// The optional the deployment is expected to be completed at. + /// The repository GitHub owner, if any. + /// The repository GitHub name, if any. + /// if the local deployment commit was pushed to the remote repository. + /// The for the operation. + /// A resulting in a to call to update the message at the deployment's conclusion. Parameters: Error message if any, DreamMaker output if any. + Task> SendDeploymentMessage( + Models.RevisionInformation revisionInformation, + Version byondVersion, + DateTimeOffset? estimatedCompletionTime, + string gitHubOwner, + string gitHubRepo, + bool localCommitPushed, + CancellationToken cancellationToken); /// /// Start tracking s and s. diff --git a/src/Tgstation.Server.Host/Components/Chat/Providers/DiscordProvider.cs b/src/Tgstation.Server.Host/Components/Chat/Providers/DiscordProvider.cs index 9e7052bcfb..ba23173db1 100644 --- a/src/Tgstation.Server.Host/Components/Chat/Providers/DiscordProvider.cs +++ b/src/Tgstation.Server.Host/Components/Chat/Providers/DiscordProvider.cs @@ -6,6 +6,8 @@ using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; +using Tgstation.Server.Host.Models; +using Tgstation.Server.Host.System; namespace Tgstation.Server.Host.Components.Chat.Providers { @@ -28,6 +30,11 @@ namespace Tgstation.Server.Host.Components.Chat.Providers } } + /// + /// The for the . + /// + readonly IAssemblyInformationProvider assemblyInformationProvider; + /// /// The for the /// @@ -53,12 +60,18 @@ namespace Tgstation.Server.Host.Components.Chat.Providers /// /// Construct a /// + /// The value of . /// The value of /// The value of /// The initial reconnect interval in minutes. - public DiscordProvider(ILogger logger, string botToken, uint reconnectInterval) + public DiscordProvider( + IAssemblyInformationProvider assemblyInformationProvider, + ILogger logger, + string botToken, + uint reconnectInterval) : base(logger, reconnectInterval) { + this.assemblyInformationProvider = assemblyInformationProvider ?? throw new ArgumentNullException(nameof(assemblyInformationProvider)); this.botToken = botToken ?? throw new ArgumentNullException(nameof(botToken)); client = new DiscordSocketClient(); client.MessageReceived += Client_MessageReceived; @@ -260,5 +273,138 @@ namespace Tgstation.Server.Host.Components.Chat.Providers Logger.LogWarning("Error sending discord message: {0}", e); } } + + /// + public override async Task> SendUpdateMessage( + RevisionInformation revisionInformation, + Version byondVersion, + DateTimeOffset? estimatedCompletionTime, + string gitHubOwner, + string gitHubRepo, + ulong channelId, + bool localCommitPushed, + CancellationToken cancellationToken) + { + bool gitHub = gitHubOwner != null && gitHubRepo != null; + + localCommitPushed |= revisionInformation.CommitSha == revisionInformation.OriginCommitSha; + + var fields = new List + { + new EmbedFieldBuilder + { + Name = "BYOND Version", + Value = $"{byondVersion.Major}.{byondVersion.Minor}", + IsInline = true + }, + new EmbedFieldBuilder + { + Name = "Local Commit", + Value = localCommitPushed && gitHub + ? $"[{revisionInformation.CommitSha.Substring(0, 7)}](https://github.com/{gitHubOwner}/{gitHubRepo}/commit/{revisionInformation.CommitSha})" + : revisionInformation.CommitSha.Substring(0, 7), + IsInline = true + }, + new EmbedFieldBuilder + { + Name = "Branch Commit", + Value = gitHub + ? $"[{revisionInformation.OriginCommitSha.Substring(0, 7)}](https://github.com/{gitHubOwner}/{gitHubRepo}/commit/{revisionInformation.OriginCommitSha})" + : revisionInformation.OriginCommitSha.Substring(0, 7), + IsInline = true + } + }; + + fields.AddRange((revisionInformation.ActiveTestMerges ?? Enumerable.Empty()) + .Select(x => x.TestMerge) + .Select(x => new EmbedFieldBuilder + { + Name = $"#{x.Number}", + Value = $"[{x.TitleAtMerge}]({x.Url}) by _[@{x.Author}](https://github.com/{x.Author})_{Environment.NewLine}Commit: [{x.PullRequestRevision.Substring(0, 7)}](https://github.com/{gitHubOwner}/{gitHubRepo}/commit/{x.PullRequestRevision}){(String.IsNullOrWhiteSpace(x.Comment) ? String.Empty : $"{Environment.NewLine}_**{x.Comment}**_")}" + })); + + var builder = new EmbedBuilder + { + Author = new EmbedAuthorBuilder + { + Name = assemblyInformationProvider.VersionPrefix, + Url = "https://github.com/tgstation/tgstation-server", + IconUrl = "https://avatars0.githubusercontent.com/u/1363778?s=280&v=4" + }, + Color = Color.Gold, + Description = "TGS has begun deploying active repository code to production.", + Fields = fields, + Title = "Code Deployment", + Footer = new EmbedFooterBuilder + { + Text = "In progress... ETA" + }, + Timestamp = estimatedCompletionTime + }; + + Logger.LogTrace("Attempting to post deploy embed to channel {0}...", channelId); + if (!(client.GetChannel(channelId) is IMessageChannel channel)) + { + Logger.LogTrace("Channel ID {0} does not exist or is not an IMessageChannel!", channelId); + return (errorMessage, dreamMakerOutput) => Task.CompletedTask; + } + + var message = await channel.SendMessageAsync( + String.Empty, + false, + builder.Build(), + new RequestOptions + { + CancelToken = cancellationToken + }) + .ConfigureAwait(false); + + return async (errorMessage, dreamMakerOutput) => + { + builder.Footer.Text = errorMessage == null ? "Succeeded" : "Failed"; + builder.Color = errorMessage == null ? Color.Green : Color.Red; + builder.Timestamp = DateTimeOffset.Now; + builder.Description = errorMessage == null + ? "The deployment completed successfully and will be available at the next server reboot." + : "The deployment failed."; + + if (dreamMakerOutput != null) + builder.AddField(new EmbedFieldBuilder + { + Name = "DreamMaker Output", + Value = $"```{Environment.NewLine}{dreamMakerOutput}{Environment.NewLine}```" + }); + + if (errorMessage != null) + builder.AddField(new EmbedFieldBuilder + { + Name = "Error Message", + Value = errorMessage + }); + + try + { + await message.ModifyAsync( + props => props.Embed = builder.Build()) + .ConfigureAwait(false); + } + catch (Exception ex) + { + Logger.LogWarning("Updating deploy embed {0} failed, attempting new post! Exception: {1}", message.Id, ex); + try + { + await channel.SendMessageAsync( + String.Empty, + false, + builder.Build()) + .ConfigureAwait(false); + } + catch (Exception ex2) + { + Logger.LogWarning("Posting completion deploy embed failed! Exception: {0}", ex2); + } + } + }; + } } } diff --git a/src/Tgstation.Server.Host/Components/Chat/Providers/IProvider.cs b/src/Tgstation.Server.Host/Components/Chat/Providers/IProvider.cs index d4dd515150..c06c22844f 100644 --- a/src/Tgstation.Server.Host/Components/Chat/Providers/IProvider.cs +++ b/src/Tgstation.Server.Host/Components/Chat/Providers/IProvider.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; +using Tgstation.Server.Host.Models; namespace Tgstation.Server.Host.Components.Chat.Providers { @@ -65,5 +66,27 @@ namespace Tgstation.Server.Host.Components.Chat.Providers /// The reconnection interval in minutes. /// A representing the running operation. Task SetReconnectInterval(uint reconnectInterval); + + /// + /// Send the message for a deployment. + /// + /// The of the deployment. + /// The BYOND of the deployment. + /// The optional the deployment is expected to be completed at. + /// The repository GitHub owner, if any. + /// The repository GitHub name, if any. + /// The to send to + /// if the local deployment commit was pushed to the remote repository. + /// The for the operation. + /// A resulting in a to call to update the message at the deployment's conclusion. Parameters: Error message if any, DreamMaker output if any. + Task> SendUpdateMessage( + RevisionInformation revisionInformation, + Version byondVersion, + DateTimeOffset? estimatedCompletionTime, + string gitHubOwner, + string gitHubRepo, + ulong channelId, + bool localCommitPushed, + CancellationToken cancellationToken); } } diff --git a/src/Tgstation.Server.Host/Components/Chat/Providers/IrcProvider.cs b/src/Tgstation.Server.Host/Components/Chat/Providers/IrcProvider.cs index 9d66fc0f38..67e85e152a 100644 --- a/src/Tgstation.Server.Host/Components/Chat/Providers/IrcProvider.cs +++ b/src/Tgstation.Server.Host/Components/Chat/Providers/IrcProvider.cs @@ -467,5 +467,55 @@ namespace Tgstation.Server.Host.Components.Chat.Providers Logger.LogWarning("Unable to send to channel: {0}", e); } }, cancellationToken, TaskCreationOptions.LongRunning, TaskScheduler.Current); + + /// + public override async Task> SendUpdateMessage( + Models.RevisionInformation revisionInformation, + Version byondVersion, + DateTimeOffset? estimatedCompletionTime, + string gitHubOwner, + string gitHubRepo, + ulong channelId, + bool localCommitPushed, + CancellationToken cancellationToken) + { + var commitInsert = revisionInformation.CommitSha.Substring(0, 7); + string remoteCommitInsert; + if (revisionInformation.CommitSha == revisionInformation.OriginCommitSha) + { + commitInsert = String.Format(CultureInfo.InvariantCulture, localCommitPushed ? "^{0}" : "{0}", commitInsert); + remoteCommitInsert = String.Empty; + } + else + remoteCommitInsert = String.Format(CultureInfo.InvariantCulture, ". Remote commit: ^{0}", revisionInformation.OriginCommitSha.Substring(0, 7)); + + var testmergeInsert = (revisionInformation.ActiveTestMerges?.Count ?? 0) == 0 ? String.Empty : String.Format(CultureInfo.InvariantCulture, " (Test Merges: {0})", + String.Join(", ", revisionInformation.ActiveTestMerges.Select(x => x.TestMerge).Select(x => + { + var result = String.Format(CultureInfo.InvariantCulture, "#{0} at {1}", x.Number, x.PullRequestRevision.Substring(0, 7)); + if (x.Comment != null) + result += String.Format(CultureInfo.InvariantCulture, " ({0})", x.Comment); + return result; + }))); + + await SendMessage( + channelId, + String.Format( + CultureInfo.InvariantCulture, + "DM: Deploying revision: {0}{1}{2} BYOND Version: {3}{4}", + commitInsert, + testmergeInsert, + remoteCommitInsert, + byondVersion, + estimatedCompletionTime.HasValue + ? $" ETA: {estimatedCompletionTime - DateTimeOffset.Now}" + : String.Empty), + cancellationToken).ConfigureAwait(false); + + return (errorMessage, dreamMakerOutput) => SendMessage( + channelId, + $"DM: Deployment {(errorMessage == null ? "complete" : "failed")}!", + cancellationToken); + } } } diff --git a/src/Tgstation.Server.Host/Components/Chat/Providers/Provider.cs b/src/Tgstation.Server.Host/Components/Chat/Providers/Provider.cs index 9a4fcae488..9f686bedca 100644 --- a/src/Tgstation.Server.Host/Components/Chat/Providers/Provider.cs +++ b/src/Tgstation.Server.Host/Components/Chat/Providers/Provider.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; +using Tgstation.Server.Host.Models; namespace Tgstation.Server.Host.Components.Chat.Providers { @@ -180,5 +181,16 @@ namespace Tgstation.Server.Host.Components.Chat.Providers /// public abstract Task SendMessage(ulong channelId, string message, CancellationToken cancellationToken); + + /// + public abstract Task> SendUpdateMessage( + RevisionInformation revisionInformation, + Version byondVersion, + DateTimeOffset? estimatedCompletionTime, + string gitHubOwner, + string gitHubRepo, + ulong channelId, + bool localCommitPushed, + CancellationToken cancellationToken); } } diff --git a/src/Tgstation.Server.Host/Components/Chat/Providers/ProviderFactory.cs b/src/Tgstation.Server.Host/Components/Chat/Providers/ProviderFactory.cs index 8d0bc2c832..3a23549cba 100644 --- a/src/Tgstation.Server.Host/Components/Chat/Providers/ProviderFactory.cs +++ b/src/Tgstation.Server.Host/Components/Chat/Providers/ProviderFactory.cs @@ -56,7 +56,11 @@ namespace Tgstation.Server.Host.Components.Chat.Providers return new IrcProvider(assemblyInformationProvider, asyncDelayer, loggerFactory.CreateLogger(), ircBuilder.Address, ircBuilder.Port.Value, ircBuilder.Nickname, ircBuilder.Password, ircBuilder.PasswordType, settings.ReconnectionInterval.Value, ircBuilder.UseSsl.Value); case ChatProvider.Discord: var discordBuilder = (DiscordConnectionStringBuilder)builder; - return new DiscordProvider(loggerFactory.CreateLogger(), discordBuilder.BotToken, settings.ReconnectionInterval.Value); + return new DiscordProvider( + assemblyInformationProvider, + loggerFactory.CreateLogger(), + discordBuilder.BotToken, + settings.ReconnectionInterval.Value); default: throw new InvalidOperationException(String.Format(CultureInfo.InvariantCulture, "Invalid ChatProvider: {0}", settings.Provider)); } diff --git a/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs b/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs index a10d0cb2ef..07d01b1bd7 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs @@ -14,7 +14,6 @@ using Tgstation.Server.Host.Components.Byond; using Tgstation.Server.Host.Components.Chat; using Tgstation.Server.Host.Components.Repository; using Tgstation.Server.Host.Components.Session; -using Tgstation.Server.Host.Components.Watchdog; using Tgstation.Server.Host.Core; using Tgstation.Server.Host.Database; using Tgstation.Server.Host.IO; @@ -82,11 +81,6 @@ namespace Tgstation.Server.Host.Components.Deployment /// readonly IProcessExecutor processExecutor; - /// - /// The for - /// - readonly IWatchdog watchdog; - /// /// The for . /// @@ -113,14 +107,18 @@ namespace Tgstation.Server.Host.Components.Deployment readonly Api.Models.Instance metadata; /// - /// for . + /// for . /// - readonly object compilingLock; + readonly object deploymentLock; + + Func currentChatCallback; + + string currentDreamMakerOutput; /// /// If a compile job is running /// - bool compiling; + bool deploying; /// /// Construct @@ -132,7 +130,6 @@ namespace Tgstation.Server.Host.Components.Deployment /// The value of /// The value of /// The value of - /// The value of /// The value of . /// The value of . /// The value of . @@ -146,7 +143,6 @@ namespace Tgstation.Server.Host.Components.Deployment IEventConsumer eventConsumer, IChatManager chatManager, IProcessExecutor processExecutor, - IWatchdog watchdog, IGitHubClientFactory gitHubClientFactory, ICompileJobSink compileJobConsumer, IRepositoryManager repositoryManager, @@ -160,14 +156,13 @@ namespace Tgstation.Server.Host.Components.Deployment this.eventConsumer = eventConsumer ?? throw new ArgumentNullException(nameof(eventConsumer)); this.chatManager = chatManager ?? throw new ArgumentNullException(nameof(chatManager)); this.processExecutor = processExecutor ?? throw new ArgumentNullException(nameof(processExecutor)); - this.watchdog = watchdog ?? throw new ArgumentNullException(nameof(watchdog)); this.gitHubClientFactory = gitHubClientFactory ?? throw new ArgumentNullException(nameof(gitHubClientFactory)); this.compileJobConsumer = compileJobConsumer ?? throw new ArgumentNullException(nameof(compileJobConsumer)); this.repositoryManager = repositoryManager ?? throw new ArgumentNullException(nameof(repositoryManager)); this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); this.metadata = metadata ?? throw new ArgumentNullException(nameof(metadata)); - compilingLock = new object(); + deploymentLock = new object(); } /// @@ -289,7 +284,7 @@ namespace Tgstation.Server.Host.Components.Deployment cancellationToken.ThrowIfCancellationRequested(); logger.LogDebug("DreamMaker exit code: {0}", exitCode); - job.Output = dm.GetCombinedOutput(); + currentDreamMakerOutput = job.Output = dm.GetCombinedOutput(); logger.LogDebug("DreamMaker output: {0}{1}", Environment.NewLine, job.Output); return exitCode; } @@ -352,13 +347,10 @@ namespace Tgstation.Server.Host.Components.Deployment /// Cleans up a failed compile /// /// The running - /// If the was cancelled - /// The for the operation /// A representing the running operation - async Task CleanupFailedCompile(Models.CompileJob job, bool cancelled, CancellationToken cancellationToken) + async Task CleanupFailedCompile(Models.CompileJob job) { logger.LogTrace("Cleaning compile directory..."); - var chatTask = chatManager.SendUpdateMessage(cancelled ? "Deploy cancelled!" : "Deploy failed!", cancellationToken); var jobPath = job.DirectoryName.ToString(); try { @@ -368,66 +360,6 @@ namespace Tgstation.Server.Host.Components.Deployment { logger.LogWarning("Error cleaning up compile directory {0}! Exception: {1}", ioManager.ResolvePath(jobPath), e); } - - await chatTask.ConfigureAwait(false); - } - - /// - /// Send a message to about a deployment - /// - /// The for the deployment - /// The for the deployment - /// The for the operation - /// A representing the running operation - async Task SendDeploymentMessage(Models.RevisionInformation revisionInformation, IByondExecutableLock byondLock, CancellationToken cancellationToken) - { - var commitInsert = revisionInformation.CommitSha.Substring(0, 7); - string remoteCommitInsert; - if (revisionInformation.CommitSha == revisionInformation.OriginCommitSha) - { - commitInsert = String.Format(CultureInfo.InvariantCulture, "^{0}", commitInsert); - remoteCommitInsert = String.Empty; - } - else - remoteCommitInsert = String.Format(CultureInfo.InvariantCulture, ". Remote commit: ^{0}", revisionInformation.OriginCommitSha.Substring(0, 7)); - - var testmergeInsert = (revisionInformation.ActiveTestMerges?.Count ?? 0) == 0 - ? String.Empty - : String.Format( - CultureInfo.InvariantCulture, - "{0}Test Merges:{1}", - Environment.NewLine, - String.Join( - Environment.NewLine, - revisionInformation - .ActiveTestMerges - .Select(x => x.TestMerge) - .Select(x => - { - var result = String.Format( - CultureInfo.InvariantCulture, - "- #{0} at {1}", - x.Number, - x.PullRequestRevision.Substring(0, 7)); - - if (x.Comment != null) - result += $": {x.Comment}"; - - return result; - }))); - - await chatManager.SendUpdateMessage( - String.Format( - CultureInfo.InvariantCulture, - "*Deployment Triggered*{0}Revision: {1}{2}{3}{0}BYOND Version: {4}.{5}", - Environment.NewLine, - commitInsert, - testmergeInsert, - remoteCommitInsert, - byondLock.Version.Major, - byondLock.Version.Minor), - cancellationToken) - .ConfigureAwait(false); } /// @@ -523,9 +455,9 @@ namespace Tgstation.Server.Host.Components.Deployment logger.LogDebug("Compile complete!"); } - catch (Exception e) + catch { - await CleanupFailedCompile(job, e is OperationCanceledException, cancellationToken).ConfigureAwait(false); + await CleanupFailedCompile(job).ConfigureAwait(false); throw; } } @@ -547,174 +479,201 @@ namespace Tgstation.Server.Host.Components.Deployment if (progressReporter == null) throw new ArgumentNullException(nameof(progressReporter)); - string repoOwner = null; - string repoName = null; - TimeSpan? averageSpan = null; - Models.RepositorySettings repositorySettings = null; - Models.DreamDaemonSettings ddSettings = null; - DreamMakerSettings dreamMakerSettings = null; - IRepository repo = null; + lock (deploymentLock) + { + if (deploying) + throw new JobException(ErrorCode.DreamMakerCompileJobInProgress); + deploying = true; + } + + currentChatCallback = null; + currentDreamMakerOutput = null; Models.CompileJob compileJob = null; - Models.RevisionInformation revInfo = null; - await databaseContextFactory.UseContext( - async databaseContext => - { - averageSpan = await CalculateExpectedDeploymentTime(databaseContext, cancellationToken).ConfigureAwait(false); - - ddSettings = await databaseContext - .DreamDaemonSettings - .Where(x => x.InstanceId == metadata.Id) - .Select(x => new Models.DreamDaemonSettings - { - StartupTimeout = x.StartupTimeout, - }) - .FirstOrDefaultAsync(cancellationToken) - .ConfigureAwait(false); - if (ddSettings == default) - throw new JobException(ErrorCode.InstanceMissingDreamDaemonSettings); - - dreamMakerSettings = await databaseContext.DreamMakerSettings.Where(x => x.InstanceId == metadata.Id).FirstAsync(cancellationToken).ConfigureAwait(false); - if (dreamMakerSettings == default) - throw new JobException(ErrorCode.InstanceMissingDreamMakerSettings); - - repositorySettings = await databaseContext - .RepositorySettings - .Where(x => x.InstanceId == metadata.Id) - .Select(x => new Models.RepositorySettings - { - AccessToken = x.AccessToken, - ShowTestMergeCommitters = x.ShowTestMergeCommitters, - PushTestMergeCommits = x.PushTestMergeCommits, - PostTestMergeComment = x.PostTestMergeComment - }) - .FirstOrDefaultAsync(cancellationToken) - .ConfigureAwait(false); - if (repositorySettings == default) - throw new JobException(ErrorCode.InstanceMissingRepositorySettings); - - repo = await repositoryManager.LoadRepository(cancellationToken).ConfigureAwait(false); - try - { - if (repo == null) - throw new JobException(ErrorCode.RepoMissing); - - if (repo.IsGitHubRepository) - { - repoOwner = repo.GitHubOwner; - repoName = repo.GitHubRepoName; - } - - var repoSha = repo.Head; - revInfo = await databaseContext - .RevisionInformations - .Where(x => x.CommitSha == repoSha && x.Instance.Id == metadata.Id) - .Include(x => x.ActiveTestMerges) - .ThenInclude(x => x.TestMerge) - .ThenInclude(x => x.MergedBy) - .FirstOrDefaultAsync(cancellationToken) - .ConfigureAwait(false); - - if (revInfo == default) - { - revInfo = new Models.RevisionInformation - { - CommitSha = repoSha, - OriginCommitSha = repoSha, - Instance = new Models.Instance - { - Id = metadata.Id - }, - ActiveTestMerges = new List() - }; - - logger.LogWarning(Repository.Repository.OriginTrackingErrorTemplate, repoSha); - databaseContext.Instances.Attach(revInfo.Instance); - await databaseContext.Save(cancellationToken).ConfigureAwait(false); - } - } - catch - { - repo.Dispose(); - throw; - } - }) - .ConfigureAwait(false); - - using (repo) - compileJob = await Compile( - revInfo, - dreamMakerSettings, - ddSettings.StartupTimeout.Value, - repo, - progressReporter, - averageSpan, - cancellationToken) - .ConfigureAwait(false); - - var activeCompileJob = compileJobConsumer.LatestCompileJob(); try { + string repoOwner = null; + string repoName = null; + TimeSpan? averageSpan = null; + Models.RepositorySettings repositorySettings = null; + Models.DreamDaemonSettings ddSettings = null; + DreamMakerSettings dreamMakerSettings = null; + IRepository repo = null; + Models.RevisionInformation revInfo = null; await databaseContextFactory.UseContext( async databaseContext => { - compileJob.Job = new Models.Job - { - Id = job.Id - }; - compileJob.RevisionInformation = new Models.RevisionInformation - { - Id = revInfo.Id - }; + averageSpan = await CalculateExpectedDeploymentTime(databaseContext, cancellationToken).ConfigureAwait(false); - databaseContext.Jobs.Attach(compileJob.Job); - databaseContext.RevisionInformations.Attach(compileJob.RevisionInformation); - databaseContext.CompileJobs.Add(compileJob); + ddSettings = await databaseContext + .DreamDaemonSettings + .Where(x => x.InstanceId == metadata.Id) + .Select(x => new Models.DreamDaemonSettings + { + StartupTimeout = x.StartupTimeout, + }) + .FirstOrDefaultAsync(cancellationToken) + .ConfigureAwait(false); + if (ddSettings == default) + throw new JobException(ErrorCode.InstanceMissingDreamDaemonSettings); - // The difficulty with compile jobs is they have a two part commit - await databaseContext.Save(cancellationToken).ConfigureAwait(false); + dreamMakerSettings = await databaseContext.DreamMakerSettings.Where(x => x.InstanceId == metadata.Id).FirstAsync(cancellationToken).ConfigureAwait(false); + if (dreamMakerSettings == default) + throw new JobException(ErrorCode.InstanceMissingDreamMakerSettings); + + repositorySettings = await databaseContext + .RepositorySettings + .Where(x => x.InstanceId == metadata.Id) + .Select(x => new Models.RepositorySettings + { + AccessToken = x.AccessToken, + ShowTestMergeCommitters = x.ShowTestMergeCommitters, + PushTestMergeCommits = x.PushTestMergeCommits, + PostTestMergeComment = x.PostTestMergeComment + }) + .FirstOrDefaultAsync(cancellationToken) + .ConfigureAwait(false); + if (repositorySettings == default) + throw new JobException(ErrorCode.InstanceMissingRepositorySettings); + + repo = await repositoryManager.LoadRepository(cancellationToken).ConfigureAwait(false); try { - await compileJobConsumer.LoadCompileJob(compileJob, cancellationToken).ConfigureAwait(false); + if (repo == null) + throw new JobException(ErrorCode.RepoMissing); + + if (repo.IsGitHubRepository) + { + repoOwner = repo.GitHubOwner; + repoName = repo.GitHubRepoName; + } + + var repoSha = repo.Head; + revInfo = await databaseContext + .RevisionInformations + .Where(x => x.CommitSha == repoSha && x.Instance.Id == metadata.Id) + .Include(x => x.ActiveTestMerges) + .ThenInclude(x => x.TestMerge) + .ThenInclude(x => x.MergedBy) + .FirstOrDefaultAsync(cancellationToken) + .ConfigureAwait(false); + + if (revInfo == default) + { + revInfo = new Models.RevisionInformation + { + CommitSha = repoSha, + OriginCommitSha = repoSha, + Instance = new Models.Instance + { + Id = metadata.Id + }, + ActiveTestMerges = new List() + }; + + logger.LogWarning(Repository.Repository.OriginTrackingErrorTemplate, repoSha); + databaseContext.Instances.Attach(revInfo.Instance); + await databaseContext.Save(cancellationToken).ConfigureAwait(false); + } } catch { - // So we need to un-commit the compile job if the above throws - databaseContext.CompileJobs.Remove(compileJob); - await databaseContext.Save(default).ConfigureAwait(false); + repo.Dispose(); throw; } }) .ConfigureAwait(false); + + var likelyPushedTestMergeCommit = + repositorySettings.PushTestMergeCommits.Value + && repositorySettings.AccessToken != null + && repositorySettings.AccessUser != null; + using (repo) + compileJob = await Compile( + revInfo, + dreamMakerSettings, + ddSettings.StartupTimeout.Value, + repo, + progressReporter, + averageSpan, + likelyPushedTestMergeCommit, + cancellationToken) + .ConfigureAwait(false); + + var activeCompileJob = compileJobConsumer.LatestCompileJob(); + try + { + await databaseContextFactory.UseContext( + async databaseContext => + { + compileJob.Job = new Models.Job + { + Id = job.Id + }; + compileJob.RevisionInformation = new Models.RevisionInformation + { + Id = revInfo.Id + }; + + databaseContext.Jobs.Attach(compileJob.Job); + databaseContext.RevisionInformations.Attach(compileJob.RevisionInformation); + databaseContext.CompileJobs.Add(compileJob); + + // The difficulty with compile jobs is they have a two part commit + await databaseContext.Save(cancellationToken).ConfigureAwait(false); + try + { + await compileJobConsumer.LoadCompileJob(compileJob, cancellationToken).ConfigureAwait(false); + } + catch + { + // So we need to un-commit the compile job if the above throws + databaseContext.CompileJobs.Remove(compileJob); + await databaseContext.Save(default).ConfigureAwait(false); + throw; + } + }) + .ConfigureAwait(false); + } + catch + { + await CleanupFailedCompile(compileJob).ConfigureAwait(false); + throw; + } + + var commentsTask = PostDeploymentComments( + revInfo, + activeCompileJob?.RevisionInformation, + repositorySettings, + repoOwner, + repoName); + + var eventTask = eventConsumer.HandleEvent(EventType.DeploymentComplete, null, cancellationToken); + + var chatTask = currentChatCallback(null, compileJob.Output); + currentChatCallback = null; + + try + { + await Task.WhenAll(commentsTask, eventTask, chatTask).ConfigureAwait(false); + } + catch (Exception ex) + { + throw new JobException(ErrorCode.PostDeployFailure, ex); + } } catch (Exception ex) { - await CleanupFailedCompile(compileJob, ex is OperationCanceledException, default).ConfigureAwait(false); + if (currentChatCallback != null) + await currentChatCallback( + ex.Message, + currentDreamMakerOutput) + .ConfigureAwait(false); + throw; } - - var commentsTask = PostDeploymentComments( - revInfo, - activeCompileJob?.RevisionInformation, - repositorySettings, - repoOwner, - repoName); - - var eventTask = eventConsumer.HandleEvent(EventType.DeploymentComplete, null, cancellationToken); - - var chatTask = chatManager.SendUpdateMessage( - String.Format( - CultureInfo.InvariantCulture, - "Deployment complete! Changes will be applied when DreamDaemon {0}.", - watchdog.Running ? "reboots" : "is launched"), - cancellationToken); - - try + finally { - await Task.WhenAll(commentsTask, eventTask, chatTask).ConfigureAwait(false); - } - catch (Exception ex) - { - throw new JobException(ErrorCode.PostDeployFailure, ex); + deploying = false; } } #pragma warning restore CA1506 @@ -751,23 +710,32 @@ namespace Tgstation.Server.Host.Components.Deployment return averageSpan; } - async Task Compile(Models.RevisionInformation revisionInformation, Api.Models.DreamMaker dreamMakerSettings, uint apiValidateTimeout, IRepository repository, Action progressReporter, TimeSpan? estimatedDuration, CancellationToken cancellationToken) + async Task Compile( + Models.RevisionInformation revisionInformation, + Api.Models.DreamMaker dreamMakerSettings, + uint apiValidateTimeout, + IRepository repository, + Action progressReporter, + TimeSpan? estimatedDuration, + bool localCommitExistsOnRemote, + CancellationToken cancellationToken) { logger.LogTrace("Begin Compile"); - lock (compilingLock) - { - if (compiling) - throw new JobException(ErrorCode.DreamMakerCompileJobInProgress); - compiling = true; - } - using var progressCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); var progressTask = estimatedDuration.HasValue ? ProgressTask(progressReporter, estimatedDuration.Value, cancellationToken) : Task.CompletedTask; try { using var byondLock = await byond.UseExecutables(null, cancellationToken).ConfigureAwait(false); - await SendDeploymentMessage(revisionInformation, byondLock, cancellationToken).ConfigureAwait(false); + currentChatCallback = await chatManager.SendDeploymentMessage( + revisionInformation, + byondLock.Version, + DateTimeOffset.Now + estimatedDuration, + repository.GitHubOwner, + repository.GitHubRepoName, + localCommitExistsOnRemote, + cancellationToken) + .ConfigureAwait(false); var job = new Models.CompileJob { @@ -788,7 +756,6 @@ namespace Tgstation.Server.Host.Components.Deployment } finally { - compiling = false; progressCts.Cancel(); await progressTask.ConfigureAwait(false); } diff --git a/src/Tgstation.Server.Host/Components/InstanceFactory.cs b/src/Tgstation.Server.Host/Components/InstanceFactory.cs index 9be0802cc7..23aa1da429 100644 --- a/src/Tgstation.Server.Host/Components/InstanceFactory.cs +++ b/src/Tgstation.Server.Host/Components/InstanceFactory.cs @@ -262,7 +262,6 @@ namespace Tgstation.Server.Host.Components eventConsumer, chatManager, processExecutor, - watchdog, gitHubClientFactory, dmbFactory, repoManager,