From cef860db1b29917ec813c8ed9999202431ffd66a Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Thu, 17 Aug 2023 12:42:28 -0400 Subject: [PATCH 01/22] Release notes update - Added functionality to release notes app to generate full changelog.yml from GitHub metadata. (Incremental) - Updated PULL_REQUEST_TEMPLATE.md to reflect changes (Strict component list) --- .github/PULL_REQUEST_TEMPLATE.md | 5 +- .gitignore | 1 + .../Extensions/Converters/VersionConverter.cs | 2 +- tools/Tgstation.Server.ReleaseNotes/Change.cs | 13 + .../Changelist.cs | 46 + .../Component.cs | 16 + .../Tgstation.Server.ReleaseNotes/Program.cs | 861 +++++++++++++++--- .../ReleaseNotes.cs | 9 + .../Tgstation.Server.ReleaseNotes.csproj | 6 +- 9 files changed, 838 insertions(+), 121 deletions(-) create mode 100644 tools/Tgstation.Server.ReleaseNotes/Change.cs create mode 100644 tools/Tgstation.Server.ReleaseNotes/Changelist.cs create mode 100644 tools/Tgstation.Server.ReleaseNotes/Component.cs create mode 100644 tools/Tgstation.Server.ReleaseNotes/ReleaseNotes.cs diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index b770b65721..c146ed90da 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -13,9 +13,10 @@ They will be amalgamated together in the end. /:cl: :cl: Categories -Categories are divided up in the release notes and set after the :cl: header. +Categories are used by [the release notes tool](../tools/Tgstation.Server.ReleaseNotes) to generate formatted changelists used in releases. The default category is Core. -The current standard categories are Core, DreamMaker API, HTTP API, and Host Watchdog. +Only one category may be specified for a :cl: block. +Valid categories are Core, DreamMaker API, HTTP API, Host Watchdog, Web Control Panel, Configuration, Nuget: Api, Nuget: Client, and Nuget: Common. /:cl: [Why]: # (If this does not close or work on an existing GitHub issue, please add a short description [two lines down] of why you think these changes would benefit the server. If you can't justify it in words, it might not be worth adding.) diff --git a/.gitignore b/.gitignore index d7756e60cd..8794b1a00f 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,7 @@ artifacts/ /tools/Tgstation.Server.ReleaseNotes/release_notes.md launchSettings.json release_notes.md +release_notes.yml *nupkg *.sqlite3 packaging/ diff --git a/src/Tgstation.Server.Host/Extensions/Converters/VersionConverter.cs b/src/Tgstation.Server.Host/Extensions/Converters/VersionConverter.cs index ec08e5ae7b..4ca9cdf105 100644 --- a/src/Tgstation.Server.Host/Extensions/Converters/VersionConverter.cs +++ b/src/Tgstation.Server.Host/Extensions/Converters/VersionConverter.cs @@ -13,7 +13,7 @@ namespace Tgstation.Server.Host.Extensions.Converters /// /// and for serializing s in semver format. /// - sealed class VersionConverter : JsonConverter, IYamlTypeConverter + public sealed class VersionConverter : JsonConverter, IYamlTypeConverter { /// /// Check if the supports (de)serializing a given . diff --git a/tools/Tgstation.Server.ReleaseNotes/Change.cs b/tools/Tgstation.Server.ReleaseNotes/Change.cs new file mode 100644 index 0000000000..88f5910606 --- /dev/null +++ b/tools/Tgstation.Server.ReleaseNotes/Change.cs @@ -0,0 +1,13 @@ +using System.Collections.Generic; + +using Octokit; + +namespace Tgstation.Server.ReleaseNotes +{ + sealed class Change + { + public List Descriptions { get; set; } + public string Author { get; set; } + public int PullRequest { get; set; } + } +} diff --git a/tools/Tgstation.Server.ReleaseNotes/Changelist.cs b/tools/Tgstation.Server.ReleaseNotes/Changelist.cs new file mode 100644 index 0000000000..137d018799 --- /dev/null +++ b/tools/Tgstation.Server.ReleaseNotes/Changelist.cs @@ -0,0 +1,46 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; + +namespace Tgstation.Server.ReleaseNotes +{ + sealed class Changelist + { + public Version Version { get; set; } + + public Dictionary ComponentVersions { get; set; } + + public List Changes { get; set; } + + public bool Unreleased { get; set; } + + public void DeduplicateChanges() + { + Changes = Changes + .OrderBy(x => x.PullRequest) + .GroupBy(x => x.PullRequest) + .Select(prChanges => + { + string author = null; + return new Change + { + PullRequest = prChanges.Key, + Descriptions = prChanges + .SelectMany(x => + { + if (author != null) + Debug.Assert(x.Author == author); + else + author = x.Author; + + return x.Descriptions; + }) + .ToList(), + Author = author + }; + }) + .ToList(); + } + } +} diff --git a/tools/Tgstation.Server.ReleaseNotes/Component.cs b/tools/Tgstation.Server.ReleaseNotes/Component.cs new file mode 100644 index 0000000000..f882a3a320 --- /dev/null +++ b/tools/Tgstation.Server.ReleaseNotes/Component.cs @@ -0,0 +1,16 @@ +namespace Tgstation.Server.ReleaseNotes +{ + enum Component + { + Core, + HttpApi, + Configuration, + DreamMakerApi, + InteropApi, + WebControlPanel, + HostWatchdog, + NugetCommon, + NugetApi, + NugetClient, + } +} diff --git a/tools/Tgstation.Server.ReleaseNotes/Program.cs b/tools/Tgstation.Server.ReleaseNotes/Program.cs index e0bfa45e59..e5a95795d6 100644 --- a/tools/Tgstation.Server.ReleaseNotes/Program.cs +++ b/tools/Tgstation.Server.ReleaseNotes/Program.cs @@ -1,14 +1,28 @@ -using System; +// This program is minimal effort and should be sent to remedial school + +using System; +using System.Collections.Concurrent; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Sockets; using System.Text; +using System.Text.RegularExpressions; using System.Threading.Tasks; using System.Xml.Linq; +using Newtonsoft.Json; + using Octokit; using Octokit.GraphQL; +using Tgstation.Server.Host.Extensions.Converters; + +using YamlDotNet.Serialization; + namespace Tgstation.Server.ReleaseNotes { /// @@ -34,8 +48,9 @@ namespace Tgstation.Server.ReleaseNotes var ensureRelease = versionString.Equals("--ensure-release", StringComparison.OrdinalIgnoreCase); var linkWinget = versionString.Equals("--link-winget", StringComparison.OrdinalIgnoreCase); var shaCheck = versionString.Equals("--winget-template-check", StringComparison.OrdinalIgnoreCase); + var fullNotes = versionString.Equals("--generate-full-notes", StringComparison.OrdinalIgnoreCase); - if ((!Version.TryParse(versionString, out var version) || version.Revision != -1) && !ensureRelease && !linkWinget && !shaCheck) + if ((!Version.TryParse(versionString, out var version) || version.Revision != -1) && !ensureRelease && !linkWinget && !shaCheck && !fullNotes) { Console.WriteLine("Invalid version: " + versionString); return 2; @@ -84,6 +99,11 @@ namespace Tgstation.Server.ReleaseNotes return await Winget(client, null, args[1]); } + if (fullNotes) + { + return await FullNotes(client); + } + var releasesTask = client.Repository.Release.GetAll(RepoOwner, RepoName); Console.WriteLine("Getting merged pull requests in milestone " + versionString + "..."); @@ -101,103 +121,12 @@ namespace Tgstation.Server.ReleaseNotes } Console.WriteLine(milestonePRs.Items.Count + " total pull requests"); - Task milestoneTask = null; - var milestoneTaskLock = new object(); - var releaseDictionary = new Dictionary>>(StringComparer.OrdinalIgnoreCase); - bool postControlPanelMessage = false; - async Task GetReleaseNotesFromPR(Issue pullRequest) - { - //need to check it was merged - var fullPR = await client.Repository.PullRequest.Get(RepoOwner, RepoName, pullRequest.Number).ConfigureAwait(false); + var noteTasks = new List, Dictionary, bool>>>(); - if (!fullPR.Merged) - { - if (!doNotCloseMilestone && fullPR.Milestone != null) - { - Console.WriteLine($"Removing trash PR #{fullPR.Number} from milestone..."); - await client.Issue.Update(RepoOwner, RepoName, fullPR.Number, new IssueUpdate - { - Milestone = null - }).ConfigureAwait(false); - } - - return; - } - - async Task GetMilestone() - { - if (fullPR.Milestone == null) - return null; - return await client.Issue.Milestone.Get(RepoOwner, RepoName, fullPR.Milestone.Number); - }; - - lock (milestoneTaskLock) - milestoneTask ??= GetMilestone(); - - // if (!fullPR.Merged) - //return; - - void BuildNotesFromComment(string comment, User user) - { - if (comment == null) - return; - - void CommitNotes(string component, List notes) - { - lock (releaseDictionary) - { - foreach (var I in notes) - Console.WriteLine(component + " #" + fullPR.Number + " - " + I + " (@" + user.Login + ")"); - - var tupleSelector = notes.Select(note => Tuple.Create(note, fullPR.Number, user.Login)); - if (releaseDictionary.TryGetValue(component, out var currentValues)) - currentValues.AddRange(tupleSelector); - else - releaseDictionary.Add(component, tupleSelector.ToList()); - } - } - - var commentSplits = comment.Split('\n'); - string targetComponent = null; - var notes = new List(); - foreach (var line in commentSplits) - { - var trimmedLine = line.Trim(); - if (targetComponent == null) - { - if (trimmedLine.StartsWith(":cl:", StringComparison.Ordinal)) - { - targetComponent = trimmedLine[4..].Trim(); - if (targetComponent.Length == 0) - targetComponent = "Core"; - } - continue; - } - if (trimmedLine.StartsWith("/:cl:", StringComparison.Ordinal)) - { - CommitNotes(targetComponent, notes); - targetComponent = null; - notes.Clear(); - continue; - } - if (trimmedLine.Length == 0) - continue; - - notes.Add(trimmedLine); - } - } - - var comments = await client.Issue.Comment.GetAllForIssue(RepoOwner, RepoName, fullPR.Number).ConfigureAwait(false); - BuildNotesFromComment(fullPR.Body, fullPR.User); - foreach(var x in comments) - BuildNotesFromComment(x.Body, x.User); - } - - var tasks = new List(); foreach (var I in milestonePRs.Items) - tasks.Add(GetReleaseNotesFromPR(I)); + noteTasks.Add(GetReleaseNotesFromPR(client, I, doNotCloseMilestone, false, false)); var releases = await releasesTask.ConfigureAwait(false); @@ -299,19 +228,21 @@ namespace Tgstation.Server.ReleaseNotes newNotes.Append(version.Build); } newNotes.Append("]("); - var milestone = await milestoneTask.ConfigureAwait(false); + + var milestone = await milestoneTasks.Single().Value.ConfigureAwait(false); if (milestone == null) { Console.WriteLine("Unable to detemine milestone!"); return 9; } + var allTasks = new List(noteTasks); if (doNotCloseMilestone) Console.WriteLine("Not closing milestone due to parameter!"); else { Console.WriteLine("Closing milestone..."); - tasks.Add(client.Issue.Milestone.Update(RepoOwner, RepoName, milestone.Number, new MilestoneUpdate + allTasks.Add(client.Issue.Milestone.Update(RepoOwner, RepoName, milestone.Number, new MilestoneUpdate { State = ItemState.Closed })); @@ -358,14 +289,16 @@ namespace Tgstation.Server.ReleaseNotes if (I.PullRequest != null) { Console.WriteLine($"Adding additional merged PR #{I.Number}..."); - tasks.Add(GetReleaseNotesFromPR(I)); + var task = GetReleaseNotesFromPR(client, I, doNotCloseMilestone, false, false); + noteTasks.Add(task); + allTasks.Add(task); } } await Task.WhenAll(issueUpdateTasks).ConfigureAwait(false); } - tasks.Add(client.Issue.Milestone.Delete(RepoOwner, RepoName, milestoneToDelete.Number)); + allTasks.Add(client.Issue.Milestone.Delete(RepoOwner, RepoName, milestoneToDelete.Number)); } // Create the next minor milestone @@ -378,7 +311,7 @@ namespace Tgstation.Server.ReleaseNotes { Description = "Next minor version" }); - tasks.Add(nextMinorMilestoneTask); + allTasks.Add(nextMinorMilestoneTask); // Move unfinished stuff to new minor milestone Console.WriteLine($"Moving {milestone.OpenIssues} abandoned issue(s) from previous milestone to new one..."); @@ -393,7 +326,7 @@ namespace Tgstation.Server.ReleaseNotes { var nextMinorMilestone = await nextMinorMilestoneTask.ConfigureAwait(false); foreach (var I in abandonedIssues.Items) - tasks.Add(client.Issue.Update(RepoOwner, RepoName, I.Number, new IssueUpdate + allTasks.Add(client.Issue.Update(RepoOwner, RepoName, I.Number, new IssueUpdate { Milestone = nextMinorMilestone.Number })); @@ -405,7 +338,40 @@ namespace Tgstation.Server.ReleaseNotes newNotes.Append("?closed=1)"); newNotes.Append(Environment.NewLine); - await Task.WhenAll(tasks).ConfigureAwait(false); + await Task.WhenAll(allTasks).ConfigureAwait(false); + + var componentVersionDict = new Dictionary + { + { Component.Configuration, configVersion }, + { Component.HttpApi, apiVersion }, + { Component.DreamMakerApi, dmApiVersion }, + { Component.InteropApi, interopVersion }, + { Component.WebControlPanel, webControlVersion }, + { Component.HostWatchdog, hostWatchdogVersion }, + }; + + var releaseDictionary = new Dictionary( + noteTasks + .SelectMany(task => task.Result.Item1) + .GroupBy(kvp => kvp.Key) + .Select(grouping => + { + var component = grouping.Key; + var changelist = new Changelist + { + Changes = grouping.SelectMany(kvp => kvp.Value.Changes).ToList() + }; + + if (component == Component.Core) + { + changelist.Version = coreVersion; + changelist.ComponentVersions = componentVersionDict; + } + else + changelist.Version = componentVersionDict[component]; + + return new KeyValuePair(component, changelist); + })); if (releaseDictionary.Count == 0) { @@ -417,20 +383,31 @@ namespace Tgstation.Server.ReleaseNotes { newNotes.Append(Environment.NewLine); newNotes.Append("#### "); - newNotes.Append(I.Key); - - - foreach (var noteTuple in I.Value) + string componentName = I.Key switch { - newNotes.Append(Environment.NewLine); - newNotes.Append("- "); - newNotes.Append(noteTuple.Item1); - newNotes.Append(" (#"); - newNotes.Append(noteTuple.Item2); - newNotes.Append(" @"); - newNotes.Append(noteTuple.Item3); - newNotes.Append(')'); - } + Component.HttpApi => "HTTP API", + Component.InteropApi => "Interop API", + Component.Configuration => "**Configuration**", + Component.DreamMakerApi => "DreamMaker API", + Component.HostWatchdog => "Host Watchdog", + Component.Core => "Core", + Component.WebControlPanel => "Web Control Panel", + _ => throw new Exception($"Unknown Component: {I.Key}"), + }; + newNotes.Append(componentName); + + foreach (var change in I.Value.Changes) + foreach (var line in change.Descriptions) + { + newNotes.Append(Environment.NewLine); + newNotes.Append("- "); + newNotes.Append(line); + newNotes.Append(" (#"); + newNotes.Append(change.PullRequest); + newNotes.Append(" @"); + newNotes.Append(change.Author); + newNotes.Append(')'); + } newNotes.Append(Environment.NewLine); } @@ -473,6 +450,237 @@ namespace Tgstation.Server.ReleaseNotes } } + static ConcurrentDictionary> milestoneTasks = new ConcurrentDictionary>(); + static Task GetMilestone(IGitHubClient client, int number) + => milestoneTasks.GetOrAdd(number, localNumber => client.Issue.Milestone.Get(RepoOwner, RepoName, localNumber)); + + static async Task, Dictionary, bool>> GetReleaseNotesFromPR(IGitHubClient client, Issue pullRequest, bool doNotCloseMilestone, bool needComponentExactVersions, bool forAllComponents) + { + //need to check it was merged + var fullPR = await RLR(() => client.Repository.PullRequest.Get(RepoOwner, RepoName, pullRequest.Number)); + + if (!fullPR.Merged) + { + if (!doNotCloseMilestone && fullPR.Milestone != null) + { + Console.WriteLine($"Removing trash PR #{fullPR.Number} from milestone..."); + await RLR(() => client.Issue.Update(RepoOwner, RepoName, fullPR.Number, new IssueUpdate + { + Milestone = null + })); + } + + return null; + } + + if (fullPR.Milestone == null) + { + return null; + } + + var commentsTask = TripleCheckGitHubPagination(apiOptions => client.Issue.Comment.GetAllForIssue(fullPR.Base.Repository.Id, pullRequest.Number, apiOptions), comment => comment.Id); + + bool isReleasePR = false; + async Task ShouldGetExtendedComponentVersions() + { + if (forAllComponents) + return true; + + var commit = await RLR(() => client.Repository.Commit.Get(fullPR.Base.Repository.Id, fullPR.MergeCommitSha)); + + return isReleasePR = commit.Commit.Message.Contains("[TGSDeploy]"); + } + + Task needExtendedComponentVersions = null; + async Task> GetComponentVersions() + { + var mergeCommit = fullPR.MergeCommitSha; + // we don't care about unreleased web control panel changes + + try + { + needExtendedComponentVersions = ShouldGetExtendedComponentVersions(); + + var versionsBytes = await RLR(() => client.Repository.Content.GetRawContentByRef(RepoOwner, RepoName, "build/Version.props", mergeCommit)); + + XDocument doc; + using (var ms = new MemoryStream(versionsBytes)) + doc = XDocument.Load(ms); + + var project = doc.Root; + var xmlNamespace = project.GetDefaultNamespace(); + var versionsPropertyGroup = project.Elements().First(x => x.Name == xmlNamespace + "PropertyGroup"); + + Version Parse(string elemName, bool controlPanel = false) + { + var element = versionsPropertyGroup.Element(xmlNamespace + elemName); + if (element == null) + return null; + + return Version.Parse(element.Value); + } + + var dict = new Dictionary + { + { Component.Core, Parse("TgsCoreVersion") }, + { Component.HttpApi, Parse("TgsApiVersion") }, + { Component.DreamMakerApi, Parse("TgsDmapiVersion") }, + }; + + if (await needExtendedComponentVersions) + { + // only grab some versions at release time + // we aggregate later + dict.Add(Component.Configuration, Parse("TgsConfigVersion")); + dict.Add(Component.InteropApi, Parse("TgsInteropVersion")); + dict.Add(Component.HostWatchdog, Parse("TgsHostWatchdogVersion")); + dict.Add(Component.NugetCommon, Parse("TgsCommonLibraryVersion")); + dict.Add(Component.NugetApi, Parse("TgsApiLibraryVersion")); + dict.Add(Component.NugetClient, Parse("TgsClientVersion")); + + var webVersion = Parse("TgsControlPanelVersion"); + if (webVersion != null) + { + dict.Add(Component.WebControlPanel, webVersion); + } + else + { + var controlPanelVersionBytes = await RLR(() => client.Repository.Content.GetRawContentByRef(RepoOwner, RepoName, "build/ControlPanelVersion.props", mergeCommit)); + using (var ms = new MemoryStream(controlPanelVersionBytes)) + doc = XDocument.Load(ms); + + + project = doc.Root; + var controlPanelXmlNamespace = project.GetDefaultNamespace(); + var controlPanelVersionsPropertyGroup = project.Elements().First(x => x.Name == controlPanelXmlNamespace + "PropertyGroup"); + dict.Add(Component.WebControlPanel, Version.Parse(controlPanelVersionsPropertyGroup.Element(controlPanelXmlNamespace + "TgsControlPanelVersion").Value)); + } + } + + return dict; + } + catch + { + return new Dictionary(); + } + } + + var componentVersions = needComponentExactVersions ? GetComponentVersions() : Task.FromResult>(null); + var changelists = new ConcurrentDictionary(); + async Task BuildNotesFromComment(string comment, User user, Task localPreviousTask) + { + await localPreviousTask; + if (comment == null) + return; + + async Task CommitNotes(Component component, List notes) + { + foreach (var I in notes) + Console.WriteLine(component + " #" + fullPR.Number + " - " + I + " (@" + user.Login + ")"); + + var tupleSelector = notes.Select(note => new Change + { + Descriptions = new List { note }, + PullRequest = fullPR.Number, + Author = user.Login + }); + + var useExtendedComponentVersions = await needExtendedComponentVersions; + var componentVersionsResult = await componentVersions; + lock (changelists) + if (changelists.TryGetValue(component, out var currentChangelist)) + currentChangelist.Changes.AddRange(tupleSelector); + else + Debug.Assert(changelists.TryAdd(component, new Changelist + { + Changes = tupleSelector.ToList(), + Unreleased = false, + Version = needComponentExactVersions && componentVersionsResult.TryGetValue(component, out var componentVersion) + ? componentVersion + : null, + ComponentVersions = component == Component.Core && needComponentExactVersions && useExtendedComponentVersions + ? new Dictionary(componentVersionsResult.Where(kvp => kvp.Key != Component.Core)) + : null + })); + } + + var commentSplits = comment.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + string targetComponent = null; + var notes = new List(); + foreach (var line in commentSplits) + { + var trimmedLine = line.Trim(); + if (targetComponent == null) + { + if (trimmedLine.StartsWith(":cl:", StringComparison.Ordinal)) + { + targetComponent = trimmedLine[4..].Trim(); + if (targetComponent.Length == 0) + targetComponent = "Core"; + } + continue; + } + if (trimmedLine.StartsWith("/:cl:", StringComparison.Ordinal)) + { + if(!Enum.TryParse(targetComponent, out var component)) + switch (targetComponent.ToUpperInvariant()) + { + case "**CONFIGURATION**": + case "CONFIGURATION": + case "CONFIG": + component = Component.Configuration; + break; + case "HTTP API": + component = Component.HttpApi; + break; + case "WEB CONTROL PANEL": + component = Component.WebControlPanel; + break; + case "DMAPI": + case "DREAMMAKER API": + component = Component.DreamMakerApi; + break; + case "INTEROP API": + component = Component.InteropApi; + break; + case "HOST WATCHDOG": + component = Component.HostWatchdog; + break; + case "NUGET: API": + component = Component.NugetApi; + break; + case "NUGET: COMMON": + component = Component.NugetCommon; + break; + case "NUGET: CLIENT": + component = Component.NugetClient; + break; + default: + throw new Exception($"Unknown component: \"{targetComponent}\""); + } + + await CommitNotes(component, notes); + targetComponent = null; + notes.Clear(); + continue; + } + if (trimmedLine.Length == 0) + continue; + + notes.Add(trimmedLine); + } + } + + var previousTask = BuildNotesFromComment(fullPR.Body, fullPR.User, Task.CompletedTask); + var comments = await commentsTask; + foreach (var x in comments) + previousTask = BuildNotesFromComment(x.Body, x.User, previousTask); + + await previousTask; + + return Tuple.Create(changelists.ToDictionary(kvp => kvp.Key, kvp => kvp.Value), await componentVersions, isReleasePR); + } + class ExtendedReleaseUpdate : ReleaseUpdate { public bool? MakeLatest { get; set; } @@ -570,8 +778,429 @@ The user account that created this pull request is available to correct any issu { Body = prBody, }); - return 0; } + + static async Task RLR(Func> func) + { + while (true) + try + { + return await func(); + } + catch (HttpRequestException ex) when (ex.InnerException is IOException ioEx && ioEx.InnerException is SocketException sockEx && sockEx.ErrorCode == 10053) + { + await Task.Delay(15000); + } + catch (SecondaryRateLimitExceededException) + { + await Task.Delay(15000); + } + catch (RateLimitExceededException ex) + { + var now = DateTimeOffset.UtcNow.AddSeconds(-10); + if (ex.Reset > now) + { + var delay = ex.Reset - now; + await Task.Delay(delay); + } + } + } + + static async Task> TripleCheckGitHubPagination(Func>> apiCall, Func idSelector) + { + // I've seen GitHub pagination return incomplete result sets in the past + // It has an in-built pagination limit of 100 + var apiOptions = new ApiOptions + { + PageSize = 100 + }; + var results = await RLR(() => apiCall(apiOptions)); + Dictionary distinctEntries = new Dictionary(results.Count); + foreach (var result in results) + distinctEntries.Add(idSelector(result), result); + + if (results.Count > 100) + { + results = await RLR(() => apiCall(apiOptions)); + foreach (var result in results) + distinctEntries.TryAdd(idSelector(result), result); + + results = await RLR(() => apiCall(apiOptions)); + foreach (var result in results) + distinctEntries.TryAdd(idSelector(result), result); + } + + return distinctEntries.Values.ToList(); + } + + static async Task> ProcessMilestone(IGitHubClient client, Milestone milestone) + { + // have to trust this works + SearchIssuesResult results; + + var milestoneTask = Task.FromResult(milestone); + var pullRequests = new Dictionary(); + var iteration = 0; + while (true) + { + results = await RLR(() => client.Search.SearchIssues(new SearchIssuesRequest + { + Type = IssueTypeQualifier.PullRequest, + Milestone = milestone.Title, + Repos = new RepositoryCollection + { + { RepoOwner, RepoName }, + }, + })); + + foreach (var result in results.Items) + if (pullRequests.TryAdd(result.Number, result)) + milestoneTasks.TryAdd(result.Number, milestoneTask); + + if (results.IncompleteResults) + continue; + + if (results.TotalCount <= 100 || ++iteration == 3) + break; + } + + async Task RunPRs() + { + var milestoneVersion = Version.Parse(milestone.Title[1..]); + var prTasks = pullRequests.Select( + kvp => GetReleaseNotesFromPR(client, kvp.Value, true, true, milestone.State.Value == ItemState.Open)) + .ToList(); + + await Task.WhenAll(prTasks); + + var prResults = prTasks.Select(x => x.Result).Where(result => result != null).ToList(); + + var releasePRResult = prResults.FirstOrDefault(x => x.Item3); + + Dictionary releasedComponentVersions; + if (releasePRResult != null) + releasedComponentVersions = releasePRResult.Item2; + else + { + releasedComponentVersions = new Dictionary( + prResults + .SelectMany(result => result.Item2) + .GroupBy(kvp => kvp.Key) + .Select(grouping => new KeyValuePair(grouping.Key, grouping.Max(kvp => kvp.Value)))); + + foreach(var maxVersionKvp in prResults.SelectMany(x => x.Item1) + .Where(x => !releasedComponentVersions.ContainsKey(x.Key)) + .GroupBy(x => x.Key) + .Select(group => { + var versions = group + .Where(x => x.Value.Version != null) + .ToList(); + + if (versions.Count == 0) + return new KeyValuePair(group.Key, null); + + return new KeyValuePair(group.Key, versions.Max(x => x.Value.Version)); + }) + .Where(kvp => kvp.Value != null) + .ToList()) + { + releasedComponentVersions.Add(maxVersionKvp.Key, maxVersionKvp.Value); + } + } + + var finalResults = new Dictionary>(); + foreach (var componentKvp in releasedComponentVersions) + { + var component = componentKvp.Key; + var list = new List(); + + foreach(var changelistDict in prResults.Select(x => x.Item1)) + { + if (!changelistDict.TryGetValue(component, out var changelist)) + continue; + + Version componentVersion = milestoneVersion; + var unreleased = milestone.State.Value == ItemState.Open; + if (component != Component.Core) + { + componentVersion = changelist.Version ?? componentKvp.Value; + if (releasedNonCoreVersions != null + && releasedNonCoreVersions.TryGetValue(component, out var releasedVersions) + && !releasedVersions.Any(x => x == componentVersion)) + { + // roll forward + var newList = releasedVersions + .ToList(); + newList.Add(componentVersion); + newList = newList.OrderBy(x => x).ToList(); + + var index = newList.IndexOf(componentVersion); + Debug.Assert(index != -1); + if (index != (newList.Count - 1)) + { + componentVersion = newList[index + 1]; + unreleased = false; + } + else + unreleased = true; + } + } + + var entry = list.FirstOrDefault(x => x.Version == componentVersion); + if (entry == null) + { + entry = changelist; + entry.Version = componentVersion; + entry.Unreleased = unreleased; + list.Add(entry); + } + else + entry.Changes.AddRange(changelist.Changes); + } + + Debug.Assert(list.Select(x => x.Version.ToString()).Distinct().Count() == list.Count); + if (component == Component.Core) + { + Debug.Assert(list.All(x => x.Version == milestoneVersion)); + } + + list = list.OrderByDescending(x => x.Version).ToList(); + finalResults.Add(component, list); + } + + if (!finalResults.ContainsKey(Component.Core) || finalResults[Component.Core].Count == 0) + { + finalResults.Remove(Component.Core); + finalResults.Add(Component.Core, new List + { + new Changelist + { + Changes = new List(), + ComponentVersions = releasedComponentVersions, + Unreleased = milestone.State.Value == ItemState.Open, + Version = milestoneVersion, + } + }); + } + else + Debug.Assert(finalResults[Component.Core].All(x => x.Version == milestoneVersion)); + + var notes = new ReleaseNotes + { + Components = new SortedDictionary>(finalResults), + }; + + return notes; + } + + return RunPRs(); + } + + static async Task FullNotes(IGitHubClient client) + { + var startRateLimit = (client.GetLastApiInfo()?.RateLimit ?? (await client.RateLimit.GetRateLimits()).Rate).Remaining; + + ReleaseNotes existingNotes = null; + if (File.Exists("changelog.yml")) + { + var existingYml = await File.ReadAllTextAsync("changelog.yml"); + var deserializer = new DeserializerBuilder() + .Build(); + + existingNotes = deserializer.Deserialize(existingYml); + } + + var releaseNotes = await GenerateNotes(client, existingNotes); + + Console.WriteLine($"Generating all release notes took {startRateLimit - client.GetLastApiInfo().RateLimit.Remaining} requests."); + + var serializer = new SerializerBuilder() + .ConfigureDefaultValuesHandling(DefaultValuesHandling.OmitDefaults) + .WithTypeConverter(new VersionConverter()) + .Build(); + + var serializedYaml = serializer.Serialize(releaseNotes); + await File.WriteAllTextAsync("changelog.yml", serializedYaml).ConfigureAwait(false); + return 0; + } + + static HttpClient httpClient = new HttpClient(new HttpClientHandler() + { + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate + }); + static async Task> EnumerateNugetVersions(string package) + { + var url = new Uri($"https://api.nuget.org/v3/registration5-gz-semver2/{package.ToLowerInvariant()}/index.json"); + + using var req = new HttpRequestMessage(); + req.Headers.UserAgent.Add(new System.Net.Http.Headers.ProductInfoHeaderValue("Tgstation.Server.ReleaseNotes", "0.1.0")); + req.Method = HttpMethod.Get; + req.RequestUri = url; + + using var resp = await httpClient.SendAsync(req); + resp.EnsureSuccessStatusCode(); + + var json = await resp.Content.ReadAsStringAsync(); + + dynamic dynamicJson = JsonConvert.DeserializeObject(json); + + var versions = (IEnumerable)dynamicJson.items[0].items; + var results = versions + .Select(x => Version.TryParse((string)x.catalogEntry.version, out var version) ? version : null) + .Where(version => version != null) + .OrderBy(x => x) + .ToHashSet(); + return results; + } + + static IReadOnlyDictionary> releasedNonCoreVersions; + + static async Task GenerateNotes(IGitHubClient client, ReleaseNotes previousNotes) + { + var releasesTask = TripleCheckGitHubPagination( + apiOptions => client.Repository.Release.GetAll(RepoOwner, RepoName, apiOptions), + release => release.Id); + + var milestones = await TripleCheckGitHubPagination( + apiOptions => client.Issue.Milestone.GetAllForRepository(RepoOwner, RepoName, new MilestoneRequest { + State = ItemStateFilter.All + }, apiOptions), + milestone => milestone.Id); + + var versionMilestones = milestones + .Where(milestone => Regex.IsMatch(milestone.Title, @"v[1-9][0-9]*\.[1-9]*[0-9]+\.[1-9]*[0-9]+$")) + .ToList(); + + var releases = await releasesTask; + + var nugetCommonVersions = EnumerateNugetVersions("Tgstation.Server.Common"); + var nugetApiVersions = EnumerateNugetVersions("Tgstation.Server.Api"); + var nugetClientVersions = EnumerateNugetVersions("Tgstation.Server.Client"); + + releasedNonCoreVersions = new Dictionary> { + { Component.HttpApi, releases + .Where(x => x.TagName.StartsWith("api-v")) + .Select(x => Version.Parse(x.TagName[5..])) + .OrderBy(x => x) + .ToHashSet() }, + { Component.DreamMakerApi, releases + .Where(x => x.TagName.StartsWith("dmapi-v")) + .Select(x => Version.Parse(x.TagName[7..])) + .OrderBy(x => x) + .ToHashSet() }, + { Component.NugetCommon, await nugetCommonVersions }, + { Component.NugetApi, await nugetApiVersions }, + { Component.NugetClient, await nugetClientVersions } + }; + + var milestonesToProcess = versionMilestones; + if (previousNotes != null) + { + var releasedVersions = previousNotes.Components[Component.Core].Where(cl => !cl.Unreleased).ToList(); + milestonesToProcess = milestonesToProcess + .Where(x => !releasedVersions.Any( + version => version.Version == Version.Parse(x.Title.AsSpan(1)))) + .ToList(); + + foreach (var kvp in previousNotes.Components) + if (releasedNonCoreVersions.TryGetValue(kvp.Key, out var releasedComponentVersions)) + kvp.Value.RemoveAll(x => x.Unreleased = !releasedComponentVersions.Any(y => y == x.Version)); + else + kvp.Value.RemoveAll(x => x.Unreleased); + } + + var milestonePRTasks = milestonesToProcess + .Select(milestone => ProcessMilestone(client, milestone)) + .ToList(); + + await Task.WhenAll(milestonePRTasks); + + await Task.WhenAll(milestonePRTasks.Select(task => task.Result)); + + var coreCls = milestonePRTasks + .SelectMany(task => task.Result.Result.Components) + .Where(x => x.Key == Component.Core) + .ToList(); + + Debug.Assert( + coreCls.Count == milestonesToProcess.Count); + + var distinctCoreVersions = coreCls + .SelectMany(x => x.Value) + .Select(x => x.Version.ToString()) + .Distinct() + .Select(Version.Parse) + .OrderBy(x => x) + .ToList(); + + var missingCoreVersions = milestonesToProcess + .Where(x => !distinctCoreVersions.Any(y => Version.Parse(x.Title.Substring(1)) == y)) + .ToList(); + + Debug.Assert(missingCoreVersions.Count == 0); + foreach (var missingCoreVersion in missingCoreVersions) + await await ProcessMilestone(client, missingCoreVersion); + + var changelistsGroupedByComponent = + milestonePRTasks + .SelectMany(task => task.Result.Result.Components) + .GroupBy(kvp => kvp.Key) + .ToDictionary(grouping => grouping.Key, grouping => grouping.SelectMany(kvp => kvp.Value)); + + var releaseNotes = new ReleaseNotes + { + Components = new SortedDictionary>( + changelistsGroupedByComponent + .ToDictionary( + kvp => kvp.Key, + kvp => kvp + .Value + .GroupBy(changelist => changelist.Version) + .Select(grouping => + { + var firstEntry = grouping.First(); + return new Changelist + { + Changes = grouping.SelectMany(cl => cl.Changes).ToList(), + ComponentVersions = firstEntry.ComponentVersions, + Unreleased = firstEntry.Unreleased, + Version = grouping.Key + }; + }) + .OrderByDescending(cl => cl.Version) + .ToList())) + }; + + Debug.Assert(releaseNotes.Components.ContainsKey(Component.Core) && releaseNotes.Components[Component.Core].Count == milestonesToProcess.Count); + + if (previousNotes != null) + { + foreach (var component in Enum.GetValues()) + { + if (!previousNotes.Components.ContainsKey(component)) + continue; + + if (releaseNotes.Components.TryGetValue(component, out var changelists)) + releaseNotes.Components[component] = changelists + .Concat(previousNotes.Components[component]) + .OrderByDescending(cl => cl.Version) + .ToList(); + else + releaseNotes.Components[component] = previousNotes.Components[component]; + } + } + + foreach (var kvp in releaseNotes.Components) + { + var distinctCount = kvp.Value.Select(changelist => changelist.Version.ToString()).Distinct().Count(); + Debug.Assert(distinctCount == kvp.Value.Count); + + foreach (var cl in kvp.Value) + cl.DeduplicateChanges(); + } + + return releaseNotes; + } } } diff --git a/tools/Tgstation.Server.ReleaseNotes/ReleaseNotes.cs b/tools/Tgstation.Server.ReleaseNotes/ReleaseNotes.cs new file mode 100644 index 0000000000..8c10c880ce --- /dev/null +++ b/tools/Tgstation.Server.ReleaseNotes/ReleaseNotes.cs @@ -0,0 +1,9 @@ +using System.Collections.Generic; + +namespace Tgstation.Server.ReleaseNotes +{ + sealed class ReleaseNotes + { + public SortedDictionary> Components { get; set; } + } +} diff --git a/tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj b/tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj index d95f0139e3..35312949ce 100644 --- a/tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj +++ b/tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj @@ -7,10 +7,12 @@ - - + + + + From 34dbf590d9d0ed4ee36ecc19f0c5630946fa5dab Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 18 Aug 2023 10:58:29 -0400 Subject: [PATCH 02/22] Workflow to check that a PR has a milestone set --- .github/workflows/check-pr-has-milestone.yml | 32 ++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 .github/workflows/check-pr-has-milestone.yml diff --git a/.github/workflows/check-pr-has-milestone.yml b/.github/workflows/check-pr-has-milestone.yml new file mode 100644 index 0000000000..c360fb6397 --- /dev/null +++ b/.github/workflows/check-pr-has-milestone.yml @@ -0,0 +1,32 @@ +name: "Check PR Has Milestone" + +on: + pull_request_target: + types: [ opened, edited, synchronize, reopened ] + branches: + - dev + - master + - V6 + +concurrency: + group: "check-pr-milestone-${{ github.head_ref || github.run_id }}-${{ github.event_name }}" + cancel-in-progress: true + +jobs: + fail-on-bad-milestone: + name: Fail if Pull Request has no Associated Version Milestone + runs-on: ubuntu-latest + steps: + - name: Fail if no Milestone Set + if: github.event.pull_request.milestone == null + run: | + echo "::error::Pull request must have milestone set!" + exit 1 + + - name: Fail if Invalid Milestone Set + run: | + if [[ ${{ github.event.pull_request.milestone }} =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + exit 0 + fi + echo "::error::Pull request has non-standard milestone!" + exit 1 From 4ed976168261da2e332c1363603d044a452b6b86 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 18 Aug 2023 11:22:56 -0400 Subject: [PATCH 03/22] Build Changelog as part of CI Renaming `dox-build` job to `pages-build` --- .github/workflows/ci-pipeline.yml | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 3e3b3945da..9d09e6334e 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -163,12 +163,17 @@ jobs: fi exit $retval - dox-build: - name: Build Doxygen Site + pages-build: + name: Build gh-pages runs-on: ubuntu-latest needs: start-ci-run-gate if: "!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success'" steps: + - name: Setup dotnet + uses: actions/setup-dotnet@v2 + with: + dotnet-version: ${{ env.TGS_DOTNET_VERSION }} + - name: Checkout (Branch) uses: actions/checkout@v3 if: github.event_name == 'push' || github.event_name == 'schedule' @@ -179,6 +184,20 @@ jobs: with: ref: "refs/pull/${{ github.event.number }}/merge" + - name: gh-pages Clone + run: git clone -b gh-pages --single-branch "https://git@github.com/tgstation/tgstation-server" $HOME/tgsdox + + - name: Restore + run: dotnet restore + + - name: Build ReleaseNotes + run: dotnet build -c Release tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj + + - name: Build Changelog (Incremental) + run: | + cp $HOME/tgsdox/changelog.yml ./ 2>/dev/null + dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --generate-full-notes + - name: Patch Doxyfile run: | VERSION=$(cat "build/Version.props" | grep -oPm1 "(?<=)[^<]+") @@ -189,13 +208,13 @@ jobs: with: doxyfile-path: 'docs/Doxyfile' - - name: gh-pages push + - name: gh-pages Push if: github.event_name == 'push' && github.event.ref == 'refs/heads/dev' && env.TGS_RELEASE_NOTES_TOKEN != '' run: | - git clone -b gh-pages --single-branch "https://git@github.com/tgstation/tgstation-server" $HOME/tgsdox pushd $HOME/tgsdox rm -r * popd + sudo mv changlog.yml $HOME/tgsdox/ echo ./doxout/* | xargs -n 10 sudo mv -t $HOME/tgsdox cd $HOME/tgsdox git config --global push.default simple @@ -1211,9 +1230,9 @@ jobs: ci-completion-gate: # This job exists so there isn't a moving target for branch protections name: CI Completion Gate - needs: [ dox-build, docker-build, build-deb, build-msi, validate-openapi-spec, upload-code-coverage, check-winget-pr-template ] + needs: [ pages-build, docker-build, build-deb, build-msi, validate-openapi-spec, upload-code-coverage, check-winget-pr-template ] runs-on: ubuntu-latest - if: "!(cancelled() || failure()) && needs.dox-build.result == 'success' && needs.docker-build.result == 'success' && needs.build-deb.result == 'success' && needs.build-msi.result == 'success' && needs.validate-openapi-spec.result == 'success' && needs.upload-code-coverage.result == 'success' && needs.check-winget-pr-template.result == 'success'" + if: "!(cancelled() || failure()) && needs.pages-build.result == 'success' && needs.docker-build.result == 'success' && needs.build-deb.result == 'success' && needs.build-msi.result == 'success' && needs.validate-openapi-spec.result == 'success' && needs.upload-code-coverage.result == 'success' && needs.check-winget-pr-template.result == 'success'" steps: - name: GitHub Requires at Least One Step for a Job run: exit 0 From a165d393f46a89862b54eb2699c0e3b572fe8d2b Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 18 Aug 2023 11:32:48 -0400 Subject: [PATCH 04/22] Allow host to be built without webpanel - Allows for a speedup when just building ReleaseNotes in CI --- .github/workflows/ci-pipeline.yml | 22 +++++++++++++------ .../ControlPanelConfiguration.cs | 13 ++++++++++- src/Tgstation.Server.Host/Core/Application.cs | 4 ++++ .../Tgstation.Server.Host.csproj | 8 +++++-- 4 files changed, 37 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 9d09e6334e..9450a014a1 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -191,7 +191,9 @@ jobs: run: dotnet restore - name: Build ReleaseNotes - run: dotnet build -c Release tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj + run: | + export TGS_HOST_NO_WEBPANEL=true + dotnet build -c Release tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj - name: Build Changelog (Incremental) run: | @@ -1223,7 +1225,9 @@ jobs: run: dotnet restore - name: Build ReleaseNotes - run: dotnet build -c Release tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj + run: | + export TGS_HOST_NO_WEBPANEL=true + dotnet build -c Release tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj - name: Run ReleaseNotes Check run: dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --winget-template-check ${{ steps.get-sha.outputs.pr_template_sha }} @@ -1387,7 +1391,9 @@ jobs: run: dotnet restore - name: Build ReleaseNotes - run: dotnet build -c Release tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj + run: | + export TGS_HOST_NO_WEBPANEL=true + dotnet build -c Release tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj - name: Run ReleaseNotes with --ensure-release run: dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --ensure-release @@ -1414,9 +1420,6 @@ jobs: cd build/package/winget dotnet tool restore - - name: Build ReleaseNotes - run: dotnet build -c Release tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj - # We need to rebuild the installer.exe so it can be properly signed - name: Restore run: dotnet restore @@ -1427,6 +1430,9 @@ jobs: - name: Build Service run: dotnet build -c Release src/Tgstation.Server.Host.Service/Tgstation.Server.Host.Service.csproj + - name: Build ReleaseNotes + run: dotnet build -c Release tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj + - name: Prepare Artifacts shell: powershell run: build/package/winget/prepare_installer_input_artifacts.ps1 @@ -1653,7 +1659,9 @@ jobs: uses: actions/checkout@v3 - name: Build ReleaseNotes - run: dotnet build -c Release tools/Tgstation.Server.ReleaseNotes + run: | + $Env:TGS_HOST_NO_WEBPANEL=true + dotnet build -c Release tools/Tgstation.Server.ReleaseNotes - name: Execute Push Script shell: powershell diff --git a/src/Tgstation.Server.Host/Configuration/ControlPanelConfiguration.cs b/src/Tgstation.Server.Host/Configuration/ControlPanelConfiguration.cs index b54c471f65..19dfa9cace 100644 --- a/src/Tgstation.Server.Host/Configuration/ControlPanelConfiguration.cs +++ b/src/Tgstation.Server.Host/Configuration/ControlPanelConfiguration.cs @@ -15,7 +15,18 @@ namespace Tgstation.Server.Host.Configuration /// /// If the control panel is enabled. /// - public bool Enable { get; set; } + public bool Enable +#if NO_WEBPANEL + { + get => false; + set => _ = value; + } +#else + { + get; + set; + } +#endif /// /// If any origin is allowed for CORS requests. This overrides . diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index 591953c0d8..ed1cb97cad 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -508,7 +508,11 @@ namespace Tgstation.Server.Host.Core }); } else +#if NO_WEBPANEL + logger.LogTrace("Web control panel was not included in TGS build!"); +#else logger.LogTrace("Web control panel disabled!"); +#endif // Do not cache a single thing beyond this point, it's all API applicationBuilder.UseDisabledClientCache(); diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index 7808673608..dc49c33df0 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -18,7 +18,11 @@ $(DefineConstants);WATCHDOG_FREE_RESTART - + + $(DefineConstants);NO_WEBPANEL + + + @@ -28,7 +32,7 @@ - + From bc14c2de07b380e4e6fc14950107828474c5e043 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 18 Aug 2023 11:33:10 -0400 Subject: [PATCH 05/22] Remove extra restore step in `tgs-deploy` job --- .github/workflows/ci-pipeline.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 9450a014a1..0139ecf6e5 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -1421,8 +1421,6 @@ jobs: dotnet tool restore # We need to rebuild the installer.exe so it can be properly signed - - name: Restore - run: dotnet restore - name: Build Host run: dotnet build -c Release src/Tgstation.Server.Host/Tgstation.Server.Host.csproj From 29929d2978bf789712272c004466597ddea86415 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 18 Aug 2023 11:35:42 -0400 Subject: [PATCH 06/22] Switch to MariaDB stable to fix build Note they have a REST API for releases which we will likely need to use to avoid more 404s in the future: https://mariadb.org/downloads-rest-api/ --- build/Version.props | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/Version.props b/build/Version.props index f8653e54ae..49a571b3de 100644 --- a/build/Version.props +++ b/build/Version.props @@ -18,8 +18,8 @@ 6 https://dotnetcli.azureedge.net/dotnet/aspnetcore/Runtime/6.0.21/dotnet-hosting-6.0.21-win.exe - 11.0.2 + 10.11.5 - https://ftp.osuosl.org/pub/mariadb/mariadb-11.0.2/winx64-packages/mariadb-11.0.2-winx64.msi + https://ftp.osuosl.org/pub/mariadb//mariadb-10.11.5/winx64-packages/mariadb-10.11.5-winx64.msi From f82463337adda20a21fcef2019f545c48d022423 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 18 Aug 2023 11:38:43 -0400 Subject: [PATCH 07/22] Remove unnecessary CA1506 suppression --- src/Tgstation.Server.Host/Core/Application.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index ed1cb97cad..d3e09adf7d 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -59,7 +59,6 @@ namespace Tgstation.Server.Host.Core /// /// Sets up dependency injection. /// -#pragma warning disable CA1506 public sealed class Application : SetupApplication { /// From f33e682d2163f1718b9db1405ec09c0c36966165 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 18 Aug 2023 11:40:46 -0400 Subject: [PATCH 08/22] Add link to changelog.yml in GitHub release --- tools/Tgstation.Server.ReleaseNotes/Program.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/Tgstation.Server.ReleaseNotes/Program.cs b/tools/Tgstation.Server.ReleaseNotes/Program.cs index e5a95795d6..f565e04dc1 100644 --- a/tools/Tgstation.Server.ReleaseNotes/Program.cs +++ b/tools/Tgstation.Server.ReleaseNotes/Program.cs @@ -204,7 +204,7 @@ namespace Tgstation.Server.ReleaseNotes if (webControlVersion.Major == 0) postControlPanelMessage = true; - prefix = $"Please refer to the [README](https://github.com/tgstation/tgstation-server#setup) for setup instructions.{Environment.NewLine}{Environment.NewLine}#### Component Versions\nCore: {coreVersion}\nConfiguration: {configVersion}\nHTTP API: {apiVersion}\nDreamMaker API: {dmApiVersion} (Interop: {interopVersion})\n[Web Control Panel](https://github.com/tgstation/tgstation-server-webpanel): {webControlVersion}\nHost Watchdog: {hostWatchdogVersion}"; + prefix = $"Please refer to the [README](https://github.com/tgstation/tgstation-server#setup) for setup instructions. Full changelog can be found [here](https://raw.githubusercontent.com/tgstation/tgstation-server/gh-pages/changelog.yml).{Environment.NewLine}{Environment.NewLine}#### Component Versions\nCore: {coreVersion}\nConfiguration: {configVersion}\nHTTP API: {apiVersion}\nDreamMaker API: {dmApiVersion} (Interop: {interopVersion})\n[Web Control Panel](https://github.com/tgstation/tgstation-server-webpanel): {webControlVersion}\nHost Watchdog: {hostWatchdogVersion}"; var newNotes = new StringBuilder(prefix); if (postControlPanelMessage) From 0bb479f100b0764db7c16d4f188c717cff3c7c1e Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 18 Aug 2023 11:49:20 -0400 Subject: [PATCH 09/22] Change milestone check to `pull_request` `pull_request_target` would only be needed if it accessed secrets --- .github/workflows/check-pr-has-milestone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-pr-has-milestone.yml b/.github/workflows/check-pr-has-milestone.yml index c360fb6397..11c87a22db 100644 --- a/.github/workflows/check-pr-has-milestone.yml +++ b/.github/workflows/check-pr-has-milestone.yml @@ -1,7 +1,7 @@ name: "Check PR Has Milestone" on: - pull_request_target: + pull_request: types: [ opened, edited, synchronize, reopened ] branches: - dev From 3d6065f4c40594ffc1ee04e2b7262a5ba5556535 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 18 Aug 2023 12:29:55 -0400 Subject: [PATCH 10/22] Fix bad field reference in milestone check --- .github/workflows/check-pr-has-milestone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-pr-has-milestone.yml b/.github/workflows/check-pr-has-milestone.yml index 11c87a22db..954d539927 100644 --- a/.github/workflows/check-pr-has-milestone.yml +++ b/.github/workflows/check-pr-has-milestone.yml @@ -25,7 +25,7 @@ jobs: - name: Fail if Invalid Milestone Set run: | - if [[ ${{ github.event.pull_request.milestone }} =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + if [[ ${{ github.event.pull_request.milestone.title }} =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then exit 0 fi echo "::error::Pull request has non-standard milestone!" From 53ff35efb47aa188a1b859744a4ab7d957558e60 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 18 Aug 2023 12:30:21 -0400 Subject: [PATCH 11/22] Revert "Remove unnecessary CA1506 suppression" This reverts commit f82463337adda20a21fcef2019f545c48d022423. --- src/Tgstation.Server.Host/Core/Application.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index d3e09adf7d..ed1cb97cad 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -59,6 +59,7 @@ namespace Tgstation.Server.Host.Core /// /// Sets up dependency injection. /// +#pragma warning disable CA1506 public sealed class Application : SetupApplication { /// From 12250239e0e104e2a6141384710ed17329d3a709 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Fri, 18 Aug 2023 13:02:32 -0400 Subject: [PATCH 12/22] Add missing `ArgumentNullException` check --- .../Extensions/Converters/VersionConverter.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Tgstation.Server.Host/Extensions/Converters/VersionConverter.cs b/src/Tgstation.Server.Host/Extensions/Converters/VersionConverter.cs index 4ca9cdf105..0373dc5e20 100644 --- a/src/Tgstation.Server.Host/Extensions/Converters/VersionConverter.cs +++ b/src/Tgstation.Server.Host/Extensions/Converters/VersionConverter.cs @@ -35,6 +35,8 @@ namespace Tgstation.Server.Host.Extensions.Converters /// public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) { + ArgumentNullException.ThrowIfNull(writer); + if (value == null) { writer.WriteNull(); From 85d66b6ece1f660379bb7e70e8c44d9e7a9c27a7 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 20 Aug 2023 02:24:00 -0400 Subject: [PATCH 13/22] Add DMAPI/HTTPAPI release notes generation --- .github/workflows/ci-pipeline.yml | 52 ++++- .../Tgstation.Server.ReleaseNotes/Program.cs | 195 +++++++++++------- 2 files changed, 166 insertions(+), 81 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index b0492e230c..6e74f72fcf 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -1256,9 +1256,22 @@ jobs: runs-on: windows-latest if: (!(cancelled() || failure()) && needs.deployment-gate.result == 'success' && github.event_name == 'push' && contains(github.event.head_commit.message, '[APIDeploy]')) steps: + - name: Setup dotnet + uses: actions/setup-dotnet@v2 + with: + dotnet-version: ${{ env.TGS_DOTNET_VERSION }} + - name: Checkout uses: actions/checkout@v3 + - name: Restore + run: dotnet restore + + - name: Build ReleaseNotes + run: | + export TGS_HOST_NO_WEBPANEL=true + dotnet build -c Release tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj + - name: Parse API version shell: powershell run: | @@ -1272,15 +1285,24 @@ jobs: name: openapi-spec path: swagger + - name: Grab Most Recent Changelog + shell: powershell + run: | + $ProgressPreference = 'SilentlyContinue' + Invoke-WebRequest -Uri https://raw.githubusercontent.com/tgstation/tgstation-server/gh-pages/changelog.yml -OutFile changelog.yml + + - name: Generate Release Notes + run: dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes ${{ env.TGS_API_VERSION }} --httpapi + - name: Create GitHub Release - uses: juitnow/github-action-create-release@80dc88a9d09c0ee394a4a31b0450e8cbe62996ef # v1 + uses: actions/create-release@v1 id: create_release env: GITHUB_TOKEN: ${{ secrets.DEV_PUSH_TOKEN }} with: tag_name: api-v${{ env.TGS_API_VERSION }} release_name: tgstation-server API v${{ env.TGS_API_VERSION }} - body: The TGS HTTP API + body_path: release_notes.md commitish: ${{ github.event.head_commit.id }} - name: Upload OpenApi Spec @@ -1299,9 +1321,22 @@ jobs: runs-on: windows-latest if: (!(cancelled() || failure()) && needs.deployment-gate.result == 'success' && github.event_name == 'push' && contains(github.event.head_commit.message, '[DMDeploy]')) steps: + - name: Setup dotnet + uses: actions/setup-dotnet@v2 + with: + dotnet-version: ${{ env.TGS_DOTNET_VERSION }} + - name: Checkout uses: actions/checkout@v3 + - name: Restore + run: dotnet restore + + - name: Build ReleaseNotes + run: | + export TGS_HOST_NO_WEBPANEL=true + dotnet build -c Release tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj + - name: Parse DMAPI version shell: powershell run: | @@ -1314,15 +1349,24 @@ jobs: run: | &"C:/Program Files/7-Zip/7z.exe" a DMAPI.zip ./src/DMAPI/* -tzip + - name: Grab Most Recent Changelog + shell: powershell + run: | + $ProgressPreference = 'SilentlyContinue' + Invoke-WebRequest -Uri https://raw.githubusercontent.com/tgstation/tgstation-server/gh-pages/changelog.yml -OutFile changelog.yml + + - name: Generate Release Notes + run: dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes ${{ env.TGS_DM_VERSION }} --dmapi + - name: Create GitHub Release - uses: juitnow/github-action-create-release@v1 + uses: actions/create-release@v1 id: create_release env: GITHUB_TOKEN: ${{ secrets.DEV_PUSH_TOKEN }} with: tag_name: dmapi-v${{ env.TGS_DM_VERSION }} release_name: tgstation-server DMAPI v${{ env.TGS_DM_VERSION }} - body: The TGS DMAPI \#tgs-dmapi-release + body_path: release_notes.md commitish: ${{ github.event.head_commit.id }} - name: Upload DMAPI Artifact diff --git a/tools/Tgstation.Server.ReleaseNotes/Program.cs b/tools/Tgstation.Server.ReleaseNotes/Program.cs index f565e04dc1..d336cd8367 100644 --- a/tools/Tgstation.Server.ReleaseNotes/Program.cs +++ b/tools/Tgstation.Server.ReleaseNotes/Program.cs @@ -30,6 +30,7 @@ namespace Tgstation.Server.ReleaseNotes /// static class Program { + const string OutputPath = "release_notes.md"; const string RepoOwner = "tgstation"; const string RepoName = "tgstation-server"; @@ -50,13 +51,34 @@ namespace Tgstation.Server.ReleaseNotes var shaCheck = versionString.Equals("--winget-template-check", StringComparison.OrdinalIgnoreCase); var fullNotes = versionString.Equals("--generate-full-notes", StringComparison.OrdinalIgnoreCase); - if ((!Version.TryParse(versionString, out var version) || version.Revision != -1) && !ensureRelease && !linkWinget && !shaCheck && !fullNotes) + if ((!Version.TryParse(versionString, out var version) || version.Revision != -1) + && !ensureRelease + && !linkWinget + && !shaCheck + && !fullNotes) { Console.WriteLine("Invalid version: " + versionString); return 2; } - var doNotCloseMilestone = args.Length > 1 && args[1].ToUpperInvariant() == "--NO-CLOSE"; + var doNotCloseMilestone = false; + Component? componentRelease = null; + if (args.Length > 1) + switch (args[1].ToUpperInvariant()) + { + case "--NO-CLOSE": + doNotCloseMilestone = true; + break; + case "--HTTPAPI": + componentRelease = Component.HttpApi; + break; + case "--INTEROPAPI": + componentRelease = Component.InteropApi; + break; + case "--DMAPI": + componentRelease = Component.DreamMakerApi; + break; + } const string ReleaseNotesEnvVar = "TGS_RELEASE_NOTES_TOKEN"; var githubToken = Environment.GetEnvironmentVariable(ReleaseNotesEnvVar); @@ -100,9 +122,10 @@ namespace Tgstation.Server.ReleaseNotes } if (fullNotes) - { return await FullNotes(client); - } + + if (componentRelease.HasValue) + return await ReleaseComponent(client, version, componentRelease.Value); var releasesTask = client.Repository.Release.GetAll(RepoOwner, RepoName); @@ -396,18 +419,7 @@ namespace Tgstation.Server.ReleaseNotes }; newNotes.Append(componentName); - foreach (var change in I.Value.Changes) - foreach (var line in change.Descriptions) - { - newNotes.Append(Environment.NewLine); - newNotes.Append("- "); - newNotes.Append(line); - newNotes.Append(" (#"); - newNotes.Append(change.PullRequest); - newNotes.Append(" @"); - newNotes.Append(change.Author); - newNotes.Append(')'); - } + PrintChanges(newNotes, I.Value); newNotes.Append(Environment.NewLine); } @@ -417,7 +429,6 @@ namespace Tgstation.Server.ReleaseNotes if (version.Minor != 0 && version.Build != 0) newNotes.Append(oldNotes); - const string OutputPath = "release_notes.md"; Console.WriteLine($"Writing out new release notes to {Path.GetFullPath(OutputPath)}..."); var releaseNotes = newNotes.ToString(); await File.WriteAllTextAsync(OutputPath, releaseNotes).ConfigureAwait(false); @@ -450,9 +461,7 @@ namespace Tgstation.Server.ReleaseNotes } } - static ConcurrentDictionary> milestoneTasks = new ConcurrentDictionary>(); - static Task GetMilestone(IGitHubClient client, int number) - => milestoneTasks.GetOrAdd(number, localNumber => client.Issue.Milestone.Get(RepoOwner, RepoName, localNumber)); + static readonly ConcurrentDictionary> milestoneTasks = new (); static async Task, Dictionary, bool>> GetReleaseNotesFromPR(IGitHubClient client, Issue pullRequest, bool doNotCloseMilestone, bool needComponentExactVersions, bool forAllComponents) { @@ -623,42 +632,19 @@ namespace Tgstation.Server.ReleaseNotes if (trimmedLine.StartsWith("/:cl:", StringComparison.Ordinal)) { if(!Enum.TryParse(targetComponent, out var component)) - switch (targetComponent.ToUpperInvariant()) + component = targetComponent.ToUpperInvariant() switch { - case "**CONFIGURATION**": - case "CONFIGURATION": - case "CONFIG": - component = Component.Configuration; - break; - case "HTTP API": - component = Component.HttpApi; - break; - case "WEB CONTROL PANEL": - component = Component.WebControlPanel; - break; - case "DMAPI": - case "DREAMMAKER API": - component = Component.DreamMakerApi; - break; - case "INTEROP API": - component = Component.InteropApi; - break; - case "HOST WATCHDOG": - component = Component.HostWatchdog; - break; - case "NUGET: API": - component = Component.NugetApi; - break; - case "NUGET: COMMON": - component = Component.NugetCommon; - break; - case "NUGET: CLIENT": - component = Component.NugetClient; - break; - default: - throw new Exception($"Unknown component: \"{targetComponent}\""); - } - + "**CONFIGURATION**" or "CONFIGURATION" or "CONFIG" => Component.Configuration, + "HTTP API" => Component.HttpApi, + "WEB CONTROL PANEL" => Component.WebControlPanel, + "DMAPI" or "DREAMMAKER API" => Component.DreamMakerApi, + "INTEROP API" => Component.InteropApi, + "HOST WATCHDOG" => Component.HostWatchdog, + "NUGET: API" => Component.NugetApi, + "NUGET: COMMON" => Component.NugetCommon, + "NUGET: CLIENT" => Component.NugetClient, + _ => throw new Exception($"Unknown component: \"{targetComponent}\""), + }; await CommitNotes(component, notes); targetComponent = null; notes.Clear(); @@ -816,7 +802,7 @@ The user account that created this pull request is available to correct any issu PageSize = 100 }; var results = await RLR(() => apiCall(apiOptions)); - Dictionary distinctEntries = new Dictionary(results.Count); + var distinctEntries = new Dictionary(results.Count); foreach (var result in results) distinctEntries.Add(idSelector(result), result); @@ -1001,17 +987,7 @@ The user account that created this pull request is available to correct any issu { var startRateLimit = (client.GetLastApiInfo()?.RateLimit ?? (await client.RateLimit.GetRateLimits()).Rate).Remaining; - ReleaseNotes existingNotes = null; - if (File.Exists("changelog.yml")) - { - var existingYml = await File.ReadAllTextAsync("changelog.yml"); - var deserializer = new DeserializerBuilder() - .Build(); - - existingNotes = deserializer.Deserialize(existingYml); - } - - var releaseNotes = await GenerateNotes(client, existingNotes); + var releaseNotes = await GenerateNotes(client); Console.WriteLine($"Generating all release notes took {startRateLimit - client.GetLastApiInfo().RateLimit.Remaining} requests."); @@ -1025,10 +1001,11 @@ The user account that created this pull request is available to correct any issu return 0; } - static HttpClient httpClient = new HttpClient(new HttpClientHandler() - { - AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate - }); + static readonly HttpClient httpClient = new ( + new HttpClientHandler() + { + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate + }); static async Task> EnumerateNugetVersions(string package) { var url = new Uri($"https://api.nuget.org/v3/registration5-gz-semver2/{package.ToLowerInvariant()}/index.json"); @@ -1056,8 +1033,18 @@ The user account that created this pull request is available to correct any issu static IReadOnlyDictionary> releasedNonCoreVersions; - static async Task GenerateNotes(IGitHubClient client, ReleaseNotes previousNotes) + static async Task GenerateNotes(IGitHubClient client, Tuple forceReleaseVersion = null) { + ReleaseNotes previousNotes = null; + if (File.Exists("changelog.yml")) + { + var existingYml = await File.ReadAllTextAsync("changelog.yml"); + var deserializer = new DeserializerBuilder() + .Build(); + + previousNotes = deserializer.Deserialize(existingYml); + } + var releasesTask = TripleCheckGitHubPagination( apiOptions => client.Repository.Release.GetAll(RepoOwner, RepoName, apiOptions), release => release.Id); @@ -1078,7 +1065,7 @@ The user account that created this pull request is available to correct any issu var nugetApiVersions = EnumerateNugetVersions("Tgstation.Server.Api"); var nugetClientVersions = EnumerateNugetVersions("Tgstation.Server.Client"); - releasedNonCoreVersions = new Dictionary> { + var newDic = new Dictionary> { { Component.HttpApi, releases .Where(x => x.TagName.StartsWith("api-v")) .Select(x => Version.Parse(x.TagName[5..])) @@ -1094,6 +1081,14 @@ The user account that created this pull request is available to correct any issu { Component.NugetClient, await nugetClientVersions } }; + if (forceReleaseVersion != null && !newDic[forceReleaseVersion.Item1].Any(x => x == forceReleaseVersion.Item2)) + newDic[forceReleaseVersion.Item1] = newDic[forceReleaseVersion.Item1] + .Concat(new List { forceReleaseVersion.Item2 }) + .OrderBy(x => x) + .ToHashSet(); + + releasedNonCoreVersions = newDic; + var milestonesToProcess = versionMilestones; if (previousNotes != null) { @@ -1135,7 +1130,7 @@ The user account that created this pull request is available to correct any issu .ToList(); var missingCoreVersions = milestonesToProcess - .Where(x => !distinctCoreVersions.Any(y => Version.Parse(x.Title.Substring(1)) == y)) + .Where(x => !distinctCoreVersions.Any(y => Version.Parse(x.Title.AsSpan(1)) == y)) .ToList(); Debug.Assert(missingCoreVersions.Count == 0); @@ -1181,11 +1176,27 @@ The user account that created this pull request is available to correct any issu if (!previousNotes.Components.ContainsKey(component)) continue; - if (releaseNotes.Components.TryGetValue(component, out var changelists)) - releaseNotes.Components[component] = changelists - .Concat(previousNotes.Components[component]) + if (releaseNotes.Components.TryGetValue(component, out var newChangelists)) + { + var missingVersions = previousNotes.Components[component] + .Where(olderVersion => + { + var newerVersion = newChangelists.SingleOrDefault(y => olderVersion.Version == y.Version); + if (newerVersion != null) + { + newerVersion.Changes.AddRange( + olderVersion.Changes.Where(x => !newerVersion.Changes.Any(y => x.PullRequest == y.PullRequest))); + return false; + } + + return true; + }); + + releaseNotes.Components[component] = newChangelists + .Concat(missingVersions) .OrderByDescending(cl => cl.Version) .ToList(); + } else releaseNotes.Components[component] = previousNotes.Components[component]; } @@ -1202,5 +1213,35 @@ The user account that created this pull request is available to correct any issu return releaseNotes; } + + static void PrintChanges(StringBuilder newNotes, Changelist changelist) + { + foreach (var change in changelist.Changes) + foreach (var line in change.Descriptions) + { + newNotes.Append(Environment.NewLine); + newNotes.Append("- "); + newNotes.Append(line); + newNotes.Append(" (#"); + newNotes.Append(change.PullRequest); + newNotes.Append(" @"); + newNotes.Append(change.Author); + newNotes.Append(')'); + } + } + + static async Task ReleaseComponent(IGitHubClient client, Version version, Component component) + { + var releaseNotes = await GenerateNotes(client, Tuple.Create(component, version)); + var relevantChangelog = releaseNotes.Components[component].First(x => x.Version == version); + + var newNotes = new StringBuilder("Full changelog can be found [here](https://raw.githubusercontent.com/tgstation/tgstation-server/gh-pages/changelog.yml)."); + newNotes.AppendLine(); + PrintChanges(newNotes, relevantChangelog); + + var markdown = newNotes.ToString(); + await File.WriteAllTextAsync(OutputPath, markdown); + return 0; + } } } From 0f65522efd1738e3a70030e7d976ff089bb1ad7d Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 20 Aug 2023 02:49:34 -0400 Subject: [PATCH 14/22] Setup autogenerated Nuget release notes --- .github/workflows/ci-pipeline.yml | 14 +++ .../Tgstation.Server.Api.csproj | 2 +- .../Tgstation.Server.Client.csproj | 2 +- .../Tgstation.Server.Common.csproj | 2 +- .../Tgstation.Server.ReleaseNotes/Program.cs | 93 ++++++++++++++++--- 5 files changed, 97 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 6e74f72fcf..714ea46c77 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -1393,6 +1393,20 @@ jobs: - name: Checkout uses: actions/checkout@v3 + - name: Restore + run: dotnet restore + + - name: Build ReleaseNotes + run: | + export TGS_HOST_NO_WEBPANEL=true + dotnet build -c Release tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj + + - name: Grab Most Recent Changelog + run: curl -L https://raw.githubusercontent.com/tgstation/tgstation-server/gh-pages/changelog.yml -o changelog.yml + + - name: Generate Release Notes + run: dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes ${{ env.TGS_DM_VERSION }} --nuget + - name: Publish Tgstation.Server.Common to NuGet uses: alirezanet/publish-nuget@e276c40afeb2a154046f0997820f2a9ea74832d9 # v3.1.0 with: diff --git a/src/Tgstation.Server.Api/Tgstation.Server.Api.csproj b/src/Tgstation.Server.Api/Tgstation.Server.Api.csproj index 913f4db126..582c079b6e 100644 --- a/src/Tgstation.Server.Api/Tgstation.Server.Api.csproj +++ b/src/Tgstation.Server.Api/Tgstation.Server.Api.csproj @@ -6,7 +6,7 @@ $(TgsApiLibraryVersion) API definitions for tgstation-server. json web api tgstation-server tgstation ss13 byond http - Added support for -map-threads DreamDaemon parameter and UploadZip Administration update option. + $(TGS_NUGET_RELEASE_NOTES_API) CA1028 diff --git a/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj b/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj index 67bd6448a3..af89800112 100644 --- a/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj +++ b/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj @@ -6,7 +6,7 @@ $(TgsClientVersion) Client library for tgstation-server. json web api tgstation-server tgstation ss13 byond client http - Added support for .zip uploads for server updates. Added missing Dispose() call to the StringContents for requests with bodies and added missing ConfigureAwait(false) to async call. + $(TGS_NUGET_RELEASE_NOTES_CLIENT) diff --git a/src/Tgstation.Server.Common/Tgstation.Server.Common.csproj b/src/Tgstation.Server.Common/Tgstation.Server.Common.csproj index 29d40742bc..405226e89f 100644 --- a/src/Tgstation.Server.Common/Tgstation.Server.Common.csproj +++ b/src/Tgstation.Server.Common/Tgstation.Server.Common.csproj @@ -6,7 +6,7 @@ $(TgsCommonLibraryVersion) Common functions for tgstation-server. web tgstation-server tgstation ss13 byond client http - Added CachedResponseStream. + $(TGS_NUGET_RELEASE_NOTES_COMMON) diff --git a/tools/Tgstation.Server.ReleaseNotes/Program.cs b/tools/Tgstation.Server.ReleaseNotes/Program.cs index d336cd8367..c02128c7db 100644 --- a/tools/Tgstation.Server.ReleaseNotes/Program.cs +++ b/tools/Tgstation.Server.ReleaseNotes/Program.cs @@ -9,6 +9,7 @@ using System.Linq; using System.Net; using System.Net.Http; using System.Net.Sockets; +using System.Security; using System.Text; using System.Text.RegularExpressions; using System.Threading.Tasks; @@ -50,12 +51,14 @@ namespace Tgstation.Server.ReleaseNotes var linkWinget = versionString.Equals("--link-winget", StringComparison.OrdinalIgnoreCase); var shaCheck = versionString.Equals("--winget-template-check", StringComparison.OrdinalIgnoreCase); var fullNotes = versionString.Equals("--generate-full-notes", StringComparison.OrdinalIgnoreCase); + var nuget = versionString.Equals("--nuget", StringComparison.OrdinalIgnoreCase); if ((!Version.TryParse(versionString, out var version) || version.Revision != -1) && !ensureRelease && !linkWinget && !shaCheck - && !fullNotes) + && !fullNotes + && !nuget) { Console.WriteLine("Invalid version: " + versionString); return 2; @@ -127,6 +130,9 @@ namespace Tgstation.Server.ReleaseNotes if (componentRelease.HasValue) return await ReleaseComponent(client, version, componentRelease.Value); + if (nuget) + return await ReleaseNuget(client); + var releasesTask = client.Repository.Release.GetAll(RepoOwner, RepoName); Console.WriteLine("Getting merged pull requests in milestone " + versionString + "..."); @@ -1033,7 +1039,7 @@ The user account that created this pull request is available to correct any issu static IReadOnlyDictionary> releasedNonCoreVersions; - static async Task GenerateNotes(IGitHubClient client, Tuple forceReleaseVersion = null) + static async Task GenerateNotes(IGitHubClient client, Dictionary forceReleaseVersions = null) { ReleaseNotes previousNotes = null; if (File.Exists("changelog.yml")) @@ -1081,11 +1087,13 @@ The user account that created this pull request is available to correct any issu { Component.NugetClient, await nugetClientVersions } }; - if (forceReleaseVersion != null && !newDic[forceReleaseVersion.Item1].Any(x => x == forceReleaseVersion.Item2)) - newDic[forceReleaseVersion.Item1] = newDic[forceReleaseVersion.Item1] - .Concat(new List { forceReleaseVersion.Item2 }) - .OrderBy(x => x) - .ToHashSet(); + if (forceReleaseVersions != null) + foreach (var kvp in forceReleaseVersions) + if (!newDic[kvp.Key].Any(x => x == kvp.Value)) + newDic[kvp.Key] = newDic[kvp.Key] + .Concat(new List { kvp.Value }) + .OrderBy(x => x) + .ToHashSet(); releasedNonCoreVersions = newDic; @@ -1230,17 +1238,76 @@ The user account that created this pull request is available to correct any issu } } - static async Task ReleaseComponent(IGitHubClient client, Version version, Component component) + static string GenerateComponentNotes(ReleaseNotes releaseNotes, Component component, Version version) { - var releaseNotes = await GenerateNotes(client, Tuple.Create(component, version)); - var relevantChangelog = releaseNotes.Components[component].First(x => x.Version == version); + var relevantChangelog = releaseNotes.Components[component].FirstOrDefault(x => x.Version == version); var newNotes = new StringBuilder("Full changelog can be found [here](https://raw.githubusercontent.com/tgstation/tgstation-server/gh-pages/changelog.yml)."); - newNotes.AppendLine(); - PrintChanges(newNotes, relevantChangelog); + if (relevantChangelog != null) + { + newNotes.AppendLine(); + PrintChanges(newNotes, relevantChangelog); + } var markdown = newNotes.ToString(); - await File.WriteAllTextAsync(OutputPath, markdown); + return markdown; + } + + static async Task ReleaseComponent(IGitHubClient client, Version version, Component component) + { + var releaseNotes = await GenerateNotes(client, new Dictionary { { component, version } }); + await File.WriteAllTextAsync(OutputPath, GenerateComponentNotes(releaseNotes, component, version)); + return 0; + } + + // must run from repo root + static async Task ReleaseNuget(IGitHubClient client) + { + const string PropsPath = "build/Version.props"; + + var doc = XDocument.Load(PropsPath); + var project = doc.Root; + var xmlNamespace = project.GetDefaultNamespace(); + var versionsPropertyGroup = project.Elements().First(x => x.Name == xmlNamespace + "PropertyGroup"); + + var commonVersion = Version.Parse(versionsPropertyGroup.Element(xmlNamespace + "TgsCommonLibraryVersion").Value); + var apiVersion = Version.Parse(versionsPropertyGroup.Element(xmlNamespace + "TgsApiLibraryVersion").Value); + var clientVersion = Version.Parse(versionsPropertyGroup.Element(xmlNamespace + "TgsClientVersion").Value); + + var componentVersions = new Dictionary + { + { Component.NugetCommon, commonVersion }, + { Component.NugetApi, apiVersion }, + { Component.NugetClient, clientVersion }, + }; + + var releaseNotes = await GenerateNotes( + client, + componentVersions); + + const string CsprojSubstitution = "src/Tgstation.Server.$PROJECT$/Tgstation.Server.$PROJECT$.csproj"; + var csprojNameMap = new Dictionary + { + { Component.NugetCommon, "Common" }, + { Component.NugetApi, "Api" }, + { Component.NugetClient, "Client" }, + }; + + foreach(var kvp in csprojNameMap) + { + var component = kvp.Key; + var csprojPath = CsprojSubstitution.Replace("$PROJECT$", kvp.Value); + + var markdown = GenerateComponentNotes(releaseNotes, component, componentVersions[component]); + + var escapedMarkdown = SecurityElement.Escape(markdown); + + var originalCsproj = await File.ReadAllTextAsync(csprojPath); + var substitutedCsproj = originalCsproj.Replace($"$(TGS_NUGET_RELEASE_NOTES_{kvp.Value.ToUpperInvariant()})", escapedMarkdown); + + await File.WriteAllTextAsync(csprojPath, substitutedCsproj); + } + return 0; } } From d084080e7891303e2b666504c9143a0c170cab66 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 20 Aug 2023 02:49:43 -0400 Subject: [PATCH 15/22] Fix .gitignore for new release notes --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 8794b1a00f..0c71765597 100644 --- a/.gitignore +++ b/.gitignore @@ -23,7 +23,7 @@ artifacts/ /tools/Tgstation.Server.ReleaseNotes/release_notes.md launchSettings.json release_notes.md -release_notes.yml +changelog.yml *nupkg *.sqlite3 packaging/ From 45de81500eb740816dc227812e371d777258aea0 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 20 Aug 2023 02:50:40 -0400 Subject: [PATCH 16/22] Fix setting env vars when building on Windows --- .github/workflows/ci-pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 714ea46c77..76d2947efa 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -1269,7 +1269,7 @@ jobs: - name: Build ReleaseNotes run: | - export TGS_HOST_NO_WEBPANEL=true + $Env:TGS_HOST_NO_WEBPANEL=true dotnet build -c Release tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj - name: Parse API version @@ -1334,7 +1334,7 @@ jobs: - name: Build ReleaseNotes run: | - export TGS_HOST_NO_WEBPANEL=true + $Env:TGS_HOST_NO_WEBPANEL=true dotnet build -c Release tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj - name: Parse DMAPI version From 0b35e29d612226d1a700c06a86c69226525ce523 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 20 Aug 2023 04:18:13 -0400 Subject: [PATCH 17/22] Add debian changelog support Fix some other issues as well --- .github/workflows/ci-pipeline.yml | 3 + build/package/deb/build_package.sh | 7 +- .../Changelist.cs | 7 + .../Component.cs | 8 +- .../Tgstation.Server.ReleaseNotes/Program.cs | 340 +++++++++++++++--- 5 files changed, 309 insertions(+), 56 deletions(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 76d2947efa..d47b979f61 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -1015,6 +1015,9 @@ jobs: run: | echo "TGS_VERSION=$(xmlstarlet sel -N X="http://schemas.microsoft.com/developer/msbuild/2003" --template --value-of /X:Project/X:PropertyGroup/X:TgsCoreVersion build/Version.props)" >> $GITHUB_ENV + - name: Grab Most Recent Changelog + run: curl -L https://raw.githubusercontent.com/tgstation/tgstation-server/gh-pages/changelog.yml -o changelog.yml + - name: Execute Build Script (Unsigned) if: (!(github.event_name == 'push' && contains(github.event.head_commit.message, '[TGSDeploy]') && (github.event.ref == 'refs/heads/master' || github.event.ref == 'refs/heads/dev'))) run: sudo build/package/deb/build_package.sh diff --git a/build/package/deb/build_package.sh b/build/package/deb/build_package.sh index 54063487dc..74e66793c2 100755 --- a/build/package/deb/build_package.sh +++ b/build/package/deb/build_package.sh @@ -31,8 +31,13 @@ dh_make -p tgstation-server_$TGS_VERSION -y --createorig -s rm -f debian/README* debian/changelog debian/*.ex debian/upstream/*.ex +pushd .. +export TGS_HOST_NO_WEBPANEL=true +dotnet run -c Release --project tools/Tgstation.Server.ReleaseNotes $TGS_VERSION --debian packaging/debian/changelog $(git rev-parse HEAD) +export TGS_HOST_NO_WEBPANEL=false +popd + cp -r build/package/deb/debian/* debian/ -sed -i "s/~!VERSION!~/$TGS_VERSION/g" debian/changelog cp build/tgstation-server.service debian/ diff --git a/tools/Tgstation.Server.ReleaseNotes/Changelist.cs b/tools/Tgstation.Server.ReleaseNotes/Changelist.cs index 137d018799..feac94bc26 100644 --- a/tools/Tgstation.Server.ReleaseNotes/Changelist.cs +++ b/tools/Tgstation.Server.ReleaseNotes/Changelist.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.Linq; +using System.Text.RegularExpressions; namespace Tgstation.Server.ReleaseNotes { @@ -42,5 +43,11 @@ namespace Tgstation.Server.ReleaseNotes }) .ToList(); } + + public void StripConfigVersionMessage() + { + foreach (var change in Changes) + change.Descriptions.RemoveAll(desc => Regex.IsMatch(desc, "The new config.* version is")); + } } } diff --git a/tools/Tgstation.Server.ReleaseNotes/Component.cs b/tools/Tgstation.Server.ReleaseNotes/Component.cs index f882a3a320..d1baa3fcda 100644 --- a/tools/Tgstation.Server.ReleaseNotes/Component.cs +++ b/tools/Tgstation.Server.ReleaseNotes/Component.cs @@ -2,13 +2,13 @@ { enum Component { - Core, - HttpApi, Configuration, + Core, + HostWatchdog, + WebControlPanel, + HttpApi, DreamMakerApi, InteropApi, - WebControlPanel, - HostWatchdog, NugetCommon, NugetApi, NugetClient, diff --git a/tools/Tgstation.Server.ReleaseNotes/Program.cs b/tools/Tgstation.Server.ReleaseNotes/Program.cs index c02128c7db..ef97ac22ef 100644 --- a/tools/Tgstation.Server.ReleaseNotes/Program.cs +++ b/tools/Tgstation.Server.ReleaseNotes/Program.cs @@ -65,10 +65,26 @@ namespace Tgstation.Server.ReleaseNotes } var doNotCloseMilestone = false; + var debianMode = false; Component? componentRelease = null; if (args.Length > 1) switch (args[1].ToUpperInvariant()) { + case "--DEBIAN": + debianMode = true; + doNotCloseMilestone = true; + if (args.Length < 3) + { + Console.WriteLine("Missing output path!"); + return 238; + } + + if (args.Length < 4) + { + Console.WriteLine("Missing current SHA!"); + return 239; + } + break; case "--NO-CLOSE": doNotCloseMilestone = true; break; @@ -133,6 +149,9 @@ namespace Tgstation.Server.ReleaseNotes if (nuget) return await ReleaseNuget(client); + if (debianMode) + return await GenDebianChangelog(client, version, args[2], args[3]); + var releasesTask = client.Repository.Release.GetAll(RepoOwner, RepoName); Console.WriteLine("Getting merged pull requests in milestone " + versionString + "..."); @@ -258,7 +277,9 @@ namespace Tgstation.Server.ReleaseNotes } newNotes.Append("]("); - var milestone = await milestoneTasks.Single().Value.ConfigureAwait(false); + await Task.WhenAll(noteTasks); + + var milestone = milestones.Single().Value; if (milestone == null) { Console.WriteLine("Unable to detemine milestone!"); @@ -379,28 +400,31 @@ namespace Tgstation.Server.ReleaseNotes { Component.HostWatchdog, hostWatchdogVersion }, }; - var releaseDictionary = new Dictionary( - noteTasks - .SelectMany(task => task.Result.Item1) - .GroupBy(kvp => kvp.Key) - .Select(grouping => - { - var component = grouping.Key; - var changelist = new Changelist + var releaseDictionary = new SortedDictionary( + new Dictionary( + noteTasks + .Where(task => task.Result != null) + .SelectMany(task => task.Result.Item1) + .Where(kvp => kvp.Key == Component.Core || componentVersionDict.ContainsKey(kvp.Key)) + .GroupBy(kvp => kvp.Key) + .Select(grouping => { - Changes = grouping.SelectMany(kvp => kvp.Value.Changes).ToList() - }; + var component = grouping.Key; + var changelist = new Changelist + { + Changes = grouping.SelectMany(kvp => kvp.Value.Changes).ToList() + }; - if (component == Component.Core) - { - changelist.Version = coreVersion; - changelist.ComponentVersions = componentVersionDict; - } - else - changelist.Version = componentVersionDict[component]; + if (component == Component.Core) + { + changelist.Version = coreVersion; + changelist.ComponentVersions = componentVersionDict; + } + else + changelist.Version = componentVersionDict[component]; - return new KeyValuePair(component, changelist); - })); + return new KeyValuePair(component, changelist); + }))); if (releaseDictionary.Count == 0) { @@ -408,23 +432,22 @@ namespace Tgstation.Server.ReleaseNotes return 8; } - foreach (var I in releaseDictionary.OrderBy(kvp => kvp.Key)) + foreach (var I in releaseDictionary) { newNotes.Append(Environment.NewLine); newNotes.Append("#### "); - string componentName = I.Key switch - { - Component.HttpApi => "HTTP API", - Component.InteropApi => "Interop API", - Component.Configuration => "**Configuration**", - Component.DreamMakerApi => "DreamMaker API", - Component.HostWatchdog => "Host Watchdog", - Component.Core => "Core", - Component.WebControlPanel => "Web Control Panel", - _ => throw new Exception($"Unknown Component: {I.Key}"), - }; + string componentName = GetComponentDisplayName(I.Key, false); newNotes.Append(componentName); + if (I.Key == Component.Configuration) + { + I.Value.StripConfigVersionMessage(); + newNotes.AppendLine(); + newNotes.Append("- **The new configuration version is `"); + newNotes.Append(I.Value.Version); + newNotes.Append("`. Please update your `General:ConfigVersion` setting appropriately.**"); + } + PrintChanges(newNotes, I.Value); newNotes.Append(Environment.NewLine); @@ -467,12 +490,28 @@ namespace Tgstation.Server.ReleaseNotes } } - static readonly ConcurrentDictionary> milestoneTasks = new (); + static string GetComponentDisplayName(Component component, bool debian) => component switch + { + Component.HttpApi => debian ? "the HTTP API" : "HTTP API", + Component.InteropApi => debian ? "the Interop API" : "Interop API", + Component.Configuration => debian ? "the TGS configuration" : "**Configuration**", + Component.DreamMakerApi => debian ? "the DreamMaker API" : "DreamMaker API", + Component.HostWatchdog => debian ? "the Host Watchdog" : "Host Watchdog", + Component.Core => debian ? "the main server" : "Core", + Component.WebControlPanel => debian ? "the Web Control Panel" : "Web Control Panel", + _ => throw new Exception($"Unnamed Component: {component}"), + }; + + static readonly ConcurrentDictionary milestones = new(); + static readonly ConcurrentDictionary> pullRequests = new(); + + static Task GetPR(IGitHubClient client, int pr) => pullRequests.GetOrAdd(pr, x => RLR(() => client.Repository.PullRequest.Get(RepoOwner, RepoName, x))); static async Task, Dictionary, bool>> GetReleaseNotesFromPR(IGitHubClient client, Issue pullRequest, bool doNotCloseMilestone, bool needComponentExactVersions, bool forAllComponents) { //need to check it was merged - var fullPR = await RLR(() => client.Repository.PullRequest.Get(RepoOwner, RepoName, pullRequest.Number)); + var prTask = GetPR(client, pullRequest.Number); + var fullPR = await prTask; if (!fullPR.Merged) { @@ -493,6 +532,8 @@ namespace Tgstation.Server.ReleaseNotes return null; } + milestones.TryAdd(fullPR.Milestone.Number, fullPR.Milestone); + var commentsTask = TripleCheckGitHubPagination(apiOptions => client.Issue.Comment.GetAllForIssue(fullPR.Base.Repository.Id, pullRequest.Number, apiOptions), comment => comment.Id); bool isReleasePR = false; @@ -503,10 +544,24 @@ namespace Tgstation.Server.ReleaseNotes var commit = await RLR(() => client.Repository.Commit.Get(fullPR.Base.Repository.Id, fullPR.MergeCommitSha)); - return isReleasePR = commit.Commit.Message.Contains("[TGSDeploy]"); + isReleasePR = commit.Commit.Message.Contains("[TGSDeploy]") + || fullPR.Number == 966 + || fullPR.Number == 1048 + || fullPR.Number == 1435 + || fullPR.Number == 1263 + || fullPR.Number == 1087 + || fullPR.Number == 1441 + || fullPR.Number == 1437 + || fullPR.Number == 1443 + || fullPR.Number == 1311 + || fullPR.Number == 1598 + || fullPR.Number == 1463 + || fullPR.Number == 1209; // some special tactics from before we were more stingent + + return isReleasePR; } - Task needExtendedComponentVersions = null; + Task needExtendedComponentVersions = Task.FromResult(false); async Task> GetComponentVersions() { var mergeCommit = fullPR.MergeCommitSha; @@ -670,6 +725,8 @@ namespace Tgstation.Server.ReleaseNotes await previousTask; + Debug.Assert(!(await needExtendedComponentVersions) || changelists.Where(x => x.Key == Component.Core).All(x => x.Value.ComponentVersions != null && x.Value.ComponentVersions.Count > 3)); + return Tuple.Create(changelists.ToDictionary(kvp => kvp.Key, kvp => kvp.Value), await componentVersions, isReleasePR); } @@ -799,7 +856,7 @@ The user account that created this pull request is available to correct any issu } } - static async Task> TripleCheckGitHubPagination(Func>> apiCall, Func idSelector) + static async Task> TripleCheckGitHubPagination(Func>> apiCall, Func idSelector) { // I've seen GitHub pagination return incomplete result sets in the past // It has an in-built pagination limit of 100 @@ -808,19 +865,19 @@ The user account that created this pull request is available to correct any issu PageSize = 100 }; var results = await RLR(() => apiCall(apiOptions)); - var distinctEntries = new Dictionary(results.Count); + var distinctEntries = new Dictionary(results.Count); foreach (var result in results) - distinctEntries.Add(idSelector(result), result); + distinctEntries.Add(idSelector(result).ToString(), result); if (results.Count > 100) { results = await RLR(() => apiCall(apiOptions)); foreach (var result in results) - distinctEntries.TryAdd(idSelector(result), result); + distinctEntries.TryAdd(idSelector(result).ToString(), result); results = await RLR(() => apiCall(apiOptions)); foreach (var result in results) - distinctEntries.TryAdd(idSelector(result), result); + distinctEntries.TryAdd(idSelector(result).ToString(), result); } return distinctEntries.Values.ToList(); @@ -844,11 +901,11 @@ The user account that created this pull request is available to correct any issu { { RepoOwner, RepoName }, }, + Merged = DateRange.GreaterThan(new DateTimeOffset(2018, 9, 27, 0, 0, 0, TimeSpan.Zero)), })); foreach (var result in results.Items) - if (pullRequests.TryAdd(result.Number, result)) - milestoneTasks.TryAdd(result.Number, milestoneTask); + pullRequests.TryAdd(result.Number, result); if (results.IncompleteResults) continue; @@ -866,10 +923,12 @@ The user account that created this pull request is available to correct any issu await Task.WhenAll(prTasks); - var prResults = prTasks.Select(x => x.Result).Where(result => result != null).ToList(); + var prResults = prTasks.Select(x => x.Result).ToList(); var releasePRResult = prResults.FirstOrDefault(x => x.Item3); + prResults = prResults.Where(result => result != null).ToList(); + Dictionary releasedComponentVersions; if (releasePRResult != null) releasedComponentVersions = releasePRResult.Item2; @@ -945,6 +1004,9 @@ The user account that created this pull request is available to correct any issu entry = changelist; entry.Version = componentVersion; entry.Unreleased = unreleased; + if (component == Component.Core && entry.ComponentVersions == null) + entry.ComponentVersions = releasedComponentVersions; + list.Add(entry); } else @@ -976,7 +1038,7 @@ The user account that created this pull request is available to correct any issu }); } else - Debug.Assert(finalResults[Component.Core].All(x => x.Version == milestoneVersion)); + Debug.Assert(finalResults[Component.Core].All(x => x.Version == milestoneVersion && x.ComponentVersions != null && x.ComponentVersions.Count > 3)); var notes = new ReleaseNotes { @@ -991,7 +1053,8 @@ The user account that created this pull request is available to correct any issu static async Task FullNotes(IGitHubClient client) { - var startRateLimit = (client.GetLastApiInfo()?.RateLimit ?? (await client.RateLimit.GetRateLimits()).Rate).Remaining; + var rateLimitInfo = client.GetLastApiInfo()?.RateLimit ?? (await client.RateLimit.GetRateLimits()).Rate; + var startRateLimit = rateLimitInfo.Remaining; var releaseNotes = await GenerateNotes(client); @@ -1142,8 +1205,6 @@ The user account that created this pull request is available to correct any issu .ToList(); Debug.Assert(missingCoreVersions.Count == 0); - foreach (var missingCoreVersion in missingCoreVersions) - await await ProcessMilestone(client, missingCoreVersion); var changelistsGroupedByComponent = milestonePRTasks @@ -1216,19 +1277,28 @@ The user account that created this pull request is available to correct any issu Debug.Assert(distinctCount == kvp.Value.Count); foreach (var cl in kvp.Value) + { cl.DeduplicateChanges(); + + if (kvp.Key == Component.Configuration) + cl.StripConfigVersionMessage(); + } } return releaseNotes; } - static void PrintChanges(StringBuilder newNotes, Changelist changelist) + static void PrintChanges(StringBuilder newNotes, Changelist changelist, bool debianMode = false) { foreach (var change in changelist.Changes) foreach (var line in change.Descriptions) { - newNotes.Append(Environment.NewLine); - newNotes.Append("- "); + newNotes.AppendLine(); + if (debianMode) + newNotes.Append(" * "); + else + newNotes.Append("- "); + newNotes.Append(line); newNotes.Append(" (#"); newNotes.Append(change.PullRequest); @@ -1310,5 +1380,173 @@ The user account that created this pull request is available to correct any issu return 0; } + + static async Task GenDebianChangelog(IGitHubClient client, Version version, string outputPath, string currentSha) + { + var tagsTask = RLR(() => TripleCheckGitHubPagination( + apiOptions => client.Repository.GetAllTags(RepoOwner, RepoName, apiOptions), + x => x.Name)); + var currentRefTask = client.Repository.Commit.Get(RepoOwner, RepoName, currentSha); + var releaseNotes = await GenerateNotes(client); + + // https://www.debian.org/doc/manuals/maint-guide/dreq.en.html#changelog + // https://www.debian.org/doc/debian-policy/ch-source.html#s-dpkgchangelog + + /* +package (version) distribution(s); urgency=urgency + [optional blank line(s), stripped] + * change details + more change details + [blank line(s), included in output of dpkg-parsechangelog] + * even more change details + [optional blank line(s), stripped] + -- maintainer name [two spaces] date + */ + + // debian package did not exist before uhhh... + var debianPackageFirstRelease = new Version(5, 13, 0); + + var coreChangelists = releaseNotes + .Components[Component.Core] + .Where(x => x.Version >= debianPackageFirstRelease && (!x.Unreleased || x.Version == version)) + .OrderByDescending(x => x.Version) + .ToList(); + + var currentReleaseChangelists = new List>(); + + for (var i = 0; i < coreChangelists.Count; ++i) + { + var currentDic = new SortedDictionary(); + currentReleaseChangelists.Insert(0, currentDic); + var nowRelease = coreChangelists[i]; + var previousRelease = (i + 1) < coreChangelists.Count + ? coreChangelists[i + 1] + : releaseNotes + .Components[Component.Core] + .First(x => x.Version == new Version(5, 12, 7)); + + currentDic.Add(Component.Core, nowRelease); + foreach (var componentKvp in nowRelease.ComponentVersions) + { + try + { + var component = componentKvp.Key; + if (component == Component.Core + || component == Component.NugetClient + || component == Component.NugetApi + || component == Component.NugetCommon) + continue; + + var takeNotesFrom = previousRelease.ComponentVersions[componentKvp.Key]; + var changesEnumerator = releaseNotes + .Components[component] + .Where(changelist => changelist.Version > takeNotesFrom && changelist.Version <= componentKvp.Value) + .SelectMany(x => x.Changes) + .OrderBy(x => x.PullRequest); + var changelist = new Changelist + { + Version = componentKvp.Value, + Changes = changesEnumerator + .ToList(), + }; + + currentDic.Add(component, changelist); + } + catch when (Debugger.IsAttached) + { + Debugger.Break(); + } + } + } + + var builder = new StringBuilder(); + foreach (var releaseDictionary in currentReleaseChangelists) + { + var allPrNumbers = releaseDictionary.Values.SelectMany(x => x.Changes.Select(y => y.PullRequest)).Distinct().OrderBy(x => x).ToList(); + var allPrTasks = allPrNumbers + .Select(x => GetPR(client, x)) + .ToList(); + + await Task.WhenAll(allPrTasks); + + var prDict = allPrTasks.ToDictionary(x => x.Result.Number, x => x.Result); + + bool AnyPRHasLabel(string labelName) => prDict.Values.Any(x => x.Labels.Any(y => y.Name == labelName)); + + // determine urgency + + string urgency; + if (AnyPRHasLabel("Priority: CRITICAL")) + urgency = "critical"; + else if (AnyPRHasLabel("Priority: High")) + urgency = "high"; + else if (AnyPRHasLabel("Fix")) + urgency = "medium"; + else + urgency = "low"; + + builder.Append($"tgstation-server ("); + + builder.Append(releaseDictionary[Component.Core].Version); + builder.Append("-1) unstable; urgency="); + builder.Append(urgency); + + foreach (var kvp in releaseDictionary) + { + builder.AppendLine(); + builder.AppendLine(); + builder.Append(" * The following changes are for "); + builder.Append(GetComponentDisplayName(kvp.Key, true)); + if(kvp.Key == Component.Configuration) + { + builder.Append(". You "); + if (kvp.Value.Version.Minor == 0 && kvp.Value.Version.Build == 0) + builder.Append("will need to"); + else + builder.Append("should"); + builder.Append(" update your `General:ConfigVersion` setting in `/etc/tgstation-server/appsettings.Production.yml` to this new version"); + } + + builder.Append(':'); + + PrintChanges(builder, kvp.Value, true); + } + + builder.AppendLine(); + builder.Append(" -- "); + + GitHubCommit currentRef; + var tags = await tagsTask; + var releaseTag = tags.FirstOrDefault(x => x.Name == $"tgstation-server-v{releaseDictionary[Component.Core].Version}"); + + if (releaseTag != null) + currentRef = await client.Repository.Commit.Get(RepoOwner, RepoName, releaseTag.Commit.Sha); + else + currentRef = await currentRefTask; + + var committer = currentRef.Commit.Committer; + if (committer.Name == "GitHub" && committer.Email == "noreply@github.com") + committer = currentRef.Commit.Author; + + builder.Append(committer.Name); + builder.Append(" <"); + builder.Append(committer.Email); + builder.Append("> "); + + var commitTime = currentRef.Commit.Committer.Date; + + builder.Append(commitTime.ToString("ddd").TrimEnd('.')); + builder.Append(", "); + builder.Append(commitTime.ToString("dd")); + builder.Append(' '); + builder.Append(commitTime.ToString("MMM").TrimEnd('.')); + builder.Append(' '); + builder.AppendLine(commitTime.ToString("yyyy HH:mm:sszz00")); + } + + var changelog = builder.ToString().Replace("\r", String.Empty); + await File.WriteAllTextAsync(outputPath, changelog); + return 0; + } } } From aacccfa64d7861c5d019ebfe0f48311ecab3c729 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 20 Aug 2023 16:18:24 -0400 Subject: [PATCH 18/22] Pass in missing env vars to `build_package.sh` --- .github/workflows/ci-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 73b72d91a0..2f184995e6 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -1020,7 +1020,7 @@ jobs: - name: Execute Build Script (Unsigned) if: (!(github.event_name == 'push' && contains(github.event.head_commit.message, '[TGSDeploy]') && (github.event.ref == 'refs/heads/master' || github.event.ref == 'refs/heads/dev'))) - run: sudo build/package/deb/build_package.sh + run: sudo -E build/package/deb/build_package.sh - name: Execute Build Script (Signed) if: (github.event_name == 'push' && contains(github.event.head_commit.message, '[TGSDeploy]') && (github.event.ref == 'refs/heads/master' || github.event.ref == 'refs/heads/dev')) From a00f2e0a4a2fb5024cbe21f76f5654407161f962 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 20 Aug 2023 16:34:00 -0400 Subject: [PATCH 19/22] Actually delete stub changelog --- build/package/deb/debian/changelog | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 build/package/deb/debian/changelog diff --git a/build/package/deb/debian/changelog b/build/package/deb/debian/changelog deleted file mode 100644 index 61eeab7d19..0000000000 --- a/build/package/deb/debian/changelog +++ /dev/null @@ -1,5 +0,0 @@ -tgstation-server (~!VERSION!~-1) unstable; urgency=medium - - * Package changelogs are currently stubs. They will be auto-generated in the future (https://github.com/tgstation/tgstation-server/issues/1147). Refer to GitHub release for changelog: https://github.com/tgstation/tgstation-server/releases/tag/tgstation-server-v~!VERSION!~ - - -- Jordan Dominion Mon, 19 Jun 2023 16:14:13 +0000 From 83da107dbf5a3214b5e051383d89ef955e788d11 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 20 Aug 2023 16:59:01 -0400 Subject: [PATCH 20/22] Fix debian changelog terminating lines --- tools/Tgstation.Server.ReleaseNotes/Program.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/Tgstation.Server.ReleaseNotes/Program.cs b/tools/Tgstation.Server.ReleaseNotes/Program.cs index ef97ac22ef..f24db1155d 100644 --- a/tools/Tgstation.Server.ReleaseNotes/Program.cs +++ b/tools/Tgstation.Server.ReleaseNotes/Program.cs @@ -1541,7 +1541,7 @@ package (version) distribution(s); urgency=urgency builder.Append(' '); builder.Append(commitTime.ToString("MMM").TrimEnd('.')); builder.Append(' '); - builder.AppendLine(commitTime.ToString("yyyy HH:mm:sszz00")); + builder.AppendLine(commitTime.ToString("yyyy HH:mm:ss zz00")); } var changelog = builder.ToString().Replace("\r", String.Empty); From dd57d0a9b744e6363e5b32da10ee8d8b81eee81d Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 20 Aug 2023 17:05:06 -0400 Subject: [PATCH 21/22] Remove unnecessary second `rev-parse` --- build/package/deb/build_package.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/package/deb/build_package.sh b/build/package/deb/build_package.sh index 74e66793c2..dc653cdea0 100755 --- a/build/package/deb/build_package.sh +++ b/build/package/deb/build_package.sh @@ -33,7 +33,7 @@ rm -f debian/README* debian/changelog debian/*.ex debian/upstream/*.ex pushd .. export TGS_HOST_NO_WEBPANEL=true -dotnet run -c Release --project tools/Tgstation.Server.ReleaseNotes $TGS_VERSION --debian packaging/debian/changelog $(git rev-parse HEAD) +dotnet run -c Release --project tools/Tgstation.Server.ReleaseNotes $TGS_VERSION --debian packaging/debian/changelog $CURRENT_COMMIT export TGS_HOST_NO_WEBPANEL=false popd From fe88394051b30f114a1d7596e174336b26925549 Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Sun, 20 Aug 2023 17:32:04 -0400 Subject: [PATCH 22/22] Fix bad ordering of debian changelog No notes before 5.14.0 because of issues --- .../Tgstation.Server.ReleaseNotes/Program.cs | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/tools/Tgstation.Server.ReleaseNotes/Program.cs b/tools/Tgstation.Server.ReleaseNotes/Program.cs index f24db1155d..978e5276bc 100644 --- a/tools/Tgstation.Server.ReleaseNotes/Program.cs +++ b/tools/Tgstation.Server.ReleaseNotes/Program.cs @@ -1290,9 +1290,11 @@ The user account that created this pull request is available to correct any issu static void PrintChanges(StringBuilder newNotes, Changelist changelist, bool debianMode = false) { + var none = true; foreach (var change in changelist.Changes) foreach (var line in change.Descriptions) { + none = false; newNotes.AppendLine(); if (debianMode) newNotes.Append(" * "); @@ -1306,6 +1308,9 @@ The user account that created this pull request is available to correct any issu newNotes.Append(change.Author); newNotes.Append(')'); } + + if (debianMode && none) + throw new Exception($"Changlist {changelist.Version} has no changes!"); } static string GenerateComponentNotes(ReleaseNotes releaseNotes, Component component, Version version) @@ -1404,11 +1409,14 @@ package (version) distribution(s); urgency=urgency */ // debian package did not exist before uhhh... - var debianPackageFirstRelease = new Version(5, 13, 0); + // var debianPackageFirstRelease = new Version(5, 13, 0); + // can't use that, there are irreconcilable changelog/version errors + // keep it straight going forwards + var noChangelogsBeforeVersion = new Version(5, 14, 0); var coreChangelists = releaseNotes .Components[Component.Core] - .Where(x => x.Version >= debianPackageFirstRelease && (!x.Unreleased || x.Version == version)) + .Where(x => x.Version >= noChangelogsBeforeVersion && (!x.Unreleased || x.Version == version)) .OrderByDescending(x => x.Version) .ToList(); @@ -1417,13 +1425,13 @@ package (version) distribution(s); urgency=urgency for (var i = 0; i < coreChangelists.Count; ++i) { var currentDic = new SortedDictionary(); - currentReleaseChangelists.Insert(0, currentDic); + currentReleaseChangelists.Add(currentDic); var nowRelease = coreChangelists[i]; var previousRelease = (i + 1) < coreChangelists.Count ? coreChangelists[i + 1] : releaseNotes .Components[Component.Core] - .First(x => x.Version == new Version(5, 12, 7)); + .First(x => x.Version == new Version(5, 13, 7)); currentDic.Add(Component.Core, nowRelease); foreach (var componentKvp in nowRelease.ComponentVersions) @@ -1450,7 +1458,8 @@ package (version) distribution(s); urgency=urgency .ToList(), }; - currentDic.Add(component, changelist); + if (changelist.Changes.Any()) + currentDic.Add(component, changelist); } catch when (Debugger.IsAttached) { @@ -1491,7 +1500,7 @@ package (version) distribution(s); urgency=urgency builder.Append("-1) unstable; urgency="); builder.Append(urgency); - foreach (var kvp in releaseDictionary) + foreach (var kvp in releaseDictionary.Where(x => x.Value.Changes.Count > 0 || x.Key == Component.Configuration)) { builder.AppendLine(); builder.AppendLine();