From cef860db1b29917ec813c8ed9999202431ffd66a Mon Sep 17 00:00:00 2001 From: Jordan Dominion Date: Thu, 17 Aug 2023 12:42:28 -0400 Subject: [PATCH] Release notes update - Added functionality to release notes app to generate full changelog.yml from GitHub metadata. (Incremental) - Updated PULL_REQUEST_TEMPLATE.md to reflect changes (Strict component list) --- .github/PULL_REQUEST_TEMPLATE.md | 5 +- .gitignore | 1 + .../Extensions/Converters/VersionConverter.cs | 2 +- tools/Tgstation.Server.ReleaseNotes/Change.cs | 13 + .../Changelist.cs | 46 + .../Component.cs | 16 + .../Tgstation.Server.ReleaseNotes/Program.cs | 861 +++++++++++++++--- .../ReleaseNotes.cs | 9 + .../Tgstation.Server.ReleaseNotes.csproj | 6 +- 9 files changed, 838 insertions(+), 121 deletions(-) create mode 100644 tools/Tgstation.Server.ReleaseNotes/Change.cs create mode 100644 tools/Tgstation.Server.ReleaseNotes/Changelist.cs create mode 100644 tools/Tgstation.Server.ReleaseNotes/Component.cs create mode 100644 tools/Tgstation.Server.ReleaseNotes/ReleaseNotes.cs diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index b770b65721..c146ed90da 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -13,9 +13,10 @@ They will be amalgamated together in the end. /:cl: :cl: Categories -Categories are divided up in the release notes and set after the :cl: header. +Categories are used by [the release notes tool](../tools/Tgstation.Server.ReleaseNotes) to generate formatted changelists used in releases. The default category is Core. -The current standard categories are Core, DreamMaker API, HTTP API, and Host Watchdog. +Only one category may be specified for a :cl: block. +Valid categories are Core, DreamMaker API, HTTP API, Host Watchdog, Web Control Panel, Configuration, Nuget: Api, Nuget: Client, and Nuget: Common. /:cl: [Why]: # (If this does not close or work on an existing GitHub issue, please add a short description [two lines down] of why you think these changes would benefit the server. If you can't justify it in words, it might not be worth adding.) diff --git a/.gitignore b/.gitignore index d7756e60cd..8794b1a00f 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,7 @@ artifacts/ /tools/Tgstation.Server.ReleaseNotes/release_notes.md launchSettings.json release_notes.md +release_notes.yml *nupkg *.sqlite3 packaging/ diff --git a/src/Tgstation.Server.Host/Extensions/Converters/VersionConverter.cs b/src/Tgstation.Server.Host/Extensions/Converters/VersionConverter.cs index ec08e5ae7b..4ca9cdf105 100644 --- a/src/Tgstation.Server.Host/Extensions/Converters/VersionConverter.cs +++ b/src/Tgstation.Server.Host/Extensions/Converters/VersionConverter.cs @@ -13,7 +13,7 @@ namespace Tgstation.Server.Host.Extensions.Converters /// /// and for serializing s in semver format. /// - sealed class VersionConverter : JsonConverter, IYamlTypeConverter + public sealed class VersionConverter : JsonConverter, IYamlTypeConverter { /// /// Check if the supports (de)serializing a given . diff --git a/tools/Tgstation.Server.ReleaseNotes/Change.cs b/tools/Tgstation.Server.ReleaseNotes/Change.cs new file mode 100644 index 0000000000..88f5910606 --- /dev/null +++ b/tools/Tgstation.Server.ReleaseNotes/Change.cs @@ -0,0 +1,13 @@ +using System.Collections.Generic; + +using Octokit; + +namespace Tgstation.Server.ReleaseNotes +{ + sealed class Change + { + public List Descriptions { get; set; } + public string Author { get; set; } + public int PullRequest { get; set; } + } +} diff --git a/tools/Tgstation.Server.ReleaseNotes/Changelist.cs b/tools/Tgstation.Server.ReleaseNotes/Changelist.cs new file mode 100644 index 0000000000..137d018799 --- /dev/null +++ b/tools/Tgstation.Server.ReleaseNotes/Changelist.cs @@ -0,0 +1,46 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; + +namespace Tgstation.Server.ReleaseNotes +{ + sealed class Changelist + { + public Version Version { get; set; } + + public Dictionary ComponentVersions { get; set; } + + public List Changes { get; set; } + + public bool Unreleased { get; set; } + + public void DeduplicateChanges() + { + Changes = Changes + .OrderBy(x => x.PullRequest) + .GroupBy(x => x.PullRequest) + .Select(prChanges => + { + string author = null; + return new Change + { + PullRequest = prChanges.Key, + Descriptions = prChanges + .SelectMany(x => + { + if (author != null) + Debug.Assert(x.Author == author); + else + author = x.Author; + + return x.Descriptions; + }) + .ToList(), + Author = author + }; + }) + .ToList(); + } + } +} diff --git a/tools/Tgstation.Server.ReleaseNotes/Component.cs b/tools/Tgstation.Server.ReleaseNotes/Component.cs new file mode 100644 index 0000000000..f882a3a320 --- /dev/null +++ b/tools/Tgstation.Server.ReleaseNotes/Component.cs @@ -0,0 +1,16 @@ +namespace Tgstation.Server.ReleaseNotes +{ + enum Component + { + Core, + HttpApi, + Configuration, + DreamMakerApi, + InteropApi, + WebControlPanel, + HostWatchdog, + NugetCommon, + NugetApi, + NugetClient, + } +} diff --git a/tools/Tgstation.Server.ReleaseNotes/Program.cs b/tools/Tgstation.Server.ReleaseNotes/Program.cs index e0bfa45e59..e5a95795d6 100644 --- a/tools/Tgstation.Server.ReleaseNotes/Program.cs +++ b/tools/Tgstation.Server.ReleaseNotes/Program.cs @@ -1,14 +1,28 @@ -using System; +// This program is minimal effort and should be sent to remedial school + +using System; +using System.Collections.Concurrent; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Sockets; using System.Text; +using System.Text.RegularExpressions; using System.Threading.Tasks; using System.Xml.Linq; +using Newtonsoft.Json; + using Octokit; using Octokit.GraphQL; +using Tgstation.Server.Host.Extensions.Converters; + +using YamlDotNet.Serialization; + namespace Tgstation.Server.ReleaseNotes { /// @@ -34,8 +48,9 @@ namespace Tgstation.Server.ReleaseNotes var ensureRelease = versionString.Equals("--ensure-release", StringComparison.OrdinalIgnoreCase); var linkWinget = versionString.Equals("--link-winget", StringComparison.OrdinalIgnoreCase); var shaCheck = versionString.Equals("--winget-template-check", StringComparison.OrdinalIgnoreCase); + var fullNotes = versionString.Equals("--generate-full-notes", StringComparison.OrdinalIgnoreCase); - if ((!Version.TryParse(versionString, out var version) || version.Revision != -1) && !ensureRelease && !linkWinget && !shaCheck) + if ((!Version.TryParse(versionString, out var version) || version.Revision != -1) && !ensureRelease && !linkWinget && !shaCheck && !fullNotes) { Console.WriteLine("Invalid version: " + versionString); return 2; @@ -84,6 +99,11 @@ namespace Tgstation.Server.ReleaseNotes return await Winget(client, null, args[1]); } + if (fullNotes) + { + return await FullNotes(client); + } + var releasesTask = client.Repository.Release.GetAll(RepoOwner, RepoName); Console.WriteLine("Getting merged pull requests in milestone " + versionString + "..."); @@ -101,103 +121,12 @@ namespace Tgstation.Server.ReleaseNotes } Console.WriteLine(milestonePRs.Items.Count + " total pull requests"); - Task milestoneTask = null; - var milestoneTaskLock = new object(); - var releaseDictionary = new Dictionary>>(StringComparer.OrdinalIgnoreCase); - bool postControlPanelMessage = false; - async Task GetReleaseNotesFromPR(Issue pullRequest) - { - //need to check it was merged - var fullPR = await client.Repository.PullRequest.Get(RepoOwner, RepoName, pullRequest.Number).ConfigureAwait(false); + var noteTasks = new List, Dictionary, bool>>>(); - if (!fullPR.Merged) - { - if (!doNotCloseMilestone && fullPR.Milestone != null) - { - Console.WriteLine($"Removing trash PR #{fullPR.Number} from milestone..."); - await client.Issue.Update(RepoOwner, RepoName, fullPR.Number, new IssueUpdate - { - Milestone = null - }).ConfigureAwait(false); - } - - return; - } - - async Task GetMilestone() - { - if (fullPR.Milestone == null) - return null; - return await client.Issue.Milestone.Get(RepoOwner, RepoName, fullPR.Milestone.Number); - }; - - lock (milestoneTaskLock) - milestoneTask ??= GetMilestone(); - - // if (!fullPR.Merged) - //return; - - void BuildNotesFromComment(string comment, User user) - { - if (comment == null) - return; - - void CommitNotes(string component, List notes) - { - lock (releaseDictionary) - { - foreach (var I in notes) - Console.WriteLine(component + " #" + fullPR.Number + " - " + I + " (@" + user.Login + ")"); - - var tupleSelector = notes.Select(note => Tuple.Create(note, fullPR.Number, user.Login)); - if (releaseDictionary.TryGetValue(component, out var currentValues)) - currentValues.AddRange(tupleSelector); - else - releaseDictionary.Add(component, tupleSelector.ToList()); - } - } - - var commentSplits = comment.Split('\n'); - string targetComponent = null; - var notes = new List(); - foreach (var line in commentSplits) - { - var trimmedLine = line.Trim(); - if (targetComponent == null) - { - if (trimmedLine.StartsWith(":cl:", StringComparison.Ordinal)) - { - targetComponent = trimmedLine[4..].Trim(); - if (targetComponent.Length == 0) - targetComponent = "Core"; - } - continue; - } - if (trimmedLine.StartsWith("/:cl:", StringComparison.Ordinal)) - { - CommitNotes(targetComponent, notes); - targetComponent = null; - notes.Clear(); - continue; - } - if (trimmedLine.Length == 0) - continue; - - notes.Add(trimmedLine); - } - } - - var comments = await client.Issue.Comment.GetAllForIssue(RepoOwner, RepoName, fullPR.Number).ConfigureAwait(false); - BuildNotesFromComment(fullPR.Body, fullPR.User); - foreach(var x in comments) - BuildNotesFromComment(x.Body, x.User); - } - - var tasks = new List(); foreach (var I in milestonePRs.Items) - tasks.Add(GetReleaseNotesFromPR(I)); + noteTasks.Add(GetReleaseNotesFromPR(client, I, doNotCloseMilestone, false, false)); var releases = await releasesTask.ConfigureAwait(false); @@ -299,19 +228,21 @@ namespace Tgstation.Server.ReleaseNotes newNotes.Append(version.Build); } newNotes.Append("]("); - var milestone = await milestoneTask.ConfigureAwait(false); + + var milestone = await milestoneTasks.Single().Value.ConfigureAwait(false); if (milestone == null) { Console.WriteLine("Unable to detemine milestone!"); return 9; } + var allTasks = new List(noteTasks); if (doNotCloseMilestone) Console.WriteLine("Not closing milestone due to parameter!"); else { Console.WriteLine("Closing milestone..."); - tasks.Add(client.Issue.Milestone.Update(RepoOwner, RepoName, milestone.Number, new MilestoneUpdate + allTasks.Add(client.Issue.Milestone.Update(RepoOwner, RepoName, milestone.Number, new MilestoneUpdate { State = ItemState.Closed })); @@ -358,14 +289,16 @@ namespace Tgstation.Server.ReleaseNotes if (I.PullRequest != null) { Console.WriteLine($"Adding additional merged PR #{I.Number}..."); - tasks.Add(GetReleaseNotesFromPR(I)); + var task = GetReleaseNotesFromPR(client, I, doNotCloseMilestone, false, false); + noteTasks.Add(task); + allTasks.Add(task); } } await Task.WhenAll(issueUpdateTasks).ConfigureAwait(false); } - tasks.Add(client.Issue.Milestone.Delete(RepoOwner, RepoName, milestoneToDelete.Number)); + allTasks.Add(client.Issue.Milestone.Delete(RepoOwner, RepoName, milestoneToDelete.Number)); } // Create the next minor milestone @@ -378,7 +311,7 @@ namespace Tgstation.Server.ReleaseNotes { Description = "Next minor version" }); - tasks.Add(nextMinorMilestoneTask); + allTasks.Add(nextMinorMilestoneTask); // Move unfinished stuff to new minor milestone Console.WriteLine($"Moving {milestone.OpenIssues} abandoned issue(s) from previous milestone to new one..."); @@ -393,7 +326,7 @@ namespace Tgstation.Server.ReleaseNotes { var nextMinorMilestone = await nextMinorMilestoneTask.ConfigureAwait(false); foreach (var I in abandonedIssues.Items) - tasks.Add(client.Issue.Update(RepoOwner, RepoName, I.Number, new IssueUpdate + allTasks.Add(client.Issue.Update(RepoOwner, RepoName, I.Number, new IssueUpdate { Milestone = nextMinorMilestone.Number })); @@ -405,7 +338,40 @@ namespace Tgstation.Server.ReleaseNotes newNotes.Append("?closed=1)"); newNotes.Append(Environment.NewLine); - await Task.WhenAll(tasks).ConfigureAwait(false); + await Task.WhenAll(allTasks).ConfigureAwait(false); + + var componentVersionDict = new Dictionary + { + { Component.Configuration, configVersion }, + { Component.HttpApi, apiVersion }, + { Component.DreamMakerApi, dmApiVersion }, + { Component.InteropApi, interopVersion }, + { Component.WebControlPanel, webControlVersion }, + { Component.HostWatchdog, hostWatchdogVersion }, + }; + + var releaseDictionary = new Dictionary( + noteTasks + .SelectMany(task => task.Result.Item1) + .GroupBy(kvp => kvp.Key) + .Select(grouping => + { + var component = grouping.Key; + var changelist = new Changelist + { + Changes = grouping.SelectMany(kvp => kvp.Value.Changes).ToList() + }; + + if (component == Component.Core) + { + changelist.Version = coreVersion; + changelist.ComponentVersions = componentVersionDict; + } + else + changelist.Version = componentVersionDict[component]; + + return new KeyValuePair(component, changelist); + })); if (releaseDictionary.Count == 0) { @@ -417,20 +383,31 @@ namespace Tgstation.Server.ReleaseNotes { newNotes.Append(Environment.NewLine); newNotes.Append("#### "); - newNotes.Append(I.Key); - - - foreach (var noteTuple in I.Value) + string componentName = I.Key switch { - newNotes.Append(Environment.NewLine); - newNotes.Append("- "); - newNotes.Append(noteTuple.Item1); - newNotes.Append(" (#"); - newNotes.Append(noteTuple.Item2); - newNotes.Append(" @"); - newNotes.Append(noteTuple.Item3); - newNotes.Append(')'); - } + Component.HttpApi => "HTTP API", + Component.InteropApi => "Interop API", + Component.Configuration => "**Configuration**", + Component.DreamMakerApi => "DreamMaker API", + Component.HostWatchdog => "Host Watchdog", + Component.Core => "Core", + Component.WebControlPanel => "Web Control Panel", + _ => throw new Exception($"Unknown Component: {I.Key}"), + }; + newNotes.Append(componentName); + + foreach (var change in I.Value.Changes) + foreach (var line in change.Descriptions) + { + newNotes.Append(Environment.NewLine); + newNotes.Append("- "); + newNotes.Append(line); + newNotes.Append(" (#"); + newNotes.Append(change.PullRequest); + newNotes.Append(" @"); + newNotes.Append(change.Author); + newNotes.Append(')'); + } newNotes.Append(Environment.NewLine); } @@ -473,6 +450,237 @@ namespace Tgstation.Server.ReleaseNotes } } + static ConcurrentDictionary> milestoneTasks = new ConcurrentDictionary>(); + static Task GetMilestone(IGitHubClient client, int number) + => milestoneTasks.GetOrAdd(number, localNumber => client.Issue.Milestone.Get(RepoOwner, RepoName, localNumber)); + + static async Task, Dictionary, bool>> GetReleaseNotesFromPR(IGitHubClient client, Issue pullRequest, bool doNotCloseMilestone, bool needComponentExactVersions, bool forAllComponents) + { + //need to check it was merged + var fullPR = await RLR(() => client.Repository.PullRequest.Get(RepoOwner, RepoName, pullRequest.Number)); + + if (!fullPR.Merged) + { + if (!doNotCloseMilestone && fullPR.Milestone != null) + { + Console.WriteLine($"Removing trash PR #{fullPR.Number} from milestone..."); + await RLR(() => client.Issue.Update(RepoOwner, RepoName, fullPR.Number, new IssueUpdate + { + Milestone = null + })); + } + + return null; + } + + if (fullPR.Milestone == null) + { + return null; + } + + var commentsTask = TripleCheckGitHubPagination(apiOptions => client.Issue.Comment.GetAllForIssue(fullPR.Base.Repository.Id, pullRequest.Number, apiOptions), comment => comment.Id); + + bool isReleasePR = false; + async Task ShouldGetExtendedComponentVersions() + { + if (forAllComponents) + return true; + + var commit = await RLR(() => client.Repository.Commit.Get(fullPR.Base.Repository.Id, fullPR.MergeCommitSha)); + + return isReleasePR = commit.Commit.Message.Contains("[TGSDeploy]"); + } + + Task needExtendedComponentVersions = null; + async Task> GetComponentVersions() + { + var mergeCommit = fullPR.MergeCommitSha; + // we don't care about unreleased web control panel changes + + try + { + needExtendedComponentVersions = ShouldGetExtendedComponentVersions(); + + var versionsBytes = await RLR(() => client.Repository.Content.GetRawContentByRef(RepoOwner, RepoName, "build/Version.props", mergeCommit)); + + XDocument doc; + using (var ms = new MemoryStream(versionsBytes)) + doc = XDocument.Load(ms); + + var project = doc.Root; + var xmlNamespace = project.GetDefaultNamespace(); + var versionsPropertyGroup = project.Elements().First(x => x.Name == xmlNamespace + "PropertyGroup"); + + Version Parse(string elemName, bool controlPanel = false) + { + var element = versionsPropertyGroup.Element(xmlNamespace + elemName); + if (element == null) + return null; + + return Version.Parse(element.Value); + } + + var dict = new Dictionary + { + { Component.Core, Parse("TgsCoreVersion") }, + { Component.HttpApi, Parse("TgsApiVersion") }, + { Component.DreamMakerApi, Parse("TgsDmapiVersion") }, + }; + + if (await needExtendedComponentVersions) + { + // only grab some versions at release time + // we aggregate later + dict.Add(Component.Configuration, Parse("TgsConfigVersion")); + dict.Add(Component.InteropApi, Parse("TgsInteropVersion")); + dict.Add(Component.HostWatchdog, Parse("TgsHostWatchdogVersion")); + dict.Add(Component.NugetCommon, Parse("TgsCommonLibraryVersion")); + dict.Add(Component.NugetApi, Parse("TgsApiLibraryVersion")); + dict.Add(Component.NugetClient, Parse("TgsClientVersion")); + + var webVersion = Parse("TgsControlPanelVersion"); + if (webVersion != null) + { + dict.Add(Component.WebControlPanel, webVersion); + } + else + { + var controlPanelVersionBytes = await RLR(() => client.Repository.Content.GetRawContentByRef(RepoOwner, RepoName, "build/ControlPanelVersion.props", mergeCommit)); + using (var ms = new MemoryStream(controlPanelVersionBytes)) + doc = XDocument.Load(ms); + + + project = doc.Root; + var controlPanelXmlNamespace = project.GetDefaultNamespace(); + var controlPanelVersionsPropertyGroup = project.Elements().First(x => x.Name == controlPanelXmlNamespace + "PropertyGroup"); + dict.Add(Component.WebControlPanel, Version.Parse(controlPanelVersionsPropertyGroup.Element(controlPanelXmlNamespace + "TgsControlPanelVersion").Value)); + } + } + + return dict; + } + catch + { + return new Dictionary(); + } + } + + var componentVersions = needComponentExactVersions ? GetComponentVersions() : Task.FromResult>(null); + var changelists = new ConcurrentDictionary(); + async Task BuildNotesFromComment(string comment, User user, Task localPreviousTask) + { + await localPreviousTask; + if (comment == null) + return; + + async Task CommitNotes(Component component, List notes) + { + foreach (var I in notes) + Console.WriteLine(component + " #" + fullPR.Number + " - " + I + " (@" + user.Login + ")"); + + var tupleSelector = notes.Select(note => new Change + { + Descriptions = new List { note }, + PullRequest = fullPR.Number, + Author = user.Login + }); + + var useExtendedComponentVersions = await needExtendedComponentVersions; + var componentVersionsResult = await componentVersions; + lock (changelists) + if (changelists.TryGetValue(component, out var currentChangelist)) + currentChangelist.Changes.AddRange(tupleSelector); + else + Debug.Assert(changelists.TryAdd(component, new Changelist + { + Changes = tupleSelector.ToList(), + Unreleased = false, + Version = needComponentExactVersions && componentVersionsResult.TryGetValue(component, out var componentVersion) + ? componentVersion + : null, + ComponentVersions = component == Component.Core && needComponentExactVersions && useExtendedComponentVersions + ? new Dictionary(componentVersionsResult.Where(kvp => kvp.Key != Component.Core)) + : null + })); + } + + var commentSplits = comment.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + string targetComponent = null; + var notes = new List(); + foreach (var line in commentSplits) + { + var trimmedLine = line.Trim(); + if (targetComponent == null) + { + if (trimmedLine.StartsWith(":cl:", StringComparison.Ordinal)) + { + targetComponent = trimmedLine[4..].Trim(); + if (targetComponent.Length == 0) + targetComponent = "Core"; + } + continue; + } + if (trimmedLine.StartsWith("/:cl:", StringComparison.Ordinal)) + { + if(!Enum.TryParse(targetComponent, out var component)) + switch (targetComponent.ToUpperInvariant()) + { + case "**CONFIGURATION**": + case "CONFIGURATION": + case "CONFIG": + component = Component.Configuration; + break; + case "HTTP API": + component = Component.HttpApi; + break; + case "WEB CONTROL PANEL": + component = Component.WebControlPanel; + break; + case "DMAPI": + case "DREAMMAKER API": + component = Component.DreamMakerApi; + break; + case "INTEROP API": + component = Component.InteropApi; + break; + case "HOST WATCHDOG": + component = Component.HostWatchdog; + break; + case "NUGET: API": + component = Component.NugetApi; + break; + case "NUGET: COMMON": + component = Component.NugetCommon; + break; + case "NUGET: CLIENT": + component = Component.NugetClient; + break; + default: + throw new Exception($"Unknown component: \"{targetComponent}\""); + } + + await CommitNotes(component, notes); + targetComponent = null; + notes.Clear(); + continue; + } + if (trimmedLine.Length == 0) + continue; + + notes.Add(trimmedLine); + } + } + + var previousTask = BuildNotesFromComment(fullPR.Body, fullPR.User, Task.CompletedTask); + var comments = await commentsTask; + foreach (var x in comments) + previousTask = BuildNotesFromComment(x.Body, x.User, previousTask); + + await previousTask; + + return Tuple.Create(changelists.ToDictionary(kvp => kvp.Key, kvp => kvp.Value), await componentVersions, isReleasePR); + } + class ExtendedReleaseUpdate : ReleaseUpdate { public bool? MakeLatest { get; set; } @@ -570,8 +778,429 @@ The user account that created this pull request is available to correct any issu { Body = prBody, }); - return 0; } + + static async Task RLR(Func> func) + { + while (true) + try + { + return await func(); + } + catch (HttpRequestException ex) when (ex.InnerException is IOException ioEx && ioEx.InnerException is SocketException sockEx && sockEx.ErrorCode == 10053) + { + await Task.Delay(15000); + } + catch (SecondaryRateLimitExceededException) + { + await Task.Delay(15000); + } + catch (RateLimitExceededException ex) + { + var now = DateTimeOffset.UtcNow.AddSeconds(-10); + if (ex.Reset > now) + { + var delay = ex.Reset - now; + await Task.Delay(delay); + } + } + } + + static async Task> TripleCheckGitHubPagination(Func>> apiCall, Func idSelector) + { + // I've seen GitHub pagination return incomplete result sets in the past + // It has an in-built pagination limit of 100 + var apiOptions = new ApiOptions + { + PageSize = 100 + }; + var results = await RLR(() => apiCall(apiOptions)); + Dictionary distinctEntries = new Dictionary(results.Count); + foreach (var result in results) + distinctEntries.Add(idSelector(result), result); + + if (results.Count > 100) + { + results = await RLR(() => apiCall(apiOptions)); + foreach (var result in results) + distinctEntries.TryAdd(idSelector(result), result); + + results = await RLR(() => apiCall(apiOptions)); + foreach (var result in results) + distinctEntries.TryAdd(idSelector(result), result); + } + + return distinctEntries.Values.ToList(); + } + + static async Task> ProcessMilestone(IGitHubClient client, Milestone milestone) + { + // have to trust this works + SearchIssuesResult results; + + var milestoneTask = Task.FromResult(milestone); + var pullRequests = new Dictionary(); + var iteration = 0; + while (true) + { + results = await RLR(() => client.Search.SearchIssues(new SearchIssuesRequest + { + Type = IssueTypeQualifier.PullRequest, + Milestone = milestone.Title, + Repos = new RepositoryCollection + { + { RepoOwner, RepoName }, + }, + })); + + foreach (var result in results.Items) + if (pullRequests.TryAdd(result.Number, result)) + milestoneTasks.TryAdd(result.Number, milestoneTask); + + if (results.IncompleteResults) + continue; + + if (results.TotalCount <= 100 || ++iteration == 3) + break; + } + + async Task RunPRs() + { + var milestoneVersion = Version.Parse(milestone.Title[1..]); + var prTasks = pullRequests.Select( + kvp => GetReleaseNotesFromPR(client, kvp.Value, true, true, milestone.State.Value == ItemState.Open)) + .ToList(); + + await Task.WhenAll(prTasks); + + var prResults = prTasks.Select(x => x.Result).Where(result => result != null).ToList(); + + var releasePRResult = prResults.FirstOrDefault(x => x.Item3); + + Dictionary releasedComponentVersions; + if (releasePRResult != null) + releasedComponentVersions = releasePRResult.Item2; + else + { + releasedComponentVersions = new Dictionary( + prResults + .SelectMany(result => result.Item2) + .GroupBy(kvp => kvp.Key) + .Select(grouping => new KeyValuePair(grouping.Key, grouping.Max(kvp => kvp.Value)))); + + foreach(var maxVersionKvp in prResults.SelectMany(x => x.Item1) + .Where(x => !releasedComponentVersions.ContainsKey(x.Key)) + .GroupBy(x => x.Key) + .Select(group => { + var versions = group + .Where(x => x.Value.Version != null) + .ToList(); + + if (versions.Count == 0) + return new KeyValuePair(group.Key, null); + + return new KeyValuePair(group.Key, versions.Max(x => x.Value.Version)); + }) + .Where(kvp => kvp.Value != null) + .ToList()) + { + releasedComponentVersions.Add(maxVersionKvp.Key, maxVersionKvp.Value); + } + } + + var finalResults = new Dictionary>(); + foreach (var componentKvp in releasedComponentVersions) + { + var component = componentKvp.Key; + var list = new List(); + + foreach(var changelistDict in prResults.Select(x => x.Item1)) + { + if (!changelistDict.TryGetValue(component, out var changelist)) + continue; + + Version componentVersion = milestoneVersion; + var unreleased = milestone.State.Value == ItemState.Open; + if (component != Component.Core) + { + componentVersion = changelist.Version ?? componentKvp.Value; + if (releasedNonCoreVersions != null + && releasedNonCoreVersions.TryGetValue(component, out var releasedVersions) + && !releasedVersions.Any(x => x == componentVersion)) + { + // roll forward + var newList = releasedVersions + .ToList(); + newList.Add(componentVersion); + newList = newList.OrderBy(x => x).ToList(); + + var index = newList.IndexOf(componentVersion); + Debug.Assert(index != -1); + if (index != (newList.Count - 1)) + { + componentVersion = newList[index + 1]; + unreleased = false; + } + else + unreleased = true; + } + } + + var entry = list.FirstOrDefault(x => x.Version == componentVersion); + if (entry == null) + { + entry = changelist; + entry.Version = componentVersion; + entry.Unreleased = unreleased; + list.Add(entry); + } + else + entry.Changes.AddRange(changelist.Changes); + } + + Debug.Assert(list.Select(x => x.Version.ToString()).Distinct().Count() == list.Count); + if (component == Component.Core) + { + Debug.Assert(list.All(x => x.Version == milestoneVersion)); + } + + list = list.OrderByDescending(x => x.Version).ToList(); + finalResults.Add(component, list); + } + + if (!finalResults.ContainsKey(Component.Core) || finalResults[Component.Core].Count == 0) + { + finalResults.Remove(Component.Core); + finalResults.Add(Component.Core, new List + { + new Changelist + { + Changes = new List(), + ComponentVersions = releasedComponentVersions, + Unreleased = milestone.State.Value == ItemState.Open, + Version = milestoneVersion, + } + }); + } + else + Debug.Assert(finalResults[Component.Core].All(x => x.Version == milestoneVersion)); + + var notes = new ReleaseNotes + { + Components = new SortedDictionary>(finalResults), + }; + + return notes; + } + + return RunPRs(); + } + + static async Task FullNotes(IGitHubClient client) + { + var startRateLimit = (client.GetLastApiInfo()?.RateLimit ?? (await client.RateLimit.GetRateLimits()).Rate).Remaining; + + ReleaseNotes existingNotes = null; + if (File.Exists("changelog.yml")) + { + var existingYml = await File.ReadAllTextAsync("changelog.yml"); + var deserializer = new DeserializerBuilder() + .Build(); + + existingNotes = deserializer.Deserialize(existingYml); + } + + var releaseNotes = await GenerateNotes(client, existingNotes); + + Console.WriteLine($"Generating all release notes took {startRateLimit - client.GetLastApiInfo().RateLimit.Remaining} requests."); + + var serializer = new SerializerBuilder() + .ConfigureDefaultValuesHandling(DefaultValuesHandling.OmitDefaults) + .WithTypeConverter(new VersionConverter()) + .Build(); + + var serializedYaml = serializer.Serialize(releaseNotes); + await File.WriteAllTextAsync("changelog.yml", serializedYaml).ConfigureAwait(false); + return 0; + } + + static HttpClient httpClient = new HttpClient(new HttpClientHandler() + { + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate + }); + static async Task> EnumerateNugetVersions(string package) + { + var url = new Uri($"https://api.nuget.org/v3/registration5-gz-semver2/{package.ToLowerInvariant()}/index.json"); + + using var req = new HttpRequestMessage(); + req.Headers.UserAgent.Add(new System.Net.Http.Headers.ProductInfoHeaderValue("Tgstation.Server.ReleaseNotes", "0.1.0")); + req.Method = HttpMethod.Get; + req.RequestUri = url; + + using var resp = await httpClient.SendAsync(req); + resp.EnsureSuccessStatusCode(); + + var json = await resp.Content.ReadAsStringAsync(); + + dynamic dynamicJson = JsonConvert.DeserializeObject(json); + + var versions = (IEnumerable)dynamicJson.items[0].items; + var results = versions + .Select(x => Version.TryParse((string)x.catalogEntry.version, out var version) ? version : null) + .Where(version => version != null) + .OrderBy(x => x) + .ToHashSet(); + return results; + } + + static IReadOnlyDictionary> releasedNonCoreVersions; + + static async Task GenerateNotes(IGitHubClient client, ReleaseNotes previousNotes) + { + var releasesTask = TripleCheckGitHubPagination( + apiOptions => client.Repository.Release.GetAll(RepoOwner, RepoName, apiOptions), + release => release.Id); + + var milestones = await TripleCheckGitHubPagination( + apiOptions => client.Issue.Milestone.GetAllForRepository(RepoOwner, RepoName, new MilestoneRequest { + State = ItemStateFilter.All + }, apiOptions), + milestone => milestone.Id); + + var versionMilestones = milestones + .Where(milestone => Regex.IsMatch(milestone.Title, @"v[1-9][0-9]*\.[1-9]*[0-9]+\.[1-9]*[0-9]+$")) + .ToList(); + + var releases = await releasesTask; + + var nugetCommonVersions = EnumerateNugetVersions("Tgstation.Server.Common"); + var nugetApiVersions = EnumerateNugetVersions("Tgstation.Server.Api"); + var nugetClientVersions = EnumerateNugetVersions("Tgstation.Server.Client"); + + releasedNonCoreVersions = new Dictionary> { + { Component.HttpApi, releases + .Where(x => x.TagName.StartsWith("api-v")) + .Select(x => Version.Parse(x.TagName[5..])) + .OrderBy(x => x) + .ToHashSet() }, + { Component.DreamMakerApi, releases + .Where(x => x.TagName.StartsWith("dmapi-v")) + .Select(x => Version.Parse(x.TagName[7..])) + .OrderBy(x => x) + .ToHashSet() }, + { Component.NugetCommon, await nugetCommonVersions }, + { Component.NugetApi, await nugetApiVersions }, + { Component.NugetClient, await nugetClientVersions } + }; + + var milestonesToProcess = versionMilestones; + if (previousNotes != null) + { + var releasedVersions = previousNotes.Components[Component.Core].Where(cl => !cl.Unreleased).ToList(); + milestonesToProcess = milestonesToProcess + .Where(x => !releasedVersions.Any( + version => version.Version == Version.Parse(x.Title.AsSpan(1)))) + .ToList(); + + foreach (var kvp in previousNotes.Components) + if (releasedNonCoreVersions.TryGetValue(kvp.Key, out var releasedComponentVersions)) + kvp.Value.RemoveAll(x => x.Unreleased = !releasedComponentVersions.Any(y => y == x.Version)); + else + kvp.Value.RemoveAll(x => x.Unreleased); + } + + var milestonePRTasks = milestonesToProcess + .Select(milestone => ProcessMilestone(client, milestone)) + .ToList(); + + await Task.WhenAll(milestonePRTasks); + + await Task.WhenAll(milestonePRTasks.Select(task => task.Result)); + + var coreCls = milestonePRTasks + .SelectMany(task => task.Result.Result.Components) + .Where(x => x.Key == Component.Core) + .ToList(); + + Debug.Assert( + coreCls.Count == milestonesToProcess.Count); + + var distinctCoreVersions = coreCls + .SelectMany(x => x.Value) + .Select(x => x.Version.ToString()) + .Distinct() + .Select(Version.Parse) + .OrderBy(x => x) + .ToList(); + + var missingCoreVersions = milestonesToProcess + .Where(x => !distinctCoreVersions.Any(y => Version.Parse(x.Title.Substring(1)) == y)) + .ToList(); + + Debug.Assert(missingCoreVersions.Count == 0); + foreach (var missingCoreVersion in missingCoreVersions) + await await ProcessMilestone(client, missingCoreVersion); + + var changelistsGroupedByComponent = + milestonePRTasks + .SelectMany(task => task.Result.Result.Components) + .GroupBy(kvp => kvp.Key) + .ToDictionary(grouping => grouping.Key, grouping => grouping.SelectMany(kvp => kvp.Value)); + + var releaseNotes = new ReleaseNotes + { + Components = new SortedDictionary>( + changelistsGroupedByComponent + .ToDictionary( + kvp => kvp.Key, + kvp => kvp + .Value + .GroupBy(changelist => changelist.Version) + .Select(grouping => + { + var firstEntry = grouping.First(); + return new Changelist + { + Changes = grouping.SelectMany(cl => cl.Changes).ToList(), + ComponentVersions = firstEntry.ComponentVersions, + Unreleased = firstEntry.Unreleased, + Version = grouping.Key + }; + }) + .OrderByDescending(cl => cl.Version) + .ToList())) + }; + + Debug.Assert(releaseNotes.Components.ContainsKey(Component.Core) && releaseNotes.Components[Component.Core].Count == milestonesToProcess.Count); + + if (previousNotes != null) + { + foreach (var component in Enum.GetValues()) + { + if (!previousNotes.Components.ContainsKey(component)) + continue; + + if (releaseNotes.Components.TryGetValue(component, out var changelists)) + releaseNotes.Components[component] = changelists + .Concat(previousNotes.Components[component]) + .OrderByDescending(cl => cl.Version) + .ToList(); + else + releaseNotes.Components[component] = previousNotes.Components[component]; + } + } + + foreach (var kvp in releaseNotes.Components) + { + var distinctCount = kvp.Value.Select(changelist => changelist.Version.ToString()).Distinct().Count(); + Debug.Assert(distinctCount == kvp.Value.Count); + + foreach (var cl in kvp.Value) + cl.DeduplicateChanges(); + } + + return releaseNotes; + } } } diff --git a/tools/Tgstation.Server.ReleaseNotes/ReleaseNotes.cs b/tools/Tgstation.Server.ReleaseNotes/ReleaseNotes.cs new file mode 100644 index 0000000000..8c10c880ce --- /dev/null +++ b/tools/Tgstation.Server.ReleaseNotes/ReleaseNotes.cs @@ -0,0 +1,9 @@ +using System.Collections.Generic; + +namespace Tgstation.Server.ReleaseNotes +{ + sealed class ReleaseNotes + { + public SortedDictionary> Components { get; set; } + } +} diff --git a/tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj b/tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj index d95f0139e3..35312949ce 100644 --- a/tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj +++ b/tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj @@ -7,10 +7,12 @@ - - + + + +