diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index b770b65721..c146ed90da 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -13,9 +13,10 @@ They will be amalgamated together in the end.
/:cl:
:cl: Categories
-Categories are divided up in the release notes and set after the :cl: header.
+Categories are used by [the release notes tool](../tools/Tgstation.Server.ReleaseNotes) to generate formatted changelists used in releases.
The default category is Core.
-The current standard categories are Core, DreamMaker API, HTTP API, and Host Watchdog.
+Only one category may be specified for a :cl: block.
+Valid categories are Core, DreamMaker API, HTTP API, Host Watchdog, Web Control Panel, Configuration, Nuget: Api, Nuget: Client, and Nuget: Common.
/:cl:
[Why]: # (If this does not close or work on an existing GitHub issue, please add a short description [two lines down] of why you think these changes would benefit the server. If you can't justify it in words, it might not be worth adding.)
diff --git a/.gitignore b/.gitignore
index d7756e60cd..8794b1a00f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -23,6 +23,7 @@ artifacts/
/tools/Tgstation.Server.ReleaseNotes/release_notes.md
launchSettings.json
release_notes.md
+release_notes.yml
*nupkg
*.sqlite3
packaging/
diff --git a/src/Tgstation.Server.Host/Extensions/Converters/VersionConverter.cs b/src/Tgstation.Server.Host/Extensions/Converters/VersionConverter.cs
index ec08e5ae7b..4ca9cdf105 100644
--- a/src/Tgstation.Server.Host/Extensions/Converters/VersionConverter.cs
+++ b/src/Tgstation.Server.Host/Extensions/Converters/VersionConverter.cs
@@ -13,7 +13,7 @@ namespace Tgstation.Server.Host.Extensions.Converters
///
/// and for serializing s in semver format.
///
- sealed class VersionConverter : JsonConverter, IYamlTypeConverter
+ public sealed class VersionConverter : JsonConverter, IYamlTypeConverter
{
///
/// Check if the supports (de)serializing a given .
diff --git a/tools/Tgstation.Server.ReleaseNotes/Change.cs b/tools/Tgstation.Server.ReleaseNotes/Change.cs
new file mode 100644
index 0000000000..88f5910606
--- /dev/null
+++ b/tools/Tgstation.Server.ReleaseNotes/Change.cs
@@ -0,0 +1,13 @@
+using System.Collections.Generic;
+
+using Octokit;
+
+namespace Tgstation.Server.ReleaseNotes
+{
+ sealed class Change
+ {
+ public List Descriptions { get; set; }
+ public string Author { get; set; }
+ public int PullRequest { get; set; }
+ }
+}
diff --git a/tools/Tgstation.Server.ReleaseNotes/Changelist.cs b/tools/Tgstation.Server.ReleaseNotes/Changelist.cs
new file mode 100644
index 0000000000..137d018799
--- /dev/null
+++ b/tools/Tgstation.Server.ReleaseNotes/Changelist.cs
@@ -0,0 +1,46 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+
+namespace Tgstation.Server.ReleaseNotes
+{
+ sealed class Changelist
+ {
+ public Version Version { get; set; }
+
+ public Dictionary ComponentVersions { get; set; }
+
+ public List Changes { get; set; }
+
+ public bool Unreleased { get; set; }
+
+ public void DeduplicateChanges()
+ {
+ Changes = Changes
+ .OrderBy(x => x.PullRequest)
+ .GroupBy(x => x.PullRequest)
+ .Select(prChanges =>
+ {
+ string author = null;
+ return new Change
+ {
+ PullRequest = prChanges.Key,
+ Descriptions = prChanges
+ .SelectMany(x =>
+ {
+ if (author != null)
+ Debug.Assert(x.Author == author);
+ else
+ author = x.Author;
+
+ return x.Descriptions;
+ })
+ .ToList(),
+ Author = author
+ };
+ })
+ .ToList();
+ }
+ }
+}
diff --git a/tools/Tgstation.Server.ReleaseNotes/Component.cs b/tools/Tgstation.Server.ReleaseNotes/Component.cs
new file mode 100644
index 0000000000..f882a3a320
--- /dev/null
+++ b/tools/Tgstation.Server.ReleaseNotes/Component.cs
@@ -0,0 +1,16 @@
+namespace Tgstation.Server.ReleaseNotes
+{
+ enum Component
+ {
+ Core,
+ HttpApi,
+ Configuration,
+ DreamMakerApi,
+ InteropApi,
+ WebControlPanel,
+ HostWatchdog,
+ NugetCommon,
+ NugetApi,
+ NugetClient,
+ }
+}
diff --git a/tools/Tgstation.Server.ReleaseNotes/Program.cs b/tools/Tgstation.Server.ReleaseNotes/Program.cs
index e0bfa45e59..e5a95795d6 100644
--- a/tools/Tgstation.Server.ReleaseNotes/Program.cs
+++ b/tools/Tgstation.Server.ReleaseNotes/Program.cs
@@ -1,14 +1,28 @@
-using System;
+// This program is minimal effort and should be sent to remedial school
+
+using System;
+using System.Collections.Concurrent;
using System.Collections.Generic;
+using System.Diagnostics;
using System.IO;
using System.Linq;
+using System.Net;
+using System.Net.Http;
+using System.Net.Sockets;
using System.Text;
+using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Xml.Linq;
+using Newtonsoft.Json;
+
using Octokit;
using Octokit.GraphQL;
+using Tgstation.Server.Host.Extensions.Converters;
+
+using YamlDotNet.Serialization;
+
namespace Tgstation.Server.ReleaseNotes
{
///
@@ -34,8 +48,9 @@ namespace Tgstation.Server.ReleaseNotes
var ensureRelease = versionString.Equals("--ensure-release", StringComparison.OrdinalIgnoreCase);
var linkWinget = versionString.Equals("--link-winget", StringComparison.OrdinalIgnoreCase);
var shaCheck = versionString.Equals("--winget-template-check", StringComparison.OrdinalIgnoreCase);
+ var fullNotes = versionString.Equals("--generate-full-notes", StringComparison.OrdinalIgnoreCase);
- if ((!Version.TryParse(versionString, out var version) || version.Revision != -1) && !ensureRelease && !linkWinget && !shaCheck)
+ if ((!Version.TryParse(versionString, out var version) || version.Revision != -1) && !ensureRelease && !linkWinget && !shaCheck && !fullNotes)
{
Console.WriteLine("Invalid version: " + versionString);
return 2;
@@ -84,6 +99,11 @@ namespace Tgstation.Server.ReleaseNotes
return await Winget(client, null, args[1]);
}
+ if (fullNotes)
+ {
+ return await FullNotes(client);
+ }
+
var releasesTask = client.Repository.Release.GetAll(RepoOwner, RepoName);
Console.WriteLine("Getting merged pull requests in milestone " + versionString + "...");
@@ -101,103 +121,12 @@ namespace Tgstation.Server.ReleaseNotes
}
Console.WriteLine(milestonePRs.Items.Count + " total pull requests");
- Task milestoneTask = null;
- var milestoneTaskLock = new object();
- var releaseDictionary = new Dictionary>>(StringComparer.OrdinalIgnoreCase);
-
bool postControlPanelMessage = false;
- async Task GetReleaseNotesFromPR(Issue pullRequest)
- {
- //need to check it was merged
- var fullPR = await client.Repository.PullRequest.Get(RepoOwner, RepoName, pullRequest.Number).ConfigureAwait(false);
+ var noteTasks = new List, Dictionary, bool>>>();
- if (!fullPR.Merged)
- {
- if (!doNotCloseMilestone && fullPR.Milestone != null)
- {
- Console.WriteLine($"Removing trash PR #{fullPR.Number} from milestone...");
- await client.Issue.Update(RepoOwner, RepoName, fullPR.Number, new IssueUpdate
- {
- Milestone = null
- }).ConfigureAwait(false);
- }
-
- return;
- }
-
- async Task GetMilestone()
- {
- if (fullPR.Milestone == null)
- return null;
- return await client.Issue.Milestone.Get(RepoOwner, RepoName, fullPR.Milestone.Number);
- };
-
- lock (milestoneTaskLock)
- milestoneTask ??= GetMilestone();
-
- // if (!fullPR.Merged)
- //return;
-
- void BuildNotesFromComment(string comment, User user)
- {
- if (comment == null)
- return;
-
- void CommitNotes(string component, List notes)
- {
- lock (releaseDictionary)
- {
- foreach (var I in notes)
- Console.WriteLine(component + " #" + fullPR.Number + " - " + I + " (@" + user.Login + ")");
-
- var tupleSelector = notes.Select(note => Tuple.Create(note, fullPR.Number, user.Login));
- if (releaseDictionary.TryGetValue(component, out var currentValues))
- currentValues.AddRange(tupleSelector);
- else
- releaseDictionary.Add(component, tupleSelector.ToList());
- }
- }
-
- var commentSplits = comment.Split('\n');
- string targetComponent = null;
- var notes = new List();
- foreach (var line in commentSplits)
- {
- var trimmedLine = line.Trim();
- if (targetComponent == null)
- {
- if (trimmedLine.StartsWith(":cl:", StringComparison.Ordinal))
- {
- targetComponent = trimmedLine[4..].Trim();
- if (targetComponent.Length == 0)
- targetComponent = "Core";
- }
- continue;
- }
- if (trimmedLine.StartsWith("/:cl:", StringComparison.Ordinal))
- {
- CommitNotes(targetComponent, notes);
- targetComponent = null;
- notes.Clear();
- continue;
- }
- if (trimmedLine.Length == 0)
- continue;
-
- notes.Add(trimmedLine);
- }
- }
-
- var comments = await client.Issue.Comment.GetAllForIssue(RepoOwner, RepoName, fullPR.Number).ConfigureAwait(false);
- BuildNotesFromComment(fullPR.Body, fullPR.User);
- foreach(var x in comments)
- BuildNotesFromComment(x.Body, x.User);
- }
-
- var tasks = new List();
foreach (var I in milestonePRs.Items)
- tasks.Add(GetReleaseNotesFromPR(I));
+ noteTasks.Add(GetReleaseNotesFromPR(client, I, doNotCloseMilestone, false, false));
var releases = await releasesTask.ConfigureAwait(false);
@@ -299,19 +228,21 @@ namespace Tgstation.Server.ReleaseNotes
newNotes.Append(version.Build);
}
newNotes.Append("](");
- var milestone = await milestoneTask.ConfigureAwait(false);
+
+ var milestone = await milestoneTasks.Single().Value.ConfigureAwait(false);
if (milestone == null)
{
Console.WriteLine("Unable to detemine milestone!");
return 9;
}
+ var allTasks = new List(noteTasks);
if (doNotCloseMilestone)
Console.WriteLine("Not closing milestone due to parameter!");
else
{
Console.WriteLine("Closing milestone...");
- tasks.Add(client.Issue.Milestone.Update(RepoOwner, RepoName, milestone.Number, new MilestoneUpdate
+ allTasks.Add(client.Issue.Milestone.Update(RepoOwner, RepoName, milestone.Number, new MilestoneUpdate
{
State = ItemState.Closed
}));
@@ -358,14 +289,16 @@ namespace Tgstation.Server.ReleaseNotes
if (I.PullRequest != null)
{
Console.WriteLine($"Adding additional merged PR #{I.Number}...");
- tasks.Add(GetReleaseNotesFromPR(I));
+ var task = GetReleaseNotesFromPR(client, I, doNotCloseMilestone, false, false);
+ noteTasks.Add(task);
+ allTasks.Add(task);
}
}
await Task.WhenAll(issueUpdateTasks).ConfigureAwait(false);
}
- tasks.Add(client.Issue.Milestone.Delete(RepoOwner, RepoName, milestoneToDelete.Number));
+ allTasks.Add(client.Issue.Milestone.Delete(RepoOwner, RepoName, milestoneToDelete.Number));
}
// Create the next minor milestone
@@ -378,7 +311,7 @@ namespace Tgstation.Server.ReleaseNotes
{
Description = "Next minor version"
});
- tasks.Add(nextMinorMilestoneTask);
+ allTasks.Add(nextMinorMilestoneTask);
// Move unfinished stuff to new minor milestone
Console.WriteLine($"Moving {milestone.OpenIssues} abandoned issue(s) from previous milestone to new one...");
@@ -393,7 +326,7 @@ namespace Tgstation.Server.ReleaseNotes
{
var nextMinorMilestone = await nextMinorMilestoneTask.ConfigureAwait(false);
foreach (var I in abandonedIssues.Items)
- tasks.Add(client.Issue.Update(RepoOwner, RepoName, I.Number, new IssueUpdate
+ allTasks.Add(client.Issue.Update(RepoOwner, RepoName, I.Number, new IssueUpdate
{
Milestone = nextMinorMilestone.Number
}));
@@ -405,7 +338,40 @@ namespace Tgstation.Server.ReleaseNotes
newNotes.Append("?closed=1)");
newNotes.Append(Environment.NewLine);
- await Task.WhenAll(tasks).ConfigureAwait(false);
+ await Task.WhenAll(allTasks).ConfigureAwait(false);
+
+ var componentVersionDict = new Dictionary
+ {
+ { Component.Configuration, configVersion },
+ { Component.HttpApi, apiVersion },
+ { Component.DreamMakerApi, dmApiVersion },
+ { Component.InteropApi, interopVersion },
+ { Component.WebControlPanel, webControlVersion },
+ { Component.HostWatchdog, hostWatchdogVersion },
+ };
+
+ var releaseDictionary = new Dictionary(
+ noteTasks
+ .SelectMany(task => task.Result.Item1)
+ .GroupBy(kvp => kvp.Key)
+ .Select(grouping =>
+ {
+ var component = grouping.Key;
+ var changelist = new Changelist
+ {
+ Changes = grouping.SelectMany(kvp => kvp.Value.Changes).ToList()
+ };
+
+ if (component == Component.Core)
+ {
+ changelist.Version = coreVersion;
+ changelist.ComponentVersions = componentVersionDict;
+ }
+ else
+ changelist.Version = componentVersionDict[component];
+
+ return new KeyValuePair(component, changelist);
+ }));
if (releaseDictionary.Count == 0)
{
@@ -417,20 +383,31 @@ namespace Tgstation.Server.ReleaseNotes
{
newNotes.Append(Environment.NewLine);
newNotes.Append("#### ");
- newNotes.Append(I.Key);
-
-
- foreach (var noteTuple in I.Value)
+ string componentName = I.Key switch
{
- newNotes.Append(Environment.NewLine);
- newNotes.Append("- ");
- newNotes.Append(noteTuple.Item1);
- newNotes.Append(" (#");
- newNotes.Append(noteTuple.Item2);
- newNotes.Append(" @");
- newNotes.Append(noteTuple.Item3);
- newNotes.Append(')');
- }
+ Component.HttpApi => "HTTP API",
+ Component.InteropApi => "Interop API",
+ Component.Configuration => "**Configuration**",
+ Component.DreamMakerApi => "DreamMaker API",
+ Component.HostWatchdog => "Host Watchdog",
+ Component.Core => "Core",
+ Component.WebControlPanel => "Web Control Panel",
+ _ => throw new Exception($"Unknown Component: {I.Key}"),
+ };
+ newNotes.Append(componentName);
+
+ foreach (var change in I.Value.Changes)
+ foreach (var line in change.Descriptions)
+ {
+ newNotes.Append(Environment.NewLine);
+ newNotes.Append("- ");
+ newNotes.Append(line);
+ newNotes.Append(" (#");
+ newNotes.Append(change.PullRequest);
+ newNotes.Append(" @");
+ newNotes.Append(change.Author);
+ newNotes.Append(')');
+ }
newNotes.Append(Environment.NewLine);
}
@@ -473,6 +450,237 @@ namespace Tgstation.Server.ReleaseNotes
}
}
+ static ConcurrentDictionary> milestoneTasks = new ConcurrentDictionary>();
+ static Task GetMilestone(IGitHubClient client, int number)
+ => milestoneTasks.GetOrAdd(number, localNumber => client.Issue.Milestone.Get(RepoOwner, RepoName, localNumber));
+
+ static async Task, Dictionary, bool>> GetReleaseNotesFromPR(IGitHubClient client, Issue pullRequest, bool doNotCloseMilestone, bool needComponentExactVersions, bool forAllComponents)
+ {
+ //need to check it was merged
+ var fullPR = await RLR(() => client.Repository.PullRequest.Get(RepoOwner, RepoName, pullRequest.Number));
+
+ if (!fullPR.Merged)
+ {
+ if (!doNotCloseMilestone && fullPR.Milestone != null)
+ {
+ Console.WriteLine($"Removing trash PR #{fullPR.Number} from milestone...");
+ await RLR(() => client.Issue.Update(RepoOwner, RepoName, fullPR.Number, new IssueUpdate
+ {
+ Milestone = null
+ }));
+ }
+
+ return null;
+ }
+
+ if (fullPR.Milestone == null)
+ {
+ return null;
+ }
+
+ var commentsTask = TripleCheckGitHubPagination(apiOptions => client.Issue.Comment.GetAllForIssue(fullPR.Base.Repository.Id, pullRequest.Number, apiOptions), comment => comment.Id);
+
+ bool isReleasePR = false;
+ async Task ShouldGetExtendedComponentVersions()
+ {
+ if (forAllComponents)
+ return true;
+
+ var commit = await RLR(() => client.Repository.Commit.Get(fullPR.Base.Repository.Id, fullPR.MergeCommitSha));
+
+ return isReleasePR = commit.Commit.Message.Contains("[TGSDeploy]");
+ }
+
+ Task needExtendedComponentVersions = null;
+ async Task> GetComponentVersions()
+ {
+ var mergeCommit = fullPR.MergeCommitSha;
+ // we don't care about unreleased web control panel changes
+
+ try
+ {
+ needExtendedComponentVersions = ShouldGetExtendedComponentVersions();
+
+ var versionsBytes = await RLR(() => client.Repository.Content.GetRawContentByRef(RepoOwner, RepoName, "build/Version.props", mergeCommit));
+
+ XDocument doc;
+ using (var ms = new MemoryStream(versionsBytes))
+ doc = XDocument.Load(ms);
+
+ var project = doc.Root;
+ var xmlNamespace = project.GetDefaultNamespace();
+ var versionsPropertyGroup = project.Elements().First(x => x.Name == xmlNamespace + "PropertyGroup");
+
+ Version Parse(string elemName, bool controlPanel = false)
+ {
+ var element = versionsPropertyGroup.Element(xmlNamespace + elemName);
+ if (element == null)
+ return null;
+
+ return Version.Parse(element.Value);
+ }
+
+ var dict = new Dictionary
+ {
+ { Component.Core, Parse("TgsCoreVersion") },
+ { Component.HttpApi, Parse("TgsApiVersion") },
+ { Component.DreamMakerApi, Parse("TgsDmapiVersion") },
+ };
+
+ if (await needExtendedComponentVersions)
+ {
+ // only grab some versions at release time
+ // we aggregate later
+ dict.Add(Component.Configuration, Parse("TgsConfigVersion"));
+ dict.Add(Component.InteropApi, Parse("TgsInteropVersion"));
+ dict.Add(Component.HostWatchdog, Parse("TgsHostWatchdogVersion"));
+ dict.Add(Component.NugetCommon, Parse("TgsCommonLibraryVersion"));
+ dict.Add(Component.NugetApi, Parse("TgsApiLibraryVersion"));
+ dict.Add(Component.NugetClient, Parse("TgsClientVersion"));
+
+ var webVersion = Parse("TgsControlPanelVersion");
+ if (webVersion != null)
+ {
+ dict.Add(Component.WebControlPanel, webVersion);
+ }
+ else
+ {
+ var controlPanelVersionBytes = await RLR(() => client.Repository.Content.GetRawContentByRef(RepoOwner, RepoName, "build/ControlPanelVersion.props", mergeCommit));
+ using (var ms = new MemoryStream(controlPanelVersionBytes))
+ doc = XDocument.Load(ms);
+
+
+ project = doc.Root;
+ var controlPanelXmlNamespace = project.GetDefaultNamespace();
+ var controlPanelVersionsPropertyGroup = project.Elements().First(x => x.Name == controlPanelXmlNamespace + "PropertyGroup");
+ dict.Add(Component.WebControlPanel, Version.Parse(controlPanelVersionsPropertyGroup.Element(controlPanelXmlNamespace + "TgsControlPanelVersion").Value));
+ }
+ }
+
+ return dict;
+ }
+ catch
+ {
+ return new Dictionary();
+ }
+ }
+
+ var componentVersions = needComponentExactVersions ? GetComponentVersions() : Task.FromResult>(null);
+ var changelists = new ConcurrentDictionary();
+ async Task BuildNotesFromComment(string comment, User user, Task localPreviousTask)
+ {
+ await localPreviousTask;
+ if (comment == null)
+ return;
+
+ async Task CommitNotes(Component component, List notes)
+ {
+ foreach (var I in notes)
+ Console.WriteLine(component + " #" + fullPR.Number + " - " + I + " (@" + user.Login + ")");
+
+ var tupleSelector = notes.Select(note => new Change
+ {
+ Descriptions = new List { note },
+ PullRequest = fullPR.Number,
+ Author = user.Login
+ });
+
+ var useExtendedComponentVersions = await needExtendedComponentVersions;
+ var componentVersionsResult = await componentVersions;
+ lock (changelists)
+ if (changelists.TryGetValue(component, out var currentChangelist))
+ currentChangelist.Changes.AddRange(tupleSelector);
+ else
+ Debug.Assert(changelists.TryAdd(component, new Changelist
+ {
+ Changes = tupleSelector.ToList(),
+ Unreleased = false,
+ Version = needComponentExactVersions && componentVersionsResult.TryGetValue(component, out var componentVersion)
+ ? componentVersion
+ : null,
+ ComponentVersions = component == Component.Core && needComponentExactVersions && useExtendedComponentVersions
+ ? new Dictionary(componentVersionsResult.Where(kvp => kvp.Key != Component.Core))
+ : null
+ }));
+ }
+
+ var commentSplits = comment.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
+ string targetComponent = null;
+ var notes = new List();
+ foreach (var line in commentSplits)
+ {
+ var trimmedLine = line.Trim();
+ if (targetComponent == null)
+ {
+ if (trimmedLine.StartsWith(":cl:", StringComparison.Ordinal))
+ {
+ targetComponent = trimmedLine[4..].Trim();
+ if (targetComponent.Length == 0)
+ targetComponent = "Core";
+ }
+ continue;
+ }
+ if (trimmedLine.StartsWith("/:cl:", StringComparison.Ordinal))
+ {
+ if(!Enum.TryParse(targetComponent, out var component))
+ switch (targetComponent.ToUpperInvariant())
+ {
+ case "**CONFIGURATION**":
+ case "CONFIGURATION":
+ case "CONFIG":
+ component = Component.Configuration;
+ break;
+ case "HTTP API":
+ component = Component.HttpApi;
+ break;
+ case "WEB CONTROL PANEL":
+ component = Component.WebControlPanel;
+ break;
+ case "DMAPI":
+ case "DREAMMAKER API":
+ component = Component.DreamMakerApi;
+ break;
+ case "INTEROP API":
+ component = Component.InteropApi;
+ break;
+ case "HOST WATCHDOG":
+ component = Component.HostWatchdog;
+ break;
+ case "NUGET: API":
+ component = Component.NugetApi;
+ break;
+ case "NUGET: COMMON":
+ component = Component.NugetCommon;
+ break;
+ case "NUGET: CLIENT":
+ component = Component.NugetClient;
+ break;
+ default:
+ throw new Exception($"Unknown component: \"{targetComponent}\"");
+ }
+
+ await CommitNotes(component, notes);
+ targetComponent = null;
+ notes.Clear();
+ continue;
+ }
+ if (trimmedLine.Length == 0)
+ continue;
+
+ notes.Add(trimmedLine);
+ }
+ }
+
+ var previousTask = BuildNotesFromComment(fullPR.Body, fullPR.User, Task.CompletedTask);
+ var comments = await commentsTask;
+ foreach (var x in comments)
+ previousTask = BuildNotesFromComment(x.Body, x.User, previousTask);
+
+ await previousTask;
+
+ return Tuple.Create(changelists.ToDictionary(kvp => kvp.Key, kvp => kvp.Value), await componentVersions, isReleasePR);
+ }
+
class ExtendedReleaseUpdate : ReleaseUpdate
{
public bool? MakeLatest { get; set; }
@@ -570,8 +778,429 @@ The user account that created this pull request is available to correct any issu
{
Body = prBody,
});
-
return 0;
}
+
+ static async Task RLR(Func> func)
+ {
+ while (true)
+ try
+ {
+ return await func();
+ }
+ catch (HttpRequestException ex) when (ex.InnerException is IOException ioEx && ioEx.InnerException is SocketException sockEx && sockEx.ErrorCode == 10053)
+ {
+ await Task.Delay(15000);
+ }
+ catch (SecondaryRateLimitExceededException)
+ {
+ await Task.Delay(15000);
+ }
+ catch (RateLimitExceededException ex)
+ {
+ var now = DateTimeOffset.UtcNow.AddSeconds(-10);
+ if (ex.Reset > now)
+ {
+ var delay = ex.Reset - now;
+ await Task.Delay(delay);
+ }
+ }
+ }
+
+ static async Task> TripleCheckGitHubPagination(Func>> apiCall, Func idSelector)
+ {
+ // I've seen GitHub pagination return incomplete result sets in the past
+ // It has an in-built pagination limit of 100
+ var apiOptions = new ApiOptions
+ {
+ PageSize = 100
+ };
+ var results = await RLR(() => apiCall(apiOptions));
+ Dictionary distinctEntries = new Dictionary(results.Count);
+ foreach (var result in results)
+ distinctEntries.Add(idSelector(result), result);
+
+ if (results.Count > 100)
+ {
+ results = await RLR(() => apiCall(apiOptions));
+ foreach (var result in results)
+ distinctEntries.TryAdd(idSelector(result), result);
+
+ results = await RLR(() => apiCall(apiOptions));
+ foreach (var result in results)
+ distinctEntries.TryAdd(idSelector(result), result);
+ }
+
+ return distinctEntries.Values.ToList();
+ }
+
+ static async Task> ProcessMilestone(IGitHubClient client, Milestone milestone)
+ {
+ // have to trust this works
+ SearchIssuesResult results;
+
+ var milestoneTask = Task.FromResult(milestone);
+ var pullRequests = new Dictionary();
+ var iteration = 0;
+ while (true)
+ {
+ results = await RLR(() => client.Search.SearchIssues(new SearchIssuesRequest
+ {
+ Type = IssueTypeQualifier.PullRequest,
+ Milestone = milestone.Title,
+ Repos = new RepositoryCollection
+ {
+ { RepoOwner, RepoName },
+ },
+ }));
+
+ foreach (var result in results.Items)
+ if (pullRequests.TryAdd(result.Number, result))
+ milestoneTasks.TryAdd(result.Number, milestoneTask);
+
+ if (results.IncompleteResults)
+ continue;
+
+ if (results.TotalCount <= 100 || ++iteration == 3)
+ break;
+ }
+
+ async Task RunPRs()
+ {
+ var milestoneVersion = Version.Parse(milestone.Title[1..]);
+ var prTasks = pullRequests.Select(
+ kvp => GetReleaseNotesFromPR(client, kvp.Value, true, true, milestone.State.Value == ItemState.Open))
+ .ToList();
+
+ await Task.WhenAll(prTasks);
+
+ var prResults = prTasks.Select(x => x.Result).Where(result => result != null).ToList();
+
+ var releasePRResult = prResults.FirstOrDefault(x => x.Item3);
+
+ Dictionary releasedComponentVersions;
+ if (releasePRResult != null)
+ releasedComponentVersions = releasePRResult.Item2;
+ else
+ {
+ releasedComponentVersions = new Dictionary(
+ prResults
+ .SelectMany(result => result.Item2)
+ .GroupBy(kvp => kvp.Key)
+ .Select(grouping => new KeyValuePair(grouping.Key, grouping.Max(kvp => kvp.Value))));
+
+ foreach(var maxVersionKvp in prResults.SelectMany(x => x.Item1)
+ .Where(x => !releasedComponentVersions.ContainsKey(x.Key))
+ .GroupBy(x => x.Key)
+ .Select(group => {
+ var versions = group
+ .Where(x => x.Value.Version != null)
+ .ToList();
+
+ if (versions.Count == 0)
+ return new KeyValuePair(group.Key, null);
+
+ return new KeyValuePair(group.Key, versions.Max(x => x.Value.Version));
+ })
+ .Where(kvp => kvp.Value != null)
+ .ToList())
+ {
+ releasedComponentVersions.Add(maxVersionKvp.Key, maxVersionKvp.Value);
+ }
+ }
+
+ var finalResults = new Dictionary>();
+ foreach (var componentKvp in releasedComponentVersions)
+ {
+ var component = componentKvp.Key;
+ var list = new List();
+
+ foreach(var changelistDict in prResults.Select(x => x.Item1))
+ {
+ if (!changelistDict.TryGetValue(component, out var changelist))
+ continue;
+
+ Version componentVersion = milestoneVersion;
+ var unreleased = milestone.State.Value == ItemState.Open;
+ if (component != Component.Core)
+ {
+ componentVersion = changelist.Version ?? componentKvp.Value;
+ if (releasedNonCoreVersions != null
+ && releasedNonCoreVersions.TryGetValue(component, out var releasedVersions)
+ && !releasedVersions.Any(x => x == componentVersion))
+ {
+ // roll forward
+ var newList = releasedVersions
+ .ToList();
+ newList.Add(componentVersion);
+ newList = newList.OrderBy(x => x).ToList();
+
+ var index = newList.IndexOf(componentVersion);
+ Debug.Assert(index != -1);
+ if (index != (newList.Count - 1))
+ {
+ componentVersion = newList[index + 1];
+ unreleased = false;
+ }
+ else
+ unreleased = true;
+ }
+ }
+
+ var entry = list.FirstOrDefault(x => x.Version == componentVersion);
+ if (entry == null)
+ {
+ entry = changelist;
+ entry.Version = componentVersion;
+ entry.Unreleased = unreleased;
+ list.Add(entry);
+ }
+ else
+ entry.Changes.AddRange(changelist.Changes);
+ }
+
+ Debug.Assert(list.Select(x => x.Version.ToString()).Distinct().Count() == list.Count);
+ if (component == Component.Core)
+ {
+ Debug.Assert(list.All(x => x.Version == milestoneVersion));
+ }
+
+ list = list.OrderByDescending(x => x.Version).ToList();
+ finalResults.Add(component, list);
+ }
+
+ if (!finalResults.ContainsKey(Component.Core) || finalResults[Component.Core].Count == 0)
+ {
+ finalResults.Remove(Component.Core);
+ finalResults.Add(Component.Core, new List
+ {
+ new Changelist
+ {
+ Changes = new List(),
+ ComponentVersions = releasedComponentVersions,
+ Unreleased = milestone.State.Value == ItemState.Open,
+ Version = milestoneVersion,
+ }
+ });
+ }
+ else
+ Debug.Assert(finalResults[Component.Core].All(x => x.Version == milestoneVersion));
+
+ var notes = new ReleaseNotes
+ {
+ Components = new SortedDictionary>(finalResults),
+ };
+
+ return notes;
+ }
+
+ return RunPRs();
+ }
+
+ static async Task FullNotes(IGitHubClient client)
+ {
+ var startRateLimit = (client.GetLastApiInfo()?.RateLimit ?? (await client.RateLimit.GetRateLimits()).Rate).Remaining;
+
+ ReleaseNotes existingNotes = null;
+ if (File.Exists("changelog.yml"))
+ {
+ var existingYml = await File.ReadAllTextAsync("changelog.yml");
+ var deserializer = new DeserializerBuilder()
+ .Build();
+
+ existingNotes = deserializer.Deserialize(existingYml);
+ }
+
+ var releaseNotes = await GenerateNotes(client, existingNotes);
+
+ Console.WriteLine($"Generating all release notes took {startRateLimit - client.GetLastApiInfo().RateLimit.Remaining} requests.");
+
+ var serializer = new SerializerBuilder()
+ .ConfigureDefaultValuesHandling(DefaultValuesHandling.OmitDefaults)
+ .WithTypeConverter(new VersionConverter())
+ .Build();
+
+ var serializedYaml = serializer.Serialize(releaseNotes);
+ await File.WriteAllTextAsync("changelog.yml", serializedYaml).ConfigureAwait(false);
+ return 0;
+ }
+
+ static HttpClient httpClient = new HttpClient(new HttpClientHandler()
+ {
+ AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate
+ });
+ static async Task> EnumerateNugetVersions(string package)
+ {
+ var url = new Uri($"https://api.nuget.org/v3/registration5-gz-semver2/{package.ToLowerInvariant()}/index.json");
+
+ using var req = new HttpRequestMessage();
+ req.Headers.UserAgent.Add(new System.Net.Http.Headers.ProductInfoHeaderValue("Tgstation.Server.ReleaseNotes", "0.1.0"));
+ req.Method = HttpMethod.Get;
+ req.RequestUri = url;
+
+ using var resp = await httpClient.SendAsync(req);
+ resp.EnsureSuccessStatusCode();
+
+ var json = await resp.Content.ReadAsStringAsync();
+
+ dynamic dynamicJson = JsonConvert.DeserializeObject(json);
+
+ var versions = (IEnumerable)dynamicJson.items[0].items;
+ var results = versions
+ .Select(x => Version.TryParse((string)x.catalogEntry.version, out var version) ? version : null)
+ .Where(version => version != null)
+ .OrderBy(x => x)
+ .ToHashSet();
+ return results;
+ }
+
+ static IReadOnlyDictionary> releasedNonCoreVersions;
+
+ static async Task GenerateNotes(IGitHubClient client, ReleaseNotes previousNotes)
+ {
+ var releasesTask = TripleCheckGitHubPagination(
+ apiOptions => client.Repository.Release.GetAll(RepoOwner, RepoName, apiOptions),
+ release => release.Id);
+
+ var milestones = await TripleCheckGitHubPagination(
+ apiOptions => client.Issue.Milestone.GetAllForRepository(RepoOwner, RepoName, new MilestoneRequest {
+ State = ItemStateFilter.All
+ }, apiOptions),
+ milestone => milestone.Id);
+
+ var versionMilestones = milestones
+ .Where(milestone => Regex.IsMatch(milestone.Title, @"v[1-9][0-9]*\.[1-9]*[0-9]+\.[1-9]*[0-9]+$"))
+ .ToList();
+
+ var releases = await releasesTask;
+
+ var nugetCommonVersions = EnumerateNugetVersions("Tgstation.Server.Common");
+ var nugetApiVersions = EnumerateNugetVersions("Tgstation.Server.Api");
+ var nugetClientVersions = EnumerateNugetVersions("Tgstation.Server.Client");
+
+ releasedNonCoreVersions = new Dictionary> {
+ { Component.HttpApi, releases
+ .Where(x => x.TagName.StartsWith("api-v"))
+ .Select(x => Version.Parse(x.TagName[5..]))
+ .OrderBy(x => x)
+ .ToHashSet() },
+ { Component.DreamMakerApi, releases
+ .Where(x => x.TagName.StartsWith("dmapi-v"))
+ .Select(x => Version.Parse(x.TagName[7..]))
+ .OrderBy(x => x)
+ .ToHashSet() },
+ { Component.NugetCommon, await nugetCommonVersions },
+ { Component.NugetApi, await nugetApiVersions },
+ { Component.NugetClient, await nugetClientVersions }
+ };
+
+ var milestonesToProcess = versionMilestones;
+ if (previousNotes != null)
+ {
+ var releasedVersions = previousNotes.Components[Component.Core].Where(cl => !cl.Unreleased).ToList();
+ milestonesToProcess = milestonesToProcess
+ .Where(x => !releasedVersions.Any(
+ version => version.Version == Version.Parse(x.Title.AsSpan(1))))
+ .ToList();
+
+ foreach (var kvp in previousNotes.Components)
+ if (releasedNonCoreVersions.TryGetValue(kvp.Key, out var releasedComponentVersions))
+ kvp.Value.RemoveAll(x => x.Unreleased = !releasedComponentVersions.Any(y => y == x.Version));
+ else
+ kvp.Value.RemoveAll(x => x.Unreleased);
+ }
+
+ var milestonePRTasks = milestonesToProcess
+ .Select(milestone => ProcessMilestone(client, milestone))
+ .ToList();
+
+ await Task.WhenAll(milestonePRTasks);
+
+ await Task.WhenAll(milestonePRTasks.Select(task => task.Result));
+
+ var coreCls = milestonePRTasks
+ .SelectMany(task => task.Result.Result.Components)
+ .Where(x => x.Key == Component.Core)
+ .ToList();
+
+ Debug.Assert(
+ coreCls.Count == milestonesToProcess.Count);
+
+ var distinctCoreVersions = coreCls
+ .SelectMany(x => x.Value)
+ .Select(x => x.Version.ToString())
+ .Distinct()
+ .Select(Version.Parse)
+ .OrderBy(x => x)
+ .ToList();
+
+ var missingCoreVersions = milestonesToProcess
+ .Where(x => !distinctCoreVersions.Any(y => Version.Parse(x.Title.Substring(1)) == y))
+ .ToList();
+
+ Debug.Assert(missingCoreVersions.Count == 0);
+ foreach (var missingCoreVersion in missingCoreVersions)
+ await await ProcessMilestone(client, missingCoreVersion);
+
+ var changelistsGroupedByComponent =
+ milestonePRTasks
+ .SelectMany(task => task.Result.Result.Components)
+ .GroupBy(kvp => kvp.Key)
+ .ToDictionary(grouping => grouping.Key, grouping => grouping.SelectMany(kvp => kvp.Value));
+
+ var releaseNotes = new ReleaseNotes
+ {
+ Components = new SortedDictionary>(
+ changelistsGroupedByComponent
+ .ToDictionary(
+ kvp => kvp.Key,
+ kvp => kvp
+ .Value
+ .GroupBy(changelist => changelist.Version)
+ .Select(grouping =>
+ {
+ var firstEntry = grouping.First();
+ return new Changelist
+ {
+ Changes = grouping.SelectMany(cl => cl.Changes).ToList(),
+ ComponentVersions = firstEntry.ComponentVersions,
+ Unreleased = firstEntry.Unreleased,
+ Version = grouping.Key
+ };
+ })
+ .OrderByDescending(cl => cl.Version)
+ .ToList()))
+ };
+
+ Debug.Assert(releaseNotes.Components.ContainsKey(Component.Core) && releaseNotes.Components[Component.Core].Count == milestonesToProcess.Count);
+
+ if (previousNotes != null)
+ {
+ foreach (var component in Enum.GetValues())
+ {
+ if (!previousNotes.Components.ContainsKey(component))
+ continue;
+
+ if (releaseNotes.Components.TryGetValue(component, out var changelists))
+ releaseNotes.Components[component] = changelists
+ .Concat(previousNotes.Components[component])
+ .OrderByDescending(cl => cl.Version)
+ .ToList();
+ else
+ releaseNotes.Components[component] = previousNotes.Components[component];
+ }
+ }
+
+ foreach (var kvp in releaseNotes.Components)
+ {
+ var distinctCount = kvp.Value.Select(changelist => changelist.Version.ToString()).Distinct().Count();
+ Debug.Assert(distinctCount == kvp.Value.Count);
+
+ foreach (var cl in kvp.Value)
+ cl.DeduplicateChanges();
+ }
+
+ return releaseNotes;
+ }
}
}
diff --git a/tools/Tgstation.Server.ReleaseNotes/ReleaseNotes.cs b/tools/Tgstation.Server.ReleaseNotes/ReleaseNotes.cs
new file mode 100644
index 0000000000..8c10c880ce
--- /dev/null
+++ b/tools/Tgstation.Server.ReleaseNotes/ReleaseNotes.cs
@@ -0,0 +1,9 @@
+using System.Collections.Generic;
+
+namespace Tgstation.Server.ReleaseNotes
+{
+ sealed class ReleaseNotes
+ {
+ public SortedDictionary> Components { get; set; }
+ }
+}
diff --git a/tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj b/tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj
index d95f0139e3..35312949ce 100644
--- a/tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj
+++ b/tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj
@@ -7,10 +7,12 @@
-
-
+
+
+
+