Merge pull request #828 from tgstation/755-ReleaseNotes

Release notes generation tool

.github/CONTRIBUTING.md

@@ -149,7 +149,32 @@ There is no strict process when it comes to merging pull requests. Pull requests
 
 * Commits MUST be properly titled and commented as we only use merge commits for the pull request process
 
+## Deployment process
+
+Every issue/pull request in a release should share a common milestone named with the release version i.e. `4.5.3.5`
+
+When every issue and PR in the milestone is closed. Create a version bump PR that changes the version numbers. At the time of this writing they exist in the following files
+
+- `/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj`
+- `/src/Tgstation.Server.Host.Console/Tgstation.Server.Host.Console.csproj`
+- 2 in `/src/Tgstation.Server.Host.Service/Properties/AssemblyInfo.cs`
+- `/src/Tgstation.Server.Host.Watchdog/Tgstation.Server.Host.Watchdog.csproj`
+
+Merge the pull request with `[TGSDeploy]` somewhere in the commit title. The scripts will handle amalgamating release notes, building, closing the milestone, and publishing the release. This will also trigger update notifications on existing TGS deployments.
+
+### API/Client Deployment
+
+The Api/Client project versions must be updated on nuget when changed. The numbers exist in the following files:
+
+- `/src/Tgstation.Server.Api/Tgstation.Server.Api.csproj`
+- `/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj`
+
+These should not be the same numbers as the main suite. When bumping the API version the client should only receive a minor (3rd number) version bump unless major changes to CLIENT code were made.
+
+Merge these alongside regular deployments with `[NugetDeploy]` in the commit title (Won't work without an accompanying `[TGSDeploy]`). This will handle the nuget publishing.
+
 ## Banned content
+
 Do not add any of the following in a Pull Request or risk getting the PR closed:
 * National Socialist Party of Germany content, National Socialist Party of Germany related content, or National Socialist Party of Germany references
 

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,13 @@
+[Release Notes]: # (Your PR should contain a detailed list of notable changes, titled appropriately. This includes any observable changes to the server or DMAPI. See example below)
+
+:cl:
+Description of your change
+Each newline corresponds to a release note in the upcoming sprint
+/:cl:
+
+:cl:
+You can also have multiple sets of release notes per pull request
+They will be amalgamated together in the end
+/:cl:
+
+[Why]: # (Please add a short description [two lines down] of why you think these changes would benefit the game. If you can't justify it in words, it might not be worth adding.)

.gitignore

@@ -15,3 +15,4 @@ artifacts/
 /tests/DMAPI/travistester.int
 /tests/DMAPI/travistester.dmb
 /src/Tgstation.Server.Host/appsettings.Development.json
+/tools/ReleaseNotes/release_notes.md

appveyor.yml

@@ -6,6 +6,8 @@ environment:
   TGS4_TEST_CONNECTION_STRING: Server=(local)\SQL2017;Initial Catalog=TGS_Test;User ID=sa;Password=Password12!
   TGS4_TEST_GITHUB_TOKEN:
     secure: lJNGAXwiB5HlWdthz3K4PetqpTG5IEAyRgKaiKxFMQ8HW8CcOjRtB97B05op7BsK
+  TGS_RELEASE_NOTES_TOKEN:
+    secure: lJNGAXwiB5HlWdthz3K4PetqpTG5IEAyRgKaiKxFMQ8HW8CcOjRtB97B05op7BsK
 branches:
   only:
     - master
@@ -77,17 +79,16 @@ after_test:
   - ps: Move-Item -path artifacts/ServerService/lib/Default/appsettings.json -destination artifacts/ServerService/
   - ps: Remove-Item artifacts/ServerHost/appsettings.json
   #deploy stuff
-  - ps: $env:TGSVersion = [System.Diagnostics.FileVersionInfo]::GetVersionInfo("$env:APPVEYOR_BUILD_FOLDER/artifacts/ServerHost/Tgstation.Server.Host.dll").FileVersion
-  - ps: if($env:APPVEYOR_REPO_COMMIT_MESSAGE -match "\[TGSDeploy\]"){ if($env:APPVEYOR_REPO_BRANCH -match "master"){ if($env:CONFIGURATION -match "Release"){ $env:TGSDeploy = "Do it." }}}
-  - ps: if($env:APPVEYOR_REPO_COMMIT_MESSAGE -match "\[NugetDeploy\]"){ if($env:APPVEYOR_REPO_BRANCH -match "master"){ if($env:CONFIGURATION -match "Release"){ $env:NugetDeploy = "Do it." }}}
+  - ps: build/prep_deployment.ps1
 deploy:
   - provider: GitHub
     release: "tgstation-server-v$(TGSVersion)"
-    description: 'The /tg/station server suite'
+    description: "$(TGSReleaseNotes)"
     auth_token:
       secure: lJNGAXwiB5HlWdthz3K4PetqpTG5IEAyRgKaiKxFMQ8HW8CcOjRtB97B05op7BsK
     artifact: ServerConsole,ServerService,ServerUpdatePackage,DMAPI
-    draft: true
+    draft: $(TGSDraftNotes)
+    prerelease: true
     on:
       TGSDeploy: "Do it."
   - provider: NuGet

build/prep_deployment.ps1

@@ -0,0 +1,25 @@
+$env:TGSVersion = [System.Diagnostics.FileVersionInfo]::GetVersionInfo("$env:APPVEYOR_BUILD_FOLDER/artifacts/ServerHost/Tgstation.Server.Host.dll").FileVersion
+
+Write-Host "TGS Version: $env:TGSVersion"
+
+if (($env:CONFIGURATION -match "Release") -And ($env:APPVEYOR_REPO_BRANCH -match "master") -And ($env:APPVEYOR_REPO_COMMIT_MESSAGE -match "\[TGSDeploy\]")) {
+    Write-Host "Deploying..."
+    $env:TGSDeploy = "Do it." 
+
+    Write-Host "Generating release notes..."
+    dotnet run -p "$env:APPVEYOR_BUILD_FOLDER/tools/ReleaseNotes" $env:TGSVersion
+    $env:TGSDraftNotes = !($?)
+    $releaseNotesPath = "$env:APPVEYOR_BUILD_FOLDER/tools/ReleaseNotes/release_notes.md"
+    if (Test-Path $releaseNotesPath -PathType Leaf) {
+        $env:TGSReleaseNotes = [IO.File]::ReadAllText($releaseNotesPath)
+    }
+    else {
+        Write-Host "Release note generation failed, release will be created as a draft!"
+        $env:TGSReleaseNotes = "Automatic generation failed, please fill manually!"
+    }      
+    
+    if ($env:APPVEYOR_REPO_COMMIT_MESSAGE -match "\[NugetDeploy\]") {
+        $env:NugetDeploy = "Do it."
+        Write-Host "Nuget deployment enabled"
+    }  
+}

tgstation-server.sln

@@ -114,6 +114,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".github", ".github", "{E821
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tgstation.Server.Tests", "tests\Tgstation.Server.Tests\Tgstation.Server.Tests.csproj", "{09056964-1C74-445A-96EC-33F6DFC07916}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ReleaseNotes", "tools\ReleaseNotes\ReleaseNotes.csproj", "{5CB51532-55F0-4255-B6E5-69ED5CCD14CD}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -172,6 +174,10 @@ Global
 		{09056964-1C74-445A-96EC-33F6DFC07916}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{09056964-1C74-445A-96EC-33F6DFC07916}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{09056964-1C74-445A-96EC-33F6DFC07916}.Release|Any CPU.Build.0 = Release|Any CPU
+		{5CB51532-55F0-4255-B6E5-69ED5CCD14CD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{5CB51532-55F0-4255-B6E5-69ED5CCD14CD}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{5CB51532-55F0-4255-B6E5-69ED5CCD14CD}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{5CB51532-55F0-4255-B6E5-69ED5CCD14CD}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -183,6 +189,7 @@ Global
 		{3BB10856-AA01-43D1-AAB0-A013085426D6} = {7E4E7CF6-A48D-410E-8191-C49C6B5AB9CC}
 		{82066812-6C73-4360-943B-B23F2F491261} = {7E4E7CF6-A48D-410E-8191-C49C6B5AB9CC}
 		{057FAC33-CC31-4948-91C6-B0977C335890} = {F7765A4B-021C-454E-ABB1-2AB1B85F91B4}
+		{5CB51532-55F0-4255-B6E5-69ED5CCD14CD} = {A55C1117-5808-4AB2-BEA6-4D4A3E66A2F2}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {DFD36C95-3E49-41C7-ACDB-86BAF5B18A79}

tools/ReleaseNotes/Program.cs

@@ -0,0 +1,309 @@
+using Octokit;
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace ReleaseNotes
+{
+	/// <summary>
+	/// Contains the application entrypoint
+	/// </summary>
+	static class Program
+	{
+		/// <summary>
+		/// The entrypoint for the <see cref="Program"/>
+		/// </summary>
+		static async Task<int> Main(string[] args)
+		{
+			if (args.Length < 1)
+			{
+				Console.WriteLine("Missing version argument!");
+				return 1;
+			}
+
+			var versionString = args[0];
+			if (!Version.TryParse(versionString, out var version))
+			{
+				Console.WriteLine("Invalid version: " + versionString);
+				return 2;
+			}
+
+			var doNotCloseMilestone = args.Length >= 2 && args[1].ToUpperInvariant() == "--NO-CLOSE";
+
+			const string ReleaseNotesEnvVar = "TGS_RELEASE_NOTES_TOKEN";
+			var githubToken = Environment.GetEnvironmentVariable(ReleaseNotesEnvVar);
+			if (String.IsNullOrWhiteSpace(githubToken))
+			{
+				Console.WriteLine("Missing " + ReleaseNotesEnvVar + " environment variable!");
+				return 3;
+			}
+
+			try
+			{
+				var client = new GitHubClient(new ProductHeaderValue("tgs_release_notes"));
+				client.Credentials = new Credentials(githubToken);
+
+				const string RepoOwner = "tgstation";
+				const string RepoName = "tgstation-server";
+
+				var releasesTask = client.Repository.Release.GetAll(RepoOwner, RepoName);
+
+				Console.WriteLine("Getting pull requests in milestone " + versionString + "...");
+				var milestonePRs = await client.Search.SearchIssues(new SearchIssuesRequest
+				{
+					Milestone = versionString,
+					Type = IssueTypeQualifier.PullRequest,
+					Repos = { { RepoOwner, RepoName } }
+				}).ConfigureAwait(false);
+
+				if (milestonePRs.IncompleteResults)
+				{
+					Console.WriteLine("Incomplete results for milestone PRs query!");
+					return 5;
+				}
+				Console.WriteLine(milestonePRs.Items.Count + " total pull requests");
+
+				Task<Milestone> milestoneTask = null;
+				var milestoneTaskLock = new object();
+				var releaseDictionary = new Dictionary<int, List<string>>();
+				var authorizedUsers = new Dictionary<long, Task<bool>>();
+
+				async Task GetReleaseNotesFromPR(Issue pullRequest)
+				{
+					//need to check it was merged
+					var fullPR = await client.Repository.PullRequest.Get(RepoOwner, RepoName, pullRequest.Number).ConfigureAwait(false);
+
+					async Task<Milestone> GetMilestone()
+					{
+						if (fullPR.Milestone == null)
+							return null;
+						return await client.Issue.Milestone.Get(RepoOwner, RepoName, fullPR.Milestone.Number);
+					};
+
+					lock (milestoneTaskLock)
+						if (milestoneTask == null)
+							milestoneTask = GetMilestone();
+
+					if (!fullPR.Merged)
+						return;
+
+					async Task BuildNotesFromComment(string comment, User user)
+					{
+						var commentSplits = comment.Split('\n');
+						var notesOpen = false;
+						var notesClosed = false;
+						var notes = new List<string>();
+						foreach (var line in commentSplits)
+						{
+							var trimmedLine = line.Trim();
+							if (!notesOpen)
+							{
+								notesOpen = trimmedLine.StartsWith(":cl:", StringComparison.Ordinal);
+								notesClosed = false;
+								continue;
+							}
+							if (trimmedLine.StartsWith("/:cl:", StringComparison.Ordinal))
+							{
+								notesClosed = true;
+								notesOpen = false;
+								continue;
+							}
+							if (trimmedLine.Length == 0)
+								continue;
+							notes.Add(trimmedLine);
+						}
+						if (!notesClosed || notes.Count == 0)
+							return;
+
+						Task<bool> authTask;
+						TaskCompletionSource<bool> ourTcs = null;
+						lock (authorizedUsers)
+						{
+							if (!authorizedUsers.TryGetValue(user.Id, out authTask))
+							{
+								ourTcs = new TaskCompletionSource<bool>();
+								authTask = ourTcs.Task;
+								authorizedUsers.Add(user.Id, authTask);
+							}
+						}
+
+						if (ourTcs != null)
+							try
+							{
+								//check if the user has access
+								var perm = await client.Repository.Collaborator.ReviewPermission(RepoOwner, RepoName, user.Login).ConfigureAwait(false);
+								ourTcs.SetResult(perm.Permission == PermissionLevel.Write || perm.Permission == PermissionLevel.Admin);
+							}
+							catch
+							{
+								ourTcs.SetResult(false);
+								throw;
+							}
+
+						var authorized = await authTask.ConfigureAwait(false);
+						if (!authorized)
+							return;
+
+						lock (releaseDictionary)
+						{
+							foreach (var I in notes)
+								Console.WriteLine("#" + fullPR.Number + " - " + I + " (@" + user.Login + ")");
+							if (releaseDictionary.TryGetValue(fullPR.Number, out var currentValues))
+								currentValues.AddRange(notes);
+							else
+								releaseDictionary.Add(fullPR.Number, notes);
+						}
+					}
+
+					var comments = await client.Issue.Comment.GetAllForIssue(RepoOwner, RepoName, fullPR.Number).ConfigureAwait(false);
+					await Task.WhenAll(BuildNotesFromComment(fullPR.Body, fullPR.User), Task.WhenAll(comments.Select(x => BuildNotesFromComment(x.Body, x.User)))).ConfigureAwait(false);
+				}
+
+				var tasks = new List<Task>();
+				foreach (var I in milestonePRs.Items)
+					tasks.Add(GetReleaseNotesFromPR(I));
+
+				var releases = await releasesTask.ConfigureAwait(false);
+
+				var releasingSuite = version.Major;
+
+				Version highestReleaseVersion = null;
+				Release highestRelease = null;
+				foreach (var I in releases)
+				{
+					if (!Version.TryParse(I.TagName.Replace("tgstation-server-v", String.Empty), out var currentReleaseVersion))
+					{
+						Console.WriteLine("WARNING: Unable to determine version of release " + I.HtmlUrl);
+						continue;
+					}
+
+					if (currentReleaseVersion.Major == releasingSuite && (highestReleaseVersion == null || currentReleaseVersion > highestReleaseVersion))
+					{
+						highestReleaseVersion = currentReleaseVersion;
+						highestRelease = I;
+					}
+				}
+
+				if (highestReleaseVersion == null)
+				{
+					Console.WriteLine("Unable to determine highest release version for suite " + releasingSuite + "!");
+					return 6;
+				}
+
+				var oldNotes = highestRelease.Body;
+
+				var splits = new List<string>(oldNotes.Split('\n'));
+				//trim away all the lines that don't start with #
+
+				string keepThisRelease;
+				if (version.Revision == 0)
+					if (version.Build == 0)
+						keepThisRelease = "# ";
+					else
+						keepThisRelease = "## ";
+				else
+					keepThisRelease = "### ";
+
+				for (; !splits[0].StartsWith(keepThisRelease, StringComparison.Ordinal); splits.RemoveAt(0))
+					if (splits.Count == 1)
+					{
+						Console.WriteLine("Error formatting release notes: Can't detemine notes start!");
+						return 7;
+					}
+
+				oldNotes = String.Join('\n', splits);
+
+				string prefix;
+				switch (releasingSuite)
+				{
+					case 3:
+						prefix = "The /tg/station server suite";
+						break;
+					default:
+						prefix = "See https://tgstation.github.io/tgstation-server for installation instructions";
+						break;
+				}
+
+				var newNotes = new StringBuilder(prefix);
+				newNotes.Append(Environment.NewLine);
+				newNotes.Append(Environment.NewLine);
+				if (version.Revision == 0)
+					if (version.Build == 0)
+					{
+						newNotes.Append("# [Version ");
+						newNotes.Append(version.Minor);
+					}
+					else
+					{
+						newNotes.Append("## [Changelog for ");
+						newNotes.Append(version.Build);
+						newNotes.Append(".x");
+					}
+				else
+				{
+					newNotes.Append("### [Patch ");
+					newNotes.Append(version.Revision);
+				}
+				newNotes.Append("](");
+				var milestone = await milestoneTask.ConfigureAwait(false);
+				if (milestone == null)
+				{
+					Console.WriteLine("Unable to detemine milestone!");
+					return 9;
+				}
+				newNotes.Append(milestone.HtmlUrl);
+				newNotes.Append("?closed=1)");
+				newNotes.Append(Environment.NewLine);
+
+				await Task.WhenAll(tasks).ConfigureAwait(false);
+
+				if (releaseDictionary.Count == 0)
+				{
+					Console.WriteLine("No release notes for this milestone!");
+					return 8;
+				}
+
+				foreach (var I in releaseDictionary)
+					foreach (var note in I.Value)
+					{
+						newNotes.Append(Environment.NewLine);
+						newNotes.Append("- ");
+						newNotes.Append(note);
+						newNotes.Append(" (#");
+						newNotes.Append(I.Key);
+						newNotes.Append(')');
+					}
+
+				newNotes.Append(Environment.NewLine);
+				newNotes.Append(Environment.NewLine);
+				newNotes.Append(oldNotes);
+
+				Console.WriteLine("Writing out new release notes...");
+				var releaseNotes = newNotes.ToString();
+				await File.WriteAllTextAsync("release_notes.md", releaseNotes).ConfigureAwait(false);
+
+				if (doNotCloseMilestone)
+					Console.WriteLine("Not closing milestone due to parameter!");
+				else
+				{
+					Console.WriteLine("Closing milestone...");
+					await client.Issue.Milestone.Update(RepoOwner, RepoName, milestone.Number, new MilestoneUpdate
+					{
+						State = ItemState.Closed
+					}).ConfigureAwait(false);
+				}
+
+				return 0;
+			}
+			catch (Exception e)
+			{
+				Console.WriteLine(e);
+				return 4;
+			}
+		}
+	}
+}

tools/ReleaseNotes/README.md

@@ -0,0 +1,7 @@
+This is a small tool to generate TGS release notes from PR descriptions
+
+Requires environment variable `TGS_RELEASE_NOTES_TOKEN`
+
+Run it with `dotnet run <version> [--no-close (optionally doesn't close the milestone, USE WHILE DEBUGGING)]`
+
+Will close the release milestone and output `release_notes.md` with the updated release notes

tools/ReleaseNotes/ReleaseNotes.csproj

@@ -0,0 +1,13 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>netcoreapp2.1</TargetFramework>
+    <LangVersion>latest</LangVersion>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="octokit" Version="0.32.0" />
+  </ItemGroup>
+
+</Project>