diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index b770b65721..c146ed90da 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -13,9 +13,10 @@ They will be amalgamated together in the end. /:cl: :cl: Categories -Categories are divided up in the release notes and set after the :cl: header. +Categories are used by [the release notes tool](../tools/Tgstation.Server.ReleaseNotes) to generate formatted changelists used in releases. The default category is Core. -The current standard categories are Core, DreamMaker API, HTTP API, and Host Watchdog. +Only one category may be specified for a :cl: block. +Valid categories are Core, DreamMaker API, HTTP API, Host Watchdog, Web Control Panel, Configuration, Nuget: Api, Nuget: Client, and Nuget: Common. /:cl: [Why]: # (If this does not close or work on an existing GitHub issue, please add a short description [two lines down] of why you think these changes would benefit the server. If you can't justify it in words, it might not be worth adding.) diff --git a/.github/workflows/check-pr-has-milestone.yml b/.github/workflows/check-pr-has-milestone.yml new file mode 100644 index 0000000000..954d539927 --- /dev/null +++ b/.github/workflows/check-pr-has-milestone.yml @@ -0,0 +1,32 @@ +name: "Check PR Has Milestone" + +on: + pull_request: + types: [ opened, edited, synchronize, reopened ] + branches: + - dev + - master + - V6 + +concurrency: + group: "check-pr-milestone-${{ github.head_ref || github.run_id }}-${{ github.event_name }}" + cancel-in-progress: true + +jobs: + fail-on-bad-milestone: + name: Fail if Pull Request has no Associated Version Milestone + runs-on: ubuntu-latest + steps: + - name: Fail if no Milestone Set + if: github.event.pull_request.milestone == null + run: | + echo "::error::Pull request must have milestone set!" + exit 1 + + - name: Fail if Invalid Milestone Set + run: | + if [[ ${{ github.event.pull_request.milestone.title }} =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + exit 0 + fi + echo "::error::Pull request has non-standard milestone!" + exit 1 diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 20c5fa804e..e44a8489a7 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -164,12 +164,17 @@ jobs: fi exit $retval - dox-build: - name: Build Doxygen Site + pages-build: + name: Build gh-pages runs-on: ubuntu-latest needs: start-ci-run-gate if: (!(cancelled() || failure()) && needs.start-ci-run-gate.result == 'success') steps: + - name: Setup dotnet + uses: actions/setup-dotnet@v2 + with: + dotnet-version: ${{ env.TGS_DOTNET_VERSION }} + - name: Checkout (Branch) uses: actions/checkout@v3 if: github.event_name == 'push' || github.event_name == 'schedule' @@ -180,6 +185,22 @@ jobs: with: ref: "refs/pull/${{ github.event.number }}/merge" + - name: gh-pages Clone + run: git clone -b gh-pages --single-branch "https://git@github.com/tgstation/tgstation-server" $HOME/tgsdox + + - name: Restore + run: dotnet restore + + - name: Build ReleaseNotes + run: | + export TGS_HOST_NO_WEBPANEL=true + dotnet build -c Release tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj + + - name: Build Changelog (Incremental) + run: | + cp $HOME/tgsdox/changelog.yml ./ 2>/dev/null + dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --generate-full-notes + - name: Patch Doxyfile run: | VERSION=$(cat "build/Version.props" | grep -oPm1 "(?<=)[^<]+") @@ -190,13 +211,13 @@ jobs: with: doxyfile-path: 'docs/Doxyfile' - - name: gh-pages push + - name: gh-pages Push if: github.event_name == 'push' && github.event.ref == 'refs/heads/dev' && env.TGS_RELEASE_NOTES_TOKEN != '' run: | - git clone -b gh-pages --single-branch "https://git@github.com/tgstation/tgstation-server" $HOME/tgsdox pushd $HOME/tgsdox rm -r * popd + sudo mv changlog.yml $HOME/tgsdox/ echo ./doxout/* | xargs -n 10 sudo mv -t $HOME/tgsdox cd $HOME/tgsdox git config --global push.default simple @@ -997,9 +1018,12 @@ jobs: run: | echo "TGS_VERSION=$(xmlstarlet sel -N X="http://schemas.microsoft.com/developer/msbuild/2003" --template --value-of /X:Project/X:PropertyGroup/X:TgsCoreVersion build/Version.props)" >> $GITHUB_ENV + - name: Grab Most Recent Changelog + run: curl -L https://raw.githubusercontent.com/tgstation/tgstation-server/gh-pages/changelog.yml -o changelog.yml + - name: Execute Build Script (Unsigned) if: (!(github.event_name == 'push' && contains(github.event.head_commit.message, '[TGSDeploy]') && (github.event.ref == 'refs/heads/master' || github.event.ref == 'refs/heads/dev'))) - run: sudo build/package/deb/build_package.sh + run: sudo -E build/package/deb/build_package.sh - name: Execute Build Script (Signed) if: (github.event_name == 'push' && contains(github.event.head_commit.message, '[TGSDeploy]') && (github.event.ref == 'refs/heads/master' || github.event.ref == 'refs/heads/dev')) @@ -1207,16 +1231,18 @@ jobs: run: dotnet restore - name: Build ReleaseNotes - run: dotnet build -c Release tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj + run: | + export TGS_HOST_NO_WEBPANEL=true + dotnet build -c Release tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj - name: Run ReleaseNotes Check run: dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --winget-template-check ${{ steps.get-sha.outputs.pr_template_sha }} ci-completion-gate: # This job exists so there isn't a moving target for branch protections name: CI Completion Gate - needs: [ dox-build, docker-build, build-deb, build-msi, validate-openapi-spec, upload-code-coverage, check-winget-pr-template ] + needs: [ pages-build, docker-build, build-deb, build-msi, validate-openapi-spec, upload-code-coverage, check-winget-pr-template ] runs-on: ubuntu-latest - if: (!(cancelled() || failure()) && needs.dox-build.result == 'success' && needs.docker-build.result == 'success' && needs.build-deb.result == 'success' && needs.build-msi.result == 'success' && needs.validate-openapi-spec.result == 'success' && needs.upload-code-coverage.result == 'success' && needs.check-winget-pr-template.result == 'success') + if: (!(cancelled() || failure()) && needs.pages-build.result == 'success' && needs.docker-build.result == 'success' && needs.build-deb.result == 'success' && needs.build-msi.result == 'success' && needs.validate-openapi-spec.result == 'success' && needs.upload-code-coverage.result == 'success' && needs.check-winget-pr-template.result == 'success') steps: - name: GitHub Requires at Least One Step for a Job run: exit 0 @@ -1236,9 +1262,22 @@ jobs: runs-on: windows-latest if: (!(cancelled() || failure()) && needs.deployment-gate.result == 'success' && contains(github.event.head_commit.message, '[APIDeploy]')) steps: + - name: Setup dotnet + uses: actions/setup-dotnet@v2 + with: + dotnet-version: ${{ env.TGS_DOTNET_VERSION }} + - name: Checkout uses: actions/checkout@v3 + - name: Restore + run: dotnet restore + + - name: Build ReleaseNotes + run: | + $Env:TGS_HOST_NO_WEBPANEL=true + dotnet build -c Release tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj + - name: Parse API version shell: powershell run: | @@ -1252,15 +1291,24 @@ jobs: name: openapi-spec path: swagger + - name: Grab Most Recent Changelog + shell: powershell + run: | + $ProgressPreference = 'SilentlyContinue' + Invoke-WebRequest -Uri https://raw.githubusercontent.com/tgstation/tgstation-server/gh-pages/changelog.yml -OutFile changelog.yml + + - name: Generate Release Notes + run: dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes ${{ env.TGS_API_VERSION }} --httpapi + - name: Create GitHub Release - uses: juitnow/github-action-create-release@80dc88a9d09c0ee394a4a31b0450e8cbe62996ef # v1 + uses: actions/create-release@v1 id: create_release env: GITHUB_TOKEN: ${{ secrets.DEV_PUSH_TOKEN }} with: tag_name: api-v${{ env.TGS_API_VERSION }} release_name: tgstation-server API v${{ env.TGS_API_VERSION }} - body: The TGS HTTP API + body_path: release_notes.md commitish: ${{ github.event.head_commit.id }} - name: Upload OpenApi Spec @@ -1279,9 +1327,22 @@ jobs: runs-on: windows-latest if: (!(cancelled() || failure()) && needs.deployment-gate.result == 'success' && contains(github.event.head_commit.message, '[DMDeploy]')) steps: + - name: Setup dotnet + uses: actions/setup-dotnet@v2 + with: + dotnet-version: ${{ env.TGS_DOTNET_VERSION }} + - name: Checkout uses: actions/checkout@v3 + - name: Restore + run: dotnet restore + + - name: Build ReleaseNotes + run: | + $Env:TGS_HOST_NO_WEBPANEL=true + dotnet build -c Release tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj + - name: Parse DMAPI version shell: powershell run: | @@ -1294,15 +1355,24 @@ jobs: run: | &"C:/Program Files/7-Zip/7z.exe" a DMAPI.zip ./src/DMAPI/* -tzip + - name: Grab Most Recent Changelog + shell: powershell + run: | + $ProgressPreference = 'SilentlyContinue' + Invoke-WebRequest -Uri https://raw.githubusercontent.com/tgstation/tgstation-server/gh-pages/changelog.yml -OutFile changelog.yml + + - name: Generate Release Notes + run: dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes ${{ env.TGS_DM_VERSION }} --dmapi + - name: Create GitHub Release - uses: juitnow/github-action-create-release@v1 + uses: actions/create-release@v1 id: create_release env: GITHUB_TOKEN: ${{ secrets.DEV_PUSH_TOKEN }} with: tag_name: dmapi-v${{ env.TGS_DM_VERSION }} release_name: tgstation-server DMAPI v${{ env.TGS_DM_VERSION }} - body: The TGS DMAPI \#tgs-dmapi-release + body_path: release_notes.md commitish: ${{ github.event.head_commit.id }} - name: Upload DMAPI Artifact @@ -1329,6 +1399,20 @@ jobs: - name: Checkout uses: actions/checkout@v3 + - name: Restore + run: dotnet restore + + - name: Build ReleaseNotes + run: | + export TGS_HOST_NO_WEBPANEL=true + dotnet build -c Release tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj + + - name: Grab Most Recent Changelog + run: curl -L https://raw.githubusercontent.com/tgstation/tgstation-server/gh-pages/changelog.yml -o changelog.yml + + - name: Generate Release Notes + run: dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes ${{ env.TGS_DM_VERSION }} --nuget + - name: Publish Tgstation.Server.Common to NuGet uses: alirezanet/publish-nuget@e276c40afeb2a154046f0997820f2a9ea74832d9 # v3.1.0 with: @@ -1371,7 +1455,9 @@ jobs: run: dotnet restore - name: Build ReleaseNotes - run: dotnet build -c Release tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj + run: | + export TGS_HOST_NO_WEBPANEL=true + dotnet build -c Release tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj - name: Run ReleaseNotes with --ensure-release run: dotnet run -c Release --no-build --project tools/Tgstation.Server.ReleaseNotes --ensure-release @@ -1398,12 +1484,7 @@ jobs: cd build/package/winget dotnet tool restore - - name: Build ReleaseNotes - run: dotnet build -c Release tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj - # We need to rebuild the installer.exe so it can be properly signed - - name: Restore - run: dotnet restore - name: Build Host run: dotnet build -c Release src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -1411,6 +1492,9 @@ jobs: - name: Build Service run: dotnet build -c Release src/Tgstation.Server.Host.Service/Tgstation.Server.Host.Service.csproj + - name: Build ReleaseNotes + run: dotnet build -c Release tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj + - name: Prepare Artifacts shell: powershell run: build/package/winget/prepare_installer_input_artifacts.ps1 @@ -1637,7 +1721,9 @@ jobs: uses: actions/checkout@v3 - name: Build ReleaseNotes - run: dotnet build -c Release tools/Tgstation.Server.ReleaseNotes + run: | + $Env:TGS_HOST_NO_WEBPANEL=true + dotnet build -c Release tools/Tgstation.Server.ReleaseNotes - name: Execute Push Script shell: powershell diff --git a/.gitignore b/.gitignore index d7756e60cd..0c71765597 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,7 @@ artifacts/ /tools/Tgstation.Server.ReleaseNotes/release_notes.md launchSettings.json release_notes.md +changelog.yml *nupkg *.sqlite3 packaging/ diff --git a/build/Version.props b/build/Version.props index 982e77cb93..eeef128f26 100644 --- a/build/Version.props +++ b/build/Version.props @@ -18,8 +18,8 @@ 8 https://download.visualstudio.microsoft.com/download/pr/e465df26-2f50-432f-a588-51c7682fd9b1/9efca17f575afd387aa9b02fb3dfb7e1/dotnet-hosting-8.0.0-preview.7.23375.9-win.exe - 11.0.2 + 10.11.5 - https://ftp.osuosl.org/pub/mariadb/mariadb-11.0.2/winx64-packages/mariadb-11.0.2-winx64.msi + https://ftp.osuosl.org/pub/mariadb//mariadb-10.11.5/winx64-packages/mariadb-10.11.5-winx64.msi diff --git a/build/package/deb/build_package.sh b/build/package/deb/build_package.sh index 91524707e9..c1a3cd088d 100755 --- a/build/package/deb/build_package.sh +++ b/build/package/deb/build_package.sh @@ -40,8 +40,13 @@ dh_make -p tgstation-server_$TGS_VERSION -y --createorig -s rm -f debian/README* debian/changelog debian/*.ex debian/upstream/*.ex +pushd .. +export TGS_HOST_NO_WEBPANEL=true +dotnet run -c Release --project tools/Tgstation.Server.ReleaseNotes $TGS_VERSION --debian packaging/debian/changelog $CURRENT_COMMIT +export TGS_HOST_NO_WEBPANEL=false +popd + cp -r build/package/deb/debian/* debian/ -sed -i "s/~!VERSION!~/$TGS_VERSION/g" debian/changelog cp build/tgstation-server.service debian/ diff --git a/build/package/deb/debian/changelog b/build/package/deb/debian/changelog deleted file mode 100644 index 61eeab7d19..0000000000 --- a/build/package/deb/debian/changelog +++ /dev/null @@ -1,5 +0,0 @@ -tgstation-server (~!VERSION!~-1) unstable; urgency=medium - - * Package changelogs are currently stubs. They will be auto-generated in the future (https://github.com/tgstation/tgstation-server/issues/1147). Refer to GitHub release for changelog: https://github.com/tgstation/tgstation-server/releases/tag/tgstation-server-v~!VERSION!~ - - -- Jordan Dominion Mon, 19 Jun 2023 16:14:13 +0000 diff --git a/src/Tgstation.Server.Api/Tgstation.Server.Api.csproj b/src/Tgstation.Server.Api/Tgstation.Server.Api.csproj index 913f4db126..582c079b6e 100644 --- a/src/Tgstation.Server.Api/Tgstation.Server.Api.csproj +++ b/src/Tgstation.Server.Api/Tgstation.Server.Api.csproj @@ -6,7 +6,7 @@ $(TgsApiLibraryVersion) API definitions for tgstation-server. json web api tgstation-server tgstation ss13 byond http - Added support for -map-threads DreamDaemon parameter and UploadZip Administration update option. + $(TGS_NUGET_RELEASE_NOTES_API) CA1028 diff --git a/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj b/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj index 67bd6448a3..af89800112 100644 --- a/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj +++ b/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj @@ -6,7 +6,7 @@ $(TgsClientVersion) Client library for tgstation-server. json web api tgstation-server tgstation ss13 byond client http - Added support for .zip uploads for server updates. Added missing Dispose() call to the StringContents for requests with bodies and added missing ConfigureAwait(false) to async call. + $(TGS_NUGET_RELEASE_NOTES_CLIENT) diff --git a/src/Tgstation.Server.Common/Tgstation.Server.Common.csproj b/src/Tgstation.Server.Common/Tgstation.Server.Common.csproj index 29d40742bc..405226e89f 100644 --- a/src/Tgstation.Server.Common/Tgstation.Server.Common.csproj +++ b/src/Tgstation.Server.Common/Tgstation.Server.Common.csproj @@ -6,7 +6,7 @@ $(TgsCommonLibraryVersion) Common functions for tgstation-server. web tgstation-server tgstation ss13 byond client http - Added CachedResponseStream. + $(TGS_NUGET_RELEASE_NOTES_COMMON) diff --git a/src/Tgstation.Server.Host/Configuration/ControlPanelConfiguration.cs b/src/Tgstation.Server.Host/Configuration/ControlPanelConfiguration.cs index b54c471f65..19dfa9cace 100644 --- a/src/Tgstation.Server.Host/Configuration/ControlPanelConfiguration.cs +++ b/src/Tgstation.Server.Host/Configuration/ControlPanelConfiguration.cs @@ -15,7 +15,18 @@ namespace Tgstation.Server.Host.Configuration /// /// If the control panel is enabled. /// - public bool Enable { get; set; } + public bool Enable +#if NO_WEBPANEL + { + get => false; + set => _ = value; + } +#else + { + get; + set; + } +#endif /// /// If any origin is allowed for CORS requests. This overrides . diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index 5718d2cd1d..da0a804468 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -503,7 +503,11 @@ namespace Tgstation.Server.Host.Core }); } else +#if NO_WEBPANEL + logger.LogTrace("Web control panel was not included in TGS build!"); +#else logger.LogTrace("Web control panel disabled!"); +#endif // Do not cache a single thing beyond this point, it's all API applicationBuilder.UseDisabledClientCache(); diff --git a/src/Tgstation.Server.Host/Extensions/Converters/VersionConverter.cs b/src/Tgstation.Server.Host/Extensions/Converters/VersionConverter.cs index ec08e5ae7b..0373dc5e20 100644 --- a/src/Tgstation.Server.Host/Extensions/Converters/VersionConverter.cs +++ b/src/Tgstation.Server.Host/Extensions/Converters/VersionConverter.cs @@ -13,7 +13,7 @@ namespace Tgstation.Server.Host.Extensions.Converters /// /// and for serializing s in semver format. /// - sealed class VersionConverter : JsonConverter, IYamlTypeConverter + public sealed class VersionConverter : JsonConverter, IYamlTypeConverter { /// /// Check if the supports (de)serializing a given . @@ -35,6 +35,8 @@ namespace Tgstation.Server.Host.Extensions.Converters /// public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) { + ArgumentNullException.ThrowIfNull(writer); + if (value == null) { writer.WriteNull(); diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index fdac8b2b1c..86f7d74dd6 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -18,7 +18,11 @@ $(DefineConstants);WATCHDOG_FREE_RESTART - + + $(DefineConstants);NO_WEBPANEL + + + @@ -28,7 +32,7 @@ - + diff --git a/tools/Tgstation.Server.ReleaseNotes/Change.cs b/tools/Tgstation.Server.ReleaseNotes/Change.cs new file mode 100644 index 0000000000..88f5910606 --- /dev/null +++ b/tools/Tgstation.Server.ReleaseNotes/Change.cs @@ -0,0 +1,13 @@ +using System.Collections.Generic; + +using Octokit; + +namespace Tgstation.Server.ReleaseNotes +{ + sealed class Change + { + public List Descriptions { get; set; } + public string Author { get; set; } + public int PullRequest { get; set; } + } +} diff --git a/tools/Tgstation.Server.ReleaseNotes/Changelist.cs b/tools/Tgstation.Server.ReleaseNotes/Changelist.cs new file mode 100644 index 0000000000..feac94bc26 --- /dev/null +++ b/tools/Tgstation.Server.ReleaseNotes/Changelist.cs @@ -0,0 +1,53 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text.RegularExpressions; + +namespace Tgstation.Server.ReleaseNotes +{ + sealed class Changelist + { + public Version Version { get; set; } + + public Dictionary ComponentVersions { get; set; } + + public List Changes { get; set; } + + public bool Unreleased { get; set; } + + public void DeduplicateChanges() + { + Changes = Changes + .OrderBy(x => x.PullRequest) + .GroupBy(x => x.PullRequest) + .Select(prChanges => + { + string author = null; + return new Change + { + PullRequest = prChanges.Key, + Descriptions = prChanges + .SelectMany(x => + { + if (author != null) + Debug.Assert(x.Author == author); + else + author = x.Author; + + return x.Descriptions; + }) + .ToList(), + Author = author + }; + }) + .ToList(); + } + + public void StripConfigVersionMessage() + { + foreach (var change in Changes) + change.Descriptions.RemoveAll(desc => Regex.IsMatch(desc, "The new config.* version is")); + } + } +} diff --git a/tools/Tgstation.Server.ReleaseNotes/Component.cs b/tools/Tgstation.Server.ReleaseNotes/Component.cs new file mode 100644 index 0000000000..d1baa3fcda --- /dev/null +++ b/tools/Tgstation.Server.ReleaseNotes/Component.cs @@ -0,0 +1,16 @@ +namespace Tgstation.Server.ReleaseNotes +{ + enum Component + { + Configuration, + Core, + HostWatchdog, + WebControlPanel, + HttpApi, + DreamMakerApi, + InteropApi, + NugetCommon, + NugetApi, + NugetClient, + } +} diff --git a/tools/Tgstation.Server.ReleaseNotes/Program.cs b/tools/Tgstation.Server.ReleaseNotes/Program.cs index e0bfa45e59..978e5276bc 100644 --- a/tools/Tgstation.Server.ReleaseNotes/Program.cs +++ b/tools/Tgstation.Server.ReleaseNotes/Program.cs @@ -1,14 +1,29 @@ -using System; +// This program is minimal effort and should be sent to remedial school + +using System; +using System.Collections.Concurrent; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Sockets; +using System.Security; using System.Text; +using System.Text.RegularExpressions; using System.Threading.Tasks; using System.Xml.Linq; +using Newtonsoft.Json; + using Octokit; using Octokit.GraphQL; +using Tgstation.Server.Host.Extensions.Converters; + +using YamlDotNet.Serialization; + namespace Tgstation.Server.ReleaseNotes { /// @@ -16,6 +31,7 @@ namespace Tgstation.Server.ReleaseNotes /// static class Program { + const string OutputPath = "release_notes.md"; const string RepoOwner = "tgstation"; const string RepoName = "tgstation-server"; @@ -34,14 +50,54 @@ namespace Tgstation.Server.ReleaseNotes var ensureRelease = versionString.Equals("--ensure-release", StringComparison.OrdinalIgnoreCase); var linkWinget = versionString.Equals("--link-winget", StringComparison.OrdinalIgnoreCase); var shaCheck = versionString.Equals("--winget-template-check", StringComparison.OrdinalIgnoreCase); + var fullNotes = versionString.Equals("--generate-full-notes", StringComparison.OrdinalIgnoreCase); + var nuget = versionString.Equals("--nuget", StringComparison.OrdinalIgnoreCase); - if ((!Version.TryParse(versionString, out var version) || version.Revision != -1) && !ensureRelease && !linkWinget && !shaCheck) + if ((!Version.TryParse(versionString, out var version) || version.Revision != -1) + && !ensureRelease + && !linkWinget + && !shaCheck + && !fullNotes + && !nuget) { Console.WriteLine("Invalid version: " + versionString); return 2; } - var doNotCloseMilestone = args.Length > 1 && args[1].ToUpperInvariant() == "--NO-CLOSE"; + var doNotCloseMilestone = false; + var debianMode = false; + Component? componentRelease = null; + if (args.Length > 1) + switch (args[1].ToUpperInvariant()) + { + case "--DEBIAN": + debianMode = true; + doNotCloseMilestone = true; + if (args.Length < 3) + { + Console.WriteLine("Missing output path!"); + return 238; + } + + if (args.Length < 4) + { + Console.WriteLine("Missing current SHA!"); + return 239; + } + break; + case "--NO-CLOSE": + doNotCloseMilestone = true; + break; + case "--HTTPAPI": + componentRelease = Component.HttpApi; + break; + case "--INTEROPAPI": + componentRelease = Component.InteropApi; + break; + case "--DMAPI": + componentRelease = Component.DreamMakerApi; + break; + } const string ReleaseNotesEnvVar = "TGS_RELEASE_NOTES_TOKEN"; var githubToken = Environment.GetEnvironmentVariable(ReleaseNotesEnvVar); @@ -84,6 +140,18 @@ namespace Tgstation.Server.ReleaseNotes return await Winget(client, null, args[1]); } + if (fullNotes) + return await FullNotes(client); + + if (componentRelease.HasValue) + return await ReleaseComponent(client, version, componentRelease.Value); + + if (nuget) + return await ReleaseNuget(client); + + if (debianMode) + return await GenDebianChangelog(client, version, args[2], args[3]); + var releasesTask = client.Repository.Release.GetAll(RepoOwner, RepoName); Console.WriteLine("Getting merged pull requests in milestone " + versionString + "..."); @@ -101,103 +169,12 @@ namespace Tgstation.Server.ReleaseNotes } Console.WriteLine(milestonePRs.Items.Count + " total pull requests"); - Task milestoneTask = null; - var milestoneTaskLock = new object(); - var releaseDictionary = new Dictionary>>(StringComparer.OrdinalIgnoreCase); - bool postControlPanelMessage = false; - async Task GetReleaseNotesFromPR(Issue pullRequest) - { - //need to check it was merged - var fullPR = await client.Repository.PullRequest.Get(RepoOwner, RepoName, pullRequest.Number).ConfigureAwait(false); + var noteTasks = new List, Dictionary, bool>>>(); - if (!fullPR.Merged) - { - if (!doNotCloseMilestone && fullPR.Milestone != null) - { - Console.WriteLine($"Removing trash PR #{fullPR.Number} from milestone..."); - await client.Issue.Update(RepoOwner, RepoName, fullPR.Number, new IssueUpdate - { - Milestone = null - }).ConfigureAwait(false); - } - - return; - } - - async Task GetMilestone() - { - if (fullPR.Milestone == null) - return null; - return await client.Issue.Milestone.Get(RepoOwner, RepoName, fullPR.Milestone.Number); - }; - - lock (milestoneTaskLock) - milestoneTask ??= GetMilestone(); - - // if (!fullPR.Merged) - //return; - - void BuildNotesFromComment(string comment, User user) - { - if (comment == null) - return; - - void CommitNotes(string component, List notes) - { - lock (releaseDictionary) - { - foreach (var I in notes) - Console.WriteLine(component + " #" + fullPR.Number + " - " + I + " (@" + user.Login + ")"); - - var tupleSelector = notes.Select(note => Tuple.Create(note, fullPR.Number, user.Login)); - if (releaseDictionary.TryGetValue(component, out var currentValues)) - currentValues.AddRange(tupleSelector); - else - releaseDictionary.Add(component, tupleSelector.ToList()); - } - } - - var commentSplits = comment.Split('\n'); - string targetComponent = null; - var notes = new List(); - foreach (var line in commentSplits) - { - var trimmedLine = line.Trim(); - if (targetComponent == null) - { - if (trimmedLine.StartsWith(":cl:", StringComparison.Ordinal)) - { - targetComponent = trimmedLine[4..].Trim(); - if (targetComponent.Length == 0) - targetComponent = "Core"; - } - continue; - } - if (trimmedLine.StartsWith("/:cl:", StringComparison.Ordinal)) - { - CommitNotes(targetComponent, notes); - targetComponent = null; - notes.Clear(); - continue; - } - if (trimmedLine.Length == 0) - continue; - - notes.Add(trimmedLine); - } - } - - var comments = await client.Issue.Comment.GetAllForIssue(RepoOwner, RepoName, fullPR.Number).ConfigureAwait(false); - BuildNotesFromComment(fullPR.Body, fullPR.User); - foreach(var x in comments) - BuildNotesFromComment(x.Body, x.User); - } - - var tasks = new List(); foreach (var I in milestonePRs.Items) - tasks.Add(GetReleaseNotesFromPR(I)); + noteTasks.Add(GetReleaseNotesFromPR(client, I, doNotCloseMilestone, false, false)); var releases = await releasesTask.ConfigureAwait(false); @@ -275,7 +252,7 @@ namespace Tgstation.Server.ReleaseNotes if (webControlVersion.Major == 0) postControlPanelMessage = true; - prefix = $"Please refer to the [README](https://github.com/tgstation/tgstation-server#setup) for setup instructions.{Environment.NewLine}{Environment.NewLine}#### Component Versions\nCore: {coreVersion}\nConfiguration: {configVersion}\nHTTP API: {apiVersion}\nDreamMaker API: {dmApiVersion} (Interop: {interopVersion})\n[Web Control Panel](https://github.com/tgstation/tgstation-server-webpanel): {webControlVersion}\nHost Watchdog: {hostWatchdogVersion}"; + prefix = $"Please refer to the [README](https://github.com/tgstation/tgstation-server#setup) for setup instructions. Full changelog can be found [here](https://raw.githubusercontent.com/tgstation/tgstation-server/gh-pages/changelog.yml).{Environment.NewLine}{Environment.NewLine}#### Component Versions\nCore: {coreVersion}\nConfiguration: {configVersion}\nHTTP API: {apiVersion}\nDreamMaker API: {dmApiVersion} (Interop: {interopVersion})\n[Web Control Panel](https://github.com/tgstation/tgstation-server-webpanel): {webControlVersion}\nHost Watchdog: {hostWatchdogVersion}"; var newNotes = new StringBuilder(prefix); if (postControlPanelMessage) @@ -299,19 +276,23 @@ namespace Tgstation.Server.ReleaseNotes newNotes.Append(version.Build); } newNotes.Append("]("); - var milestone = await milestoneTask.ConfigureAwait(false); + + await Task.WhenAll(noteTasks); + + var milestone = milestones.Single().Value; if (milestone == null) { Console.WriteLine("Unable to detemine milestone!"); return 9; } + var allTasks = new List(noteTasks); if (doNotCloseMilestone) Console.WriteLine("Not closing milestone due to parameter!"); else { Console.WriteLine("Closing milestone..."); - tasks.Add(client.Issue.Milestone.Update(RepoOwner, RepoName, milestone.Number, new MilestoneUpdate + allTasks.Add(client.Issue.Milestone.Update(RepoOwner, RepoName, milestone.Number, new MilestoneUpdate { State = ItemState.Closed })); @@ -358,14 +339,16 @@ namespace Tgstation.Server.ReleaseNotes if (I.PullRequest != null) { Console.WriteLine($"Adding additional merged PR #{I.Number}..."); - tasks.Add(GetReleaseNotesFromPR(I)); + var task = GetReleaseNotesFromPR(client, I, doNotCloseMilestone, false, false); + noteTasks.Add(task); + allTasks.Add(task); } } await Task.WhenAll(issueUpdateTasks).ConfigureAwait(false); } - tasks.Add(client.Issue.Milestone.Delete(RepoOwner, RepoName, milestoneToDelete.Number)); + allTasks.Add(client.Issue.Milestone.Delete(RepoOwner, RepoName, milestoneToDelete.Number)); } // Create the next minor milestone @@ -378,7 +361,7 @@ namespace Tgstation.Server.ReleaseNotes { Description = "Next minor version" }); - tasks.Add(nextMinorMilestoneTask); + allTasks.Add(nextMinorMilestoneTask); // Move unfinished stuff to new minor milestone Console.WriteLine($"Moving {milestone.OpenIssues} abandoned issue(s) from previous milestone to new one..."); @@ -393,7 +376,7 @@ namespace Tgstation.Server.ReleaseNotes { var nextMinorMilestone = await nextMinorMilestoneTask.ConfigureAwait(false); foreach (var I in abandonedIssues.Items) - tasks.Add(client.Issue.Update(RepoOwner, RepoName, I.Number, new IssueUpdate + allTasks.Add(client.Issue.Update(RepoOwner, RepoName, I.Number, new IssueUpdate { Milestone = nextMinorMilestone.Number })); @@ -405,7 +388,43 @@ namespace Tgstation.Server.ReleaseNotes newNotes.Append("?closed=1)"); newNotes.Append(Environment.NewLine); - await Task.WhenAll(tasks).ConfigureAwait(false); + await Task.WhenAll(allTasks).ConfigureAwait(false); + + var componentVersionDict = new Dictionary + { + { Component.Configuration, configVersion }, + { Component.HttpApi, apiVersion }, + { Component.DreamMakerApi, dmApiVersion }, + { Component.InteropApi, interopVersion }, + { Component.WebControlPanel, webControlVersion }, + { Component.HostWatchdog, hostWatchdogVersion }, + }; + + var releaseDictionary = new SortedDictionary( + new Dictionary( + noteTasks + .Where(task => task.Result != null) + .SelectMany(task => task.Result.Item1) + .Where(kvp => kvp.Key == Component.Core || componentVersionDict.ContainsKey(kvp.Key)) + .GroupBy(kvp => kvp.Key) + .Select(grouping => + { + var component = grouping.Key; + var changelist = new Changelist + { + Changes = grouping.SelectMany(kvp => kvp.Value.Changes).ToList() + }; + + if (component == Component.Core) + { + changelist.Version = coreVersion; + changelist.ComponentVersions = componentVersionDict; + } + else + changelist.Version = componentVersionDict[component]; + + return new KeyValuePair(component, changelist); + }))); if (releaseDictionary.Count == 0) { @@ -413,25 +432,24 @@ namespace Tgstation.Server.ReleaseNotes return 8; } - foreach (var I in releaseDictionary.OrderBy(kvp => kvp.Key)) + foreach (var I in releaseDictionary) { newNotes.Append(Environment.NewLine); newNotes.Append("#### "); - newNotes.Append(I.Key); + string componentName = GetComponentDisplayName(I.Key, false); + newNotes.Append(componentName); - - foreach (var noteTuple in I.Value) + if (I.Key == Component.Configuration) { - newNotes.Append(Environment.NewLine); - newNotes.Append("- "); - newNotes.Append(noteTuple.Item1); - newNotes.Append(" (#"); - newNotes.Append(noteTuple.Item2); - newNotes.Append(" @"); - newNotes.Append(noteTuple.Item3); - newNotes.Append(')'); + I.Value.StripConfigVersionMessage(); + newNotes.AppendLine(); + newNotes.Append("- **The new configuration version is `"); + newNotes.Append(I.Value.Version); + newNotes.Append("`. Please update your `General:ConfigVersion` setting appropriately.**"); } + PrintChanges(newNotes, I.Value); + newNotes.Append(Environment.NewLine); } @@ -440,7 +458,6 @@ namespace Tgstation.Server.ReleaseNotes if (version.Minor != 0 && version.Build != 0) newNotes.Append(oldNotes); - const string OutputPath = "release_notes.md"; Console.WriteLine($"Writing out new release notes to {Path.GetFullPath(OutputPath)}..."); var releaseNotes = newNotes.ToString(); await File.WriteAllTextAsync(OutputPath, releaseNotes).ConfigureAwait(false); @@ -473,6 +490,246 @@ namespace Tgstation.Server.ReleaseNotes } } + static string GetComponentDisplayName(Component component, bool debian) => component switch + { + Component.HttpApi => debian ? "the HTTP API" : "HTTP API", + Component.InteropApi => debian ? "the Interop API" : "Interop API", + Component.Configuration => debian ? "the TGS configuration" : "**Configuration**", + Component.DreamMakerApi => debian ? "the DreamMaker API" : "DreamMaker API", + Component.HostWatchdog => debian ? "the Host Watchdog" : "Host Watchdog", + Component.Core => debian ? "the main server" : "Core", + Component.WebControlPanel => debian ? "the Web Control Panel" : "Web Control Panel", + _ => throw new Exception($"Unnamed Component: {component}"), + }; + + static readonly ConcurrentDictionary milestones = new(); + static readonly ConcurrentDictionary> pullRequests = new(); + + static Task GetPR(IGitHubClient client, int pr) => pullRequests.GetOrAdd(pr, x => RLR(() => client.Repository.PullRequest.Get(RepoOwner, RepoName, x))); + + static async Task, Dictionary, bool>> GetReleaseNotesFromPR(IGitHubClient client, Issue pullRequest, bool doNotCloseMilestone, bool needComponentExactVersions, bool forAllComponents) + { + //need to check it was merged + var prTask = GetPR(client, pullRequest.Number); + var fullPR = await prTask; + + if (!fullPR.Merged) + { + if (!doNotCloseMilestone && fullPR.Milestone != null) + { + Console.WriteLine($"Removing trash PR #{fullPR.Number} from milestone..."); + await RLR(() => client.Issue.Update(RepoOwner, RepoName, fullPR.Number, new IssueUpdate + { + Milestone = null + })); + } + + return null; + } + + if (fullPR.Milestone == null) + { + return null; + } + + milestones.TryAdd(fullPR.Milestone.Number, fullPR.Milestone); + + var commentsTask = TripleCheckGitHubPagination(apiOptions => client.Issue.Comment.GetAllForIssue(fullPR.Base.Repository.Id, pullRequest.Number, apiOptions), comment => comment.Id); + + bool isReleasePR = false; + async Task ShouldGetExtendedComponentVersions() + { + if (forAllComponents) + return true; + + var commit = await RLR(() => client.Repository.Commit.Get(fullPR.Base.Repository.Id, fullPR.MergeCommitSha)); + + isReleasePR = commit.Commit.Message.Contains("[TGSDeploy]") + || fullPR.Number == 966 + || fullPR.Number == 1048 + || fullPR.Number == 1435 + || fullPR.Number == 1263 + || fullPR.Number == 1087 + || fullPR.Number == 1441 + || fullPR.Number == 1437 + || fullPR.Number == 1443 + || fullPR.Number == 1311 + || fullPR.Number == 1598 + || fullPR.Number == 1463 + || fullPR.Number == 1209; // some special tactics from before we were more stingent + + return isReleasePR; + } + + Task needExtendedComponentVersions = Task.FromResult(false); + async Task> GetComponentVersions() + { + var mergeCommit = fullPR.MergeCommitSha; + // we don't care about unreleased web control panel changes + + try + { + needExtendedComponentVersions = ShouldGetExtendedComponentVersions(); + + var versionsBytes = await RLR(() => client.Repository.Content.GetRawContentByRef(RepoOwner, RepoName, "build/Version.props", mergeCommit)); + + XDocument doc; + using (var ms = new MemoryStream(versionsBytes)) + doc = XDocument.Load(ms); + + var project = doc.Root; + var xmlNamespace = project.GetDefaultNamespace(); + var versionsPropertyGroup = project.Elements().First(x => x.Name == xmlNamespace + "PropertyGroup"); + + Version Parse(string elemName, bool controlPanel = false) + { + var element = versionsPropertyGroup.Element(xmlNamespace + elemName); + if (element == null) + return null; + + return Version.Parse(element.Value); + } + + var dict = new Dictionary + { + { Component.Core, Parse("TgsCoreVersion") }, + { Component.HttpApi, Parse("TgsApiVersion") }, + { Component.DreamMakerApi, Parse("TgsDmapiVersion") }, + }; + + if (await needExtendedComponentVersions) + { + // only grab some versions at release time + // we aggregate later + dict.Add(Component.Configuration, Parse("TgsConfigVersion")); + dict.Add(Component.InteropApi, Parse("TgsInteropVersion")); + dict.Add(Component.HostWatchdog, Parse("TgsHostWatchdogVersion")); + dict.Add(Component.NugetCommon, Parse("TgsCommonLibraryVersion")); + dict.Add(Component.NugetApi, Parse("TgsApiLibraryVersion")); + dict.Add(Component.NugetClient, Parse("TgsClientVersion")); + + var webVersion = Parse("TgsControlPanelVersion"); + if (webVersion != null) + { + dict.Add(Component.WebControlPanel, webVersion); + } + else + { + var controlPanelVersionBytes = await RLR(() => client.Repository.Content.GetRawContentByRef(RepoOwner, RepoName, "build/ControlPanelVersion.props", mergeCommit)); + using (var ms = new MemoryStream(controlPanelVersionBytes)) + doc = XDocument.Load(ms); + + + project = doc.Root; + var controlPanelXmlNamespace = project.GetDefaultNamespace(); + var controlPanelVersionsPropertyGroup = project.Elements().First(x => x.Name == controlPanelXmlNamespace + "PropertyGroup"); + dict.Add(Component.WebControlPanel, Version.Parse(controlPanelVersionsPropertyGroup.Element(controlPanelXmlNamespace + "TgsControlPanelVersion").Value)); + } + } + + return dict; + } + catch + { + return new Dictionary(); + } + } + + var componentVersions = needComponentExactVersions ? GetComponentVersions() : Task.FromResult>(null); + var changelists = new ConcurrentDictionary(); + async Task BuildNotesFromComment(string comment, User user, Task localPreviousTask) + { + await localPreviousTask; + if (comment == null) + return; + + async Task CommitNotes(Component component, List notes) + { + foreach (var I in notes) + Console.WriteLine(component + " #" + fullPR.Number + " - " + I + " (@" + user.Login + ")"); + + var tupleSelector = notes.Select(note => new Change + { + Descriptions = new List { note }, + PullRequest = fullPR.Number, + Author = user.Login + }); + + var useExtendedComponentVersions = await needExtendedComponentVersions; + var componentVersionsResult = await componentVersions; + lock (changelists) + if (changelists.TryGetValue(component, out var currentChangelist)) + currentChangelist.Changes.AddRange(tupleSelector); + else + Debug.Assert(changelists.TryAdd(component, new Changelist + { + Changes = tupleSelector.ToList(), + Unreleased = false, + Version = needComponentExactVersions && componentVersionsResult.TryGetValue(component, out var componentVersion) + ? componentVersion + : null, + ComponentVersions = component == Component.Core && needComponentExactVersions && useExtendedComponentVersions + ? new Dictionary(componentVersionsResult.Where(kvp => kvp.Key != Component.Core)) + : null + })); + } + + var commentSplits = comment.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + string targetComponent = null; + var notes = new List(); + foreach (var line in commentSplits) + { + var trimmedLine = line.Trim(); + if (targetComponent == null) + { + if (trimmedLine.StartsWith(":cl:", StringComparison.Ordinal)) + { + targetComponent = trimmedLine[4..].Trim(); + if (targetComponent.Length == 0) + targetComponent = "Core"; + } + continue; + } + if (trimmedLine.StartsWith("/:cl:", StringComparison.Ordinal)) + { + if(!Enum.TryParse(targetComponent, out var component)) + component = targetComponent.ToUpperInvariant() switch + { + "**CONFIGURATION**" or "CONFIGURATION" or "CONFIG" => Component.Configuration, + "HTTP API" => Component.HttpApi, + "WEB CONTROL PANEL" => Component.WebControlPanel, + "DMAPI" or "DREAMMAKER API" => Component.DreamMakerApi, + "INTEROP API" => Component.InteropApi, + "HOST WATCHDOG" => Component.HostWatchdog, + "NUGET: API" => Component.NugetApi, + "NUGET: COMMON" => Component.NugetCommon, + "NUGET: CLIENT" => Component.NugetClient, + _ => throw new Exception($"Unknown component: \"{targetComponent}\""), + }; + await CommitNotes(component, notes); + targetComponent = null; + notes.Clear(); + continue; + } + if (trimmedLine.Length == 0) + continue; + + notes.Add(trimmedLine); + } + } + + var previousTask = BuildNotesFromComment(fullPR.Body, fullPR.User, Task.CompletedTask); + var comments = await commentsTask; + foreach (var x in comments) + previousTask = BuildNotesFromComment(x.Body, x.User, previousTask); + + await previousTask; + + Debug.Assert(!(await needExtendedComponentVersions) || changelists.Where(x => x.Key == Component.Core).All(x => x.Value.ComponentVersions != null && x.Value.ComponentVersions.Count > 3)); + + return Tuple.Create(changelists.ToDictionary(kvp => kvp.Key, kvp => kvp.Value), await componentVersions, isReleasePR); + } + class ExtendedReleaseUpdate : ReleaseUpdate { public bool? MakeLatest { get; set; } @@ -570,7 +827,734 @@ The user account that created this pull request is available to correct any issu { Body = prBody, }); + return 0; + } + static async Task RLR(Func> func) + { + while (true) + try + { + return await func(); + } + catch (HttpRequestException ex) when (ex.InnerException is IOException ioEx && ioEx.InnerException is SocketException sockEx && sockEx.ErrorCode == 10053) + { + await Task.Delay(15000); + } + catch (SecondaryRateLimitExceededException) + { + await Task.Delay(15000); + } + catch (RateLimitExceededException ex) + { + var now = DateTimeOffset.UtcNow.AddSeconds(-10); + if (ex.Reset > now) + { + var delay = ex.Reset - now; + await Task.Delay(delay); + } + } + } + + static async Task> TripleCheckGitHubPagination(Func>> apiCall, Func idSelector) + { + // I've seen GitHub pagination return incomplete result sets in the past + // It has an in-built pagination limit of 100 + var apiOptions = new ApiOptions + { + PageSize = 100 + }; + var results = await RLR(() => apiCall(apiOptions)); + var distinctEntries = new Dictionary(results.Count); + foreach (var result in results) + distinctEntries.Add(idSelector(result).ToString(), result); + + if (results.Count > 100) + { + results = await RLR(() => apiCall(apiOptions)); + foreach (var result in results) + distinctEntries.TryAdd(idSelector(result).ToString(), result); + + results = await RLR(() => apiCall(apiOptions)); + foreach (var result in results) + distinctEntries.TryAdd(idSelector(result).ToString(), result); + } + + return distinctEntries.Values.ToList(); + } + + static async Task> ProcessMilestone(IGitHubClient client, Milestone milestone) + { + // have to trust this works + SearchIssuesResult results; + + var milestoneTask = Task.FromResult(milestone); + var pullRequests = new Dictionary(); + var iteration = 0; + while (true) + { + results = await RLR(() => client.Search.SearchIssues(new SearchIssuesRequest + { + Type = IssueTypeQualifier.PullRequest, + Milestone = milestone.Title, + Repos = new RepositoryCollection + { + { RepoOwner, RepoName }, + }, + Merged = DateRange.GreaterThan(new DateTimeOffset(2018, 9, 27, 0, 0, 0, TimeSpan.Zero)), + })); + + foreach (var result in results.Items) + pullRequests.TryAdd(result.Number, result); + + if (results.IncompleteResults) + continue; + + if (results.TotalCount <= 100 || ++iteration == 3) + break; + } + + async Task RunPRs() + { + var milestoneVersion = Version.Parse(milestone.Title[1..]); + var prTasks = pullRequests.Select( + kvp => GetReleaseNotesFromPR(client, kvp.Value, true, true, milestone.State.Value == ItemState.Open)) + .ToList(); + + await Task.WhenAll(prTasks); + + var prResults = prTasks.Select(x => x.Result).ToList(); + + var releasePRResult = prResults.FirstOrDefault(x => x.Item3); + + prResults = prResults.Where(result => result != null).ToList(); + + Dictionary releasedComponentVersions; + if (releasePRResult != null) + releasedComponentVersions = releasePRResult.Item2; + else + { + releasedComponentVersions = new Dictionary( + prResults + .SelectMany(result => result.Item2) + .GroupBy(kvp => kvp.Key) + .Select(grouping => new KeyValuePair(grouping.Key, grouping.Max(kvp => kvp.Value)))); + + foreach(var maxVersionKvp in prResults.SelectMany(x => x.Item1) + .Where(x => !releasedComponentVersions.ContainsKey(x.Key)) + .GroupBy(x => x.Key) + .Select(group => { + var versions = group + .Where(x => x.Value.Version != null) + .ToList(); + + if (versions.Count == 0) + return new KeyValuePair(group.Key, null); + + return new KeyValuePair(group.Key, versions.Max(x => x.Value.Version)); + }) + .Where(kvp => kvp.Value != null) + .ToList()) + { + releasedComponentVersions.Add(maxVersionKvp.Key, maxVersionKvp.Value); + } + } + + var finalResults = new Dictionary>(); + foreach (var componentKvp in releasedComponentVersions) + { + var component = componentKvp.Key; + var list = new List(); + + foreach(var changelistDict in prResults.Select(x => x.Item1)) + { + if (!changelistDict.TryGetValue(component, out var changelist)) + continue; + + Version componentVersion = milestoneVersion; + var unreleased = milestone.State.Value == ItemState.Open; + if (component != Component.Core) + { + componentVersion = changelist.Version ?? componentKvp.Value; + if (releasedNonCoreVersions != null + && releasedNonCoreVersions.TryGetValue(component, out var releasedVersions) + && !releasedVersions.Any(x => x == componentVersion)) + { + // roll forward + var newList = releasedVersions + .ToList(); + newList.Add(componentVersion); + newList = newList.OrderBy(x => x).ToList(); + + var index = newList.IndexOf(componentVersion); + Debug.Assert(index != -1); + if (index != (newList.Count - 1)) + { + componentVersion = newList[index + 1]; + unreleased = false; + } + else + unreleased = true; + } + } + + var entry = list.FirstOrDefault(x => x.Version == componentVersion); + if (entry == null) + { + entry = changelist; + entry.Version = componentVersion; + entry.Unreleased = unreleased; + if (component == Component.Core && entry.ComponentVersions == null) + entry.ComponentVersions = releasedComponentVersions; + + list.Add(entry); + } + else + entry.Changes.AddRange(changelist.Changes); + } + + Debug.Assert(list.Select(x => x.Version.ToString()).Distinct().Count() == list.Count); + if (component == Component.Core) + { + Debug.Assert(list.All(x => x.Version == milestoneVersion)); + } + + list = list.OrderByDescending(x => x.Version).ToList(); + finalResults.Add(component, list); + } + + if (!finalResults.ContainsKey(Component.Core) || finalResults[Component.Core].Count == 0) + { + finalResults.Remove(Component.Core); + finalResults.Add(Component.Core, new List + { + new Changelist + { + Changes = new List(), + ComponentVersions = releasedComponentVersions, + Unreleased = milestone.State.Value == ItemState.Open, + Version = milestoneVersion, + } + }); + } + else + Debug.Assert(finalResults[Component.Core].All(x => x.Version == milestoneVersion && x.ComponentVersions != null && x.ComponentVersions.Count > 3)); + + var notes = new ReleaseNotes + { + Components = new SortedDictionary>(finalResults), + }; + + return notes; + } + + return RunPRs(); + } + + static async Task FullNotes(IGitHubClient client) + { + var rateLimitInfo = client.GetLastApiInfo()?.RateLimit ?? (await client.RateLimit.GetRateLimits()).Rate; + var startRateLimit = rateLimitInfo.Remaining; + + var releaseNotes = await GenerateNotes(client); + + Console.WriteLine($"Generating all release notes took {startRateLimit - client.GetLastApiInfo().RateLimit.Remaining} requests."); + + var serializer = new SerializerBuilder() + .ConfigureDefaultValuesHandling(DefaultValuesHandling.OmitDefaults) + .WithTypeConverter(new VersionConverter()) + .Build(); + + var serializedYaml = serializer.Serialize(releaseNotes); + await File.WriteAllTextAsync("changelog.yml", serializedYaml).ConfigureAwait(false); + return 0; + } + + static readonly HttpClient httpClient = new ( + new HttpClientHandler() + { + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate + }); + static async Task> EnumerateNugetVersions(string package) + { + var url = new Uri($"https://api.nuget.org/v3/registration5-gz-semver2/{package.ToLowerInvariant()}/index.json"); + + using var req = new HttpRequestMessage(); + req.Headers.UserAgent.Add(new System.Net.Http.Headers.ProductInfoHeaderValue("Tgstation.Server.ReleaseNotes", "0.1.0")); + req.Method = HttpMethod.Get; + req.RequestUri = url; + + using var resp = await httpClient.SendAsync(req); + resp.EnsureSuccessStatusCode(); + + var json = await resp.Content.ReadAsStringAsync(); + + dynamic dynamicJson = JsonConvert.DeserializeObject(json); + + var versions = (IEnumerable)dynamicJson.items[0].items; + var results = versions + .Select(x => Version.TryParse((string)x.catalogEntry.version, out var version) ? version : null) + .Where(version => version != null) + .OrderBy(x => x) + .ToHashSet(); + return results; + } + + static IReadOnlyDictionary> releasedNonCoreVersions; + + static async Task GenerateNotes(IGitHubClient client, Dictionary forceReleaseVersions = null) + { + ReleaseNotes previousNotes = null; + if (File.Exists("changelog.yml")) + { + var existingYml = await File.ReadAllTextAsync("changelog.yml"); + var deserializer = new DeserializerBuilder() + .Build(); + + previousNotes = deserializer.Deserialize(existingYml); + } + + var releasesTask = TripleCheckGitHubPagination( + apiOptions => client.Repository.Release.GetAll(RepoOwner, RepoName, apiOptions), + release => release.Id); + + var milestones = await TripleCheckGitHubPagination( + apiOptions => client.Issue.Milestone.GetAllForRepository(RepoOwner, RepoName, new MilestoneRequest { + State = ItemStateFilter.All + }, apiOptions), + milestone => milestone.Id); + + var versionMilestones = milestones + .Where(milestone => Regex.IsMatch(milestone.Title, @"v[1-9][0-9]*\.[1-9]*[0-9]+\.[1-9]*[0-9]+$")) + .ToList(); + + var releases = await releasesTask; + + var nugetCommonVersions = EnumerateNugetVersions("Tgstation.Server.Common"); + var nugetApiVersions = EnumerateNugetVersions("Tgstation.Server.Api"); + var nugetClientVersions = EnumerateNugetVersions("Tgstation.Server.Client"); + + var newDic = new Dictionary> { + { Component.HttpApi, releases + .Where(x => x.TagName.StartsWith("api-v")) + .Select(x => Version.Parse(x.TagName[5..])) + .OrderBy(x => x) + .ToHashSet() }, + { Component.DreamMakerApi, releases + .Where(x => x.TagName.StartsWith("dmapi-v")) + .Select(x => Version.Parse(x.TagName[7..])) + .OrderBy(x => x) + .ToHashSet() }, + { Component.NugetCommon, await nugetCommonVersions }, + { Component.NugetApi, await nugetApiVersions }, + { Component.NugetClient, await nugetClientVersions } + }; + + if (forceReleaseVersions != null) + foreach (var kvp in forceReleaseVersions) + if (!newDic[kvp.Key].Any(x => x == kvp.Value)) + newDic[kvp.Key] = newDic[kvp.Key] + .Concat(new List { kvp.Value }) + .OrderBy(x => x) + .ToHashSet(); + + releasedNonCoreVersions = newDic; + + var milestonesToProcess = versionMilestones; + if (previousNotes != null) + { + var releasedVersions = previousNotes.Components[Component.Core].Where(cl => !cl.Unreleased).ToList(); + milestonesToProcess = milestonesToProcess + .Where(x => !releasedVersions.Any( + version => version.Version == Version.Parse(x.Title.AsSpan(1)))) + .ToList(); + + foreach (var kvp in previousNotes.Components) + if (releasedNonCoreVersions.TryGetValue(kvp.Key, out var releasedComponentVersions)) + kvp.Value.RemoveAll(x => x.Unreleased = !releasedComponentVersions.Any(y => y == x.Version)); + else + kvp.Value.RemoveAll(x => x.Unreleased); + } + + var milestonePRTasks = milestonesToProcess + .Select(milestone => ProcessMilestone(client, milestone)) + .ToList(); + + await Task.WhenAll(milestonePRTasks); + + await Task.WhenAll(milestonePRTasks.Select(task => task.Result)); + + var coreCls = milestonePRTasks + .SelectMany(task => task.Result.Result.Components) + .Where(x => x.Key == Component.Core) + .ToList(); + + Debug.Assert( + coreCls.Count == milestonesToProcess.Count); + + var distinctCoreVersions = coreCls + .SelectMany(x => x.Value) + .Select(x => x.Version.ToString()) + .Distinct() + .Select(Version.Parse) + .OrderBy(x => x) + .ToList(); + + var missingCoreVersions = milestonesToProcess + .Where(x => !distinctCoreVersions.Any(y => Version.Parse(x.Title.AsSpan(1)) == y)) + .ToList(); + + Debug.Assert(missingCoreVersions.Count == 0); + + var changelistsGroupedByComponent = + milestonePRTasks + .SelectMany(task => task.Result.Result.Components) + .GroupBy(kvp => kvp.Key) + .ToDictionary(grouping => grouping.Key, grouping => grouping.SelectMany(kvp => kvp.Value)); + + var releaseNotes = new ReleaseNotes + { + Components = new SortedDictionary>( + changelistsGroupedByComponent + .ToDictionary( + kvp => kvp.Key, + kvp => kvp + .Value + .GroupBy(changelist => changelist.Version) + .Select(grouping => + { + var firstEntry = grouping.First(); + return new Changelist + { + Changes = grouping.SelectMany(cl => cl.Changes).ToList(), + ComponentVersions = firstEntry.ComponentVersions, + Unreleased = firstEntry.Unreleased, + Version = grouping.Key + }; + }) + .OrderByDescending(cl => cl.Version) + .ToList())) + }; + + Debug.Assert(releaseNotes.Components.ContainsKey(Component.Core) && releaseNotes.Components[Component.Core].Count == milestonesToProcess.Count); + + if (previousNotes != null) + { + foreach (var component in Enum.GetValues()) + { + if (!previousNotes.Components.ContainsKey(component)) + continue; + + if (releaseNotes.Components.TryGetValue(component, out var newChangelists)) + { + var missingVersions = previousNotes.Components[component] + .Where(olderVersion => + { + var newerVersion = newChangelists.SingleOrDefault(y => olderVersion.Version == y.Version); + if (newerVersion != null) + { + newerVersion.Changes.AddRange( + olderVersion.Changes.Where(x => !newerVersion.Changes.Any(y => x.PullRequest == y.PullRequest))); + return false; + } + + return true; + }); + + releaseNotes.Components[component] = newChangelists + .Concat(missingVersions) + .OrderByDescending(cl => cl.Version) + .ToList(); + } + else + releaseNotes.Components[component] = previousNotes.Components[component]; + } + } + + foreach (var kvp in releaseNotes.Components) + { + var distinctCount = kvp.Value.Select(changelist => changelist.Version.ToString()).Distinct().Count(); + Debug.Assert(distinctCount == kvp.Value.Count); + + foreach (var cl in kvp.Value) + { + cl.DeduplicateChanges(); + + if (kvp.Key == Component.Configuration) + cl.StripConfigVersionMessage(); + } + } + + return releaseNotes; + } + + static void PrintChanges(StringBuilder newNotes, Changelist changelist, bool debianMode = false) + { + var none = true; + foreach (var change in changelist.Changes) + foreach (var line in change.Descriptions) + { + none = false; + newNotes.AppendLine(); + if (debianMode) + newNotes.Append(" * "); + else + newNotes.Append("- "); + + newNotes.Append(line); + newNotes.Append(" (#"); + newNotes.Append(change.PullRequest); + newNotes.Append(" @"); + newNotes.Append(change.Author); + newNotes.Append(')'); + } + + if (debianMode && none) + throw new Exception($"Changlist {changelist.Version} has no changes!"); + } + + static string GenerateComponentNotes(ReleaseNotes releaseNotes, Component component, Version version) + { + var relevantChangelog = releaseNotes.Components[component].FirstOrDefault(x => x.Version == version); + + var newNotes = new StringBuilder("Full changelog can be found [here](https://raw.githubusercontent.com/tgstation/tgstation-server/gh-pages/changelog.yml)."); + if (relevantChangelog != null) + { + newNotes.AppendLine(); + PrintChanges(newNotes, relevantChangelog); + } + + var markdown = newNotes.ToString(); + return markdown; + } + + static async Task ReleaseComponent(IGitHubClient client, Version version, Component component) + { + var releaseNotes = await GenerateNotes(client, new Dictionary { { component, version } }); + await File.WriteAllTextAsync(OutputPath, GenerateComponentNotes(releaseNotes, component, version)); + return 0; + } + + // must run from repo root + static async Task ReleaseNuget(IGitHubClient client) + { + const string PropsPath = "build/Version.props"; + + var doc = XDocument.Load(PropsPath); + var project = doc.Root; + var xmlNamespace = project.GetDefaultNamespace(); + var versionsPropertyGroup = project.Elements().First(x => x.Name == xmlNamespace + "PropertyGroup"); + + var commonVersion = Version.Parse(versionsPropertyGroup.Element(xmlNamespace + "TgsCommonLibraryVersion").Value); + var apiVersion = Version.Parse(versionsPropertyGroup.Element(xmlNamespace + "TgsApiLibraryVersion").Value); + var clientVersion = Version.Parse(versionsPropertyGroup.Element(xmlNamespace + "TgsClientVersion").Value); + + var componentVersions = new Dictionary + { + { Component.NugetCommon, commonVersion }, + { Component.NugetApi, apiVersion }, + { Component.NugetClient, clientVersion }, + }; + + var releaseNotes = await GenerateNotes( + client, + componentVersions); + + const string CsprojSubstitution = "src/Tgstation.Server.$PROJECT$/Tgstation.Server.$PROJECT$.csproj"; + var csprojNameMap = new Dictionary + { + { Component.NugetCommon, "Common" }, + { Component.NugetApi, "Api" }, + { Component.NugetClient, "Client" }, + }; + + foreach(var kvp in csprojNameMap) + { + var component = kvp.Key; + var csprojPath = CsprojSubstitution.Replace("$PROJECT$", kvp.Value); + + var markdown = GenerateComponentNotes(releaseNotes, component, componentVersions[component]); + + var escapedMarkdown = SecurityElement.Escape(markdown); + + var originalCsproj = await File.ReadAllTextAsync(csprojPath); + var substitutedCsproj = originalCsproj.Replace($"$(TGS_NUGET_RELEASE_NOTES_{kvp.Value.ToUpperInvariant()})", escapedMarkdown); + + await File.WriteAllTextAsync(csprojPath, substitutedCsproj); + } + + return 0; + } + + static async Task GenDebianChangelog(IGitHubClient client, Version version, string outputPath, string currentSha) + { + var tagsTask = RLR(() => TripleCheckGitHubPagination( + apiOptions => client.Repository.GetAllTags(RepoOwner, RepoName, apiOptions), + x => x.Name)); + var currentRefTask = client.Repository.Commit.Get(RepoOwner, RepoName, currentSha); + var releaseNotes = await GenerateNotes(client); + + // https://www.debian.org/doc/manuals/maint-guide/dreq.en.html#changelog + // https://www.debian.org/doc/debian-policy/ch-source.html#s-dpkgchangelog + + /* +package (version) distribution(s); urgency=urgency + [optional blank line(s), stripped] + * change details + more change details + [blank line(s), included in output of dpkg-parsechangelog] + * even more change details + [optional blank line(s), stripped] + -- maintainer name [two spaces] date + */ + + // debian package did not exist before uhhh... + // var debianPackageFirstRelease = new Version(5, 13, 0); + // can't use that, there are irreconcilable changelog/version errors + // keep it straight going forwards + var noChangelogsBeforeVersion = new Version(5, 14, 0); + + var coreChangelists = releaseNotes + .Components[Component.Core] + .Where(x => x.Version >= noChangelogsBeforeVersion && (!x.Unreleased || x.Version == version)) + .OrderByDescending(x => x.Version) + .ToList(); + + var currentReleaseChangelists = new List>(); + + for (var i = 0; i < coreChangelists.Count; ++i) + { + var currentDic = new SortedDictionary(); + currentReleaseChangelists.Add(currentDic); + var nowRelease = coreChangelists[i]; + var previousRelease = (i + 1) < coreChangelists.Count + ? coreChangelists[i + 1] + : releaseNotes + .Components[Component.Core] + .First(x => x.Version == new Version(5, 13, 7)); + + currentDic.Add(Component.Core, nowRelease); + foreach (var componentKvp in nowRelease.ComponentVersions) + { + try + { + var component = componentKvp.Key; + if (component == Component.Core + || component == Component.NugetClient + || component == Component.NugetApi + || component == Component.NugetCommon) + continue; + + var takeNotesFrom = previousRelease.ComponentVersions[componentKvp.Key]; + var changesEnumerator = releaseNotes + .Components[component] + .Where(changelist => changelist.Version > takeNotesFrom && changelist.Version <= componentKvp.Value) + .SelectMany(x => x.Changes) + .OrderBy(x => x.PullRequest); + var changelist = new Changelist + { + Version = componentKvp.Value, + Changes = changesEnumerator + .ToList(), + }; + + if (changelist.Changes.Any()) + currentDic.Add(component, changelist); + } + catch when (Debugger.IsAttached) + { + Debugger.Break(); + } + } + } + + var builder = new StringBuilder(); + foreach (var releaseDictionary in currentReleaseChangelists) + { + var allPrNumbers = releaseDictionary.Values.SelectMany(x => x.Changes.Select(y => y.PullRequest)).Distinct().OrderBy(x => x).ToList(); + var allPrTasks = allPrNumbers + .Select(x => GetPR(client, x)) + .ToList(); + + await Task.WhenAll(allPrTasks); + + var prDict = allPrTasks.ToDictionary(x => x.Result.Number, x => x.Result); + + bool AnyPRHasLabel(string labelName) => prDict.Values.Any(x => x.Labels.Any(y => y.Name == labelName)); + + // determine urgency + + string urgency; + if (AnyPRHasLabel("Priority: CRITICAL")) + urgency = "critical"; + else if (AnyPRHasLabel("Priority: High")) + urgency = "high"; + else if (AnyPRHasLabel("Fix")) + urgency = "medium"; + else + urgency = "low"; + + builder.Append($"tgstation-server ("); + + builder.Append(releaseDictionary[Component.Core].Version); + builder.Append("-1) unstable; urgency="); + builder.Append(urgency); + + foreach (var kvp in releaseDictionary.Where(x => x.Value.Changes.Count > 0 || x.Key == Component.Configuration)) + { + builder.AppendLine(); + builder.AppendLine(); + builder.Append(" * The following changes are for "); + builder.Append(GetComponentDisplayName(kvp.Key, true)); + if(kvp.Key == Component.Configuration) + { + builder.Append(". You "); + if (kvp.Value.Version.Minor == 0 && kvp.Value.Version.Build == 0) + builder.Append("will need to"); + else + builder.Append("should"); + builder.Append(" update your `General:ConfigVersion` setting in `/etc/tgstation-server/appsettings.Production.yml` to this new version"); + } + + builder.Append(':'); + + PrintChanges(builder, kvp.Value, true); + } + + builder.AppendLine(); + builder.Append(" -- "); + + GitHubCommit currentRef; + var tags = await tagsTask; + var releaseTag = tags.FirstOrDefault(x => x.Name == $"tgstation-server-v{releaseDictionary[Component.Core].Version}"); + + if (releaseTag != null) + currentRef = await client.Repository.Commit.Get(RepoOwner, RepoName, releaseTag.Commit.Sha); + else + currentRef = await currentRefTask; + + var committer = currentRef.Commit.Committer; + if (committer.Name == "GitHub" && committer.Email == "noreply@github.com") + committer = currentRef.Commit.Author; + + builder.Append(committer.Name); + builder.Append(" <"); + builder.Append(committer.Email); + builder.Append("> "); + + var commitTime = currentRef.Commit.Committer.Date; + + builder.Append(commitTime.ToString("ddd").TrimEnd('.')); + builder.Append(", "); + builder.Append(commitTime.ToString("dd")); + builder.Append(' '); + builder.Append(commitTime.ToString("MMM").TrimEnd('.')); + builder.Append(' '); + builder.AppendLine(commitTime.ToString("yyyy HH:mm:ss zz00")); + } + + var changelog = builder.ToString().Replace("\r", String.Empty); + await File.WriteAllTextAsync(outputPath, changelog); return 0; } } diff --git a/tools/Tgstation.Server.ReleaseNotes/ReleaseNotes.cs b/tools/Tgstation.Server.ReleaseNotes/ReleaseNotes.cs new file mode 100644 index 0000000000..8c10c880ce --- /dev/null +++ b/tools/Tgstation.Server.ReleaseNotes/ReleaseNotes.cs @@ -0,0 +1,9 @@ +using System.Collections.Generic; + +namespace Tgstation.Server.ReleaseNotes +{ + sealed class ReleaseNotes + { + public SortedDictionary> Components { get; set; } + } +} diff --git a/tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj b/tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj index d95f0139e3..35312949ce 100644 --- a/tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj +++ b/tools/Tgstation.Server.ReleaseNotes/Tgstation.Server.ReleaseNotes.csproj @@ -7,10 +7,12 @@ - - + + + +