Merge branch 'dev' of https://github.com/tgstation/tgstation-server into EFCoreUpdate

.github/workflows/ci-pipeline.yml

@@ -36,7 +36,7 @@ on:
 env:
   TGS_DOTNET_VERSION: 8
   OD_MIN_COMPAT_DOTNET_VERSION: 7
-  OD_DOTNET_VERSION: 8
+  OD_DOTNET_VERSION: 9
   TGS_DOTNET_QUALITY: ga
   TGS_WEBPANEL_NODE_VERSION: 20.x
   TGS_TEST_GITHUB_TOKEN: ${{ secrets.LIVE_TESTS_TOKEN }}
@@ -2322,9 +2322,9 @@ jobs:
 
   update-nix:
     name: Update Nix SHA
-    needs: deploy-tgs
+    needs: changelog-regen
     runs-on: ubuntu-latest
-    if: (!(cancelled() || failure())) && needs.deploy-tgs.result == 'success'
+    if: (!(cancelled() || failure())) && needs.changelog-regen.result == 'success'
     steps:
       - name: Install Native Packages # Name checked in rerunFlakyTests.js
         run: |
@@ -2345,7 +2345,7 @@ jobs:
           private-key: ${{ secrets.APP_PRIVATE_KEY }}
 
       - name: Clone
-        run: git clone -b master --single-branch "https://git@github.com/tgstation/tgstation-server" --depth 1 .
+        run: git clone -b master --single-branch "https://tgstation-server-ci:${{ steps.app-token-generation.outputs.token }}@github.com/tgstation/tgstation-server" --depth 1 .
 
       - name: Parse TGS version
         run: echo "TGS_VERSION=$(xmlstarlet sel -N X="http://schemas.microsoft.com/developer/msbuild/2003" --template --value-of /X:Project/X:PropertyGroup/X:TgsCoreVersion build/Version.props)" >> $GITHUB_ENV
@@ -2378,7 +2378,7 @@ jobs:
         run: git push "https://tgstation-server-ci:${{ steps.app-token-generation.outputs.token }}@github.com/tgstation/tgstation-server"
 
       - name: Force Push Tags
-        run: git push -f --tags "https://tgstation-server-ci:${{ steps.app-token-generation.outputs.token }}@github.com/tgstation/tgstation-server"
+        run: git push -f origin tag tgstation-server-v${{ env.TGS_VERSION }}
 
   changelog-regen:
     name: Regenerate Changelog

.github/workflows/new-pr-comments.yml

@@ -25,7 +25,7 @@ jobs:
         if: github.event.pull_request.user.id != 49699333
         uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b
         with:
-          message: Thank you for contributing to ${{ github.event.pull_request.base.repo.name }}! The workflow '${{ github.workflow }}' requires repository secrets and will not run without approval. Maintainers can add the `CI Cleared` label to allow it to run. Note that any changes to ci-security.yml and ci-pipeline.yml will not be reflected.
+          message: Thank you for contributing to ${{ github.event.pull_request.base.repo.name }}! The CI Pipeline workflow requires repository secrets and will not run without approval. Maintainers can add the `CI Cleared` label to allow it to run. Note that any changes to ci-security.yml and ci-pipeline.yml will not be reflected.
           github-token: ${{ steps.app-token-generation.outputs.token }}
 
       - name: Comment on dependabot PR

.gitignore

@@ -21,6 +21,8 @@ artifacts/
 /src/Tgstation.Server.Host/wwwroot
 /src/Tgstation.Server.Host/--applicationName
 /src/Tgstation.Server.Host/ClientApp
+/src/Tgstation.Server.Host.Utils.GitLab.GraphQL/node_modules
+/src/Tgstation.Server.Host.Utils.GitLab.GraphQL/schema.graphql
 /tools/Tgstation.Server.ReleaseNotes/release_notes.md
 launchSettings.json
 release_notes.md
@@ -29,3 +31,4 @@ changelog.yml
 *.sqlite3
 packaging/
 yarn-error.log*
+.graphqlrc.json

README.md

@@ -256,7 +256,9 @@ Create an `appsettings.Production.yml` file next to `appsettings.yml`. This will
 
 - `General:ConfigVersion`: Suppresses warnings about out of date config versions. You should change this after updating TGS to one with a new config version. The current version can be found on the releases page for your server version.
 
-- `General:MinimumPasswordLength`: Minimum password length requirement for database users
+- `General:MinimumPasswordLength`: Minimum password length requirement for database users.
+
+- `General:PrometheusPort`: Port Prometheus metrics are published on under /metrics. This can be set to the same value as the `ApiPort`, just note that accessing it does not require authentication.
 
 - `General:ValidInstancePaths`: Array meant to limit the directories in which instances may be created.
 

build/Dockerfile

@@ -58,11 +58,12 @@ RUN export TGS_TELEMETRY_KEY_FILE="../../${TGS_TELEMETRY_KEY_FILE}" \
 
 FROM mcr.microsoft.com/dotnet/aspnet:8.0-bookworm-slim
 
-#needed for byond
+#needed for byond, curl for healthchecks
 RUN apt-get update \
 	&& apt-get install -y \
 		gcc-multilib \
 		gdb \
+		curl \
 	&& rm -rf /var/lib/apt/lists/*
 
 EXPOSE 5000
@@ -78,4 +79,6 @@ COPY --from=build /repo/build/tgs.docker.sh tgs.sh
 
 VOLUME ["/config_data", "/tgs_logs", "/app/lib"]
 
+HEALTHCHECK --start-interval=60s CMD curl --fail http://localhost:5000/health || exit
+
 ENTRYPOINT ["./tgs.sh"]

build/TestCommon.props

@@ -3,7 +3,7 @@
 
   <ItemGroup>
     <!-- Usage: Code coverage collection -->
-    <PackageReference Include="coverlet.collector" Version="6.0.3">
+    <PackageReference Include="coverlet.collector" Version="6.0.4">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
@@ -13,14 +13,14 @@
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
     <!-- Usage: Hard to say what exactly this is for, but not including it removes the test icon and breaks vstest.console.exe for some reason -->
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.12.0" Condition="'$(TgsTestNoSdk)' != 'true'" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.13.0" Condition="'$(TgsTestNoSdk)' != 'true'" />
     <!-- Usage: Dependency mocking for tests -->
     <!-- Pinned: Be VERY careful about updating https://github.com/moq/moq/issues/1372 -->
     <PackageReference Include="Moq" Version="4.20.72" />
     <!-- Usage: MSTest execution -->
-    <PackageReference Include="MSTest.TestAdapter" Version="3.7.0" />
+    <PackageReference Include="MSTest.TestAdapter" Version="3.8.2" />
     <!-- Usage: MSTest asserts etc... -->
-    <PackageReference Include="MSTest.TestFramework" Version="3.7.0" />
+    <PackageReference Include="MSTest.TestFramework" Version="3.8.2" />
   </ItemGroup>
 
 </Project>

build/Version.props

@@ -3,12 +3,12 @@
   <!-- Integration tests will ensure they match across the board -->
   <Import Project="WebpanelVersion.props" />
   <PropertyGroup>
-    <TgsCoreVersion>6.12.2</TgsCoreVersion>
-    <TgsConfigVersion>5.4.0</TgsConfigVersion>
-    <TgsRestVersion>10.12.0</TgsRestVersion>
+    <TgsCoreVersion>6.14.1</TgsCoreVersion>
+    <TgsConfigVersion>5.5.0</TgsConfigVersion>
+    <TgsRestVersion>10.12.1</TgsRestVersion>
     <TgsGraphQLVersion>0.5.0</TgsGraphQLVersion>
     <TgsCommonLibraryVersion>7.0.0</TgsCommonLibraryVersion>
-    <TgsApiLibraryVersion>17.0.0</TgsApiLibraryVersion>
+    <TgsApiLibraryVersion>17.0.1</TgsApiLibraryVersion>
     <TgsClientVersion>20.0.0</TgsClientVersion>
     <TgsDmapiVersion>7.3.1</TgsDmapiVersion>
     <TgsInteropVersion>5.10.0</TgsInteropVersion>

build/package/nix/ServerConsole.sha256

@@ -1 +1 @@
-sha256-l4HG2/IYJO7YE12MkP8TGZtZt4/tMOSCP2lVzbKpvH0=
+sha256-6Wkdb1U2Xmw239zKJTbKcStdAV/G7aMa2oeCsGaSucA=

build/package/nix/tgstation-server.nix

@@ -1,6 +1,7 @@
 inputs@{
   config,
   lib,
+  systemdUtils,
   nixpkgs,
   pkgs,
   writeShellScriptBin,
@@ -53,7 +54,7 @@ in
       };
 
       username = lib.mkOption {
-        type = lib.types.str;
+        type = lib.types.nonEmptyStr;
         default = "tgstation-server";
         description = ''
           The name of the user used to execute tgstation-server.
@@ -61,7 +62,7 @@ in
       };
 
       groupname = lib.mkOption {
-        type = lib.types.str;
+        type = lib.types.nonEmptyStr;
         default = "tgstation-server";
         description = ''
           The name of group the user used to execute tgstation-server will belong to.
@@ -69,7 +70,7 @@ in
       };
 
       home-directory = lib.mkOption {
-        type = lib.types.str;
+        type = lib.types.nonEmptyStr;
         default = "/home/tgstation-server";
         description = ''
           The home directory of TGS. Should be persistent.
@@ -77,10 +78,10 @@ in
       };
 
       production-appsettings = lib.mkOption {
-        type = lib.types.lines;
+        type = lib.types.path;
         default = '''';
         description = ''
-          The contents of appsettings.Production.yml in the /etc/tgstation-server.d directory.
+          A formatted appsettings.Production.yml file.
         '';
       };
 
@@ -91,6 +92,22 @@ in
           Extra PATH entries to add to the TGS process
         '';
       };
+
+      environmentFile = lib.mkOption {
+        type = lib.types.nullOr lib.types.path;
+        default = null;
+        description = ''
+         Environment file as defined in {manpage}`systemd.exec(5)`
+        '';
+      };
+
+      wants = lib.mkOption {
+        type = lib.types.listOf systemdUtils.lib.unitNameType;
+        default = [];
+        description = ''
+          Start the specified units when this unit is started.
+        '';
+      };
     };
   };
 
@@ -111,7 +128,7 @@ in
         mode = "0644";
       };
       "tgstation-server.d/appsettings.Production.yml" = {
-        text = cfg.production-appsettings;
+        source = cfg.production-appsettings;
         group = cfg.groupname;
         mode = "0640";
       };
@@ -125,6 +142,7 @@ in
     systemd.services.tgstation-server = {
       description = "tgstation-server";
       serviceConfig = {
+        EnvironmentFile = lib.mkIf (cfg.environmentFile != null) cfg.environmentFile;
         User = cfg.username;
         Type = "notify-reload";
         NotifyAccess = "all";
@@ -138,7 +156,19 @@ in
         WatchdogSignal = "SIGTERM";
         LogsDirectory = "tgstation-server";
       };
+      reloadTriggers = [
+        (lib.mkIf (cfg.environmentFile != null) [ cfg.environmentFile ])
+        "/etc/tgstation.server.d/appsettings.Production.yml"
+      ];
+      restartIfChanged = false; # So that the TGS service doesn't just get restarted whenever it's updated, and boots players
       wantedBy = [ "multi-user.target" ];
+      after = [
+        "network.target"
+        "mysql.service"
+        "mariadb.service"
+        "postgresql.service"
+        "mssql-server.service"
+      ];
     };
   };
 }

build/stylecop.json

@@ -5,7 +5,8 @@
       "documentPrivateElements": true,
       "documentPrivateFields": true,
       "excludeFromPunctuationCheck": [
-        "remarks"
+        "remarks",
+        "example"
       ]
     },
     "indentation": {

src/Tgstation.Server.Api/ApiHeaders.cs

@@ -244,7 +244,7 @@ namespace Tgstation.Server.Api
 			else
 			{
 				var auth = authorization.First();
-				var splits = new List<string>(auth.Split(' '));
+				var splits = new List<string>(auth?.Split(' ') ?? Enumerable.Empty<string>());
 				var scheme = splits.First();
 				if (String.IsNullOrWhiteSpace(scheme))
 					AddError(HeaderErrorTypes.AuthorizationInvalid, "Missing authentication scheme!");

src/Tgstation.Server.Api/Models/ChatChannel.cs

@@ -14,6 +14,7 @@ namespace Tgstation.Server.Api.Models
 		/// For <see cref="ChatProvider.Irc"/>, it's the IRC channel name and optional password colon separated.
 		/// For <see cref="ChatProvider.Discord"/>, it's the stringified Discord channel snowflake.
 		/// </summary>
+		/// <example>124823852418</example>
 		[Required]
 		[StringLength(Limits.MaximumIndexableStringLength, MinimumLength = 1)]
 		public string? ChannelData { get; set; }

src/Tgstation.Server.Api/Models/EngineVersion.cs

@@ -26,12 +26,14 @@ namespace Tgstation.Server.Api.Models
 		/// <summary>
 		/// The <see cref="System.Version"/> of the engine. Currently only valid when <see cref="Engine"/> is <see cref="EngineType.Byond"/>.
 		/// </summary>
+		/// <example>516.1651.0</example>
 		[ResponseOptions]
 		public Version? Version { get; set; }
 
 		/// <summary>
 		/// The git commit SHA of the engine. Currently only valid when <see cref="Engine"/> is <see cref="EngineType.OpenDream"/>.
 		/// </summary>
+		/// <example>caa1e1f400c8b6a535e03cff28cf57f919e9378c</example>
 		[ResponseOptions]
 		[StringLength(Limits.MaximumCommitShaLength, MinimumLength = Limits.MaximumCommitShaLength)]
 		public string? SourceSHA { get; set; }

src/Tgstation.Server.Api/Models/EntityId.cs

@@ -8,6 +8,7 @@
 		/// <summary>
 		/// The ID of the entity.
 		/// </summary>
+		/// <example>1</example>
 		[RequestOptions(FieldPresence.Required)]
 		[RequestOptions(FieldPresence.Ignored, PutOnly = true)]
 		public virtual long? Id { get; set; }

src/Tgstation.Server.Api/Models/Internal/ChatBotSettings.cs

@@ -16,6 +16,7 @@ namespace Tgstation.Server.Api.Models.Internal
 		/// <summary>
 		/// The time interval in minutes the chat bot attempts to reconnect if <see cref="Enabled"/> and disconnected. Must not be zero.
 		/// </summary>
+		/// <example>60</example>
 		[Required]
 		[Range(1, UInt32.MaxValue)]
 		public uint? ReconnectionInterval { get; set; }
@@ -23,6 +24,7 @@ namespace Tgstation.Server.Api.Models.Internal
 		/// <summary>
 		/// The maximum number of <see cref="ChatChannel"/>s the <see cref="ChatBotSettings"/> may contain.
 		/// </summary>
+		/// <example>5</example>
 		[Required]
 		public ushort? ChannelLimit { get; set; }
 

src/Tgstation.Server.Api/Models/Internal/ChatChannelBase.cs

@@ -10,18 +10,21 @@ namespace Tgstation.Server.Api.Models.Internal
 		/// <summary>
 		/// If the <see cref="ChatChannel"/> is an admin channel.
 		/// </summary>
+		/// <example>false</example>
 		[Required]
 		public bool? IsAdminChannel { get; set; }
 
 		/// <summary>
 		/// If the <see cref="ChatChannel"/> is a watchdog channel.
 		/// </summary>
+		/// <example>false</example>
 		[Required]
 		public bool? IsWatchdogChannel { get; set; }
 
 		/// <summary>
 		/// If the <see cref="ChatChannel"/> is an updates channel.
 		/// </summary>
+		/// <example>false</example>
 		[Required]
 		public bool? IsUpdatesChannel { get; set; }
 
@@ -34,6 +37,7 @@ namespace Tgstation.Server.Api.Models.Internal
 		/// <summary>
 		/// A custom tag users can define to group channels together.
 		/// </summary>
+		/// <example>system-channel-only</example>
 		[ResponseOptions]
 		[StringLength(Limits.MaximumStringLength)]
 		public string? Tag { get; set; }

src/Tgstation.Server.Api/Models/Internal/CompileJob.cs

@@ -12,6 +12,7 @@ namespace Tgstation.Server.Api.Models.Internal
 		/// <summary>
 		/// The .dme file used for compilation.
 		/// </summary>
+		/// <example>tgstation.dme</example>
 		[Required]
 		public string? DmeName { get; set; }
 
@@ -36,6 +37,7 @@ namespace Tgstation.Server.Api.Models.Internal
 		/// <summary>
 		/// The DMAPI <see cref="Version"/>.
 		/// </summary>
+		/// <example>7.3.0</example>
 		[NotMapped]
 		[ResponseOptions]
 		public virtual Version? DMApiVersion { get; set; }

src/Tgstation.Server.Api/Models/Internal/DreamDaemonApiBase.cs

@@ -10,6 +10,7 @@ namespace Tgstation.Server.Api.Models.Internal
 		/// <summary>
 		/// An incrementing ID for representing current server execution.
 		/// </summary>
+		/// <example>1</example>
 		[ResponseOptions]
 		public long? SessionId { get; set; }
 
@@ -22,12 +23,14 @@ namespace Tgstation.Server.Api.Models.Internal
 		/// <summary>
 		/// The last known count of connected players. Requires <see cref="DreamDaemonLaunchParameters.HealthCheckSeconds"/> to not be 0 and a game server interop version >= 5.10.0 to populate.
 		/// </summary>
+		/// <example>30</example>
 		[ResponseOptions]
 		public uint? ClientCount { get; set; }
 
 		/// <summary>
 		/// If the server is undergoing a soft reset. This may be automatically set by changes to other fields.
 		/// </summary>
+		/// <example>false</example>
 		[ResponseOptions]
 		public bool? SoftRestart { get; set; }
 

src/Tgstation.Server.Api/Models/Internal/DreamDaemonLaunchParameters.cs

@@ -11,6 +11,7 @@ namespace Tgstation.Server.Api.Models.Internal
 		/// <summary>
 		/// If the BYOND web client can be used to connect to the game server. No-op for <see cref="EngineType.OpenDream"/>.
 		/// </summary>
+		/// <example>false</example>
 		[Required]
 		[ResponseOptions]
 		public bool? AllowWebClient { get; set; }
@@ -25,6 +26,7 @@ namespace Tgstation.Server.Api.Models.Internal
 		/// <summary>
 		/// The <see cref="DreamDaemonVisibility"/> level of DreamDaemon. No-op for <see cref="EngineType.OpenDream"/>.
 		/// </summary>
+		/// <example>2</example>
 		[Required]
 		[ResponseOptions]
 		[EnumDataType(typeof(DreamDaemonVisibility))]
@@ -33,6 +35,7 @@ namespace Tgstation.Server.Api.Models.Internal
 		/// <summary>
 		/// The <see cref="DreamDaemonSecurity"/> level of DreamDaemon. No-op for <see cref="EngineType.OpenDream"/>.
 		/// </summary>
+		/// <example>1</example>
 		[Required]
 		[ResponseOptions]
 		[EnumDataType(typeof(DreamDaemonSecurity))]
@@ -41,6 +44,7 @@ namespace Tgstation.Server.Api.Models.Internal
 		/// <summary>
 		/// The port DreamDaemon uses. This should be publically accessible.
 		/// </summary>
+		/// <example>1337</example>
 		[Required]
 		[ResponseOptions]
 		[Range(1, UInt16.MaxValue)]
@@ -49,6 +53,7 @@ namespace Tgstation.Server.Api.Models.Internal
 		/// <summary>
 		/// The port used by <see cref="EngineType.OpenDream"/> for its topic port.
 		/// </summary>
+		/// <example>2337</example>
 		[Required]
 		[ResponseOptions]
 		public ushort? OpenDreamTopicPort { get; set; }
@@ -56,6 +61,7 @@ namespace Tgstation.Server.Api.Models.Internal
 		/// <summary>
 		/// The DreamDaemon startup timeout in seconds.
 		/// </summary>
+		/// <example>5</example>
 		[Required]
 		[ResponseOptions]
 		[Range(1, UInt32.MaxValue)]
@@ -64,6 +70,7 @@ namespace Tgstation.Server.Api.Models.Internal
 		/// <summary>
 		/// The number of seconds between each watchdog health check. 0 disables.
 		/// </summary>
+		/// <example>5</example>
 		[Required]
 		[ResponseOptions]
 		public uint? HealthCheckSeconds { get; set; }
@@ -78,6 +85,7 @@ namespace Tgstation.Server.Api.Models.Internal
 		/// <summary>
 		/// The timeout for sending and receiving BYOND topics in milliseconds.
 		/// </summary>
+		/// <example>500</example>
 		[Required]
 		[ResponseOptions]
 		[Range(1, UInt32.MaxValue)]

src/Tgstation.Server.Api/Models/Internal/Job.cs

@@ -20,6 +20,7 @@ namespace Tgstation.Server.Api.Models.Internal
 		/// English description of the <see cref="Job"/>.
 		/// </summary>
 		/// <remarks>May not match the <see cref="System.ComponentModel.DescriptionAttribute"/> listed on the <see cref="Models.JobCode"/>.</remarks>
+		/// <example>Installing and configuring important objects.</example>
 		[Required]
 		public string? Description { get; set; }
 
@@ -50,6 +51,7 @@ namespace Tgstation.Server.Api.Models.Internal
 		/// <summary>
 		/// If the <see cref="Job"/> was cancelled.
 		/// </summary>
+		/// <example>false</example>
 		[Required]
 		public bool? Cancelled { get; set; }
 

src/Tgstation.Server.Api/Models/Internal/ServerInformationBase.cs

@@ -10,26 +10,31 @@ namespace Tgstation.Server.Api.Models.Internal
 		/// <summary>
 		/// Minimum length of database user passwords.
 		/// </summary>
+		/// <example>20</example>
 		public uint MinimumPasswordLength { get; set; }
 
 		/// <summary>
 		/// The maximum number of <see cref="Instance"/>s allowed.
 		/// </summary>
+		/// <example>100</example>
 		public uint InstanceLimit { get; set; }
 
 		/// <summary>
 		/// The maximum number of users allowed.
 		/// </summary>
+		/// <example>100</example>
 		public uint UserLimit { get; set; }
 
 		/// <summary>
 		/// The maximum number of user groups allowed.
 		/// </summary>
+		/// <example>50</example>
 		public uint UserGroupLimit { get; set; }
 
 		/// <summary>
 		/// Limits the locations instances may be created or attached from.
 		/// </summary>
+		/// <example>["/home/tgstation-server/my-server-1", "/home/tgstation-server/my-server-2"]</example>
 		[ResponseOptions]
 		public List<string>? ValidInstancePaths { get; set; }
 	}

src/Tgstation.Server.Api/Models/Internal/SwarmServer.cs

@@ -22,6 +22,7 @@ namespace Tgstation.Server.Api.Models.Internal
 		/// <summary>
 		/// The server's identifier.
 		/// </summary>
+		/// <example>myserver-us-east</example>
 		[Required]
 		public string? Identifier { get; set; }
 

src/Tgstation.Server.Api/Models/Internal/SwarmServerInformation.cs

@@ -10,6 +10,7 @@ namespace Tgstation.Server.Api.Models.Internal
 		/// <summary>
 		/// If the <see cref="SwarmServerResponse"/> is the controller.
 		/// </summary>
+		/// <example>false</example>
 		public bool Controller { get; set; }
 
 		/// <summary>

src/Tgstation.Server.Api/Models/Internal/TestMergeApiBase.cs

@@ -11,6 +11,7 @@ namespace Tgstation.Server.Api.Models.Internal
 		/// <summary>
 		/// The ID of the <see cref="TestMergeApiBase"/>.
 		/// </summary>
+		/// <example>1</example>
 		public long Id { get; set; }
 
 		/// <summary>

src/Tgstation.Server.Api/Models/Internal/TestMergeModelBase.cs

@@ -11,18 +11,21 @@ namespace Tgstation.Server.Api.Models.Internal
 		/// <summary>
 		/// The title of the test merge source.
 		/// </summary>
+		/// <example>Fixes and Breaks everything</example>
 		[Required]
 		public string? TitleAtMerge { get; set; }
 
 		/// <summary>
 		/// The body of the test merge source.
 		/// </summary>
+		/// <example># GitHub markdown\n\rI assume?</example>
 		[Required]
 		public string? BodyAtMerge { get; set; }
 
 		/// <summary>
 		/// The URL of the test merge source.
 		/// </summary>
+		/// <example>https://github.com/tgstation/tgstation/pull/31026</example>
 		[Required]
 #pragma warning disable CA1056 // Uri properties should not be strings
 		public string? Url { get; set; }
@@ -31,6 +34,7 @@ namespace Tgstation.Server.Api.Models.Internal
 		/// <summary>
 		/// The author of the test merge source.
 		/// </summary>
+		/// <example>MrStonedOne</example>
 		[Required]
 		public string? Author { get; set; }
 

src/Tgstation.Server.Api/Models/Internal/UpdateInformation.cs

@@ -10,6 +10,7 @@ namespace Tgstation.Server.Api.Models.Internal
 		/// <summary>
 		/// The latest available version of the Tgstation.Server.Host assembly from the upstream repository. If <see cref="Version.Major"/> is less than 4 the update cannot be applied due to API changes.
 		/// </summary>
+		/// <example>1.0.0</example>
 		public Version? LatestVersion { get; set; }
 
 		/// <summary>

src/Tgstation.Server.Api/Models/NamedEntity.cs

@@ -10,6 +10,7 @@ namespace Tgstation.Server.Api.Models
 		/// <summary>
 		/// The name of the entity represented by the <see cref="NamedEntity"/>.
 		/// </summary>
+		/// <example>MyThingyName</example>
 		[Required]
 		[RequestOptions(FieldPresence.Required, PutOnly = true)]
 		[StringLength(Limits.MaximumIndexableStringLength, MinimumLength = 1)]

src/Tgstation.Server.Api/Models/Request/ServerUpdateRequest.cs

@@ -10,6 +10,7 @@ namespace Tgstation.Server.Api.Models.Request
 		/// <summary>
 		/// Changes the version of tgstation-server to the given version from the upstream repository.
 		/// </summary>
+		/// <example>6.12.3</example>
 		[RequestOptions(FieldPresence.Required)]
 		public Version? NewVersion { get; set; }
 

src/Tgstation.Server.Api/Models/Response/AdministrationResponse.cs

@@ -12,6 +12,7 @@ namespace Tgstation.Server.Api.Models.Response
 		/// <summary>
 		/// The GitHub repository the server is built to receive updates from.
 		/// </summary>
+		/// <example>https://github.com/tgstation/tgstation</example>
 		public Uri? TrackedRepositoryUrl { get; set; }
 	}
 }

src/Tgstation.Server.Api/Models/Response/DreamDaemonResponse.cs

@@ -24,6 +24,7 @@ namespace Tgstation.Server.Api.Models.Response
 		/// <summary>
 		/// The current <see cref="WatchdogStatus"/>.
 		/// </summary>
+		/// <example>2</example>
 		[EnumDataType(typeof(WatchdogStatus))]
 		[ResponseOptions]
 		public WatchdogStatus? Status { get; set; }
@@ -31,6 +32,7 @@ namespace Tgstation.Server.Api.Models.Response
 		/// <summary>
 		/// The current <see cref="DreamDaemonSecurity"/>. May be upgraded. due to requirements of <see cref="ActiveCompileJob"/>.
 		/// </summary>
+		/// <example>1</example>
 		[EnumDataType(typeof(DreamDaemonSecurity))]
 		[ResponseOptions]
 		public DreamDaemonSecurity? CurrentSecurity { get; set; }
@@ -38,6 +40,7 @@ namespace Tgstation.Server.Api.Models.Response
 		/// <summary>
 		/// The current <see cref="DreamDaemonVisibility"/>.
 		/// </summary>
+		/// <example>2</example>
 		[EnumDataType(typeof(DreamDaemonVisibility))]
 		[ResponseOptions]
 		public DreamDaemonVisibility? CurrentVisibility { get; set; }
@@ -45,24 +48,28 @@ namespace Tgstation.Server.Api.Models.Response
 		/// <summary>
 		/// The port the running <see cref="DreamDaemonResponse"/> instance is set to.
 		/// </summary>
+		/// <example>1337</example>
 		[ResponseOptions]
 		public ushort? CurrentPort { get; set; }
 
 		/// <summary>
 		/// The <see cref="EngineType.OpenDream"/> topic port the running <see cref="DreamDaemonResponse"/> instance is set to.
 		/// </summary>
+		/// <example>3000</example>
 		[ResponseOptions]
 		public ushort? CurrentTopicPort { get; set; }
 
 		/// <summary>
 		/// The webclient status the running <see cref="DreamDaemonResponse"/> instance is set to.
 		/// </summary>
+		/// <example>false</example>
 		[ResponseOptions]
 		public bool? CurrentAllowWebclient { get; set; }
 
 		/// <summary>
 		/// The amount of RAM in use by the game server in bytes.
 		/// </summary>
+		/// <example>1073741824</example>
 		[ResponseOptions]
 		public long? ImmediateMemoryUsage { get; set; }
 	}

src/Tgstation.Server.Api/Models/Response/ErrorMessageResponse.cs

@@ -13,16 +13,19 @@ namespace Tgstation.Server.Api.Models.Response
 		/// <summary>
 		/// The version of the API the server is using.
 		/// </summary>
+		/// <example>9.3.0</example>
 		public Version? ServerApiVersion { get; set; }
 
 		/// <summary>
-		/// A human readable description of the error.
+		/// A human-readable description of the error.
 		/// </summary>
+		/// <example>Oopsie woopsie, we did a fucky wucky!</example>
 		public string? Message { get; set; }
 
 		/// <summary>
 		/// Additional data associated with the error message.
 		/// </summary>
+		/// <example>Error at thing.app.dependency.class.function in line 32</example>
 		[ResponseOptions]
 		public string? AdditionalData { get; set; }
 

src/Tgstation.Server.Api/Models/Response/JobResponse.cs

@@ -8,6 +8,7 @@
 		/// <summary>
 		/// The <see cref="EntityId.Id"/> of the <see cref="Instance"/>.
 		/// </summary>
+		/// <example>1</example>
 		public long? InstanceId { get; set; }
 
 		/// <summary>
@@ -24,12 +25,14 @@
 		/// <summary>
 		/// Optional progress between 0 and 100 inclusive.
 		/// </summary>
+		/// <example>25</example>
 		[ResponseOptions]
 		public int? Progress { get; set; }
 
 		/// <summary>
-		/// Optional description of the job's current .
+		/// Optional description of the job's current progress.
 		/// </summary>
+		/// <example>Doing something important</example>
 		[ResponseOptions]
 		public string? Stage { get; set; }
 	}

src/Tgstation.Server.Api/Models/Response/LogFileResponse.cs

@@ -10,6 +10,7 @@ namespace Tgstation.Server.Api.Models.Response
 		/// <summary>
 		/// The name of the log file.
 		/// </summary>
+		/// <example>tgs-20250118.log</example>
 		public string? Name { get; set; }
 
 		/// <summary>

src/Tgstation.Server.Api/Models/Response/PaginatedResponse.cs

@@ -18,16 +18,19 @@ namespace Tgstation.Server.Api.Models.Response
 		/// <summary>
 		/// The total number of pages in the query.
 		/// </summary>
+		/// <example>5</example>
 		public int TotalPages { get; set; }
 
 		/// <summary>
 		/// The current size of pages in the query.
 		/// </summary>
+		/// <example>20</example>
 		public int PageSize { get; set; }
 
 		/// <summary>
 		/// The total items across all pages.
 		/// </summary>
+		/// <example>100</example>
 		public int TotalItems { get; set; }
 	}
 }

src/Tgstation.Server.Api/Models/Response/ServerInformationResponse.cs

@@ -11,16 +11,19 @@ namespace Tgstation.Server.Api.Models.Response
 		/// <summary>
 		/// The version of the host.
 		/// </summary>
+		/// <example>6.12.3</example>
 		public Version? Version { get; set; }
 
 		/// <summary>
 		/// The <see cref="Api"/> version of the host.
 		/// </summary>
+		/// <example>10.12.0</example>
 		public Version? ApiVersion { get; set; }
 
 		/// <summary>
 		/// The DMAPI interop version the server uses.
 		/// </summary>
+		/// <example>7.3.0</example>
 		public Version? DMApiVersion { get; set; }
 
 		/// <summary>
@@ -36,6 +39,7 @@ namespace Tgstation.Server.Api.Models.Response
 		/// <summary>
 		/// If there is a server update in progress.
 		/// </summary>
+		/// <example>false</example>
 		public bool UpdateInProgress { get; set; }
 
 		/// <summary>

src/Tgstation.Server.Api/Models/Response/ServerUpdateResponse.cs

@@ -12,6 +12,7 @@ namespace Tgstation.Server.Api.Models.Response
 		/// <summary>
 		/// The version of tgstation-server pending update.
 		/// </summary>
+		/// <example>6.12.3</example>
 		public Version NewVersion { get; }
 
 		/// <summary>

src/Tgstation.Server.Api/Models/Response/TokenResponse.cs

@@ -10,6 +10,7 @@ namespace Tgstation.Server.Api.Models.Response
 		/// <summary>
 		/// The value of the JWT.
 		/// </summary>
+		/// <example>eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxIiwibmJmIjoxNzM3MDkzNDgwLCJleHAiOjE3MzcwOTQzODAsImlhdCI6MTczNzA5MzQ4MCwiaXNzIjoiVGdzdGF0aW9uLlNlcnZlci5Ib3N0IiwiYXVkIjoiVGdzdGF0aW9uLlNlcnZlci5BcGkifQ.v64KX34_YOpH-HCbwqlx1p8u-MNbb4L6a9qEyXhcNcU</example>
 		public string? Bearer { get; set; }
 
 		/// <summary>

src/Tgstation.Server.Api/Models/TestMergeParameters.cs

@@ -10,18 +10,21 @@ namespace Tgstation.Server.Api.Models
 		/// <summary>
 		/// The number of the test merge source.
 		/// </summary>
+		/// <example>31026</example>
 		public int Number { get; set; }
 
 		/// <summary>
 		/// The sha of the test merge revision to merge. If not specified, the latest commit from the source will be used.
 		/// </summary>
+		/// <example>caa1e1f400c8b6a535e03cff28cf57f919e9378c</example>
 		[Required]
-		[StringLength(40)]
+		[StringLength(Limits.MaximumCommitShaLength, MinimumLength = Limits.MaximumCommitShaLength)]
 		public virtual string? TargetCommitSha { get; set; }
 
 		/// <summary>
 		/// Optional comment about the test.
 		/// </summary>
+		/// <example>this will fix everything -Admin</example>
 		[ResponseOptions]
 		[StringLength(Limits.MaximumStringLength)]
 		public string? Comment { get; set; }

src/Tgstation.Server.Api/Models/UserName.cs

@@ -6,6 +6,7 @@
 	public class UserName : NamedEntity
 	{
 		/// <inheritdoc />
+		/// <example>Admin</example>
 		[RequestOptions(FieldPresence.Optional)]
 		public override string? Name
 		{

src/Tgstation.Server.Api/Tgstation.Server.Api.csproj

@@ -26,12 +26,11 @@
 
   <ItemGroup>
     <!-- Usage: HTTP constants reference -->
-    <PackageReference Include="Microsoft.AspNetCore.Http.Extensions" Version="2.2.0" />
+    <PackageReference Include="Microsoft.AspNetCore.Http.Extensions" Version="2.3.0" />
     <!-- Usage: Decoding the 'nbf' property of JWTs -->
-    <PackageReference Include="Microsoft.IdentityModel.JsonWebTokens" Version="8.3.0" />
+    <PackageReference Include="Microsoft.IdentityModel.JsonWebTokens" Version="8.6.1" />
     <!-- Usage: Data model annotating -->
     <PackageReference Include="System.ComponentModel.Annotations" Version="5.0.0" />
-    <PackageReference Include="System.Text.Encodings.Web" Version="9.0.0" />
   </ItemGroup>
 
   <ItemGroup>

src/Tgstation.Server.Client.GraphQL/Tgstation.Server.Client.GraphQL.csproj

@@ -9,8 +9,8 @@
 
   <ItemGroup>
     <!-- GraphQL connector and code generator -->
-    <PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="9.0.0" />
-    <PackageReference Include="StrawberryShake.Server" Version="14.3.0" />
+    <PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="9.0.2" />
+    <PackageReference Include="StrawberryShake.Server" Version="15.0.3" />
   </ItemGroup>
 
   <ItemGroup>

src/Tgstation.Server.Client/Tgstation.Server.Client.csproj

@@ -11,9 +11,9 @@
 
   <ItemGroup>
     <!-- Usage: Connecting to SignalR hubs in API -->
-    <PackageReference Include="Microsoft.AspNetCore.SignalR.Client" Version="9.0.0" />
+    <PackageReference Include="Microsoft.AspNetCore.SignalR.Client" Version="9.0.2" />
     <!-- Usage: Using target JSON serializer for API -->
-    <PackageReference Include="Microsoft.AspNetCore.SignalR.Protocols.NewtonsoftJson" Version="9.0.0" />
+    <PackageReference Include="Microsoft.AspNetCore.SignalR.Protocols.NewtonsoftJson" Version="9.0.2" />
   </ItemGroup>
 
   <ItemGroup>

src/Tgstation.Server.Host.Console/Tgstation.Server.Host.Console.csproj

@@ -13,9 +13,9 @@
 
   <ItemGroup>
     <!-- Usage: Identifying if we're running under SystemD -->
-    <PackageReference Include="Microsoft.Extensions.Hosting.Systemd" Version="9.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Hosting.Systemd" Version="9.0.2" />
     <!-- Usage: Console logging plugin -->
-    <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="9.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="9.0.2" />
   </ItemGroup>
 
   <ItemGroup>

src/Tgstation.Server.Host.Service/Tgstation.Server.Host.Service.csproj

@@ -21,21 +21,21 @@
     <!-- Usage: Command line argument support -->
     <PackageReference Include="McMaster.Extensions.CommandLineUtils" Version="4.1.1" />
     <!-- Usage: Identifies when we are running in the context of the Windows SCM -->
-    <PackageReference Include="Microsoft.Extensions.Hosting.WindowsServices" Version="9.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Hosting.WindowsServices" Version="9.0.2" />
     <!-- Usage: Windows event log logging plugin -->
-    <PackageReference Include="Microsoft.Extensions.Logging.EventLog" Version="9.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Logging.EventLog" Version="9.0.2" />
     <!-- Usage: Console logging plugin -->
-    <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="9.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="9.0.2" />
     <!-- Usage: Updated transitive dependency of Core.System.ServiceProcess -->
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
     <!-- Usage: Updated transitive dependency of Core.System.ServiceProcess -->
-    <PackageReference Include="System.Drawing.Common" Version="9.0.0" />
+    <PackageReference Include="System.Drawing.Common" Version="9.0.2" />
     <!-- Usage: Updated transitive dependency, unable to tell what of -->
     <PackageReference Include="System.Private.Uri" Version="4.3.2" />
     <!-- Usage: OS identification -->
     <PackageReference Include="System.Runtime.InteropServices" Version="4.3.0" />
     <!-- Usage: Windows Service Manager intergration -->
-    <PackageReference Include="System.ServiceProcess.ServiceController" Version="9.0.0" />
+    <PackageReference Include="System.ServiceProcess.ServiceController" Version="9.0.2" />
   </ItemGroup>
 
   <ItemGroup>

src/Tgstation.Server.Host.Utils.GitLab.GraphQL/.graphqlrc.json

@@ -0,0 +1,21 @@
+{
+  "schema": "schema.graphql",
+  "documents": "**/*.graphql",
+  "extensions": {
+    "strawberryShake": {
+      "name": "GraphQLClient",
+      "url": "../../artifacts/gitlab-api.graphql",
+      "namespace": "Tgstation.Server.Host.Utils.GitLab.GraphQL",
+      "records": {
+        "inputs": false,
+        "entities": false
+      },
+      "transportProfiles": [
+        {
+          "default": "Http",
+          "subscription": "Http"
+        }
+      ]
+    }
+  }
+}

src/Tgstation.Server.Host.Utils.GitLab.GraphQL/GQL/Mutations/CreateNote.graphql

@@ -0,0 +1,12 @@
+mutation CreateNote($id: NoteableID!, $body: String!) {
+  createNote(input: { noteableId: $id, body: $body }) {
+    note {
+      id
+      body
+      discussion {
+        id
+      }
+    }
+    errors
+  }
+}

src/Tgstation.Server.Host.Utils.GitLab.GraphQL/GQL/Mutations/ModifyNote.graphql

@@ -0,0 +1,9 @@
+mutation ModifyNote($id: NoteID!, $body: String!) {
+  updateNote(input: { id: $id, body: $body }) {
+    note {
+      id
+      body
+    }
+    errors
+  }
+}

src/Tgstation.Server.Host.Utils.GitLab.GraphQL/GQL/Queries/GetCurrentUser.graphql

@@ -0,0 +1,6 @@
+query GetCurrentUser {
+    currentUser
+    {
+        username
+    }
+}

src/Tgstation.Server.Host.Utils.GitLab.GraphQL/GQL/Queries/GetMergeRequest.graphql

@@ -0,0 +1,14 @@
+query GetMergeRequest($project: ID!, $number: String!) {
+    project(fullPath: $project) {
+        mergeRequest(iid: $number) {
+            author { username }
+            description
+            title
+            diffHeadSha
+            mergeCommitSha
+            webUrl
+            iid
+            id
+        }
+    }
+}

src/Tgstation.Server.Host.Utils.GitLab.GraphQL/GQL/Queries/GetMergeRequestNotes.graphql

@@ -0,0 +1,15 @@
+query GetMergeRequestNotes($project: ID!, $number: String!) {
+    project(fullPath: $project) {
+        mergeRequest(iid: $number) {
+            iid
+            id
+            notes {
+                nodes {
+                  author { username }
+                  body
+                  id
+                }
+            }
+        }
+    }
+}

src/Tgstation.Server.Host.Utils.GitLab.GraphQL/GQL/Queries/GetMergeRequests.graphql

@@ -0,0 +1,14 @@
+query GetMergeRequests($project: ID!, $numbers: [String!]!) {
+    project(fullPath: $project) {
+        mergeRequests(iids: $numbers) {
+            nodes {
+                state
+                diffHeadSha
+                mergeCommitSha
+                closedAt
+                iid
+                id
+            }
+        }
+    }
+}

src/Tgstation.Server.Host.Utils.GitLab.GraphQL/GraphQLGitLabClient.cs

@@ -0,0 +1,33 @@
+using System;
+using System.Threading.Tasks;
+
+using Microsoft.Extensions.DependencyInjection;
+
+namespace Tgstation.Server.Host.Utils.GitLab.GraphQL
+{
+	/// <inheritdoc />
+	sealed class GraphQLGitLabClient : IGraphQLGitLabClient
+	{
+		/// <inheritdoc />
+		public IGraphQLClient GraphQL { get; }
+
+		/// <summary>
+		/// The <see cref="ServiceProvider"/> containing the <see cref="GraphQL"/> client.
+		/// </summary>
+		readonly ServiceProvider serviceProvider;
+
+		/// <summary>
+		/// Initializes a new instance of the <see cref="GraphQLGitLabClient"/> class.
+		/// </summary>
+		/// <param name="serviceProvider">The value of <see cref="serviceProvider"/>.</param>
+		public GraphQLGitLabClient(ServiceProvider serviceProvider)
+		{
+			this.serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider));
+			GraphQL = serviceProvider.GetService<IGraphQLClient>() ?? throw new ArgumentException($"Expected an {nameof(IGraphQLClient)} service in the provider!", nameof(serviceProvider));
+		}
+
+		/// <inheritdoc />
+		public ValueTask DisposeAsync()
+			=> serviceProvider.DisposeAsync();
+	}
+}

src/Tgstation.Server.Host.Utils.GitLab.GraphQL/GraphQLGitLabClientFactory.cs

@@ -0,0 +1,49 @@
+using System;
+using System.Net.Http.Headers;
+using System.Threading.Tasks;
+
+using Microsoft.Extensions.DependencyInjection;
+
+namespace Tgstation.Server.Host.Utils.GitLab.GraphQL
+{
+	/// <summary>
+	/// Factory for creating <see cref="IGraphQLGitLabClient"/>s.
+	/// </summary>
+	public sealed class GraphQLGitLabClientFactory
+	{
+		/// <summary>
+		/// Sets up a <see cref="IGraphQLGitLabClient"/>.
+		/// </summary>
+		/// <param name="bearerToken">The token to use for authentication, if any.</param>
+		/// <returns>A <see cref="ValueTask{TResult}"/> resulting in a new <see cref="IGraphQLGitLabClient"/>.</returns>
+		public static async ValueTask<IGraphQLGitLabClient> CreateClient(string? bearerToken = null)
+		{
+			var serviceCollection = new ServiceCollection();
+
+			var clientBuilder = serviceCollection
+				.AddGraphQLClient();
+			var graphQLEndpoint = new Uri("https://gitlab.com/api/graphql");
+
+			clientBuilder.ConfigureHttpClient(
+				client =>
+				{
+					client.BaseAddress = new Uri("https://gitlab.com/api/graphql");
+					if (bearerToken != null)
+					{
+						client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", bearerToken);
+					}
+				});
+
+			var serviceProvider = serviceCollection.BuildServiceProvider();
+			try
+			{
+				return new GraphQLGitLabClient(serviceProvider);
+			}
+			catch
+			{
+				await serviceProvider.DisposeAsync();
+				throw;
+			}
+		}
+	}
+}

src/Tgstation.Server.Host.Utils.GitLab.GraphQL/IGraphQLGitLabClient.cs

@@ -0,0 +1,15 @@
+using System;
+
+namespace Tgstation.Server.Host.Utils.GitLab.GraphQL
+{
+	/// <summary>
+	/// Wrapper for using a GitLab <see cref="IGraphQLClient"/>.
+	/// </summary>
+	public interface IGraphQLGitLabClient : IAsyncDisposable
+	{
+		/// <summary>
+		/// Gets the underlying <see cref="IGraphQLClient"/>.
+		/// </summary>
+		IGraphQLClient GraphQL { get; }
+	}
+}

src/Tgstation.Server.Host.Utils.GitLab.GraphQL/Tgstation.Server.Host.Utils.GitLab.GraphQL.csproj

@@ -0,0 +1,44 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <Import Project="../../build/SrcCommon.props" />
+
+  <PropertyGroup>
+    <TargetFrameworks>$(TgsFrameworkVersion)</TargetFrameworks>
+    <Version>$(TgsCoreVersion)</Version>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <Target Name="InstallApollo" Inputs="package.json;yarn.lock" Outputs="node_modules/.bin/apollo">
+    <Message Text="Installing Apollo..." Importance="high" />
+    <Exec Command="yarn install --immutable --ignore-engines" />
+  </Target>
+
+  <Target Name="GetApi" DependsOnTargets="InstallApollo" Inputs="node_modules/.bin/apollo" Outputs="../../artifacts/gitlab-api.graphql">
+    <Message Text="Fetching GitLab GraphQL API schema..." Importance="high" />
+    <MakeDir Directories="../../artifacts" />
+    <Exec Command="node_modules/.bin/apollo client:download-schema --endpoint=https://gitlab.com/api/graphql ../../artifacts/gitlab-api.graphql" />
+  </Target>
+
+  <!-- https://github.com/ChilliCream/graphql-platform/blob/c0c8df525ca0f47bf3b3b409a8b22cbe37f7a9c0/src/StrawberryShake/MetaPackages/Common/MSBuild/StrawberryShake.targets#L20 -->
+  <Target Name="ImportGraphQLApiSchema" DependsOnTargets="GetApi" BeforeTargets="_GraphQLCodeGenerationRoot" Inputs="../../artifacts/gitlab-api.graphql" Outputs="schema.graphql">
+    <Copy SkipUnchangedFiles="true" SourceFiles="../../artifacts/gitlab-api.graphql" DestinationFiles="schema.graphql" />
+    <WriteLinesToFile File="schema.graphql" Lines="$([System.IO.File]::ReadAllText('schema.graphql').Replace('\', ''))" Overwrite="true" Encoding="UTF-8" />
+  </Target>
+
+  <Target Name="FixWarningsInGeneratedSchema" AfterTargets="GenerateGraphQLCode">
+    <PropertyGroup>
+      <InputFile>$(IntermediateOutputPath)berry/GraphQLClient.Client.cs</InputFile>
+      <OutputFile>$(IntermediateOutputPath)berry/GraphQLClient.Client.cs</OutputFile>
+    </PropertyGroup>
+    <WriteLinesToFile File="$(OutputFile)" Lines="$([System.IO.File]::ReadAllText($(InputFile)).Replace('/ &lt;auto-generated/&gt;','/ &lt;auto-generated /&gt;%0d%0a#pragma warning disable'))" Overwrite="true" Encoding="Unicode" />
+  </Target>
+
+  <ItemGroup>
+    <PackageReference Include="StrawberryShake.Server" Version="15.0.3" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Folder Include="GQL\Queries\" />
+    <Folder Include="GQL\Mutations\" />
+  </ItemGroup>
+
+</Project>

src/Tgstation.Server.Host.Utils.GitLab.GraphQL/package.json

@@ -0,0 +1,6 @@
+{
+  "devDependencies": {
+    "apollo": "^2.34.0"
+  },
+  "packageManager": "yarn@1.22.22+sha1.ac34549e6aa8e7ead463a7407e1c7390f61a6610"
+}

src/Tgstation.Server.Host.Watchdog/Tgstation.Server.Host.Watchdog.csproj

@@ -10,7 +10,7 @@
 
   <ItemGroup>
     <!-- Usage: Logging abstractions -->
-    <PackageReference Include="Microsoft.Extensions.Logging" Version="9.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Logging" Version="9.0.2" />
     <!-- Usage: POSIX support for signals -->
     <PackageReference Include="Mono.Posix.NETStandard" Version="1.0.0" />
   </ItemGroup>

src/Tgstation.Server.Host.Watchdog/Watchdog.cs

@@ -177,6 +177,10 @@ namespace Tgstation.Server.Host.Watchdog
 
 				var watchdogVersion = executingAssembly.GetName().Version?.Semver().ToString();
 
+				var serializerOptions = new JsonSerializerOptions
+				{
+					WriteIndented = true,
+				};
 				while (!cancellationToken.IsCancellationRequested)
 				{
 					if (!File.Exists(assemblyPath))
@@ -214,10 +218,7 @@ namespace Tgstation.Server.Host.Watchdog
 								bootstrapperSettingsFile,
 								JsonSerializer.Serialize(
 									bootstrapSettings,
-									new JsonSerializerOptions
-									{
-										WriteIndented = true,
-									}),
+									serializerOptions),
 								cancellationToken);
 						}
 					}
@@ -258,6 +259,7 @@ namespace Tgstation.Server.Host.Watchdog
 							process.StartInfo.UseShellExecute = false; // runs in the same console
 
 							var killedHostProcess = false;
+							var createdShutdownFile = false;
 							try
 							{
 								Task? processTask = null;
@@ -279,6 +281,7 @@ namespace Tgstation.Server.Host.Watchdog
 									{
 										logger.LogInformation("Cancellation requested! Writing shutdown lock file...");
 										File.WriteAllBytes(updateDirectory, Array.Empty<byte>());
+										createdShutdownFile = true;
 									}
 									else
 										logger.LogWarning("Cancellation requested while update directory exists!");
@@ -330,15 +333,16 @@ namespace Tgstation.Server.Host.Watchdog
 									logger.LogWarning(ex2, "Error killing host process!");
 								}
 
-								try
-								{
-									if (File.Exists(updateDirectory))
-										File.Delete(updateDirectory);
-								}
-								catch (Exception ex2)
-								{
-									logger.LogWarning(ex2, "Error deleting comms file!");
-								}
+								if (createdShutdownFile)
+									try
+									{
+										if (File.Exists(updateDirectory))
+											File.Delete(updateDirectory);
+									}
+									catch (Exception ex2)
+									{
+										logger.LogWarning(ex2, "Error deleting comms file!");
+									}
 
 								logger.LogInformation("Host exited!");
 							}

src/Tgstation.Server.Host/Components/Chat/ChatManager.cs

@@ -377,6 +377,7 @@ namespace Tgstation.Server.Host.Components.Chat
 		/// <inheritdoc />
 		public Func<string?, string, Action<bool>> QueueDeploymentMessage(
 			Models.RevisionInformation revisionInformation,
+			Models.RevisionInformation? previousRevisionInformation,
 			EngineVersion engineVersion,
 			DateTimeOffset? estimatedCompletionTime,
 			string? gitHubOwner,
@@ -407,6 +408,7 @@ namespace Tgstation.Server.Host.Components.Chat
 						{
 							var callback = await provider.SendUpdateMessage(
 								revisionInformation,
+								previousRevisionInformation,
 								engineVersion,
 								estimatedCompletionTime,
 								gitHubOwner,

src/Tgstation.Server.Host/Components/Chat/IChatManager.cs

@@ -61,6 +61,7 @@ namespace Tgstation.Server.Host.Components.Chat
 		/// Send the message for a deployment to configured deployment channels.
 		/// </summary>
 		/// <param name="revisionInformation">The <see cref="RevisionInformation"/> of the deployment.</param>
+		/// <param name="previousRevisionInformation">The optional <see cref="RevisionInformation"/> of the previous deployment.</param>
 		/// <param name="engineVersion">The <see cref="Api.Models.EngineVersion"/> of the deployment.</param>
 		/// <param name="estimatedCompletionTime">The optional <see cref="DateTimeOffset"/> the deployment is expected to be completed at.</param>
 		/// <param name="gitHubOwner">The repository GitHub owner, if any.</param>
@@ -69,6 +70,7 @@ namespace Tgstation.Server.Host.Components.Chat
 		/// <returns>A <see cref="Func{T1, T2, TResult}"/> to call to update the message at the deployment's conclusion. Parameters: Error message if any, DreamMaker output if any. Returns an <see cref="Action"/> to call to mark the deployment as active/inactive. Parameter: If the deployment is being activated or inactivated.</returns>
 		Func<string?, string, Action<bool>> QueueDeploymentMessage(
 			Models.RevisionInformation revisionInformation,
+			Models.RevisionInformation? previousRevisionInformation,
 			Api.Models.EngineVersion engineVersion,
 			DateTimeOffset? estimatedCompletionTime,
 			string? gitHubOwner,

src/Tgstation.Server.Host/Components/Chat/Providers/DiscordProvider.cs

@@ -303,6 +303,7 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
 		/// <inheritdoc />
 		public override async ValueTask<Func<string?, string, ValueTask<Func<bool, ValueTask>>>> SendUpdateMessage(
 			Models.RevisionInformation revisionInformation,
+			Models.RevisionInformation? previousRevisionInformation,
 			EngineVersion engineVersion,
 			DateTimeOffset? estimatedCompletionTime,
 			string? gitHubOwner,
@@ -316,7 +317,7 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
 
 			localCommitPushed |= revisionInformation.CommitSha == revisionInformation.OriginCommitSha;
 
-			var fields = BuildUpdateEmbedFields(revisionInformation, engineVersion, gitHubOwner, gitHubRepo, localCommitPushed);
+			var fields = BuildUpdateEmbedFields(revisionInformation, previousRevisionInformation, engineVersion, gitHubOwner, gitHubRepo, localCommitPushed);
 			Optional<IEmbedAuthor> author = new EmbedAuthor(assemblyInformationProvider.VersionPrefix)
 			{
 				Url = "https://github.com/tgstation/tgstation-server",
@@ -900,6 +901,7 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
 		/// Create a <see cref="List{T}"/> of <see cref="IEmbedField"/>s for a discord update embed.
 		/// </summary>
 		/// <param name="revisionInformation">The <see cref="RevisionInformation"/> of the deployment.</param>
+		/// <param name="previousRevisionInformation">The optional <see cref="RevisionInformation"/> of the previous deployment.</param>
 		/// <param name="engineVersion">The <see cref="EngineVersion"/> of the deployment.</param>
 		/// <param name="gitHubOwner">The repository GitHub owner, if any.</param>
 		/// <param name="gitHubRepo">The repository GitHub name, if any.</param>
@@ -907,6 +909,7 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
 		/// <returns>A new <see cref="List{T}"/> of <see cref="IEmbedField"/>s to use.</returns>
 		List<IEmbedField> BuildUpdateEmbedFields(
 			Models.RevisionInformation revisionInformation,
+			Models.RevisionInformation? previousRevisionInformation,
 			EngineVersion engineVersion,
 			string? gitHubOwner,
 			string? gitHubRepo,
@@ -936,6 +939,31 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
 			if (gitHubOwner == null || gitHubRepo == null)
 				return fields;
 
+			var previousTestMerges = (IEnumerable<RevInfoTestMerge>?)previousRevisionInformation?.ActiveTestMerges ?? Enumerable.Empty<RevInfoTestMerge>();
+			var currentTestMerges = (IEnumerable<RevInfoTestMerge>?)revisionInformation.ActiveTestMerges ?? Enumerable.Empty<RevInfoTestMerge>();
+
+			// determine what TMs were changed and how
+			var addedTestMerges = currentTestMerges
+				.Select(x => x.TestMerge)
+				.Where(x => !previousTestMerges
+					.Any(y => y.TestMerge.Number == x.Number))
+				.ToList();
+			var removedTestMerges = previousTestMerges
+				.Select(x => x.TestMerge)
+				.Where(x => !currentTestMerges
+					.Any(y => y.TestMerge.Number == x.Number))
+				.ToList();
+			var updatedTestMerges = currentTestMerges
+				.Select(x => x.TestMerge)
+				.Where(x => previousTestMerges
+					.Any(y => y.TestMerge.Number == x.Number && y.TestMerge.TargetCommitSha != x.TargetCommitSha))
+				.ToList();
+			var unchangedTestMerges = currentTestMerges
+				.Select(x => x.TestMerge)
+				.Where(x => previousTestMerges
+					.Any(y => y.TestMerge.Number == x.Number && y.TestMerge.TargetCommitSha == x.TargetCommitSha))
+				.ToList();
+
 			fields.Add(
 				new EmbedField(
 					"Local Commit",
@@ -952,13 +980,33 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
 						: revisionOriginSha[..7],
 					true));
 
-			fields.AddRange((revisionInformation.ActiveTestMerges ?? Enumerable.Empty<RevInfoTestMerge>())
-				.Select(x => x.TestMerge)
+			fields.AddRange(addedTestMerges
+				.Select(x => new EmbedField(
+					$"#{x.Number} (Added)",
+					$"[{x.TitleAtMerge}]({x.Url}) by _[@{x.Author}](https://github.com/{x.Author})_{Environment.NewLine}Commit: [{x.TargetCommitSha![..7]}](https://github.com/{gitHubOwner}/{gitHubRepo}/commit/{x.TargetCommitSha}){(String.IsNullOrWhiteSpace(x.Comment) ? String.Empty : $"{Environment.NewLine}_**{x.Comment}**_")}",
+					false)));
+
+			fields.AddRange(updatedTestMerges
+				.Select(x => new EmbedField(
+					$"#{x.Number} (Updated)",
+					$"[{x.TitleAtMerge}]({x.Url}) by _[@{x.Author}](https://github.com/{x.Author})_{Environment.NewLine}Commit: [{x.TargetCommitSha![..7]}](https://github.com/{gitHubOwner}/{gitHubRepo}/commit/{x.TargetCommitSha}){(String.IsNullOrWhiteSpace(x.Comment) ? String.Empty : $"{Environment.NewLine}_**{x.Comment}**_")}",
+					false)));
+
+			fields.AddRange(unchangedTestMerges
 				.Select(x => new EmbedField(
 					$"#{x.Number}",
 					$"[{x.TitleAtMerge}]({x.Url}) by _[@{x.Author}](https://github.com/{x.Author})_{Environment.NewLine}Commit: [{x.TargetCommitSha![..7]}](https://github.com/{gitHubOwner}/{gitHubRepo}/commit/{x.TargetCommitSha}){(String.IsNullOrWhiteSpace(x.Comment) ? String.Empty : $"{Environment.NewLine}_**{x.Comment}**_")}",
 					false)));
 
+			if (removedTestMerges.Count != 0)
+				fields.Add(
+				new EmbedField(
+					"Removed:",
+					String.Join(
+						Environment.NewLine,
+						removedTestMerges
+							.Select(x => $"- #{x.Number} [{x.TitleAtMerge}]({x.Url}) by _[@{x.Author}](https://github.com/{x.Author})_"))));
+
 			return fields;
 		}
 

src/Tgstation.Server.Host/Components/Chat/Providers/IProvider.cs

@@ -84,6 +84,7 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
 		/// Send the message for a deployment.
 		/// </summary>
 		/// <param name="revisionInformation">The <see cref="RevisionInformation"/> of the deployment.</param>
+		/// <param name="previousRevisionInformation">The <see cref="RevisionInformation"/> of the previous deployment if any.</param>
 		/// <param name="engineVersion">The <see cref="Api.Models.EngineVersion"/> of the deployment.</param>
 		/// <param name="estimatedCompletionTime">The optional <see cref="DateTimeOffset"/> the deployment is expected to be completed at.</param>
 		/// <param name="gitHubOwner">The repository GitHub owner, if any.</param>
@@ -94,6 +95,7 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
 		/// <returns>A <see cref="ValueTask{TResult}"/> resulting in a <see cref="Func{T1, T2, TResult}"/> to call to update the message at the deployment's conclusion. Parameters: Error message if any, DreamMaker output if any. Returns another callback which should be called to mark the deployment as active.</returns>
 		ValueTask<Func<string?, string, ValueTask<Func<bool, ValueTask>>>> SendUpdateMessage(
 			Models.RevisionInformation revisionInformation,
+			Models.RevisionInformation? previousRevisionInformation,
 			Api.Models.EngineVersion engineVersion,
 			DateTimeOffset? estimatedCompletionTime,
 			string? gitHubOwner,

src/Tgstation.Server.Host/Components/Chat/Providers/IrcProvider.cs

@@ -17,6 +17,7 @@ using Tgstation.Server.Host.Configuration;
 using Tgstation.Server.Host.Extensions;
 using Tgstation.Server.Host.IO;
 using Tgstation.Server.Host.Jobs;
+using Tgstation.Server.Host.Models;
 using Tgstation.Server.Host.System;
 using Tgstation.Server.Host.Utils;
 
@@ -217,6 +218,7 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
 		/// <inheritdoc />
 		public override async ValueTask<Func<string?, string, ValueTask<Func<bool, ValueTask>>>> SendUpdateMessage(
 			Models.RevisionInformation revisionInformation,
+			Models.RevisionInformation? previousRevisionInformation,
 			EngineVersion engineVersion,
 			DateTimeOffset? estimatedCompletionTime,
 			string? gitHubOwner,
@@ -227,8 +229,9 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
 		{
 			ArgumentNullException.ThrowIfNull(revisionInformation);
 			ArgumentNullException.ThrowIfNull(engineVersion);
-			ArgumentNullException.ThrowIfNull(gitHubOwner);
-			ArgumentNullException.ThrowIfNull(gitHubRepo);
+
+			var previousTestMerges = (IEnumerable<RevInfoTestMerge>?)previousRevisionInformation?.ActiveTestMerges ?? Enumerable.Empty<RevInfoTestMerge>();
+			var currentTestMerges = (IEnumerable<RevInfoTestMerge>?)revisionInformation.ActiveTestMerges ?? Enumerable.Empty<RevInfoTestMerge>();
 
 			var commitInsert = revisionInformation.CommitSha![..7];
 			string remoteCommitInsert;
@@ -240,21 +243,35 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
 			else
 				remoteCommitInsert = String.Format(CultureInfo.InvariantCulture, ". Remote commit: ^{0}", revisionInformation.OriginCommitSha![..7]);
 
-			var testmergeInsert = (revisionInformation.ActiveTestMerges?.Count ?? 0) == 0
+			var testmergeInsert = !currentTestMerges.Any()
 				? String.Empty
 				: String.Format(
 					CultureInfo.InvariantCulture,
 					" (Test Merges: {0})",
 					String.Join(
 						", ",
-						revisionInformation
-							.ActiveTestMerges!
+						currentTestMerges
 							.Select(x => x.TestMerge)
 							.Select(x =>
 							{
-								var result = String.Format(CultureInfo.InvariantCulture, "#{0} at {1}", x.Number, x.TargetCommitSha![..7]);
-								if (x.Comment != null)
-									result += String.Format(CultureInfo.InvariantCulture, " ({0})", x.Comment);
+								var status = string.Empty;
+								if (!previousTestMerges.Any(y => y.TestMerge.Number == x.Number))
+									status = "Added";
+								else if (previousTestMerges.Any(y => y.TestMerge.Number == x.Number && y.TestMerge.TargetCommitSha != x.TargetCommitSha))
+									status = "Updated";
+
+								var result = $"#{x.Number} at {x.TargetCommitSha![..7]}";
+
+								if (!string.IsNullOrEmpty(x.Comment))
+								{
+									if (!string.IsNullOrEmpty(status))
+										result += $" ({status} - {x.Comment})";
+									else
+										result += $" ({x.Comment})";
+								}
+								else if (!string.IsNullOrEmpty(status))
+									result += $" ({status})";
+
 								return result;
 							})));
 

src/Tgstation.Server.Host/Components/Chat/Providers/Provider.cs

@@ -199,6 +199,7 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
 		/// <inheritdoc />
 		public abstract ValueTask<Func<string?, string, ValueTask<Func<bool, ValueTask>>>> SendUpdateMessage(
 			RevisionInformation revisionInformation,
+			RevisionInformation? previousRevisionInformation,
 			Api.Models.EngineVersion engineVersion,
 			DateTimeOffset? estimatedCompletionTime,
 			string? gitHubOwner,

src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs

@@ -8,6 +8,8 @@ using System.Threading.Tasks;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Logging;
 
+using Prometheus;
+
 using Tgstation.Server.Api.Models;
 using Tgstation.Server.Api.Models.Internal;
 using Tgstation.Server.Common.Extensions;
@@ -105,6 +107,21 @@ namespace Tgstation.Server.Host.Components.Deployment
 		/// </summary>
 		readonly Api.Models.Instance metadata;
 
+		/// <summary>
+		/// The number of attempted deployments.
+		/// </summary>
+		readonly Counter attemptedDeployments;
+
+		/// <summary>
+		/// The number of successful deployments.
+		/// </summary>
+		readonly Counter successfulDeployments;
+
+		/// <summary>
+		/// The number of failed deployments.
+		/// </summary>
+		readonly Counter failedDeployments;
+
 		/// <summary>
 		/// <see langword="lock"/> <see cref="object"/> for <see cref="deploying"/>.
 		/// </summary>
@@ -149,6 +166,7 @@ namespace Tgstation.Server.Host.Components.Deployment
 		/// <param name="repositoryManager">The value of <see cref="repositoryManager"/>.</param>
 		/// <param name="remoteDeploymentManagerFactory">The value of <see cref="remoteDeploymentManagerFactory"/>.</param>
 		/// <param name="asyncDelayer">The value of <see cref="asyncDelayer"/>.</param>
+		/// <param name="metricFactory">The <see cref="IMetricFactory"/> to use.</param>
 		/// <param name="logger">The value of <see cref="logger"/>.</param>
 		/// <param name="sessionConfiguration">The value of <see cref="sessionConfiguration"/>.</param>
 		/// <param name="metadata">The value of <see cref="metadata"/>.</param>
@@ -164,6 +182,7 @@ namespace Tgstation.Server.Host.Components.Deployment
 			IRepositoryManager repositoryManager,
 			IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory,
 			IAsyncDelayer asyncDelayer,
+			IMetricFactory metricFactory,
 			ILogger<DreamMaker> logger,
 			SessionConfiguration sessionConfiguration,
 			Api.Models.Instance metadata)
@@ -177,12 +196,17 @@ namespace Tgstation.Server.Host.Components.Deployment
 			this.processExecutor = processExecutor ?? throw new ArgumentNullException(nameof(processExecutor));
 			this.compileJobConsumer = compileJobConsumer ?? throw new ArgumentNullException(nameof(compileJobConsumer));
 			this.repositoryManager = repositoryManager ?? throw new ArgumentNullException(nameof(repositoryManager));
-			this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer));
 			this.remoteDeploymentManagerFactory = remoteDeploymentManagerFactory ?? throw new ArgumentNullException(nameof(remoteDeploymentManagerFactory));
+			this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer));
+			ArgumentNullException.ThrowIfNull(metricFactory);
 			this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
 			this.sessionConfiguration = sessionConfiguration ?? throw new ArgumentNullException(nameof(sessionConfiguration));
 			this.metadata = metadata ?? throw new ArgumentNullException(nameof(metadata));
 
+			successfulDeployments = metricFactory.CreateCounter("tgs_successful_deployments", "The number of deployments that have completed successfully");
+			failedDeployments = metricFactory.CreateCounter("tgs_failed_deployments", "The number of deployments that have failed");
+			attemptedDeployments = metricFactory.CreateCounter("tgs_total_deployments", "The number of deployments that have been attempted");
+
 			deploymentLock = new object();
 		}
 
@@ -205,9 +229,12 @@ namespace Tgstation.Server.Host.Components.Deployment
 				deploying = true;
 			}
 
+			attemptedDeployments.Inc();
+
 			currentChatCallback = null;
 			currentDreamMakerOutput = null;
 			Models.CompileJob? compileJob = null;
+			bool success = false;
 			try
 			{
 				string? repoOwner = null;
@@ -309,13 +336,17 @@ namespace Tgstation.Server.Host.Components.Deployment
 						}
 					});
 
-				var likelyPushedTestMergeCommit =
-					repositorySettings!.PushTestMergeCommits!.Value
-					&& repositorySettings.AccessToken != null
-					&& repositorySettings.AccessUser != null;
+				Models.CompileJob? oldCompileJob;
 				using (repo)
+				{
+					var likelyPushedTestMergeCommit =
+						repositorySettings!.PushTestMergeCommits!.Value
+						&& repositorySettings.AccessToken != null
+						&& repositorySettings.AccessUser != null;
+					oldCompileJob = await compileJobConsumer.LatestCompileJob();
 					compileJob = await Compile(
 						job,
+						oldCompileJob,
 						revInfo!,
 						dreamMakerSettings!,
 						ddSettings!,
@@ -325,8 +356,8 @@ namespace Tgstation.Server.Host.Components.Deployment
 						averageSpan,
 						likelyPushedTestMergeCommit,
 						cancellationToken);
+				}
 
-				var activeCompileJob = await compileJobConsumer.LatestCompileJob();
 				try
 				{
 					await databaseContextFactory.UseContext(
@@ -351,6 +382,7 @@ namespace Tgstation.Server.Host.Components.Deployment
 							{
 								var chatNotificationAction = currentChatCallback!(null, compileJob.Output!);
 								await compileJobConsumer.LoadCompileJob(compileJob, chatNotificationAction, cancellationToken);
+								success = true;
 							}
 							catch
 							{
@@ -374,7 +406,7 @@ namespace Tgstation.Server.Host.Components.Deployment
 
 				var commentsTask = remoteDeploymentManager!.PostDeploymentComments(
 					compileJob,
-					activeCompileJob?.RevisionInformation,
+					oldCompileJob?.RevisionInformation,
 					repositorySettings,
 					repoOwner,
 					repoName,
@@ -406,6 +438,10 @@ namespace Tgstation.Server.Host.Components.Deployment
 			finally
 			{
 				deploying = false;
+				if (success)
+					successfulDeployments.Inc();
+				else
+					failedDeployments.Inc();
 			}
 		}
 #pragma warning restore CA1506
@@ -447,6 +483,7 @@ namespace Tgstation.Server.Host.Components.Deployment
 		/// Run the compile implementation.
 		/// </summary>
 		/// <param name="job">The currently running <see cref="Job"/>.</param>
+		/// <param name="oldCompileJob">The optional <see cref="CompileJob"/> of the previous deployment.</param>
 		/// <param name="revisionInformation">The <see cref="RevisionInformation"/>.</param>
 		/// <param name="dreamMakerSettings">The <see cref="Api.Models.Internal.DreamMakerSettings"/>.</param>
 		/// <param name="launchParameters">The <see cref="DreamDaemonLaunchParameters"/>.</param>
@@ -459,6 +496,7 @@ namespace Tgstation.Server.Host.Components.Deployment
 		/// <returns>A <see cref="ValueTask{TResult}"/> resulting in the completed <see cref="CompileJob"/>.</returns>
 		async ValueTask<Models.CompileJob> Compile(
 			Models.Job job,
+			Models.CompileJob? oldCompileJob,
 			Models.RevisionInformation revisionInformation,
 			Api.Models.Internal.DreamMakerSettings dreamMakerSettings,
 			DreamDaemonLaunchParameters launchParameters,
@@ -480,6 +518,7 @@ namespace Tgstation.Server.Host.Components.Deployment
 				using var engineLock = await engineManager.UseExecutables(null, null, cancellationToken);
 				currentChatCallback = chatManager.QueueDeploymentMessage(
 					revisionInformation,
+					oldCompileJob?.RevisionInformation,
 					engineLock.Version,
 					DateTimeOffset.UtcNow + estimatedDuration,
 					repository.RemoteRepositoryOwner,

src/Tgstation.Server.Host/Components/Deployment/Remote/BaseRemoteDeploymentManager.cs

@@ -18,6 +18,11 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote
 	/// </summary>
 	abstract class BaseRemoteDeploymentManager : IRemoteDeploymentManager
 	{
+		/// <summary>
+		/// The header comment that begins every deployment message comment/note.
+		/// </summary>
+		public const string DeploymentMsgHeaderStart = "<!-- tgs_test_merge_comment -->";
+
 		/// <summary>
 		/// The <see cref="Api.Models.Instance"/> for the <see cref="BaseRemoteDeploymentManager"/>.
 		/// </summary>
@@ -66,36 +71,28 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote
 			if (repositorySettings.AccessToken == null)
 				return;
 
-			var deployedRevisionInformation = compileJob.RevisionInformation;
-			if ((previousRevisionInformation != null && previousRevisionInformation.CommitSha == deployedRevisionInformation.CommitSha)
+			var revisionInformation = compileJob.RevisionInformation;
+			if ((previousRevisionInformation != null && previousRevisionInformation.CommitSha == revisionInformation.CommitSha)
 				|| !repositorySettings.PostTestMergeComment!.Value)
 				return;
 
-			previousRevisionInformation ??= new RevisionInformation();
-			previousRevisionInformation.ActiveTestMerges ??= new List<RevInfoTestMerge>();
+			var previousTestMerges = (IEnumerable<RevInfoTestMerge>?)previousRevisionInformation?.ActiveTestMerges ?? Enumerable.Empty<RevInfoTestMerge>();
+			var currentTestMerges = (IEnumerable<RevInfoTestMerge>?)revisionInformation.ActiveTestMerges ?? Enumerable.Empty<RevInfoTestMerge>();
 
-			deployedRevisionInformation.ActiveTestMerges ??= new List<RevInfoTestMerge>();
-
-			// added prs
-			var addedTestMerges = deployedRevisionInformation
-				.ActiveTestMerges
+			// determine what TMs were changed and how
+			var addedTestMerges = currentTestMerges
 				.Select(x => x.TestMerge)
-				.Where(x => !previousRevisionInformation
-					.ActiveTestMerges
+				.Where(x => !previousTestMerges
 					.Any(y => y.TestMerge.Number == x.Number))
 				.ToList();
-			var removedTestMerges = previousRevisionInformation
-				.ActiveTestMerges
+			var removedTestMerges = previousTestMerges
 				.Select(x => x.TestMerge)
-				.Where(x => !deployedRevisionInformation
-					.ActiveTestMerges
+				.Where(x => !currentTestMerges
 					.Any(y => y.TestMerge.Number == x.Number))
 				.ToList();
-			var updatedTestMerges = deployedRevisionInformation
-				.ActiveTestMerges
+			var updatedTestMerges = currentTestMerges
 				.Select(x => x.TestMerge)
-				.Where(x => previousRevisionInformation
-					.ActiveTestMerges
+				.Where(x => previousTestMerges
 					.Any(y => y.TestMerge.Number == x.Number))
 				.ToList();
 
@@ -133,7 +130,12 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote
 					repositorySettings,
 					repoOwner,
 					repoName,
-					"#### Test Merge Removed",
+					FormatTestMergeRemoval(
+						repositorySettings,
+						compileJob,
+						removedTestMerge,
+						repoOwner,
+						repoName),
 					removedTestMerge.Number,
 					cancellationToken);
 				tasks.Add(removeCommentTask);
@@ -253,6 +255,22 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote
 			string remoteRepositoryName,
 			bool updated);
 
+		/// <summary>
+		/// Formats a comment for a given <paramref name="testMerge"/> removal.
+		/// </summary>
+		/// <param name="repositorySettings">The <see cref="RepositorySettings"/> to use.</param>
+		/// <param name="compileJob">The test merge's <see cref="CompileJob"/>.</param>
+		/// <param name="testMerge">The <see cref="TestMerge"/>.</param>
+		/// <param name="remoteRepositoryOwner">The <see cref="Api.Models.Internal.IGitRemoteInformation.RemoteRepositoryOwner"/>.</param>
+		/// <param name="remoteRepositoryName">The <see cref="Api.Models.Internal.IGitRemoteInformation.RemoteRepositoryName"/>.</param>
+		/// <returns>A formatted <see cref="string"/> for posting a informative comment about the <paramref name="testMerge"/> removal.</returns>
+		protected abstract string FormatTestMergeRemoval(
+			RepositorySettings repositorySettings,
+			CompileJob compileJob,
+			TestMerge testMerge,
+			string remoteRepositoryOwner,
+			string remoteRepositoryName);
+
 		/// <summary>
 		/// Create a comment of a given <paramref name="testMergeNumber"/>'s source.
 		/// </summary>

src/Tgstation.Server.Host/Components/Deployment/Remote/GitHubRemoteDeploymentManager.cs

@@ -282,7 +282,17 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote
 
 			try
 			{
-				await gitHubService.CommentOnIssue(remoteRepositoryOwner, remoteRepositoryName, comment, testMergeNumber, cancellationToken);
+				string header = String.Format(CultureInfo.InvariantCulture, "{1}{0}## Test merge deployment history:{0}{0}", Environment.NewLine, DeploymentMsgHeaderStart);
+
+				var existingComment = await gitHubService.GetExistingCommentOnIssue(remoteRepositoryOwner, remoteRepositoryName, DeploymentMsgHeaderStart, testMergeNumber, cancellationToken);
+				if (existingComment != null)
+				{
+					await gitHubService.AppendCommentOnIssue(remoteRepositoryOwner, remoteRepositoryName, comment, existingComment, cancellationToken);
+				}
+				else
+				{
+					await gitHubService.CommentOnIssue(remoteRepositoryOwner, remoteRepositoryName, header + comment, testMergeNumber, cancellationToken);
+				}
 			}
 			catch (Exception ex) when (ex is not OperationCanceledException)
 			{
@@ -299,30 +309,43 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote
 			string remoteRepositoryName,
 			bool updated) => String.Format(
 			CultureInfo.InvariantCulture,
-			"#### Test Merge {4}{0}{0}<details><summary>Details</summary>{0}{0}##### Server Instance{0}{5}{1}{0}{0}##### Revision{0}Origin: {6}{0}Pull Request: {2}{0}Server: {7}{3}{8}{0}</details>",
-			Environment.NewLine,
+			"<details><summary>Test Merge {4} @ {8}:</summary>{0}{0}##### Server Instance{0}{5}{1}{0}{0}##### Revision{0}Origin: {6}{0}Pull Request: {2}{0}Server: {7}{3}{0}</details>{0}",
+			Environment.NewLine, // 0
 			repositorySettings.ShowTestMergeCommitters!.Value
 				? String.Format(
 					CultureInfo.InvariantCulture,
 					"{0}{0}##### Merged By{0}{1}",
 					Environment.NewLine,
 					testMerge.MergedBy!.Name)
-				: String.Empty,
-			testMerge.TargetCommitSha,
-			testMerge.Comment != null
-				? String.Format(
+				: String.Empty, // 1
+			testMerge.TargetCommitSha, // 2
+			String.IsNullOrEmpty(testMerge.Comment)
+				? String.Empty
+				: String.Format(
 					CultureInfo.InvariantCulture,
 					"{0}{0}##### Comment{0}{1}",
 					Environment.NewLine,
-					testMerge.Comment)
-				: String.Empty,
-			updated ? "Updated" : "Deployed",
-			Metadata.Name,
-			compileJob.RevisionInformation.OriginCommitSha,
-			compileJob.RevisionInformation.CommitSha,
+					testMerge.Comment), // 3
+			updated ? "Updated" : "Deployed", // 4
 			compileJob.GitHubDeploymentId.HasValue
-				? $"{Environment.NewLine}[GitHub Deployments](https://github.com/{remoteRepositoryOwner}/{remoteRepositoryName}/deployments/activity_log?environment=TGS%3A+{Metadata.Name!.Replace(" ", "+", StringComparison.Ordinal)})"
-				: String.Empty);
+				? $"{Environment.NewLine}[{Metadata.Name}](https://github.com/{remoteRepositoryOwner}/{remoteRepositoryName}/deployments/activity_log?environment=TGS%3A+{Metadata.Name!.Replace(" ", "+", StringComparison.Ordinal)})"
+				: Metadata.Name, // 5
+			compileJob.RevisionInformation.OriginCommitSha, // 6
+			compileJob.RevisionInformation.CommitSha, // 7
+			compileJob.Job.StartedAt); // 8
+
+		/// <inheritdoc />
+		protected override string FormatTestMergeRemoval(
+			RepositorySettings repositorySettings,
+			CompileJob compileJob,
+			TestMerge testMerge,
+			string remoteRepositoryOwner,
+			string remoteRepositoryName) => String.Format(
+			CultureInfo.InvariantCulture,
+			"<details><summary>Test Merge Removed @ {2}:</summary>{0}{0}##### Server Instance{0}{1}{0}</details>{0}",
+			Environment.NewLine, // 0
+			Metadata.Name, // 1
+			compileJob.Job.StartedAt); // 2
 
 		/// <summary>
 		/// Update the deployment for a given <paramref name="compileJob"/>.

src/Tgstation.Server.Host/Components/Deployment/Remote/GitLabRemoteDeploymentManager.cs

@@ -6,13 +6,13 @@ using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 
-using GitLabApiClient;
-using GitLabApiClient.Models.MergeRequests.Responses;
-using GitLabApiClient.Models.Notes.Requests;
 using Microsoft.Extensions.Logging;
 
+using StrawberryShake;
+
 using Tgstation.Server.Host.Components.Repository;
 using Tgstation.Server.Host.Models;
+using Tgstation.Server.Host.Utils.GitLab.GraphQL;
 
 namespace Tgstation.Server.Host.Components.Deployment.Remote
 {
@@ -52,49 +52,78 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote
 				return Array.Empty<TestMerge>();
 			}
 
-			var client = repositorySettings.AccessToken != null
-				? new GitLabClient(GitLabRemoteFeatures.GitLabUrl, repositorySettings.AccessToken)
-				: new GitLabClient(GitLabRemoteFeatures.GitLabUrl);
+			var newList = revisionInformation.ActiveTestMerges.Select(x => x.TestMerge).ToList();
 
-			var tasks = revisionInformation
-				.ActiveTestMerges
-				.Select(x => client
-					.MergeRequests
-					.GetAsync(
-						$"{repository.RemoteRepositoryOwner}/{repository.RemoteRepositoryName}",
-						x.TestMerge.Number)
-					.WaitAsync(cancellationToken));
+			await using var client = await GraphQLGitLabClientFactory.CreateClient(repositorySettings.AccessToken);
+			IOperationResult<IGetMergeRequestsResult> operationResult;
 			try
 			{
-				await Task.WhenAll(tasks);
+				operationResult = await client.GraphQL.GetMergeRequests.ExecuteAsync(
+					$"{repository.RemoteRepositoryOwner}/{repository.RemoteRepositoryName}",
+					revisionInformation.ActiveTestMerges.Select(revInfoTestMerge => revInfoTestMerge.TestMerge.Number.ToString(CultureInfo.InvariantCulture)).ToList(),
+					cancellationToken);
+
+				operationResult.EnsureNoErrors();
 			}
 			catch (Exception ex) when (ex is not OperationCanceledException)
 			{
 				Logger.LogWarning(ex, "Merge requests update check failed!");
+				return newList;
 			}
 
-			var newList = revisionInformation.ActiveTestMerges.Select(x => x.TestMerge).ToList();
-
-			MergeRequest? lastMerged = null;
-			async ValueTask CheckRemoveMR(Task<MergeRequest> task)
+			var data = operationResult.Data?.Project?.MergeRequests?.Nodes;
+			if (data == null)
 			{
-				var mergeRequest = await task;
+				Logger.LogWarning("GitLab MergeRequests check returned null!");
+				return newList;
+			}
+
+			async ValueTask CheckRemoveMR(IGetMergeRequests_Project_MergeRequests_Nodes? mergeRequest)
+			{
+				if (mergeRequest == null)
+				{
+					Logger.LogWarning("GitLab MergeRequest node was null!");
+					return;
+				}
+
 				if (mergeRequest.State != MergeRequestState.Merged)
 					return;
 
-				// We don't just assume, actually check the repo contains the merge commit.
-				if (await repository.CommittishIsParent(mergeRequest.MergeCommitSha, cancellationToken))
+				var mergeCommitSha = mergeRequest.MergeCommitSha;
+				if (mergeCommitSha == null)
 				{
-					if (lastMerged == null || lastMerged.ClosedAt < mergeRequest.ClosedAt)
-						lastMerged = mergeRequest;
+					Logger.LogWarning("MergeRequest #{id} had no MergeCommitSha!", mergeRequest.Iid);
+					return;
+				}
+
+				var closedAtStr = mergeRequest.ClosedAt;
+				if (closedAtStr == null)
+				{
+					Logger.LogWarning("MergeRequest #{id} had no ClosedAt!", mergeRequest.Iid);
+					return;
+				}
+
+				if (!DateTimeOffset.TryParseExact(closedAtStr, "O", CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind, out DateTimeOffset closedAt))
+				{
+					Logger.LogWarning("MergeRequest #{id} had invalid ClosedAt: {closedAt}", mergeRequest.Iid, closedAtStr);
+					return;
+				}
+
+				if (!Int64.TryParse(mergeRequest.Iid, out long number))
+				{
+					Logger.LogWarning("MergeRequest #{id} is non-numeric!", mergeRequest.Iid);
+					return;
+				}
+
+				// We don't just assume, actually check the repo contains the merge commit.
+				if (await repository.CommittishIsParent(mergeCommitSha, cancellationToken))
 					newList.Remove(
 						newList.First(
-							potential => potential.Number == mergeRequest.Id));
-				}
+							potential => potential.Number == number));
 			}
 
-			foreach (var prTask in tasks)
-				await CheckRemoveMR(prTask);
+			foreach (var mergeRequest in data)
+				await CheckRemoveMR(mergeRequest);
 
 			return newList;
 		}
@@ -131,23 +160,69 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote
 			int testMergeNumber,
 			CancellationToken cancellationToken)
 		{
-			var client = repositorySettings.AccessToken != null
-				? new GitLabClient(GitLabRemoteFeatures.GitLabUrl, repositorySettings.AccessToken)
-				: new GitLabClient(GitLabRemoteFeatures.GitLabUrl);
-
+			await using var client = await GraphQLGitLabClientFactory.CreateClient(repositorySettings.AccessToken);
 			try
 			{
-				await client
-					.MergeRequests
-					.CreateNoteAsync(
-						$"{remoteRepositoryOwner}/{remoteRepositoryName}",
-						testMergeNumber,
-						new CreateMergeRequestNoteRequest(comment))
-					.WaitAsync(cancellationToken);
+				string header = String.Format(CultureInfo.InvariantCulture, "{1}{0}## Test merge deployment history:{0}{0}", Environment.NewLine, DeploymentMsgHeaderStart);
+
+				// Try to find an existing note
+				var notesQueryResult = await client.GraphQL.GetMergeRequestNotes.ExecuteAsync(
+					$"{remoteRepositoryOwner}/{remoteRepositoryName}",
+					testMergeNumber.ToString(CultureInfo.InvariantCulture),
+					cancellationToken);
+
+				notesQueryResult.EnsureNoErrors();
+
+				var mergeRequest = notesQueryResult.Data?.Project?.MergeRequest;
+				if (mergeRequest == null)
+				{
+					Logger.LogWarning("GitLab GetMergeRequestNotes mergeRequest returned null!");
+					return;
+				}
+
+				var comments = mergeRequest.Notes?.Nodes;
+				IGetMergeRequestNotes_Project_MergeRequest_Notes_Nodes? existingComment = null;
+				if (comments != null)
+				{
+					for (int i = comments.Count - 1; i > -1; i--)
+					{
+						var currentComment = comments[i];
+						if (currentComment?.Author?.Username == repositorySettings.AccessUser && (currentComment?.Body?.StartsWith(DeploymentMsgHeaderStart) ?? false))
+						{
+							if (currentComment.Body.Length > 987856)
+							{ // Limit should be 999,999 so we'll leave a 12,143 buffer
+								break;
+							}
+
+							existingComment = currentComment;
+							break;
+						}
+					}
+				}
+
+				// Either amend or create the note
+				if (existingComment != null)
+				{
+					var noteModificationResult = await client.GraphQL.ModifyNote.ExecuteAsync(
+						existingComment.Id,
+						existingComment.Body + comment,
+						cancellationToken);
+
+					notesQueryResult.EnsureNoErrors();
+				}
+				else
+				{
+					var noteCreationResult = await client.GraphQL.CreateNote.ExecuteAsync(
+						mergeRequest.Id,
+						header + comment,
+						cancellationToken);
+
+					noteCreationResult.EnsureNoErrors();
+				}
 			}
 			catch (Exception ex) when (ex is not OperationCanceledException)
 			{
-				Logger.LogWarning(ex, "Error posting GitHub comment!");
+				Logger.LogWarning(ex, "Error posting GitLab comment!");
 			}
 		}
 
@@ -160,26 +235,40 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote
 			string remoteRepositoryName,
 			bool updated) => String.Format(
 			CultureInfo.InvariantCulture,
-			"#### Test Merge {4}{0}{0}##### Server Instance{0}{5}{1}{0}{0}##### Revision{0}Origin: {6}{0}Merge Request: {2}{0}Server: {7}{3}",
-			Environment.NewLine,
+			"<details><summary>Test Merge {4} @ {8}</summary>{0}{0}##### Server Instance{0}{5}{1}{0}{0}##### Revision{0}Origin: {6}{0}Merge Request: {2}{0}Server: {7}{3}</details>{0}",
+			Environment.NewLine, // 0
 			repositorySettings.ShowTestMergeCommitters!.Value
 				? String.Format(
 					CultureInfo.InvariantCulture,
 					"{0}{0}##### Merged By{0}{1}",
 					Environment.NewLine,
 					testMerge.MergedBy!.Name)
-				: String.Empty,
-			testMerge.TargetCommitSha,
-			testMerge.Comment != null
-				? String.Format(
+				: String.Empty, // 1
+			testMerge.TargetCommitSha, // 2
+			String.IsNullOrEmpty(testMerge.Comment)
+				? String.Empty
+				: String.Format(
 					CultureInfo.InvariantCulture,
 					"{0}{0}##### Comment{0}{1}",
 					Environment.NewLine,
-					testMerge.Comment)
-				: String.Empty,
-			updated ? "Updated" : "Deployed",
-			Metadata.Name,
-			compileJob.RevisionInformation.OriginCommitSha,
-			compileJob.RevisionInformation.CommitSha);
+					testMerge.Comment), // 3
+			updated ? "Updated" : "Deployed", // 4
+			Metadata.Name, // 5
+			compileJob.RevisionInformation.OriginCommitSha, // 6
+			compileJob.RevisionInformation.CommitSha, // 7
+			compileJob.Job.StartedAt); // 8
+
+		/// <inheritdoc />
+		protected override string FormatTestMergeRemoval(
+			RepositorySettings repositorySettings,
+			CompileJob compileJob,
+			TestMerge testMerge,
+			string remoteRepositoryOwner,
+			string remoteRepositoryName) => String.Format(
+			CultureInfo.InvariantCulture,
+			"<details><summary>Test Merge Removed @ {2}:</summary>{0}{0}##### Server Instance{0}{1}{0}</details>{0}",
+			Environment.NewLine, // 0
+			Metadata.Name, // 1
+			compileJob.Job.StartedAt); // 2
 	}
 }

src/Tgstation.Server.Host/Components/Deployment/Remote/NoOpRemoteDeploymentManager.cs

@@ -65,6 +65,15 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote
 			bool updated)
 			=> String.Empty;
 
+		/// <inheritdoc />
+		protected override string FormatTestMergeRemoval(
+			Models.RepositorySettings repositorySettings,
+			Models.CompileJob compileJob,
+			TestMerge testMerge,
+			string remoteRepositoryOwner,
+			string remoteRepositoryName)
+			=> String.Empty;
+
 		/// <inheritdoc />
 		protected override ValueTask MarkInactiveImpl(Models.CompileJob compileJob, CancellationToken cancellationToken) => ValueTask.CompletedTask;
 

src/Tgstation.Server.Host/Components/Events/EventType.cs

@@ -173,5 +173,11 @@
 		/// </summary>
 		[EventScript("EngineInstallComplete")]
 		EngineInstallComplete,
+
+		/// <summary>
+		/// Before game server process is created. No parameters.
+		/// </summary>
+		[EventScript("DreamDaemonPreLaunch")]
+		DreamDaemonPreLaunch,
 	}
 }

src/Tgstation.Server.Host/Components/InstanceFactory.cs

@@ -1,10 +1,14 @@
 using System;
+using System.Collections.Generic;
+using System.Globalization;
 using System.Threading;
 using System.Threading.Tasks;
 
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 
+using Prometheus;
+
 using Tgstation.Server.Host.Components.Chat;
 using Tgstation.Server.Host.Components.Chat.Commands;
 using Tgstation.Server.Host.Components.Deployment;
@@ -140,6 +144,11 @@ namespace Tgstation.Server.Host.Components
 		/// </summary>
 		readonly IDotnetDumpService dotnetDumpService;
 
+		/// <summary>
+		/// The <see cref="IMetricFactory"/> for the <see cref="InstanceFactory"/>.
+		/// </summary>
+		readonly IMetricFactory metricFactory;
+
 		/// <summary>
 		/// The <see cref="GeneralConfiguration"/> for the <see cref="InstanceFactory"/>.
 		/// </summary>
@@ -183,6 +192,7 @@ namespace Tgstation.Server.Host.Components
 		/// <param name="remoteDeploymentManagerFactory">The value of <see cref="remoteDeploymentManagerFactory"/>.</param>
 		/// <param name="asyncDelayer">The value of <see cref="asyncDelayer"/>.</param>
 		/// <param name="dotnetDumpService">The value of <see cref="dotnetDumpService"/>.</param>
+		/// <param name="metricFactory">The value of <see cref="metricFactory"/>.</param>
 		/// <param name="generalConfigurationOptions">The <see cref="IOptions{TOptions}"/> containing the value of <see cref="generalConfiguration"/>.</param>
 		/// <param name="sessionConfigurationOptions">The <see cref="IOptions{TOptions}"/> containing the value of <see cref="sessionConfiguration"/>.</param>
 		public InstanceFactory(
@@ -208,6 +218,7 @@ namespace Tgstation.Server.Host.Components
 			IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory,
 			IAsyncDelayer asyncDelayer,
 			IDotnetDumpService dotnetDumpService,
+			IMetricFactory metricFactory,
 			IOptions<GeneralConfiguration> generalConfigurationOptions,
 			IOptions<SessionConfiguration> sessionConfigurationOptions)
 		{
@@ -233,6 +244,7 @@ namespace Tgstation.Server.Host.Components
 			this.remoteDeploymentManagerFactory = remoteDeploymentManagerFactory ?? throw new ArgumentNullException(nameof(remoteDeploymentManagerFactory));
 			this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer));
 			this.dotnetDumpService = dotnetDumpService ?? throw new ArgumentNullException(nameof(dotnetDumpService));
+			this.metricFactory = metricFactory ?? throw new ArgumentNullException(nameof(metricFactory));
 			generalConfiguration = generalConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(generalConfigurationOptions));
 			sessionConfiguration = sessionConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(sessionConfigurationOptions));
 		}
@@ -264,6 +276,13 @@ namespace Tgstation.Server.Host.Components
 			var diagnosticsIOManager = new ResolvingIOManager(instanceIoManager, "Diagnostics");
 			var configurationIoManager = new ResolvingIOManager(instanceIoManager, "Configuration");
 
+			var metricFactory = this.metricFactory.WithLabels(
+				new Dictionary<string, string>
+				{
+					{ "instance_name", metadata.Name! },
+					{ "instance_id", metadata.Id!.Value.ToString(CultureInfo.InvariantCulture) },
+				});
+
 			var configuration = new StaticFiles.Configuration(
 				configurationIoManager,
 				synchronousIOManager,
@@ -323,6 +342,7 @@ namespace Tgstation.Server.Host.Components
 							eventConsumer,
 							asyncDelayer,
 							dotnetDumpService,
+							metricFactory,
 							loggerFactory,
 							loggerFactory.CreateLogger<SessionControllerFactory>(),
 							sessionConfiguration,
@@ -337,6 +357,7 @@ namespace Tgstation.Server.Host.Components
 							diagnosticsIOManager,
 							configuration, // watchdog doesn't need itself as an event consumer
 							remoteDeploymentManagerFactory,
+							metricFactory,
 							metadata,
 							metadata.DreamDaemonSettings!);
 						try
@@ -357,6 +378,7 @@ namespace Tgstation.Server.Host.Components
 								repoManager,
 								remoteDeploymentManagerFactory,
 								asyncDelayer,
+								metricFactory,
 								loggerFactory.CreateLogger<DreamMaker>(),
 								sessionConfiguration,
 								metadata);

src/Tgstation.Server.Host/Components/InstanceManager.cs

@@ -9,6 +9,8 @@ using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 
+using Prometheus;
+
 using Tgstation.Server.Api.Models;
 using Tgstation.Server.Common;
 using Tgstation.Server.Common.Extensions;
@@ -150,6 +152,11 @@ namespace Tgstation.Server.Host.Components
 		/// </summary>
 		readonly CancellationTokenSource shutdownCancellationTokenSource;
 
+		/// <summary>
+		/// The count of online instances.
+		/// </summary>
+		readonly Gauge onlineInstances;
+
 		/// <summary>
 		/// The original <see cref="IConsole.Title"/> of <see cref="console"/>.
 		/// </summary>
@@ -180,6 +187,8 @@ namespace Tgstation.Server.Host.Components
 		/// <param name="swarmServiceController">The value of <see cref="swarmServiceController"/>.</param>
 		/// <param name="console">The value of <see cref="console"/>.</param>
 		/// <param name="platformIdentifier">The value of <see cref="platformIdentifier"/>.</param>
+		/// <param name="metricFactory">The <see cref="IMetricFactory"/> used to create metrics.</param>
+		/// <param name="collectorRegistry">The <see cref="ICollectorRegistry"/> to use.</param>
 		/// <param name="generalConfigurationOptions">The <see cref="IOptions{TOptions}"/> containing the value of <see cref="generalConfiguration"/>.</param>
 		/// <param name="swarmConfigurationOptions">The <see cref="IOptions{TOptions}"/> containing the value of <see cref="swarmConfiguration"/>.</param>
 		/// <param name="internalConfigurationOptions">The <see cref="IOptions{TOptions}"/> containing the value of <see cref="internalConfiguration"/>.</param>
@@ -197,6 +206,8 @@ namespace Tgstation.Server.Host.Components
 			ISwarmServiceController swarmServiceController,
 			IConsole console,
 			IPlatformIdentifier platformIdentifier,
+			IMetricFactory metricFactory,
+			ICollectorRegistry collectorRegistry,
 			IOptions<GeneralConfiguration> generalConfigurationOptions,
 			IOptions<SwarmConfiguration> swarmConfigurationOptions,
 			IOptions<InternalConfiguration> internalConfigurationOptions,
@@ -214,6 +225,8 @@ namespace Tgstation.Server.Host.Components
 			this.swarmServiceController = swarmServiceController ?? throw new ArgumentNullException(nameof(swarmServiceController));
 			this.console = console ?? throw new ArgumentNullException(nameof(console));
 			this.platformIdentifier = platformIdentifier ?? throw new ArgumentNullException(nameof(platformIdentifier));
+			ArgumentNullException.ThrowIfNull(metricFactory);
+			ArgumentNullException.ThrowIfNull(collectorRegistry);
 			generalConfiguration = generalConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(generalConfigurationOptions));
 			swarmConfiguration = swarmConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(swarmConfigurationOptions));
 			internalConfiguration = internalConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(internalConfigurationOptions));
@@ -221,12 +234,21 @@ namespace Tgstation.Server.Host.Components
 
 			originalConsoleTitle = console.Title;
 
+			onlineInstances = metricFactory.CreateGauge("tgs_online_instances", "The total number of instances online");
+
 			instances = new Dictionary<long, ReferenceCountingContainer<IInstance, InstanceWrapper>>();
 			bridgeHandlers = new Dictionary<string, IBridgeHandler>();
 			readyTcs = new TaskCompletionSource();
 			instanceStateChangeSemaphore = new SemaphoreSlim(1);
 			startupCancellationTokenSource = new CancellationTokenSource();
 			shutdownCancellationTokenSource = new CancellationTokenSource();
+
+			collectorRegistry.AddBeforeCollectCallback(async cancellationToken =>
+			{
+				using (await SemaphoreSlimContext.Lock(instanceStateChangeSemaphore, cancellationToken))
+					foreach (var container in instances.Values)
+						container.Instance.Watchdog.RunMetricsScrape();
+			});
 		}
 
 		/// <inheritdoc />
@@ -396,6 +418,7 @@ namespace Tgstation.Server.Host.Components
 				finally
 				{
 					await container.Instance.DisposeAsync();
+					onlineInstances.Dec();
 				}
 			}
 		}
@@ -426,6 +449,8 @@ namespace Tgstation.Server.Host.Components
 						instances.Add(
 							instanceId,
 							new ReferenceCountingContainer<IInstance, InstanceWrapper>(instance));
+
+					onlineInstances.Inc();
 				}
 				catch (Exception ex)
 				{

src/Tgstation.Server.Host/Components/Interop/Bridge/BridgeParameters.cs

@@ -52,7 +52,7 @@ namespace Tgstation.Server.Host.Components.Interop.Bridge
 		public ushort? TopicPort { get; set; }
 
 		/// <summary>
-		/// The <see cref="Bridge.CustomEventInvocation"/> being triggered.
+		/// The <see cref="CustomEventInvocation"/> being triggered.
 		/// </summary>
 		public CustomEventInvocation? EventInvocation { get; set; }
 

src/Tgstation.Server.Host/Components/Repository/GitLabRemoteFeatures.cs

@@ -1,11 +1,14 @@
 using System;
+using System.Globalization;
 using System.Threading;
 using System.Threading.Tasks;
 
-using GitLabApiClient;
 using Microsoft.Extensions.Logging;
 
+using StrawberryShake;
+
 using Tgstation.Server.Api.Models;
+using Tgstation.Server.Host.Utils.GitLab.GraphQL;
 
 namespace Tgstation.Server.Host.Components.Repository
 {
@@ -45,30 +48,25 @@ namespace Tgstation.Server.Host.Components.Repository
 			RepositorySettings repositorySettings,
 			CancellationToken cancellationToken)
 		{
-			var client = repositorySettings.AccessToken != null
-				? new GitLabClient(GitLabUrl, repositorySettings.AccessToken)
-				: new GitLabClient(GitLabUrl);
-
+			await using var client = await GraphQLGitLabClientFactory.CreateClient(repositorySettings.AccessToken);
 			try
 			{
-				var mr = await client
-					.MergeRequests
-					.GetAsync($"{RemoteRepositoryOwner}/{RemoteRepositoryName}", parameters.Number)
-					.WaitAsync(cancellationToken);
+				var operationResult = await client.GraphQL.GetMergeRequest.ExecuteAsync(
+					$"{RemoteRepositoryOwner}/{RemoteRepositoryName}",
+					parameters.Number.ToString(CultureInfo.InvariantCulture),
+					cancellationToken);
 
-				var revisionToUse = parameters.TargetCommitSha == null
-					|| mr.Sha.StartsWith(parameters.TargetCommitSha, StringComparison.OrdinalIgnoreCase)
-					? mr.Sha
-					: parameters.TargetCommitSha;
+				operationResult.EnsureNoErrors();
+				var mr = operationResult.Data?.Project?.MergeRequest ?? throw new InvalidOperationException("GitLab MergeRequest check returned null!");
 
 				return new Models.TestMerge
 				{
-					Author = mr.Author.Username,
+					Author = mr.Author?.Username,
 					BodyAtMerge = mr.Description,
 					TitleAtMerge = mr.Title,
 					Comment = parameters.Comment,
 					Number = parameters.Number,
-					TargetCommitSha = mr.Sha,
+					TargetCommitSha = mr.DiffHeadSha,
 					Url = mr.WebUrl,
 				};
 			}

src/Tgstation.Server.Host/Components/Repository/Repository.cs

@@ -199,7 +199,7 @@ namespace Tgstation.Server.Host.Components.Repository
 
 			var commitMessage = String.Format(
 				CultureInfo.InvariantCulture,
-				"TGS Test Merge (#{0}){1}{2}",
+				"TGS Test Merge (PR {0}){1}{2}",
 				testMergeParameters.Number,
 				testMergeParameters.Comment != null
 					? Environment.NewLine

src/Tgstation.Server.Host/Components/Session/SessionController.cs

@@ -519,6 +519,10 @@ namespace Tgstation.Server.Host.Components.Session
 			return process.CreateDump(outputFile, minidump, cancellationToken);
 		}
 
+		/// <inheritdoc />
+		public double MeasureProcessorTimeDelta()
+			=> process.MeasureProcessorTimeDelta();
+
 		/// <summary>
 		/// The <see cref="Task{TResult}"/> for <see cref="LaunchResult"/>.
 		/// </summary>
@@ -1171,14 +1175,26 @@ namespace Tgstation.Server.Host.Components.Session
 			{
 				try
 				{
-					await eventTask.Value;
+					Exception? exception;
+					try
+					{
+						await eventTask.Value;
+						exception = null;
+					}
+					catch (Exception ex)
+					{
+						exception = ex;
+					}
 
 					if (notifyCompletion.Value)
 						await SendCommand(
 							new TopicParameters(eventId),
 							cancellationToken);
-					else
+					else if (exception == null)
 						Logger.LogTrace("Finished custom event {eventId}, not sending notification.", eventId);
+
+					if (exception != null)
+						throw exception;
 				}
 				catch (OperationCanceledException ex)
 				{

src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Globalization;
+using System.Linq;
 using System.Net.Sockets;
 using System.Text;
 using System.Threading;
@@ -8,6 +9,8 @@ using System.Threading.Tasks;
 
 using Microsoft.Extensions.Logging;
 
+using Prometheus;
+
 using Tgstation.Server.Api.Models;
 using Tgstation.Server.Api.Models.Internal;
 using Tgstation.Server.Common.Extensions;
@@ -126,6 +129,16 @@ namespace Tgstation.Server.Host.Components.Session
 		/// </summary>
 		readonly SessionConfiguration sessionConfiguration;
 
+		/// <summary>
+		/// The number of sessions launched.
+		/// </summary>
+		readonly Counter sessionsLaunched;
+
+		/// <summary>
+		/// The time the current session was launched.
+		/// </summary>
+		readonly Gauge lastSessionLaunch;
+
 		/// <summary>
 		/// The <see cref="Api.Models.Instance"/> for the <see cref="SessionControllerFactory"/>.
 		/// </summary>
@@ -184,6 +197,7 @@ namespace Tgstation.Server.Host.Components.Session
 		/// <param name="eventConsumer">The value of <see cref="eventConsumer"/>.</param>
 		/// <param name="asyncDelayer">The value of <see cref="asyncDelayer"/>.</param>
 		/// <param name="dotnetDumpService">The value of <see cref="dotnetDumpService"/>.</param>
+		/// <param name="metricFactory">The <see cref="IMetricFactory"/> used to create metrics.</param>
 		/// <param name="loggerFactory">The value of <see cref="loggerFactory"/>.</param>
 		/// <param name="logger">The value of <see cref="logger"/>.</param>
 		/// <param name="sessionConfiguration">The value of <see cref="sessionConfiguration"/>.</param>
@@ -203,6 +217,7 @@ namespace Tgstation.Server.Host.Components.Session
 			IEventConsumer eventConsumer,
 			IAsyncDelayer asyncDelayer,
 			IDotnetDumpService dotnetDumpService,
+			IMetricFactory metricFactory,
 			ILoggerFactory loggerFactory,
 			ILogger<SessionControllerFactory> logger,
 			SessionConfiguration sessionConfiguration,
@@ -223,10 +238,14 @@ namespace Tgstation.Server.Host.Components.Session
 			this.eventConsumer = eventConsumer ?? throw new ArgumentNullException(nameof(eventConsumer));
 			this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer));
 			this.dotnetDumpService = dotnetDumpService ?? throw new ArgumentNullException(nameof(dotnetDumpService));
+			ArgumentNullException.ThrowIfNull(metricFactory);
 			this.loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory));
 			this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
 			this.sessionConfiguration = sessionConfiguration ?? throw new ArgumentNullException(nameof(sessionConfiguration));
 			this.instance = instance ?? throw new ArgumentNullException(nameof(instance));
+
+			sessionsLaunched = metricFactory.CreateCounter("tgs_sessions_launched", "The number of game server processes created");
+			lastSessionLaunch = metricFactory.CreateGauge("tgs_session_start_time", "The UTC unix timestamp the most recent session was started");
 		}
 
 		/// <inheritdoc />
@@ -367,6 +386,12 @@ namespace Tgstation.Server.Host.Components.Session
 							false,
 							apiValidate);
 
+						if (!apiValidate)
+						{
+							sessionsLaunched.Inc();
+							lastSessionLaunch.SetToCurrentTimeUtc();
+						}
+
 						return sessionController;
 					}
 					catch
@@ -516,6 +541,14 @@ namespace Tgstation.Server.Host.Components.Session
 					? logFilePath
 					: null);
 
+			// If this isnt a staging DD (From a Deployment), fire off events
+			if (!apiValidate)
+				await eventConsumer.HandleEvent(
+					EventType.DreamDaemonPreLaunch,
+					Enumerable.Empty<string?>(),
+					false,
+					cancellationToken);
+
 			var process = await processExecutor.LaunchProcess(
 				engineLock.ServerExePath,
 				dmbProvider.Directory,
@@ -539,7 +572,6 @@ namespace Tgstation.Server.Host.Components.Session
 				if (!engineLock.HasStandardOutput)
 					networkPromptReaper.RegisterProcess(process);
 
-				// If this isnt a staging DD (From a Deployment), fire off an event
 				if (!apiValidate)
 					await eventConsumer.HandleEvent(
 						EventType.DreamDaemonLaunch,

src/Tgstation.Server.Host/Components/Watchdog/AdvancedWatchdog.cs

@@ -6,6 +6,8 @@ using System.Threading.Tasks;
 
 using Microsoft.Extensions.Logging;
 
+using Prometheus;
+
 using Tgstation.Server.Api.Models;
 using Tgstation.Server.Api.Models.Internal;
 using Tgstation.Server.Host.Components.Chat;
@@ -63,6 +65,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// <param name="diagnosticsIOManager">The 'Diagnostics' <see cref="IIOManager"/> for the <see cref="WatchdogBase"/>.</param>
 		/// <param name="eventConsumer">The <see cref="IEventConsumer"/> for the <see cref="WatchdogBase"/>.</param>
 		/// <param name="remoteDeploymentManagerFactory">The <see cref="IRemoteDeploymentManagerFactory"/> for the <see cref="WatchdogBase"/>.</param>
+		/// <param name="metricFactory">The <see cref="IMetricFactory"/> for the <see cref="WatchdogBase"/>.</param>
 		/// <param name="gameIOManager">The 'Game' <see cref="IIOManager"/> for the <see cref="WatchdogBase"/>.</param>
 		/// <param name="linkFactory">The value of <see cref="LinkFactory"/>.</param>
 		/// <param name="logger">The <see cref="ILogger"/> for the <see cref="WatchdogBase"/>.</param>
@@ -80,6 +83,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			IIOManager diagnosticsIOManager,
 			IEventConsumer eventConsumer,
 			IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory,
+			IMetricFactory metricFactory,
 			IIOManager gameIOManager,
 			IFilesystemLinkFactory linkFactory,
 			ILogger<AdvancedWatchdog> logger,
@@ -97,6 +101,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 				diagnosticsIOManager,
 				eventConsumer,
 				remoteDeploymentManagerFactory,
+				metricFactory,
 				gameIOManager,
 				logger,
 				initialLaunchParameters,

src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs

@@ -6,6 +6,8 @@ using System.Threading.Tasks;
 
 using Microsoft.Extensions.Logging;
 
+using Prometheus;
+
 using Tgstation.Server.Api.Models.Internal;
 using Tgstation.Server.Host.Components.Chat;
 using Tgstation.Server.Host.Components.Deployment;
@@ -53,6 +55,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// <param name="diagnosticsIOManager">The 'Diagnostics' <see cref="IIOManager"/> for the <see cref="WatchdogBase"/>.</param>
 		/// <param name="eventConsumer">The <see cref="IEventConsumer"/> for the <see cref="WatchdogBase"/>.</param>
 		/// <param name="remoteDeploymentManagerFactory">The <see cref="IRemoteDeploymentManagerFactory"/> for the <see cref="WatchdogBase"/>.</param>
+		/// <param name="metricFactory">The <see cref="IMetricFactory"/> for the <see cref="WatchdogBase"/>.</param>
 		/// <param name="gameIOManager">The 'Game' <see cref="IIOManager"/> for the <see cref="WatchdogBase"/>.</param>
 		/// <param name="logger">The <see cref="ILogger"/> for the <see cref="WatchdogBase"/>.</param>
 		/// <param name="initialLaunchParameters">The <see cref="DreamDaemonLaunchParameters"/> for the <see cref="WatchdogBase"/>.</param>
@@ -69,6 +72,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			IIOManager diagnosticsIOManager,
 			IEventConsumer eventConsumer,
 			IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory,
+			IMetricFactory metricFactory,
 			IIOManager gameIOManager,
 			ILogger<BasicWatchdog> logger,
 			DreamDaemonLaunchParameters initialLaunchParameters,
@@ -85,6 +89,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 				 diagnosticsIOManager,
 				 eventConsumer,
 				 remoteDeploymentManagerFactory,
+				 metricFactory,
 				 gameIOManager,
 				 logger,
 				 initialLaunchParameters,

src/Tgstation.Server.Host/Components/Watchdog/IWatchdog.cs

@@ -117,5 +117,10 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
 		/// <returns>A <see cref="ValueTask{TResult}"/> resulting in <see langword="true"/> if the broadcast succeeded., <see langword="false"/> otherwise.</returns>
 		ValueTask<bool> Broadcast(string message, CancellationToken cancellationToken);
+
+		/// <summary>
+		/// Callback to update transient metrics.
+		/// </summary>
+		void RunMetricsScrape();
 	}
 }

src/Tgstation.Server.Host/Components/Watchdog/IWatchdogFactory.cs

@@ -1,4 +1,6 @@
-using Tgstation.Server.Api.Models.Internal;
+using Prometheus;
+
+using Tgstation.Server.Api.Models.Internal;
 using Tgstation.Server.Host.Components.Chat;
 using Tgstation.Server.Host.Components.Deployment;
 using Tgstation.Server.Host.Components.Deployment.Remote;
@@ -24,6 +26,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// <param name="diagnosticsIOManager">The <see cref="IIOManager"/> pointing to the Diagnostics directory for the <see cref="IWatchdog"/>.</param>
 		/// <param name="eventConsumer">The <see cref="IEventConsumer"/> for the <see cref="IWatchdog"/>.</param>
 		/// <param name="remoteDeploymentManagerFactory">The <see cref="IRemoteDeploymentManagerFactory"/> for the <see cref="IWatchdog"/>.</param>
+		/// <param name="metricFactory">The <see cref="IMetricFactory"/> used to create metrics.</param>
 		/// <param name="instance">The <see cref="Instance"/> for the <see cref="IWatchdog"/>.</param>
 		/// <param name="settings">The initial <see cref="DreamDaemonSettings"/> for the <see cref="IWatchdog"/>.</param>
 		/// <returns>A new <see cref="IWatchdog"/>.</returns>
@@ -36,6 +39,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			IIOManager diagnosticsIOManager,
 			IEventConsumer eventConsumer,
 			IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory,
+			IMetricFactory metricFactory,
 			Api.Models.Instance instance,
 			DreamDaemonSettings settings);
 	}

src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs

@@ -5,6 +5,8 @@ using System.Threading.Tasks;
 
 using Microsoft.Extensions.Logging;
 
+using Prometheus;
+
 using Tgstation.Server.Api.Models.Internal;
 using Tgstation.Server.Host.Components.Chat;
 using Tgstation.Server.Host.Components.Deployment;
@@ -43,6 +45,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// <param name="diagnosticsIOManager">The <see cref="IIOManager"/> for the <see cref="WatchdogBase"/>.</param>
 		/// <param name="eventConsumer">The <see cref="IEventConsumer"/> for the <see cref="WatchdogBase"/>.</param>
 		/// <param name="remoteDeploymentManagerFactory">The <see cref="IRemoteDeploymentManagerFactory"/> for the <see cref="WatchdogBase"/>.</param>
+		/// <param name="metricFactory">The <see cref="IMetricFactory"/> for the <see cref="WatchdogBase"/>.</param>
 		/// <param name="gameIOManager">The <see cref="IIOManager"/> pointing to the game directory for the <see cref="AdvancedWatchdog"/>..</param>
 		/// <param name="linkFactory">The <see cref="IFilesystemLinkFactory"/> for the <see cref="AdvancedWatchdog"/>.</param>
 		/// <param name="logger">The <see cref="ILogger"/> for the <see cref="WatchdogBase"/>.</param>
@@ -61,6 +64,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			IIOManager diagnosticsIOManager,
 			IEventConsumer eventConsumer,
 			IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory,
+			IMetricFactory metricFactory,
 			IIOManager gameIOManager,
 			IFilesystemLinkFactory linkFactory,
 			ILogger<PosixWatchdog> logger,
@@ -79,6 +83,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 				  diagnosticsIOManager,
 				  eventConsumer,
 				  remoteDeploymentManagerFactory,
+				  metricFactory,
 				  gameIOManager,
 				  linkFactory,
 				  logger,

src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdogFactory.cs

@@ -4,6 +4,8 @@ using System.Runtime.Versioning;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 
+using Prometheus;
+
 using Tgstation.Server.Api.Models.Internal;
 using Tgstation.Server.Host.Components.Chat;
 using Tgstation.Server.Host.Components.Deployment;
@@ -60,6 +62,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			IIOManager diagnosticsIOManager,
 			IEventConsumer eventConsumer,
 			IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory,
+			IMetricFactory metricFactory,
 			Api.Models.Instance instance,
 			DreamDaemonSettings settings)
 			=> new PosixWatchdog(
@@ -73,6 +76,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 				diagnosticsIOManager,
 				eventConsumer,
 				remoteDeploymentManagerFactory,
+				metricFactory,
 				gameIOManager,
 				LinkFactory,
 				LoggerFactory.CreateLogger<PosixWatchdog>(),

src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs

@@ -7,6 +7,8 @@ using System.Threading.Tasks;
 
 using Microsoft.Extensions.Logging;
 
+using Prometheus;
+
 using Serilog.Context;
 
 using Tgstation.Server.Api.Models;
@@ -123,6 +125,21 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// </summary>
 		readonly SemaphoreSlim controllerDisposeSemaphore;
 
+		/// <summary>
+		/// The <see cref="Status"/> as a metric.
+		/// </summary>
+		readonly Gauge watchdogStatusMetric;
+
+		/// <summary>
+		/// Active session CPU usage as a metric.
+		/// </summary>
+		readonly Gauge cpuUsageMetric;
+
+		/// <summary>
+		/// <see cref="MemoryUsage"/> as a metric.
+		/// </summary>
+		readonly Gauge ramUsageMetric;
+
 		/// <summary>
 		/// The <see cref="IEventConsumer"/> that is not the <see cref="WatchdogBase"/>.
 		/// </summary>
@@ -201,6 +218,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// <param name="diagnosticsIOManager">The value of <see cref="diagnosticsIOManager"/>.</param>
 		/// <param name="eventConsumer">The value of <see cref="EventConsumer"/>.</param>
 		/// <param name="remoteDeploymentManagerFactory">The value of <see cref="remoteDeploymentManagerFactory"/>.</param>
+		/// <param name="metricFactory">The <see cref="IMetricFactory"/> used to create metrics.</param>
 		/// <param name="gameIOManager">The value of <see cref="GameIOManager"/>.</param>
 		/// <param name="logger">The value of <see cref="Logger"/>.</param>
 		/// <param name="initialLaunchParameters">The initial value of <see cref="ActiveLaunchParameters"/>. May be modified.</param>
@@ -217,6 +235,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			IIOManager diagnosticsIOManager,
 			IEventConsumer eventConsumer,
 			IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory,
+			IMetricFactory metricFactory,
 			IIOManager gameIOManager,
 			ILogger<WatchdogBase> logger,
 			DreamDaemonLaunchParameters initialLaunchParameters,
@@ -232,6 +251,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			this.diagnosticsIOManager = diagnosticsIOManager ?? throw new ArgumentNullException(nameof(diagnosticsIOManager));
 			this.eventConsumer = eventConsumer ?? throw new ArgumentNullException(nameof(eventConsumer));
 			this.remoteDeploymentManagerFactory = remoteDeploymentManagerFactory ?? throw new ArgumentNullException(nameof(remoteDeploymentManagerFactory));
+			ArgumentNullException.ThrowIfNull(metricFactory);
 			GameIOManager = gameIOManager ?? throw new ArgumentNullException(nameof(gameIOManager));
 			Logger = logger ?? throw new ArgumentNullException(nameof(logger));
 			ActiveLaunchParameters = initialLaunchParameters ?? throw new ArgumentNullException(nameof(initialLaunchParameters));
@@ -240,6 +260,12 @@ namespace Tgstation.Server.Host.Components.Watchdog
 
 			ArgumentNullException.ThrowIfNull(serverControl);
 
+			watchdogStatusMetric = metricFactory.CreateGauge(
+				"tgs_watchdog_status",
+				$"TGS Watchdog status: {(int)WatchdogStatus.Offline} = Offline, {(int)WatchdogStatus.Online} = Online, {(int)WatchdogStatus.Restoring} = Restoring, {(int)WatchdogStatus.DelayedRestart} = Delayed Restart");
+			cpuUsageMetric = metricFactory.CreateGauge("tgs_game_cpu_usage", "Estimated total CPU usage time for the game process from 0-1");
+			ramUsageMetric = metricFactory.CreateGauge("tgs_game_ram_usage", "Total used bytes of private memory for the game process");
+
 			chat.RegisterCommandHandler(this);
 
 			ActiveLaunchParameters = initialLaunchParameters;
@@ -272,6 +298,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			await DisposeAndNullControllersImpl();
 			controllerDisposeSemaphore.Dispose();
 			monitorCts?.Dispose();
+
 			disposed = true;
 		}
 
@@ -501,6 +528,15 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			return response != null && response.ErrorMessage == null;
 		}
 
+		/// <inheritdoc />
+		public void RunMetricsScrape()
+		{
+			watchdogStatusMetric.Set((int)Status);
+			var controller = GetActiveController();
+			ramUsageMetric.Set(controller?.MemoryUsage ?? 0);
+			cpuUsageMetric.Set(controller?.MeasureProcessorTimeDelta() ?? 0);
+		}
+
 		/// <inheritdoc />
 		async ValueTask IEventConsumer.HandleEvent(EventType eventType, IEnumerable<string?> parameters, bool deploymentPipeline, CancellationToken cancellationToken)
 		{

src/Tgstation.Server.Host/Components/Watchdog/WatchdogFactory.cs

@@ -3,6 +3,8 @@
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 
+using Prometheus;
+
 using Tgstation.Server.Api.Models.Internal;
 using Tgstation.Server.Host.Components.Chat;
 using Tgstation.Server.Host.Components.Deployment;
@@ -77,6 +79,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			IIOManager diagnosticsIOManager,
 			IEventConsumer eventConsumer,
 			IRemoteDeploymentManagerFactory remoteDeploymentManagerfactory,
+			IMetricFactory metricFactory,
 			Api.Models.Instance instance,
 			DreamDaemonSettings settings)
 			=> new BasicWatchdog(
@@ -90,6 +93,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 				diagnosticsIOManager,
 				eventConsumer,
 				remoteDeploymentManagerfactory,
+				metricFactory,
 				gameIOManager,
 				LoggerFactory.CreateLogger<BasicWatchdog>(),
 				settings,

src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs

@@ -3,6 +3,8 @@ using System.Threading.Tasks;
 
 using Microsoft.Extensions.Logging;
 
+using Prometheus;
+
 using Tgstation.Server.Api.Models;
 using Tgstation.Server.Api.Models.Internal;
 using Tgstation.Server.Host.Components.Chat;
@@ -35,6 +37,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// <param name="diagnosticsIOManager">The <see cref="IIOManager"/> for the <see cref="WatchdogBase"/>.</param>
 		/// <param name="eventConsumer">The <see cref="IEventConsumer"/> for the <see cref="WatchdogBase"/>.</param>
 		/// <param name="remoteDeploymentManagerFactory">The <see cref="IRemoteDeploymentManagerFactory"/> for the <see cref="WatchdogBase"/>.</param>
+		/// <param name="metricFactory">The <see cref="IMetricFactory"/> for the <see cref="WatchdogBase"/>.</param>
 		/// <param name="gameIOManager">The <see cref="IIOManager"/> pointing to the game directory for the <see cref="AdvancedWatchdog"/>..</param>
 		/// <param name="linkFactory">The <see cref="IFilesystemLinkFactory"/> for the <see cref="AdvancedWatchdog"/>.</param>
 		/// <param name="logger">The <see cref="ILogger"/> for the <see cref="WatchdogBase"/>.</param>
@@ -52,6 +55,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			IIOManager diagnosticsIOManager,
 			IEventConsumer eventConsumer,
 			IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory,
+			IMetricFactory metricFactory,
 			IIOManager gameIOManager,
 			IFilesystemLinkFactory linkFactory,
 			ILogger<WindowsWatchdog> logger,
@@ -69,6 +73,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 				  diagnosticsIOManager,
 				  eventConsumer,
 				  remoteDeploymentManagerFactory,
+				  metricFactory,
 				  gameIOManager,
 				  linkFactory,
 				  logger,

src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs

@@ -3,6 +3,8 @@
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 
+using Prometheus;
+
 using Tgstation.Server.Api.Models.Internal;
 using Tgstation.Server.Host.Components.Chat;
 using Tgstation.Server.Host.Components.Deployment;
@@ -63,6 +65,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			IIOManager diagnosticsIOManager,
 			IEventConsumer eventConsumer,
 			IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory,
+			IMetricFactory metricFactory,
 			Api.Models.Instance instance,
 			DreamDaemonSettings settings)
 			=> new WindowsWatchdog(
@@ -76,6 +79,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 				diagnosticsIOManager,
 				eventConsumer,
 				remoteDeploymentManagerFactory,
+				metricFactory,
 				gameIOManager,
 				LinkFactory,
 				LoggerFactory.CreateLogger<WindowsWatchdog>(),

src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs

@@ -91,6 +91,11 @@ namespace Tgstation.Server.Host.Configuration
 		/// </summary>
 		public ushort ApiPort { get; set; }
 
+		/// <summary>
+		/// The port Prometheus metrics are published on, if any.
+		/// </summary>
+		public ushort? PrometheusPort { get; set; }
+
 		/// <summary>
 		/// A classic GitHub personal access token to use for bypassing rate limits on requests. Requires no scopes.
 		/// </summary>

src/Tgstation.Server.Host/Controllers/RepositoryController.cs

@@ -7,12 +7,12 @@ using System.Reflection;
 using System.Threading;
 using System.Threading.Tasks;
 
-using GitLabApiClient;
-
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Logging;
 
+using StrawberryShake;
+
 using Tgstation.Server.Api;
 using Tgstation.Server.Api.Models;
 using Tgstation.Server.Api.Models.Request;
@@ -27,6 +27,7 @@ using Tgstation.Server.Host.Models;
 using Tgstation.Server.Host.Security;
 using Tgstation.Server.Host.Utils;
 using Tgstation.Server.Host.Utils.GitHub;
+using Tgstation.Server.Host.Utils.GitLab.GraphQL;
 
 namespace Tgstation.Server.Host.Controllers
 {
@@ -585,70 +586,84 @@ namespace Tgstation.Server.Host.Controllers
 			switch (remoteFeatures.RemoteGitProvider!.Value)
 			{
 				case RemoteGitProvider.GitHub:
-					var gitHubClient = await gitHubClientFactory.CreateClientForRepository(
+					{
+						var gitHubClient = await gitHubClientFactory.CreateClientForRepository(
 						model.AccessToken,
 						new RepositoryIdentifier(
 							remoteFeatures.RemoteRepositoryOwner!,
 							remoteFeatures.RemoteRepositoryName!),
 						cancellationToken);
-					if (gitHubClient == null)
-					{
-						return this.StatusCode(HttpStatusCode.FailedDependency, new ErrorMessageResponse(ErrorCode.RemoteApiError)
+						if (gitHubClient == null)
 						{
-							AdditionalData = "GitHub authentication failed!",
-						});
-					}
-
-					try
-					{
-						string username;
-						if (!model.AccessToken.StartsWith(Api.Models.RepositorySettings.TgsAppPrivateKeyPrefix))
-						{
-							var user = await gitHubClient.User.Current();
-							username = user.Login;
-						}
-						else
-						{
-							// we literally need to app auth again to get the damn bot username
-							var appClient = gitHubClientFactory.CreateAppClient(model.AccessToken)!;
-							var app = await appClient.GitHubApps.GetCurrent();
-							username = app.Name;
+							return this.StatusCode(HttpStatusCode.FailedDependency, new ErrorMessageResponse(ErrorCode.RemoteApiError)
+							{
+								AdditionalData = "GitHub authentication failed!",
+							});
 						}
 
-						if (username != model.AccessUser)
-							return Conflict(new ErrorMessageResponse(ErrorCode.RepoTokenUsernameMismatch));
-					}
-					catch (Exception ex)
-					{
-						return this.StatusCode(HttpStatusCode.FailedDependency, new ErrorMessageResponse(ErrorCode.RemoteApiError)
+						try
 						{
-							AdditionalData = $"GitHub Authentication Failure: {ex.Message}",
-						});
+							string username;
+							if (!model.AccessToken.StartsWith(Api.Models.RepositorySettings.TgsAppPrivateKeyPrefix))
+							{
+								var user = await gitHubClient.User.Current();
+								username = user.Login;
+							}
+							else
+							{
+								// we literally need to app auth again to get the damn bot username
+								var appClient = gitHubClientFactory.CreateAppClient(model.AccessToken)!;
+								var app = await appClient.GitHubApps.GetCurrent();
+								username = app.Name;
+							}
+
+							if (username != model.AccessUser)
+								return Conflict(new ErrorMessageResponse(ErrorCode.RepoTokenUsernameMismatch));
+						}
+						catch (Exception ex)
+						{
+							return this.StatusCode(HttpStatusCode.FailedDependency, new ErrorMessageResponse(ErrorCode.RemoteApiError)
+							{
+								AdditionalData = $"GitHub Authentication Failure: {ex.Message}",
+							});
+						}
+
+						break;
 					}
 
-					break;
 				case RemoteGitProvider.GitLab:
-					// need to abstract this eventually
-					var gitLabClient = new GitLabClient(GitLabRemoteFeatures.GitLabUrl, model.AccessToken);
-					try
 					{
-						var user = await gitLabClient.Users.GetCurrentSessionAsync();
-						if (user.Username != model.AccessUser)
-							return Conflict(new ErrorMessageResponse(ErrorCode.RepoTokenUsernameMismatch));
-					}
-					catch (Exception ex)
-					{
-						return this.StatusCode(HttpStatusCode.FailedDependency, new ErrorMessageResponse(ErrorCode.RemoteApiError)
+						// need to abstract this eventually
+						await using var gitLabClient = await GraphQLGitLabClientFactory.CreateClient(model.AccessToken);
+						try
 						{
-							AdditionalData = $"GitLab Authentication Failure: {ex.Message}",
-						});
+							var operationResult = await gitLabClient.GraphQL.GetCurrentUser.ExecuteAsync(cancellationToken);
+
+							operationResult.EnsureNoErrors();
+
+							var user = operationResult.Data?.CurrentUser;
+							if (user == null || user.Username != model.AccessUser)
+							{
+								return Conflict(new ErrorMessageResponse(ErrorCode.RepoTokenUsernameMismatch));
+							}
+						}
+						catch (Exception ex)
+						{
+							return this.StatusCode(HttpStatusCode.FailedDependency, new ErrorMessageResponse(ErrorCode.RemoteApiError)
+							{
+								AdditionalData = $"GitLab Authentication Failure: {ex.Message}",
+							});
+						}
+
+						break;
 					}
 
-					break;
 				case RemoteGitProvider.Unknown:
 				default:
-					Logger.LogWarning("RemoteGitProvider is {provider}, no auth check implemented!", remoteFeatures.RemoteGitProvider.Value);
-					break;
+					{
+						Logger.LogWarning("RemoteGitProvider is {provider}, no auth check implemented!", remoteFeatures.RemoteGitProvider.Value);
+						break;
+					}
 			}
 
 			return null;

src/Tgstation.Server.Host/Core/Application.cs

@@ -30,6 +30,8 @@ using Microsoft.Extensions.Options;
 
 using Newtonsoft.Json;
 
+using Prometheus;
+
 using Serilog;
 using Serilog.Events;
 using Serilog.Formatting.Display;
@@ -294,11 +296,29 @@ namespace Tgstation.Server.Host.Core
 			services.AddHttpClient();
 			services.AddSingleton<IAbstractHttpClientFactory, AbstractHttpClientFactory>();
 
+			// configure metrics
+			var prometheusPort = postSetupServices.GeneralConfiguration.PrometheusPort;
+
+			services.AddSingleton<IMetricFactory>(_ => Metrics.DefaultFactory);
+			services.AddSingleton<ICollectorRegistry>(_ => Metrics.DefaultRegistry);
+
+			if (prometheusPort.HasValue && prometheusPort != postSetupServices.GeneralConfiguration.ApiPort)
+				services.AddMetricServer(options => options.Port = prometheusPort.Value);
+
+			services.UseHttpClientMetrics();
+
+			var healthChecksBuilder = services
+				.AddHealthChecks()
+				.ForwardToPrometheus();
+
 			// configure graphql
 			services
 				.AddScoped<GraphQL.Subscriptions.ITopicEventReceiver, ShutdownAwareTopicEventReceiver>()
 				.AddGraphQLServer()
-				.AddAuthorization()
+				.AddAuthorization(
+					options => options.AddPolicy(
+						TgsAuthorizeAttribute.PolicyName,
+						builder => builder.RequireRole(TgsAuthorizeAttribute.UserEnabledRole)))
 				.ModifyOptions(options =>
 				{
 					options.EnsureAllNodesCanBeResolved = true;
@@ -359,6 +379,9 @@ namespace Tgstation.Server.Host.Core
 					configureAction(builder, databaseConfig);
 				});
 				services.AddScoped<IDatabaseContext>(x => x.GetRequiredService<TContext>());
+
+				healthChecksBuilder
+					.AddDbContextCheck<TContext>();
 			}
 
 			// add the correct database context type
@@ -524,6 +547,7 @@ namespace Tgstation.Server.Host.Core
 		/// <param name="assemblyInformationProvider">The <see cref="IAssemblyInformationProvider"/>.</param>
 		/// <param name="controlPanelConfigurationOptions">The <see cref="IOptions{TOptions}"/> containing the <see cref="ControlPanelConfiguration"/> to use.</param>
 		/// <param name="generalConfigurationOptions">The <see cref="IOptions{TOptions}"/> containing the <see cref="GeneralConfiguration"/> to use.</param>
+		/// <param name="databaseConfigurationOptions">The <see cref="IOptions{TOptions}"/> containing the <see cref="DatabaseConfiguration"/> to use.</param>
 		/// <param name="swarmConfigurationOptions">The <see cref="IOptions{TOptions}"/> containing the <see cref="SwarmConfiguration"/> to use.</param>
 		/// <param name="internalConfigurationOptions">The <see cref="IOptions{TOptions}"/> containing the <see cref="InternalConfiguration"/> to use.</param>
 		/// <param name="logger">The <see cref="Microsoft.Extensions.Logging.ILogger"/> for the <see cref="Application"/>.</param>
@@ -535,6 +559,7 @@ namespace Tgstation.Server.Host.Core
 			IAssemblyInformationProvider assemblyInformationProvider,
 			IOptions<ControlPanelConfiguration> controlPanelConfigurationOptions,
 			IOptions<GeneralConfiguration> generalConfigurationOptions,
+			IOptions<DatabaseConfiguration> databaseConfigurationOptions,
 			IOptions<SwarmConfiguration> swarmConfigurationOptions,
 			IOptions<InternalConfiguration> internalConfigurationOptions,
 			ILogger<Application> logger)
@@ -549,11 +574,13 @@ namespace Tgstation.Server.Host.Core
 
 			var controlPanelConfiguration = controlPanelConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(controlPanelConfigurationOptions));
 			var generalConfiguration = generalConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(generalConfigurationOptions));
+			var databaseConfiguration = databaseConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(databaseConfigurationOptions));
 			var swarmConfiguration = swarmConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(swarmConfigurationOptions));
 			var internalConfiguration = internalConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(internalConfigurationOptions));
 
 			ArgumentNullException.ThrowIfNull(logger);
 
+			logger.LogDebug("Database provider: {provider}", databaseConfiguration.DatabaseType);
 			logger.LogDebug("Content Root: {contentRoot}", hostingEnvironment.ContentRootPath);
 			logger.LogTrace("Web Root: {webRoot}", hostingEnvironment.WebRootPath);
 
@@ -564,6 +591,9 @@ namespace Tgstation.Server.Host.Core
 			// Wrap exceptions in a 500 (ErrorMessage) response
 			applicationBuilder.UseServerErrorHandling();
 
+			// metrics capture
+			applicationBuilder.UseHttpMetrics();
+
 			// Add the X-Powered-By response header
 			applicationBuilder.UseServerBranding(assemblyInformationProvider);
 
@@ -685,6 +715,19 @@ namespace Tgstation.Server.Host.Core
 						.MapGraphQL(Routes.GraphQL)
 						.WithOptions(gqlOptions);
 				}
+
+				if (generalConfiguration.PrometheusPort.HasValue)
+					if (generalConfiguration.PrometheusPort == generalConfiguration.ApiPort)
+					{
+						endpoints.MapMetrics();
+						logger.LogDebug("Prometheus being hosted alongside server");
+					}
+					else
+						logger.LogDebug("Prometheus being hosted on port {prometheusPort}", generalConfiguration.PrometheusPort);
+				else
+					logger.LogTrace("Prometheus disabled");
+
+				endpoints.MapHealthChecks("/health");
 			});
 
 			// 404 anything that gets this far

src/Tgstation.Server.Host/Extensions/WebHostBuilderExtensions.cs

@@ -65,6 +65,7 @@ namespace Tgstation.Server.Host.Extensions
 					applicationBuilder.ApplicationServices.GetRequiredService<IAssemblyInformationProvider>(),
 					applicationBuilder.ApplicationServices.GetRequiredService<IOptions<ControlPanelConfiguration>>(),
 					applicationBuilder.ApplicationServices.GetRequiredService<IOptions<GeneralConfiguration>>(),
+					applicationBuilder.ApplicationServices.GetRequiredService<IOptions<DatabaseConfiguration>>(),
 					applicationBuilder.ApplicationServices.GetRequiredService<IOptions<SwarmConfiguration>>(),
 					applicationBuilder.ApplicationServices.GetRequiredService<IOptions<InternalConfiguration>>(),
 					applicationBuilder.ApplicationServices.GetRequiredService<ILogger<Application>>());

src/Tgstation.Server.Host/GraphQL/ErrorMessageFilter.cs

@@ -38,7 +38,7 @@ namespace Tgstation.Server.Host.GraphQL
 				return error;
 
 			var errorBuilder = ErrorBuilder.FromError(error)
-				.RemoveException()
+				.SetException(null)
 				.ClearExtensions();
 
 			const string ErrorCodeFieldName = "errorCode";

src/Tgstation.Server.Host/Jobs/JobService.cs

@@ -9,6 +9,8 @@ using Microsoft.AspNetCore.SignalR;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Logging;
 
+using Prometheus;
+
 using Serilog.Context;
 
 using Tgstation.Server.Api.Extensions;
@@ -67,6 +69,16 @@ namespace Tgstation.Server.Host.Jobs
 		/// </summary>
 		readonly TaskCompletionSource<IInstanceCoreProvider> activationTcs;
 
+		/// <summary>
+		/// Total number of jobs processed.
+		/// </summary>
+		readonly Counter processedJobs;
+
+		/// <summary>
+		/// Jobs currently running.
+		/// </summary>
+		readonly Gauge runningJobs;
+
 		/// <summary>
 		/// <see langword="lock"/> <see cref="object"/> for various operations.
 		/// </summary>
@@ -87,19 +99,25 @@ namespace Tgstation.Server.Host.Jobs
 		/// </summary>
 		/// <param name="hub">The value of <see cref="hub"/>.</param>
 		/// <param name="databaseContextFactory">The value of <see cref="databaseContextFactory"/>.</param>
+		/// <param name="metricFactory">The <see cref="IMetricFactory"/> to use.</param>
 		/// <param name="loggerFactory">The value of <see cref="loggerFactory"/>.</param>
 		/// <param name="logger">The value of <see cref="logger"/>.</param>
 		public JobService(
 			IConnectionMappedHubContext<JobsHub, IJobsHub> hub,
 			IDatabaseContextFactory databaseContextFactory,
+			IMetricFactory metricFactory,
 			ILoggerFactory loggerFactory,
 			ILogger<JobService> logger)
 		{
 			this.hub = hub ?? throw new ArgumentNullException(nameof(hub));
 			this.databaseContextFactory = databaseContextFactory ?? throw new ArgumentNullException(nameof(databaseContextFactory));
 			this.loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory));
+			ArgumentNullException.ThrowIfNull(metricFactory);
 			this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
 
+			runningJobs = metricFactory.CreateGauge("tgs_jobs_running", "The number of TGS jobs running across all instances");
+			processedJobs = metricFactory.CreateCounter("tgs_jobs_processed", "The number of TGS jobs that have run previously");
+
 			jobs = new Dictionary<long, JobHandler>();
 			hubUpdateActions = new Dictionary<long, Action>();
 			activationTcs = new TaskCompletionSource<IInstanceCoreProvider>();
@@ -363,6 +381,7 @@ namespace Tgstation.Server.Host.Jobs
 		{
 			var jid = job.Require(x => x.Id);
 			using (LogContext.PushProperty(SerilogContextHelper.JobIdContextProperty, jid))
+			using (runningJobs.TrackInProgress())
 				try
 				{
 					void LogException(Exception ex) => logger.LogDebug(ex, "Job {jobId} exited with error!", jid);
@@ -553,6 +572,8 @@ namespace Tgstation.Server.Host.Jobs
 						jobs.Remove(jid);
 						handler.Dispose();
 					}
+
+					processedJobs.Inc();
 				}
 		}
 	}

src/Tgstation.Server.Host/Security/TgsAuthorizeAttribute.cs

@@ -15,10 +15,15 @@ namespace Tgstation.Server.Host.Security
 	[AttributeUsage(AttributeTargets.Class | AttributeTargets.Method, AllowMultiple = true, Inherited = true)]
 	sealed class TgsAuthorizeAttribute : AuthorizeAttribute
 	{
+		/// <summary>
+		/// Policy used to apply global requirement of <see cref="UserEnabledRole"/>.
+		/// </summary>
+		public const string PolicyName = "Policy.UserEnabled";
+
 		/// <summary>
 		/// Role used to indicate access to the server is allowed.
 		/// </summary>
-		public const string UserEnabledRole = "Core.UserEnabled";
+		public const string UserEnabledRole = "Role.UserEnabled";
 
 		/// <summary>
 		/// Gets the <see cref="Api.Rights.RightsType"/> associated with the <see cref="TgsAuthorizeAttribute"/> if any.
@@ -130,8 +135,12 @@ namespace Tgstation.Server.Host.Security
 		private TgsAuthorizeAttribute(IEnumerable<string> roles)
 		{
 			var listRoles = roles.ToList();
-			listRoles.Add(UserEnabledRole);
-			Roles = String.Join(",", listRoles);
+			if (listRoles.Count != 0)
+			{
+				Roles = String.Join(",", listRoles);
+			}
+
+			Policy = PolicyName;
 		}
 	}
 }

src/Tgstation.Server.Host/Security/TgsGraphQLAuthorizeAttribute.cs

@@ -125,8 +125,12 @@ namespace Tgstation.Server.Host.Security
 		private TgsGraphQLAuthorizeAttribute(IEnumerable<string> roleNames)
 		{
 			var listRoles = roleNames.ToList();
-			listRoles.Add(TgsAuthorizeAttribute.UserEnabledRole);
-			Roles = [.. listRoles];
+			if (listRoles.Count != 0)
+			{
+				Roles = [.. listRoles];
+			}
+
+			Policy = TgsAuthorizeAttribute.PolicyName;
 			Apply = ApplyPolicy.Validation;
 		}
 	}

src/Tgstation.Server.Host/System/IProcessBase.cs

@@ -24,6 +24,12 @@ namespace Tgstation.Server.Host.System
 		/// </summary>
 		long? MemoryUsage { get; }
 
+		/// <summary>
+		/// Gets the estimated CPU usage fraction of the process based on the last time this was called.
+		/// </summary>
+		/// <returns>The CPU's estimated usage as a value between 0 and 1.</returns>
+		double MeasureProcessorTimeDelta();
+
 		/// <summary>
 		/// Set's the owned <see cref="global::System.Diagnostics.Process.PriorityClass"/> to a non-normal value.
 		/// </summary>

src/Tgstation.Server.Host/System/Process.cs

@@ -87,10 +87,30 @@ namespace Tgstation.Server.Host.System
 		/// </summary>
 		readonly Task<string?>? readTask;
 
+		/// <summary>
+		/// <see langword="lock"/> object for measuring processor time usage.
+		/// </summary>
+		readonly object processTimeMeasureLock;
+
+		/// <summary>
+		/// The last time <see cref="MeasureProcessorTimeDelta"/> was called.
+		/// </summary>
+		DateTimeOffset lastProcessorMeasureTime;
+
+		/// <summary>
+		/// The last value of <see cref="global::System.Diagnostics.Process.TotalProcessorTime"/>.
+		/// </summary>
+		TimeSpan lastProcessorUsageTime;
+
+		/// <summary>
+		/// The last valid return value of <see cref="MeasureProcessorTimeDelta"/>.
+		/// </summary>
+		double lastProcessorUsageEstimation;
+
 		/// <summary>
 		/// If the <see cref="Process"/> was disposed.
 		/// </summary>
-		volatile int disposed;
+		int disposed;
 
 		/// <summary>
 		/// Initializes a new instance of the <see cref="Process"/> class.
@@ -123,6 +143,8 @@ namespace Tgstation.Server.Host.System
 
 			this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
 
+			processTimeMeasureLock = new object();
+
 			Lifetime = WrapLifetimeTask();
 
 			if (preExisting)
@@ -147,6 +169,8 @@ namespace Tgstation.Server.Host.System
 				DefaultIOManager.BlockingTaskCreationOptions,
 				TaskScheduler.Current);
 
+			MeasureProcessorTimeDelta();
+
 			logger.LogTrace("Created process ID: {pid}", Id);
 		}
 
@@ -267,6 +291,34 @@ namespace Tgstation.Server.Host.System
 			return processFeatures.CreateDump(handle, outputFile, minidump, cancellationToken);
 		}
 
+		/// <inheritdoc />
+		public double MeasureProcessorTimeDelta()
+		{
+			lock (processTimeMeasureLock)
+			{
+				try
+				{
+					var now = DateTimeOffset.UtcNow;
+					var newTime = handle.TotalProcessorTime;
+
+					var timeDelta = now - lastProcessorMeasureTime;
+					var newUsage = newTime - lastProcessorUsageTime;
+
+					lastProcessorMeasureTime = now;
+					lastProcessorUsageTime = newTime;
+
+					if (timeDelta != TimeSpan.Zero)
+						lastProcessorUsageEstimation = newUsage / (Environment.ProcessorCount * timeDelta);
+				}
+				catch (Exception ex)
+				{
+					logger.LogWarning(ex, "Error measuring processor time delta!");
+				}
+
+				return lastProcessorUsageEstimation;
+			}
+		}
+
 		/// <summary>
 		/// Attaches a log message to the process' exit event.
 		/// </summary>

src/Tgstation.Server.Host/Tgstation.Server.Host.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk.Web">
+<Project Sdk="Microsoft.NET.Sdk.Web">
   <Import Project="../../build/SrcCommon.props" />
 
   <PropertyGroup>
@@ -97,26 +97,24 @@
     <PackageReference Include="DotEnv.Core" Version="3.1.0" />
     <!-- Usage: Text formatter for Elasticsearch logging plugin -->
     <PackageReference Include="Elastic.CommonSchema.Serilog" Version="8.12.3" />
-    <!-- Usage: GitLab interop -->
-    <PackageReference Include="GitLabApiClient" Version="1.8.0" />
     <!-- Usage: GraphQL API Engine -->
-    <PackageReference Include="HotChocolate.AspNetCore" Version="14.3.0" />
+    <PackageReference Include="HotChocolate.AspNetCore" Version="15.0.3" />
     <!-- Usage: GraphQL Authorization Plugin -->
-    <PackageReference Include="HotChocolate.AspNetCore.Authorization" Version="14.3.0" />
+    <PackageReference Include="HotChocolate.AspNetCore.Authorization" Version="15.0.3" />
     <!-- Usage: GraphQL IDatabaseContext support -->
-    <PackageReference Include="HotChocolate.Data.EntityFramework" Version="14.3.0" />
+    <PackageReference Include="HotChocolate.Data.EntityFramework" Version="15.0.3" />
     <!-- Usage: DataLoader source generation -->
-    <PackageReference Include="HotChocolate.Types.Analyzers" Version="14.3.0" />
+    <PackageReference Include="HotChocolate.Types.Analyzers" Version="15.0.3" />
     <!-- Usage: GraphQL additional scalar type definitions -->
-    <PackageReference Include="HotChocolate.Types.Scalars" Version="14.3.0" />
+    <PackageReference Include="HotChocolate.Types.Scalars" Version="15.0.3" />
     <!-- Usage: git interop -->
     <PackageReference Include="LibGit2Sharp" Version="0.31.0" />
     <!-- Usage: Support ""legacy"" Newotonsoft.Json in HTTP pipeline. The rest of our codebase uses Newtonsoft. -->
-    <PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="8.0.11" />
+    <PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="8.0.13" />
     <!-- Usage: Using target JSON serializer for API -->
-    <PackageReference Include="Microsoft.AspNetCore.SignalR.Protocols.NewtonsoftJson" Version="9.0.0" />
+    <PackageReference Include="Microsoft.AspNetCore.SignalR.Protocols.NewtonsoftJson" Version="9.0.2" />
     <!-- Usage: Generating dumps of dotnet engine processes -->
-    <PackageReference Include="Microsoft.Diagnostics.NETCore.Client" Version="0.2.553101" />
+    <PackageReference Include="Microsoft.Diagnostics.NETCore.Client" Version="0.2.607501" />
     <!-- Usage: Database ORM -->
     <PackageReference Include="Microsoft.EntityFrameworkCore" Version="9.0.0" />
     <!-- Usage: Automatic migration generation using command line -->
@@ -127,6 +125,8 @@
     <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="9.0.0" />
     <!-- Usage: MSSQL ORM plugin -->
     <PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer" Version="9.0.0" />
+    <!-- Usage: Database connectivity health check -->
+    <PackageReference Include="Microsoft.Extensions.Diagnostics.HealthChecks.EntityFrameworkCore" Version="9.0.0" />
     <!-- Usage: POSIX support for syscalls, signals, and symlinks -->
     <PackageReference Include="Mono.Posix.NETStandard" Version="1.0.0" />
     <!-- Usage: Cron string parsing -->
@@ -137,8 +137,10 @@
     <PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="9.0.2" />
     <!-- Usage: MYSQL/MariaDB ORM plugin -->
     <PackageReference Include="Pomelo.EntityFrameworkCore.MySql" Version="9.0.0-preview.2.efcore.9.0.0" />
+    <!-- Usage: Publishing Prometheus metrics -->
+    <PackageReference Include="prometheus-net.AspNetCore.HealthChecks" Version="8.2.1" />
     <!-- Usage: Discord interop -->
-    <PackageReference Include="Remora.Discord" Version="2024.3.0" />
+    <PackageReference Include="Remora.Discord" Version="2025.1.0" />
     <!-- Usage: Rich logger builder -->
     <PackageReference Include="Serilog.Extensions.Logging" Version="9.0.0" />
     <!-- Usage: Async logging plugin -->
@@ -150,20 +152,21 @@
     <!-- Usage: File logging plugin -->
     <PackageReference Include="Serilog.Sinks.File" Version="6.0.0" />
     <!-- Usage: OpenAPI spec generator -->
-    <PackageReference Include="Swashbuckle.AspNetCore" Version="7.2.0" />
+    <PackageReference Include="Swashbuckle.AspNetCore" Version="7.3.1" />
     <!-- Usage: Newtonsoft.Json plugin for OpenAPI spec generator -->
-    <PackageReference Include="Swashbuckle.AspNetCore.Newtonsoft" Version="7.2.0" />
+    <PackageReference Include="Swashbuckle.AspNetCore.Newtonsoft" Version="7.3.1" />
     <!-- Usage: Windows authentication plugin allowing searching for users by name -->
-    <PackageReference Include="System.DirectoryServices.AccountManagement" Version="9.0.0" />
+    <PackageReference Include="System.DirectoryServices.AccountManagement" Version="9.0.2" />
     <!-- Usage: Identifying owning user of Windows Process objects -->
-    <PackageReference Include="System.Management" Version="9.0.0" />
+    <PackageReference Include="System.Management" Version="9.0.2" />
     <!-- Usage: Temporary resolution to compatibility issues with EFCore 7 and .NET 8 -->
-    <PackageReference Include="System.Security.Permissions" Version="9.0.0" />
+    <PackageReference Include="System.Security.Permissions" Version="9.0.2" />
   </ItemGroup>
 
   <ItemGroup>
     <ProjectReference Include="..\Tgstation.Server.Api\Tgstation.Server.Api.csproj" />
     <ProjectReference Include="..\Tgstation.Server.Host.Common\Tgstation.Server.Host.Common.csproj" />
+    <ProjectReference Include="..\Tgstation.Server.Host.Utils.GitLab.GraphQL\Tgstation.Server.Host.Utils.GitLab.GraphQL.csproj" />
     <ProjectReference Include="..\Tgstation.Server.Shared\Tgstation.Server.Shared.csproj" />
   </ItemGroup>
 

src/Tgstation.Server.Host/Utils/GitHub/GitHubClientFactory.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.IdentityModel.Tokens.Jwt;
 using System.Linq;
+using System.Net.Http;
 using System.Security.Cryptography;
 using System.Text;
 using System.Threading;
@@ -12,6 +13,7 @@ using Microsoft.Extensions.Options;
 using Microsoft.IdentityModel.Tokens;
 
 using Octokit;
+using Octokit.Internal;
 
 using Tgstation.Server.Api.Models;
 using Tgstation.Server.Host.Configuration;
@@ -28,6 +30,11 @@ namespace Tgstation.Server.Host.Utils.GitHub
 		/// <remarks>Set to app installation token lifetime, which is the lowest. See https://docs.github.com/en/apps/creating-github-apps/authenticating-with-a-github-app/generating-an-installation-access-token-for-a-github-app.</remarks>
 		const uint ClientCacheHours = 1;
 
+		/// <summary>
+		/// Minutes before tokens expire before not using them.
+		/// </summary>
+		const uint AppTokenExpiryGraceMinutes = 15;
+
 		/// <summary>
 		/// The <see cref="clientCache"/> <see cref="KeyValuePair{TKey, TValue}.Key"/> used in place of <see langword="null"/> when accessing a configuration-based client with no token set in <see cref="GeneralConfiguration.GitHubAccessToken"/>.
 		/// </summary>
@@ -38,6 +45,11 @@ namespace Tgstation.Server.Host.Utils.GitHub
 		/// </summary>
 		readonly IAssemblyInformationProvider assemblyInformationProvider;
 
+		/// <summary>
+		/// The <see cref="IHttpMessageHandlerFactory"/> for the <see cref="GitHubClientFactory"/>.
+		/// </summary>
+		readonly IHttpMessageHandlerFactory httpMessageHandlerFactory;
+
 		/// <summary>
 		/// The <see cref="ILogger"/> for the <see cref="GitHubClientFactory"/>.
 		/// </summary>
@@ -49,9 +61,9 @@ namespace Tgstation.Server.Host.Utils.GitHub
 		readonly GeneralConfiguration generalConfiguration;
 
 		/// <summary>
-		/// Cache of created <see cref="GitHubClient"/>s and last used times, keyed by access token.
+		/// Cache of created <see cref="GitHubClient"/>s and last used/expiry times, keyed by access token.
 		/// </summary>
-		readonly Dictionary<string, (GitHubClient Client, DateTimeOffset LastUsed)> clientCache;
+		readonly Dictionary<string, (GitHubClient Client, DateTimeOffset LastUsed, DateTimeOffset? Expiry)> clientCache;
 
 		/// <summary>
 		/// The <see cref="SemaphoreSlim"/> used to guard access to <see cref="clientCache"/>.
@@ -62,18 +74,21 @@ namespace Tgstation.Server.Host.Utils.GitHub
 		/// Initializes a new instance of the <see cref="GitHubClientFactory"/> class.
 		/// </summary>
 		/// <param name="assemblyInformationProvider">The value of <see cref="assemblyInformationProvider"/>.</param>
+		/// <param name="httpMessageHandlerFactory">The value of <see cref="httpMessageHandlerFactory"/>.</param>
 		/// <param name="logger">The value of <see cref="logger"/>.</param>
 		/// <param name="generalConfigurationOptions">The <see cref="IOptions{TOptions}"/> containing the value of <see cref="generalConfiguration"/>.</param>
 		public GitHubClientFactory(
 			IAssemblyInformationProvider assemblyInformationProvider,
+			IHttpMessageHandlerFactory httpMessageHandlerFactory,
 			ILogger<GitHubClientFactory> logger,
 			IOptions<GeneralConfiguration> generalConfigurationOptions)
 		{
 			this.assemblyInformationProvider = assemblyInformationProvider ?? throw new ArgumentNullException(nameof(assemblyInformationProvider));
+			this.httpMessageHandlerFactory = httpMessageHandlerFactory ?? throw new ArgumentNullException(nameof(httpMessageHandlerFactory));
 			this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
 			generalConfiguration = generalConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(generalConfigurationOptions));
 
-			clientCache = new Dictionary<string, (GitHubClient, DateTimeOffset)>();
+			clientCache = new Dictionary<string, (GitHubClient, DateTimeOffset, DateTimeOffset?)>();
 			clientCacheSemaphore = new SemaphoreSlim(1, 1);
 		}
 
@@ -127,13 +142,20 @@ namespace Tgstation.Server.Host.Utils.GitHub
 				else
 					cacheKey = accessString;
 
-				cacheHit = clientCache.TryGetValue(cacheKey, out var tuple);
-
 				var now = DateTimeOffset.UtcNow;
-				if (!cacheHit)
+				cacheHit = clientCache.TryGetValue(cacheKey, out var tuple);
+				var tokenValid = cacheHit && (!tuple.Expiry.HasValue || tuple.Expiry.Value <= now);
+				if (!tokenValid)
 				{
+					if (cacheHit)
+					{
+						logger.LogDebug("Previously cached GitHub token has expired!");
+						clientCache.Remove(cacheKey);
+					}
+
 					logger.LogTrace("Creating new GitHubClient...");
 
+					DateTimeOffset? expiry = null;
 					if (accessString != null)
 					{
 						if (accessString.StartsWith(RepositorySettings.TgsAppPrivateKeyPrefix))
@@ -167,6 +189,7 @@ namespace Tgstation.Server.Host.Utils.GitHub
 								var installToken = await client.GitHubApps.CreateInstallationToken(installation.Id);
 
 								client.Credentials = new Credentials(installToken.Token);
+								expiry = installToken.ExpiresAt.AddMinutes(-AppTokenExpiryGraceMinutes);
 							}
 							catch (Exception ex)
 							{
@@ -183,7 +206,7 @@ namespace Tgstation.Server.Host.Utils.GitHub
 					else
 						client = CreateUnauthenticatedClient();
 
-					clientCache.Add(cacheKey, (Client: client, LastUsed: now));
+					clientCache.Add(cacheKey, (Client: client, LastUsed: now, Expiry: expiry));
 					lastUsed = null;
 				}
 				else
@@ -203,7 +226,7 @@ namespace Tgstation.Server.Host.Utils.GitHub
 						continue; // save the hash lookup
 
 					tuple = clientCache[key];
-					if (tuple.LastUsed <= purgeAfter)
+					if (tuple.LastUsed <= purgeAfter || (tuple.Expiry.HasValue && tuple.Expiry.Value <= now))
 					{
 						clientCache.Remove(key);
 						++purgeCount;
@@ -315,10 +338,33 @@ namespace Tgstation.Server.Host.Utils.GitHub
 		GitHubClient CreateUnauthenticatedClient()
 		{
 			var product = assemblyInformationProvider.ProductInfoHeaderValue.Product!;
-			return new GitHubClient(
-				new ProductHeaderValue(
-					product.Name,
-					product.Version));
+#pragma warning disable CA2000 // Dispose objects before losing scope
+			var handler = httpMessageHandlerFactory.CreateHandler();
+			try
+			{
+				var clientAdapter = new HttpClientAdapter(() => handler);
+#pragma warning restore CA2000 // Dispose objects before losing scope
+				handler = null;
+				try
+				{
+					return new GitHubClient(
+						new Connection(
+							new ProductHeaderValue(
+								product.Name,
+								product.Version),
+							clientAdapter));
+				}
+				catch
+				{
+					clientAdapter.Dispose();
+					throw;
+				}
+			}
+			catch
+			{
+				handler?.Dispose();
+				throw;
+			}
 		}
 	}
 }