Implement server restarts and updates via GraphQL

src/Tgstation.Server.Client.GraphQL/GQL/Mutations/RepositoryBasedServerUpdate.graphql

@@ -0,0 +1,11 @@
+mutation RepositoryBasedServerUpdate($targetVersion: Semver!) {
+  changeServerVersionViaTrackedRepository(input: { targetVersion: $targetVersion }) {
+    errors {
+      ... on ErrorMessageError {
+        additionalData
+        errorCode
+        message
+      }
+    }
+  }
+}

src/Tgstation.Server.Client.GraphQL/GQL/Mutations/RestartServer.graphql

@@ -0,0 +1,11 @@
+mutation RestartServer() {
+  restartServer() {
+    errors {
+      ... on ErrorMessageError {
+        additionalData
+        errorCode
+        message
+      }
+    }
+  }
+}

src/Tgstation.Server.Client.GraphQL/GQL/Queries/GetUpdateInformation.graphql

@@ -0,0 +1,10 @@
+query GetUpdateInformation($forceFresh: Boolean!) {
+  swarm {
+    updateInformation {
+      generatedAt
+      latestVersion(forceFresh: $forceFresh)
+      updateInProgress
+      trackedRepositoryUrl(forceFresh: $forceFresh)
+    }
+  }
+}

src/Tgstation.Server.Host/GraphQL/Mutations/AdministrationMutations.cs

@@ -0,0 +1,84 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+
+using HotChocolate;
+using HotChocolate.Types;
+
+using Tgstation.Server.Api.Models.Response;
+using Tgstation.Server.Api.Rights;
+using Tgstation.Server.Host.Authority;
+using Tgstation.Server.Host.GraphQL.Scalars;
+using Tgstation.Server.Host.Security;
+
+namespace Tgstation.Server.Host.GraphQL.Mutations
+{
+	/// <summary>
+	/// <see cref="IAdministrationAuthority"/> related <see cref="Mutation"/>s.
+	/// </summary>
+	[ExtendObjectType(typeof(Mutation))]
+	[GraphQLDescription(Mutation.GraphQLDescription)]
+	public sealed class AdministrationMutations
+	{
+		/// <summary>
+		/// Restarts the server node without terminating running game instances.
+		/// </summary>
+		/// <param name="administrationAuthority">The <see cref="IGraphQLAuthorityInvoker{TAuthority}"/> for the <see cref="IAdministrationAuthority"/>.</param>
+		/// <returns>A <see cref="ValueTask"/> representing the running operation.</returns>
+		[TgsGraphQLAuthorize<IAdministrationAuthority>(nameof(IAdministrationAuthority.TriggerServerRestart))]
+		[Error(typeof(ErrorMessageException))]
+		public async ValueTask<Query> RestartServer(
+			[Service] IGraphQLAuthorityInvoker<IAdministrationAuthority> administrationAuthority)
+		{
+			ArgumentNullException.ThrowIfNull(administrationAuthority);
+			await administrationAuthority.Invoke(
+				authority => authority.TriggerServerRestart());
+
+			return new Query();
+		}
+
+		/// <summary>
+		/// Restarts the server node without terminating running game instances and changes its <paramref name="targetVersion"/>.
+		/// </summary>
+		/// <param name="targetVersion">The semver of the server <see cref="Version"/> available in the tracked repository to switch to.</param>
+		/// <param name="administrationAuthority">The <see cref="IGraphQLAuthorityInvoker{TAuthority}"/> for the <see cref="IAdministrationAuthority"/>.</param>
+		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
+		/// <returns>A <see cref="ValueTask"/> representing the running operation.</returns>
+		[TgsGraphQLAuthorize(AdministrationRights.ChangeVersion)]
+		[Error(typeof(ErrorMessageException))]
+		public async ValueTask<Query> ChangeServerVersionViaTrackedRepository(
+			Version targetVersion,
+			[Service] IGraphQLAuthorityInvoker<IAdministrationAuthority> administrationAuthority,
+			CancellationToken cancellationToken)
+		{
+			ArgumentNullException.ThrowIfNull(targetVersion);
+			ArgumentNullException.ThrowIfNull(administrationAuthority);
+			await administrationAuthority.Invoke<ServerUpdateResponse, ServerUpdateResponse>(
+				authority => authority.TriggerServerVersionChange(targetVersion, false, cancellationToken));
+			return new Query();
+		}
+
+		/// <summary>
+		/// Restarts the server node without terminating running game instances and changes its <paramref name="targetVersion"/>.
+		/// </summary>
+		/// <param name="targetVersion">The semver of the server <see cref="Version"/> available in the tracked repository to switch to.</param>
+		/// <param name="administrationAuthority">The <see cref="IGraphQLAuthorityInvoker{TAuthority}"/> for the <see cref="IAdministrationAuthority"/>.</param>
+		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
+		/// <returns>A FileTicket that should be used to upload a zip containing the update data to the file transfer service.</returns>
+		[TgsGraphQLAuthorize(AdministrationRights.UploadVersion)]
+		[Error(typeof(ErrorMessageException))]
+		[GraphQLType<FileUploadTicketType>]
+		public async ValueTask<string> ChangeServerVersionViaUpload(
+			Version targetVersion,
+			[Service] IGraphQLAuthorityInvoker<IAdministrationAuthority> administrationAuthority,
+			CancellationToken cancellationToken)
+		{
+			ArgumentNullException.ThrowIfNull(targetVersion);
+			ArgumentNullException.ThrowIfNull(administrationAuthority);
+			var response = await administrationAuthority.Invoke<ServerUpdateResponse, FileTicketResponse>(
+				authority => authority.TriggerServerVersionChange(targetVersion, true, cancellationToken));
+
+			return response.FileTicket ?? throw new InvalidOperationException("Administration authority did not generate a FileUploadTicket!");
+		}
+	}
+}

src/Tgstation.Server.Host/GraphQL/Types/ServerSwarm.cs

@@ -7,7 +7,6 @@ using HotChocolate;
 using Microsoft.Extensions.Options;
 
 using Tgstation.Server.Host.Configuration;
-using Tgstation.Server.Host.Core;
 using Tgstation.Server.Host.GraphQL.Interfaces;
 using Tgstation.Server.Host.Properties;
 using Tgstation.Server.Host.Security;
@@ -20,19 +19,6 @@ namespace Tgstation.Server.Host.GraphQL.Types
 	/// </summary>
 	public sealed class ServerSwarm
 	{
-		/// <summary>
-		/// If there is a swarm update in progress.
-		/// </summary>
-		/// <param name="serverControl">The <see cref="IServerControl"/> to use.</param>
-		/// <returns><see langword="true"/> if there is an update in progress, <see langword="false"/> otherwise.</returns>
-		[TgsGraphQLAuthorize]
-		public bool UpdateInProgress(
-			[Service] IServerControl serverControl)
-		{
-			ArgumentNullException.ThrowIfNull(serverControl);
-			return serverControl.UpdateInProgress;
-		}
-
 		/// <summary>
 		/// Gets the swarm protocol major version in use.
 		/// </summary>
@@ -78,5 +64,12 @@ namespace Tgstation.Server.Host.GraphQL.Types
 			ArgumentNullException.ThrowIfNull(swarmService);
 			return swarmService.GetSwarmServers()?.Select(x => new SwarmNode(x)).ToList();
 		}
+
+		/// <summary>
+		/// Gets the <see cref="Types.UpdateInformation"/> for the swarm.
+		/// </summary>
+		/// <returns>A new <see cref="Types.UpdateInformation"/>.</returns>
+		[TgsGraphQLAuthorize]
+		public UpdateInformation UpdateInformation() => new();
 	}
 }

src/Tgstation.Server.Host/GraphQL/Types/UpdateInformation.cs

@@ -0,0 +1,117 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+
+using HotChocolate;
+
+using Tgstation.Server.Api.Models.Response;
+using Tgstation.Server.Host.Authority;
+using Tgstation.Server.Host.Core;
+using Tgstation.Server.Host.Utils;
+
+namespace Tgstation.Server.Host.GraphQL.Types
+{
+	/// <summary>
+	/// Gets information about updates for the <see cref="ServerSwarm"/>.
+	/// </summary>
+	public sealed class UpdateInformation : IDisposable
+	{
+		/// <summary>
+		/// <see cref="SemaphoreSlim"/> to prevent duplicate cache generations in one query.
+		/// </summary>
+		readonly SemaphoreSlim cacheReadSemaphore;
+
+		/// <summary>
+		/// If the cache was already force generated this query.
+		/// </summary>
+		bool cacheForceGenerated;
+
+		/// <summary>
+		/// Initializes a new instance of the <see cref="UpdateInformation"/> class.
+		/// </summary>
+		public UpdateInformation()
+		{
+			cacheReadSemaphore = new SemaphoreSlim(1, 1);
+		}
+
+		/// <inheritdoc />
+		public void Dispose()
+			=> cacheReadSemaphore.Dispose();
+
+		/// <summary>
+		/// If there is a swarm update in progress. This is not affected by <see cref="GeneratedAt(IGraphQLAuthorityInvoker{IAdministrationAuthority}, CancellationToken)"/>.
+		/// </summary>
+		/// <param name="serverControl">The <see cref="IServerControl"/> to use.</param>
+		/// <returns><see langword="true"/> if there is an update in progress, <see langword="false"/> otherwise.</returns>
+		public bool UpdateInProgress(
+			[Service] IServerControl serverControl)
+		{
+			ArgumentNullException.ThrowIfNull(serverControl);
+			return serverControl.UpdateInProgress;
+		}
+
+		/// <summary>
+		/// Gets the <see cref="Uri"/> of the GitHub repository updates are sourced from.
+		/// </summary>
+		/// <param name="forceFresh">If <see langword="true"/> the local cache TGS keeps of this data will be bypassed.</param>
+		/// <param name="administrationAuthority">The <see cref="IGraphQLAuthorityInvoker{TAuthority}"/> for the <see cref="IAdministrationAuthority"/> to use.</param>
+		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
+		/// <returns>The <see cref="Uri"/> of the GitHub repository updates are sourced from on success. <see langword="null"/> if a GitHub API error occurred.</returns>
+		public async ValueTask<Uri?> TrackedRepositoryUrl(
+			bool forceFresh,
+			[Service] IGraphQLAuthorityInvoker<IAdministrationAuthority> administrationAuthority,
+			CancellationToken cancellationToken)
+			=> (await GetAdministrationResponseSafe(forceFresh, administrationAuthority, cancellationToken)).TrackedRepositoryUrl;
+
+		/// <summary>
+		/// Gets the time the <see cref="UpdateInformation"/> was generated.
+		/// </summary>
+		/// <param name="administrationAuthority">The <see cref="IGraphQLAuthorityInvoker{TAuthority}"/> for the <see cref="IAdministrationAuthority"/> to use.</param>
+		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
+		/// <returns>The time the <see cref="UpdateInformation"/> was generated on success. <see langword="null"/> if a GitHub API error occurred.</returns>
+		public async ValueTask<DateTimeOffset?> GeneratedAt(
+			[Service] IGraphQLAuthorityInvoker<IAdministrationAuthority> administrationAuthority,
+			CancellationToken cancellationToken)
+			=> (await GetAdministrationResponseSafe(false, administrationAuthority, cancellationToken)).GeneratedAt;
+
+		/// <summary>
+		/// Gets the latest <see cref="Version"/> of tgstation-server available on the GitHub repository updates are sourced from.
+		/// </summary>
+		/// <param name="forceFresh">If <see langword="true"/> the local cache TGS keeps of this data will be bypassed.</param>
+		/// <param name="administrationAuthority">The <see cref="IGraphQLAuthorityInvoker{TAuthority}"/> for the <see cref="IAdministrationAuthority"/> to use.</param>
+		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
+		/// <returns>The <see cref="Version"/> of the latest TGS version on success. <see langword="null"/> if a GitHub API error occurred.</returns>
+		public async ValueTask<Version?> LatestVersion(
+			bool forceFresh,
+			[Service] IGraphQLAuthorityInvoker<IAdministrationAuthority> administrationAuthority,
+			CancellationToken cancellationToken)
+			=> (await GetAdministrationResponseSafe(forceFresh, administrationAuthority, cancellationToken)).LatestVersion;
+
+		/// <summary>
+		/// Safely retrieve the <see cref="AdministrationResponse"/> from a given <paramref name="administrationAuthority"/> without generating the cache multiple times in one query.
+		/// </summary>
+		/// <param name="forceFresh">If <see langword="true"/> the local cache TGS keeps of this data will be bypassed.</param>
+		/// <param name="administrationAuthority">The <see cref="IGraphQLAuthorityInvoker{TAuthority}"/> for the <see cref="IAdministrationAuthority"/> to use.</param>
+		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
+		/// <returns>A <see cref="ValueTask{TResult}"/> resulting in the <see cref="AdministrationResponse"/> from the <paramref name="administrationAuthority"/>.</returns>
+		async ValueTask<AdministrationResponse> GetAdministrationResponseSafe(
+			bool forceFresh,
+			IGraphQLAuthorityInvoker<IAdministrationAuthority> administrationAuthority,
+			CancellationToken cancellationToken)
+		{
+			using (await SemaphoreSlimContext.Lock(cacheReadSemaphore, cancellationToken))
+			{
+				if (cacheForceGenerated)
+					forceFresh = false;
+				else
+					cacheForceGenerated |= forceFresh;
+
+				ArgumentNullException.ThrowIfNull(administrationAuthority);
+				var response = await administrationAuthority.Invoke<AdministrationResponse, AdministrationResponse>(
+					authority => authority.GetUpdateInformation(forceFresh, cancellationToken));
+
+				return response;
+			}
+		}
+	}
+}

tests/Tgstation.Server.Tests/Live/AdministrationTest.cs

@@ -13,11 +13,13 @@ namespace Tgstation.Server.Tests.Live
 {
 	sealed class AdministrationTest
 	{
-		readonly IAdministrationClient client;
+		readonly IMultiServerClient client;
+		readonly IAdministrationClient restClient;
 
-		public AdministrationTest(IAdministrationClient client)
+		public AdministrationTest(MultiServerClient client)
 		{
 			this.client = client ?? throw new ArgumentNullException(nameof(client));
+			this.restClient = client.RestClient.Administration;
 		}
 
 		public async Task Run(CancellationToken cancellationToken)
@@ -29,24 +31,24 @@ namespace Tgstation.Server.Tests.Live
 
 		async Task TestLogs(CancellationToken cancellationToken)
 		{
-			var logs = await client.ListLogs(null, cancellationToken);
+			var logs = await restClient.ListLogs(null, cancellationToken);
 			Assert.AreNotEqual(0, logs.Count);
 			var logFile = logs[0];
 			Assert.IsNotNull(logFile);
 			Assert.IsFalse(string.IsNullOrWhiteSpace(logFile.Name));
 			Assert.IsNull(logFile.FileTicket);
 
-			var downloadedTuple = await client.GetLog(logFile, cancellationToken);
+			var downloadedTuple = await restClient.GetLog(logFile, cancellationToken);
 			Assert.AreEqual(logFile.Name, downloadedTuple.Item1.Name);
 			Assert.IsTrue(logFile.LastModified <= downloadedTuple.Item1.LastModified);
 			Assert.IsNull(logFile.FileTicket);
 
-			await ApiAssert.ThrowsException<ConflictException, Tuple<LogFileResponse, Stream>>(() => client.GetLog(new LogFileResponse
+			await ApiAssert.ThrowsException<ConflictException, Tuple<LogFileResponse, Stream>>(() => restClient.GetLog(new LogFileResponse
 			{
 				Name = "very_fake_path.log"
 			}, cancellationToken), ErrorCode.IOError);
 
-			await ApiAssert.ThrowsException<InsufficientPermissionsException, Tuple<LogFileResponse, Stream>>(() => client.GetLog(new LogFileResponse
+			await ApiAssert.ThrowsException<InsufficientPermissionsException, Tuple<LogFileResponse, Stream>>(() => restClient.GetLog(new LogFileResponse
 			{
 				Name = "../out_of_bounds.file"
 			}, cancellationToken));
@@ -54,24 +56,59 @@ namespace Tgstation.Server.Tests.Live
 
 		async Task TestRead(CancellationToken cancellationToken)
 		{
-			var model = await client.Read(false, cancellationToken);
+			await client.Execute(
+				async restServerClient =>
+				{
+					var restClient = restServerClient.Administration;
 
-			//we've released a few 5.x versions now, check the release checker is at least somewhat functional
-			Assert.IsTrue(4 < model.LatestVersion.Major);
-			Assert.IsNotNull(model.TrackedRepositoryUrl);
-			Assert.IsTrue(model.GeneratedAt.HasValue);
-			Assert.IsTrue(model.GeneratedAt.Value <= DateTimeOffset.UtcNow);
+					var model = await restClient.Read(false, cancellationToken);
 
-			// test the cache
-			var newerModel = await client.Read(false, cancellationToken);
-			Assert.AreEqual(model.GeneratedAt, newerModel.GeneratedAt);
+					//we've released a few 5.x versions now, check the release checker is at least somewhat functional
+					Assert.IsNotNull(model.LatestVersion);
+					Assert.IsTrue(4 < model.LatestVersion.Major);
+					Assert.IsNotNull(model.TrackedRepositoryUrl);
+					Assert.IsTrue(model.GeneratedAt.HasValue);
+					Assert.IsTrue(model.GeneratedAt.Value <= DateTimeOffset.UtcNow);
 
-			await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken);
+					// test the cache
+					var newerModel = await restClient.Read(false, cancellationToken);
+					Assert.AreEqual(model.GeneratedAt, newerModel.GeneratedAt);
 
-			var newestModel = await client.Read(true, cancellationToken);
-			Assert.AreNotEqual(model.GeneratedAt, newestModel.GeneratedAt);
-			Assert.IsNotNull(newestModel.GeneratedAt);
-			Assert.IsTrue(model.GeneratedAt < newestModel.GeneratedAt);
+					await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken);
+
+					var newestModel = await restClient.Read(true, cancellationToken);
+					Assert.AreNotEqual(model.GeneratedAt, newestModel.GeneratedAt);
+					Assert.IsNotNull(newestModel.GeneratedAt);
+					Assert.IsTrue(model.GeneratedAt < newestModel.GeneratedAt);
+				},
+				async gqlClient =>
+				{
+					var queryResult = await gqlClient.RunQueryEnsureNoErrors(
+						gql => gql.GetUpdateInformation.ExecuteAsync(false, cancellationToken),
+						cancellationToken);
+
+					// we've released a few 5.x versions now, check the release checker is at least somewhat functional
+					Assert.IsNotNull(queryResult.Swarm.UpdateInformation.LatestVersion);
+					Assert.IsTrue(4 < queryResult.Swarm.UpdateInformation.LatestVersion.Major);
+					Assert.IsNotNull(queryResult.Swarm.UpdateInformation.TrackedRepositoryUrl);
+					Assert.IsTrue(queryResult.Swarm.UpdateInformation.GeneratedAt.HasValue);
+					Assert.IsTrue(queryResult.Swarm.UpdateInformation.GeneratedAt.Value <= DateTimeOffset.UtcNow);
+
+					// test the cache
+					var queryResult2 = await gqlClient.RunQueryEnsureNoErrors(
+						gql => gql.GetUpdateInformation.ExecuteAsync(false, cancellationToken),
+						cancellationToken);
+					Assert.AreEqual(queryResult.Swarm.UpdateInformation.GeneratedAt, queryResult2.Swarm.UpdateInformation.GeneratedAt);
+
+					await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken);
+					var queryResult3 = await gqlClient.RunQueryEnsureNoErrors(
+						gql => gql.GetUpdateInformation.ExecuteAsync(true, cancellationToken),
+						cancellationToken);
+
+					Assert.AreNotEqual(queryResult.Swarm.UpdateInformation.GeneratedAt, queryResult3.Swarm.UpdateInformation.GeneratedAt);
+					Assert.IsNotNull(queryResult3.Swarm.UpdateInformation.GeneratedAt);
+					Assert.IsTrue(queryResult.Swarm.UpdateInformation.GeneratedAt.Value < queryResult3.Swarm.UpdateInformation.GeneratedAt.Value);
+				});
 		}
 	}
 }

tests/Tgstation.Server.Tests/Live/TestLiveServer.cs

@@ -340,21 +340,29 @@ namespace Tgstation.Server.Tests.Live
 					}
 
 					//attempt to update to stable
-					var responseModel = await TestWithoutAndWithPermission(
-						() => adminClient.RestClient.Administration.Update(
-							new ServerUpdateRequest
-							{
-								NewVersion = TestUpdateVersion,
-								UploadZip = false,
-							},
-							null,
-							cancellationToken),
-						adminClient.RestClient,
-						AdministrationRights.ChangeVersion);
+					await adminClient.Execute(
+						async restClient =>
+						{
+							var responseModel = await TestWithoutAndWithPermission(
+								() => restClient.Administration.Update(
+									new ServerUpdateRequest
+									{
+										NewVersion = TestUpdateVersion,
+										UploadZip = false,
+									},
+									null,
+									cancellationToken),
+								adminClient.RestClient,
+								AdministrationRights.ChangeVersion);
 
-					Assert.IsNotNull(responseModel);
-					Assert.IsNull(responseModel.FileTicket);
-					Assert.AreEqual(TestUpdateVersion, responseModel.NewVersion);
+							Assert.IsNotNull(responseModel);
+							Assert.IsNull(responseModel.FileTicket);
+							Assert.AreEqual(TestUpdateVersion, responseModel.NewVersion);
+						},
+						async gqlClient => await gqlClient.RunMutationEnsureNoErrors(
+							gql => gql.RepositoryBasedServerUpdate.ExecuteAsync(TestUpdateVersion, cancellationToken),
+							result => result,
+							cancellationToken));
 
 					try
 					{
@@ -524,17 +532,25 @@ namespace Tgstation.Server.Tests.Live
 					CheckInfo(controllerInfo);
 
 					// test update
-					var responseModel = await controllerClient.RestClient.Administration.Update(
-						new ServerUpdateRequest
+					await controllerClient.Execute(
+						async restClient =>
 						{
-							NewVersion = TestUpdateVersion
-						},
-						null,
-						cancellationToken);
+							var responseModel = await restClient.Administration.Update(
+								new ServerUpdateRequest
+								{
+									NewVersion = TestUpdateVersion
+								},
+								null,
+								cancellationToken);
 
-					Assert.IsNotNull(responseModel);
-					Assert.IsNull(responseModel.FileTicket);
-					Assert.AreEqual(TestUpdateVersion, responseModel.NewVersion);
+							Assert.IsNotNull(responseModel);
+							Assert.IsNull(responseModel.FileTicket);
+							Assert.AreEqual(TestUpdateVersion, responseModel.NewVersion);
+						},
+						async gqlClient => await gqlClient.RunMutationEnsureNoErrors(
+							gql => gql.RepositoryBasedServerUpdate.ExecuteAsync(TestUpdateVersion, cancellationToken),
+							result => result,
+							cancellationToken));
 
 					await Task.WhenAny(Task.Delay(TimeSpan.FromMinutes(2)), serverTask);
 					Assert.IsTrue(serverTask.IsCompleted);
@@ -711,13 +727,18 @@ namespace Tgstation.Server.Tests.Live
 					await ApiAssert.ThrowsException<ConflictException, InstanceResponse>(() => node1Client.RestClient.Instances.GetId(controllerInstance, cancellationToken), Api.Models.ErrorCode.ResourceNotPresent);
 
 					// test update
-					await node1Client.RestClient.Administration.Update(
-						new ServerUpdateRequest
-						{
-							NewVersion = TestUpdateVersion
-						},
-						null,
-						cancellationToken);
+					await node1Client.Execute(
+						async restClient => await restClient.Administration.Update(
+							new ServerUpdateRequest
+							{
+								NewVersion = TestUpdateVersion
+							},
+							null,
+							cancellationToken),
+						async gqlClient => await gqlClient.RunMutationEnsureNoErrors(
+							gql => gql.RepositoryBasedServerUpdate.ExecuteAsync(TestUpdateVersion, cancellationToken),
+							result => result,
+							cancellationToken));
 					await Task.WhenAny(Task.Delay(TimeSpan.FromMinutes(2)), serverTask);
 					Assert.IsTrue(serverTask.IsCompleted);
 
@@ -752,13 +773,22 @@ namespace Tgstation.Server.Tests.Live
 					await using var controllerClient2 = await CreateAdminClient(controller.ApiUrl, cancellationToken);
 					await using var node1Client2 = await CreateAdminClient(node1.ApiUrl, cancellationToken);
 
-					await ApiAssert.ThrowsException<ApiConflictException, ServerUpdateResponse>(() => controllerClient2.RestClient.Administration.Update(
-						new ServerUpdateRequest
-						{
-							NewVersion = TestUpdateVersion
-						},
-						null,
-						cancellationToken), Api.Models.ErrorCode.SwarmIntegrityCheckFailed);
+					await controllerClient2.Execute(
+						async restClient => await ApiAssert.ThrowsException<ApiConflictException, ServerUpdateResponse>(
+							() => restClient.Administration.Update(
+								new ServerUpdateRequest
+								{
+									NewVersion = TestUpdateVersion
+								},
+								null,
+								cancellationToken),
+							Api.Models.ErrorCode.SwarmIntegrityCheckFailed),
+						async gqlClient => await ApiAssert.OperationFails(
+							gqlClient,
+							gql => gql.RepositoryBasedServerUpdate.ExecuteAsync(TestUpdateVersion, cancellationToken),
+							result => result,
+							Client.GraphQL.ErrorCode.SwarmIntegrityCheckFailed,
+							cancellationToken));
 
 					// regression: test updating also works from the controller
 					serverTask = Task.WhenAll(
@@ -954,7 +984,13 @@ namespace Tgstation.Server.Tests.Live
 					Assert.IsFalse(node2Info.SwarmServers.Any(x => x.Identifier == "node1"));
 
 					// restart the controller
-					await controllerClient.RestClient.Administration.Restart(cancellationToken);
+					await controllerClient.Execute(
+						restClient => restClient.Administration.Restart(cancellationToken),
+						async gqlClient => await gqlClient.RunMutationEnsureNoErrors(
+							gql => gql.RestartServer.ExecuteAsync(cancellationToken),
+							result => result,
+							cancellationToken));
+
 					await Task.WhenAny(
 						controllerTask,
 						Task.Delay(TimeSpan.FromMinutes(1), cancellationToken));
@@ -976,7 +1012,12 @@ namespace Tgstation.Server.Tests.Live
 					await Task.Delay(TimeSpan.FromSeconds(10), cancellationToken);
 
 					// restart node2
-					await node2Client.RestClient.Administration.Restart(cancellationToken);
+					await node2Client.Execute(
+						restClient => restClient.Administration.Restart(cancellationToken),
+						async gqlClient => await gqlClient.RunMutationEnsureNoErrors(
+							gql => gql.RestartServer.ExecuteAsync(cancellationToken),
+							result => result,
+							cancellationToken));
 					await Task.WhenAny(
 						node2Task,
 						Task.Delay(TimeSpan.FromMinutes(1)));
@@ -988,14 +1029,22 @@ namespace Tgstation.Server.Tests.Live
 					Assert.IsNull(controllerInfo.SwarmServers.SingleOrDefault(x => x.Identifier == "node2"));
 
 					// update should fail
-					await ApiAssert.ThrowsException<ApiConflictException, ServerUpdateResponse>(
-						() => controllerClient2.RestClient.Administration.Update(new ServerUpdateRequest
-						{
-							NewVersion = TestUpdateVersion
-						},
-						null,
-						cancellationToken),
-						Api.Models.ErrorCode.SwarmIntegrityCheckFailed);
+					await controllerClient2.Execute(
+						async restClient => await ApiAssert.ThrowsException<ApiConflictException, ServerUpdateResponse>(
+							() => restClient.Administration.Update(
+								new ServerUpdateRequest
+								{
+									NewVersion = TestUpdateVersion
+								},
+								null,
+								cancellationToken),
+							Api.Models.ErrorCode.SwarmIntegrityCheckFailed),
+						async gqlClient => await ApiAssert.OperationFails(
+							gqlClient,
+							gql => gql.RepositoryBasedServerUpdate.ExecuteAsync(TestUpdateVersion, cancellationToken),
+							result => result,
+							Client.GraphQL.ErrorCode.SwarmIntegrityCheckFailed,
+							cancellationToken));
 
 					node2Task = node2.Run(cancellationToken).AsTask();
 					await using var node2Client2 = await CreateAdminClient(node2.ApiUrl, cancellationToken);
@@ -1486,7 +1535,7 @@ namespace Tgstation.Server.Tests.Live
 
 						jobsHubTestTask = FailFast(await jobsHubTest.Run(cancellationToken)); // returns Task<Task>
 						var rootTest = FailFast(RawRequestTests.Run(restClientFactory, firstAdminRestClient, cancellationToken));
-						var adminTest = FailFast(new AdministrationTest(firstAdminRestClient.Administration).Run(cancellationToken));
+						var adminTest = FailFast(new AdministrationTest(firstAdminMultiClient).Run(cancellationToken));
 						var usersTest = FailFast(new UsersTest(firstAdminMultiClient).Run(cancellationToken).AsTask());
 
 						var instanceManagerTest = new InstanceManagerTest(firstAdminRestClient, server.Directory);
@@ -1632,7 +1681,12 @@ namespace Tgstation.Server.Tests.Live
 						await Task.Delay(1000, cancellationToken);
 
 						jobsHubTest.ExpectShutdown();
-						await firstAdminRestClient.Administration.Restart(cancellationToken);
+						await firstAdminMultiClient.Execute(
+							restClient => restClient.Administration.Restart(cancellationToken),
+							async gqlClient => await gqlClient.RunMutationEnsureNoErrors(
+								gql => gql.RestartServer.ExecuteAsync(cancellationToken),
+								result => result,
+								cancellationToken));
 					}
 					catch
 					{
@@ -1777,7 +1831,12 @@ namespace Tgstation.Server.Tests.Live
 					Assert.AreEqual(WatchdogStatus.Offline, dd.Status);
 
 					jobsHubTest.ExpectShutdown();
-					await adminClient.Administration.Restart(cancellationToken);
+					await multiClient.Execute(
+						restClient => restClient.Administration.Restart(cancellationToken),
+						async gqlClient => await gqlClient.RunMutationEnsureNoErrors(
+							gql => gql.RestartServer.ExecuteAsync(cancellationToken),
+							result => result,
+							cancellationToken));
 				}
 
 				await Task.WhenAny(serverTask, Task.Delay(TimeSpan.FromMinutes(1), cancellationToken));
@@ -1844,7 +1903,12 @@ namespace Tgstation.Server.Tests.Live
 					expectedStaged = compileJob.Id.Value;
 
 					jobsHubTest.ExpectShutdown();
-					await restAdminClient.Administration.Restart(cancellationToken);
+					await adminClient.Execute(
+						restClient => restClient.Administration.Restart(cancellationToken),
+						async gqlClient => await gqlClient.RunMutationEnsureNoErrors(
+							gql => gql.RestartServer.ExecuteAsync(cancellationToken),
+							result => result,
+							cancellationToken));
 				}
 
 				await Task.WhenAny(serverTask, Task.Delay(TimeSpan.FromMinutes(1), cancellationToken));