Merge pull request #1714 from tgstation/LogSpam [TGSDeploy]

v5.17.3: Fix jobs hub group mapping

README.md

@@ -385,6 +385,8 @@ System administrators will most likely have their own configuration plans, but h
 
 Once complete, test that your configuration worked by visiting your proxy site from a browser on a different computer. You should recieve a 401 Unauthorized response.
 
+_NOTE: For SignalR to function properly, make sure your reverse proxy setup supports SSE (Server-Sent Events)_
+
 #### IIS (Reccommended for Windows)
 
 1. Acquire an HTTPS certificate. The easiet free way for Windows is [win-acme](https://github.com/PKISharp/win-acme) (requires you to set up the website first)

build/Version.props

@@ -20,6 +20,6 @@
     <TgsDotnetRedistUrl>https://dotnetcli.azureedge.net/dotnet/aspnetcore/Runtime/6.0.24/dotnet-hosting-6.0.24-win.exe</TgsDotnetRedistUrl>
     <TgsMariaDBRedistVersion>10.11.5</TgsMariaDBRedistVersion>
     <!-- The two versions must match above, this is referenced by XML readers in scripts so we can't use an MSBuild property reference -->
-    <TgsMariaDBRedistUrl>https://ftp.osuosl.org/pub/mariadb//mariadb-10.11.5/winx64-packages/mariadb-10.11.5-winx64.msi</TgsMariaDBRedistUrl>
+    <TgsMariaDBRedistUrl>https://atl.mirrors.knownhost.com/mariadb//mariadb-10.11.5/winx64-packages/mariadb-10.11.5-winx64.msi</TgsMariaDBRedistUrl>
   </PropertyGroup>
 </Project>

src/Tgstation.Server.Host/Jobs/JobService.cs

@@ -369,6 +369,8 @@ namespace Tgstation.Server.Host.Jobs
 
 					var hubUpdatesTask = Task.CompletedTask;
 					var result = false;
+					var firstLogHappened = false;
+					var hubGroupName = JobsHub.HubGroupName(job);
 
 					Stopwatch stopwatch = null;
 					void QueueHubUpdate(JobResponse update, bool final)
@@ -380,10 +382,16 @@ namespace Tgstation.Server.Host.Jobs
 							{
 								await currentUpdatesTask;
 
+								if (!firstLogHappened)
+								{
+									logger.LogTrace("Sending updates for job {id} to hub group {group}", update.Id.Value, hubGroupName);
+									firstLogHappened = true;
+								}
+
 								// DCT: Cancellation token is for job, operation should always run
 								await hub
 									.Clients
-									.Group(JobsHub.HubGroupName(job))
+									.Group(hubGroupName)
 									.ReceiveJobUpdate(update, CancellationToken.None);
 							}
 

src/Tgstation.Server.Host/Jobs/JobsHubGroupMapper.cs

@@ -117,6 +117,8 @@ namespace Tgstation.Server.Host.Jobs
 		{
 			ArgumentNullException.ThrowIfNull(authenticationContext);
 
+			logger.LogTrace("MapConnectionGroups UID: {uid}", authenticationContext.User.Id.Value);
+
 			List<long> permedInstanceIds = null;
 			await databaseContextFactory.UseContext(
 				async databaseContext =>
@@ -124,7 +126,7 @@ namespace Tgstation.Server.Host.Jobs
 						.InstancePermissionSets
 						.AsQueryable()
 						.Where(ips => ips.PermissionSetId == authenticationContext.PermissionSet.Id.Value)
-						.Select(ips => ips.Id)
+						.Select(ips => ips.InstanceId)
 						.ToListAsync(cancellationToken));
 
 			await mappingFunc(

src/Tgstation.Server.Host/System/IProcess.cs

@@ -26,7 +26,7 @@ namespace Tgstation.Server.Host.System
 		/// <returns>A <see cref="Task{TResult}"/> resulting in the stderr and stdout output of the <see cref="IProcess"/>.</returns>
 		/// <remarks>
 		/// To guarantee that all data is received from the <see cref="IProcess"/> when redirecting streams to a file
-		/// the result of this function must be <see langword="await"/>ed before <see cref="IDisposable.Dispose"/> is called.
+		/// the result of this function must be <see langword="await"/>ed before <see cref="IAsyncDisposable.DisposeAsync"/> is called.
 		/// </remarks>
 		Task<string> GetCombinedOutput(CancellationToken cancellationToken);
 

src/Tgstation.Server.Host/System/Process.cs

@@ -131,7 +131,8 @@ namespace Tgstation.Server.Host.System
 		{
 			if (readTask == null)
 				throw new InvalidOperationException("Output/Error stream reading was not enabled!");
-			return readTask;
+
+			return readTask.WaitAsync(cancellationToken);
 		}
 
 		/// <inheritdoc />

src/Tgstation.Server.Host/Utils/SignalR/ComprehensiveHubContext.cs

@@ -85,9 +85,17 @@ namespace Tgstation.Server.Host.Utils.SignalR
 
 			var mappingTask = OnConnectionMapGroups?.Invoke(
 				authenticationContext,
-				mappedGroups => Task.WhenAll(
-					mappedGroups.Select(
-						group => hub.Groups.AddToGroupAsync(context.ConnectionId, group, cancellationToken))),
+				mappedGroups =>
+				{
+					mappedGroups = mappedGroups.ToList();
+					logger.LogTrace(
+						"Mapping connection ID {connectionId} with groups: {mappedGroups}",
+						context.ConnectionId,
+						String.Join(", ", mappedGroups));
+					return Task.WhenAll(
+						mappedGroups.Select(
+							group => hub.Groups.AddToGroupAsync(context.ConnectionId, group, cancellationToken)));
+				},
 				cancellationToken)
 				?? ValueTask.CompletedTask;
 			userConnections.AddOrUpdate(

tests/Tgstation.Server.Host.Tests/Jobs/TestJobsHubGroupMapper.cs

@@ -0,0 +1,120 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+using Moq;
+
+using Tgstation.Server.Api.Hubs;
+using Tgstation.Server.Api.Rights;
+using Tgstation.Server.Host.Database;
+using Tgstation.Server.Host.Jobs;
+using Tgstation.Server.Host.Models;
+using Tgstation.Server.Host.Security;
+using Tgstation.Server.Host.Utils.SignalR;
+
+namespace Tgstation.Server.Host.Tests.Jobs
+{
+	[TestClass]
+	public sealed class TestJobsHubGroupMapper
+	{
+		[TestMethod]
+		public async Task TestGroupMapping()
+		{
+			using var loggerFactory = LoggerFactory.Create(builder =>
+			{
+				builder.AddConsole();
+				builder.SetMinimumLevel(LogLevel.Trace);
+			});
+
+			var mockHub = new Mock<IConnectionMappedHubContext<JobsHub, IJobsHub>>();
+			var mockDcf = new Mock<IDatabaseContextFactory>();
+
+			
+			using var context = Utils.CreateDatabaseContext();
+			mockDcf.Setup(x => x.UseContext(It.IsNotNull<Func<IDatabaseContext, ValueTask>>())).Returns<Func<IDatabaseContext, ValueTask>>(func => func(context));
+
+			var mockPs = new PermissionSet
+			{
+				Id = 23421,
+				InstanceManagerRights = RightsHelper.AllRights<InstanceManagerRights>(),
+				AdministrationRights = RightsHelper.AllRights<AdministrationRights>(),
+			};
+			var testIps1 = new InstancePermissionSet
+			{
+				ByondRights = RightsHelper.AllRights<ByondRights>(),
+				ChatBotRights = RightsHelper.AllRights<ChatBotRights>(),
+				ConfigurationRights = RightsHelper.AllRights<ConfigurationRights>(),
+				DreamDaemonRights = RightsHelper.AllRights<DreamDaemonRights>(),
+				DreamMakerRights = RightsHelper.AllRights<DreamMakerRights>(),
+				Id = 43892849,
+				InstanceId = 348928,
+				InstancePermissionSetRights = RightsHelper.AllRights<InstancePermissionSetRights>(),
+				RepositoryRights = RightsHelper.AllRights<RepositoryRights>(),
+				PermissionSetId = mockPs.Id.Value,
+				PermissionSet = mockPs,
+			};
+
+			var testIps2 = new InstancePermissionSet
+			{
+				ByondRights = RightsHelper.AllRights<ByondRights>(),
+				ChatBotRights = RightsHelper.AllRights<ChatBotRights>(),
+				ConfigurationRights = RightsHelper.AllRights<ConfigurationRights>(),
+				DreamDaemonRights = RightsHelper.AllRights<DreamDaemonRights>(),
+				DreamMakerRights = RightsHelper.AllRights<DreamMakerRights>(),
+				Id = 454354,
+				InstanceId = 2234,
+				InstancePermissionSetRights = RightsHelper.AllRights<InstancePermissionSetRights>(),
+				RepositoryRights = RightsHelper.AllRights<RepositoryRights>(),
+				PermissionSetId = mockPs.Id.Value,
+				PermissionSet = mockPs,
+			};
+			context.InstancePermissionSets.Add(testIps1);
+			context.InstancePermissionSets.Add(testIps2);
+
+			var cancellationToken = CancellationToken.None;
+			await context.SaveChangesAsync(cancellationToken);
+
+			var mockUpdater = new Mock<IJobsHubUpdater>();
+
+			var mapper = new JobsHubGroupMapper(
+				mockHub.Object,
+				mockDcf.Object,
+				mockUpdater.Object,
+				loggerFactory.CreateLogger<JobsHubGroupMapper>());
+
+			await mapper.StartAsync(cancellationToken);
+
+			var mockAuthenticationContext = new Mock<IAuthenticationContext>();
+			var mockUser = new User
+			{
+				Id = 2134134,
+			};
+
+			mockAuthenticationContext.SetupGet(x => x.User).Returns(mockUser);
+
+			mockAuthenticationContext.SetupGet(x => x.PermissionSet).Returns(mockPs);
+
+			bool ran = false;
+			Task Callback(IEnumerable<string> results)
+			{
+				ran = true;
+				Assert.AreEqual(2, results.Count());
+				Assert.IsTrue(results.Contains(JobsHub.HubGroupName(testIps1.InstanceId)));
+				Assert.IsTrue(results.Contains(JobsHub.HubGroupName(testIps2.InstanceId)));
+				return Task.CompletedTask;
+			}
+
+			await mockHub.RaiseAsync(x => x.OnConnectionMapGroups += null, mockAuthenticationContext.Object, (Func<IEnumerable<string>, Task>)Callback, cancellationToken);
+
+			Assert.IsTrue(ran);
+
+			await mapper.StopAsync(cancellationToken);
+		}
+	}
+}

tests/Tgstation.Server.Host.Tests/MemoryDatabaseContext.cs

@@ -0,0 +1,13 @@
+using Microsoft.EntityFrameworkCore;
+
+using Tgstation.Server.Host.Database;
+
+namespace Tgstation.Server.Host.Tests
+{
+	sealed class MemoryDatabaseContext : DatabaseContext
+	{
+		public MemoryDatabaseContext(DbContextOptions dbContextOptions) : base(dbContextOptions)
+		{
+		}
+	}
+}

tests/Tgstation.Server.Host.Tests/Tgstation.Server.Host.Tests.csproj

@@ -5,6 +5,10 @@
     <TargetFramework>$(TgsFrameworkVersion)</TargetFramework>
   </PropertyGroup>
 
+  <ItemGroup>
+    <PackageReference Include="Microsoft.EntityFrameworkCore.InMemory" Version="7.0.13" />
+  </ItemGroup>
+
   <ItemGroup>
     <ProjectReference Include="..\..\src\Tgstation.Server.Host\Tgstation.Server.Host.csproj" />
   </ItemGroup>

tests/Tgstation.Server.Host.Tests/Utils.cs

@@ -0,0 +1,17 @@
+using Microsoft.EntityFrameworkCore;
+
+using Tgstation.Server.Host.Database;
+
+namespace Tgstation.Server.Host.Tests
+{
+	static class Utils
+	{
+		public static MemoryDatabaseContext CreateDatabaseContext()
+		{
+			var options = new DbContextOptionsBuilder<MemoryDatabaseContext>()
+				.UseInMemoryDatabase(databaseName: "TgsTestDB")
+				.Options;
+			return new MemoryDatabaseContext(options);
+		}
+	}
+}

tests/Tgstation.Server.Tests/Live/Instance/JobsHubTests.cs

@@ -229,13 +229,6 @@ namespace Tgstation.Server.Tests.Live.Instance
 			// force token refreshs
 			await Task.WhenAll(permedUser.Administration.Read(cancellationToken).AsTask(), permlessUser.Instances.List(null, cancellationToken).AsTask());
 
-			await Task.WhenAll(conn1.StartAsync(cancellationToken), conn2.StartAsync(cancellationToken));
-
-			Assert.AreEqual(HubConnectionState.Connected, conn1.State);
-			Assert.AreEqual(HubConnectionState.Connected, conn2.State);
-			Console.WriteLine($"New conn1: {conn1.ConnectionId}");
-			Console.WriteLine($"New conn2: {conn2.ConnectionId}");
-
 			if (!permlessPsId.HasValue)
 			{
 				var permlessUserId = long.Parse(permlessUser.Token.ParseJwt().Subject);
@@ -267,6 +260,13 @@ namespace Tgstation.Server.Tests.Live.Instance
 							PermissionSetId = permlessPsId.Value
 						}, cancellationToken);
 				}));
+
+			await Task.WhenAll(conn1.StartAsync(cancellationToken), conn2.StartAsync(cancellationToken));
+
+			Assert.AreEqual(HubConnectionState.Connected, conn1.State);
+			Assert.AreEqual(HubConnectionState.Connected, conn2.State);
+			Console.WriteLine($"New conn1: {conn1.ConnectionId}");
+			Console.WriteLine($"New conn2: {conn2.ConnectionId}");
 		}
 
 		public void CompleteNow() => finishTcs.TrySetResult();