Error Live test if an unallowed error is logged

src/Tgstation.Server.Host/Extensions/ServiceCollectionExtensions.cs

@@ -5,6 +5,7 @@ using System.Globalization;
 using Elastic.CommonSchema.Serilog;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.DependencyInjection.Extensions;
 using Microsoft.Extensions.Logging;
 
 using Serilog;
@@ -27,6 +28,11 @@ namespace Tgstation.Server.Host.Extensions
 		/// </summary>
 		static Type chatProviderFactoryType = typeof(ProviderFactory);
 
+		/// <summary>
+		/// A <see cref="ServiceDescriptor"/> for an additional <see cref="ILoggerProvider"/> to use.
+		/// </summary>
+		static ServiceDescriptor additionalLoggerProvider;
+
 		/// <summary>
 		/// Change the <see cref="Type"/> used as an implementation for calls to <see cref="AddChatProviderFactory(IServiceCollection)"/>.
 		/// </summary>
@@ -36,6 +42,17 @@ namespace Tgstation.Server.Host.Extensions
 			chatProviderFactoryType = typeof(TProviderFactory);
 		}
 
+		/// <summary>
+		/// Add an additional <see cref="ILoggerProvider"/> to <see cref="IServiceCollection"/>s that call <see cref="SetupLogging(IServiceCollection, Action{LoggerConfiguration}, Action{LoggerSinkConfiguration}, ElasticsearchConfiguration)"/>.
+		/// </summary>
+		/// <typeparam name="TLoggerProvider">The <see cref="Type"/> of <see cref="ILoggerProvider"/> to add.</typeparam>
+		public static void UseAdditionalLoggerProvider<TLoggerProvider>() where TLoggerProvider : class, ILoggerProvider
+		{
+			if (additionalLoggerProvider != null)
+				throw new InvalidOperationException("Cannot have multiple additionalLoggerProviders!");
+			additionalLoggerProvider = ServiceDescriptor.Singleton<ILoggerProvider, TLoggerProvider>();
+		}
+
 		/// <summary>
 		/// Adds a <see cref="IProviderFactory"/> implementation to the given <paramref name="serviceCollection"/>.
 		/// </summary>
@@ -133,6 +150,9 @@ namespace Tgstation.Server.Host.Extensions
 
 				if (Debugger.IsAttached)
 					builder.AddDebug();
+
+				if (additionalLoggerProvider != null)
+					builder.Services.TryAddEnumerable(additionalLoggerProvider);
 			});
 	}
 }

tests/Tgstation.Server.Tests/Live/HardFailLogger.cs

@@ -0,0 +1,29 @@
+using System;
+using System.Threading.Tasks;
+
+using Microsoft.Extensions.Logging;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace Tgstation.Server.Tests.Live
+{
+	sealed class HardFailLogger : ILogger
+	{
+		readonly Action<Exception> failureSink;
+
+		public HardFailLogger(Action<Exception> failureSink)
+		{
+			this.failureSink = failureSink ?? throw new ArgumentNullException(nameof(failureSink));
+		}
+
+		public IDisposable BeginScope<TState>(TState state) => Task.CompletedTask;
+
+		public bool IsEnabled(LogLevel logLevel) => true;
+
+		public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func<TState, Exception, string> formatter)
+		{
+			var logMessage = formatter(state, exception);
+			if (logLevel == LogLevel.Error || (logLevel == LogLevel.Critical && logMessage != "DropDatabase configuration option set! Dropping any existing database..."))
+				failureSink(new AssertFailedException(logMessage));
+		}
+	}
+}

tests/Tgstation.Server.Tests/Live/HardFailLoggerProvider.cs

@@ -0,0 +1,23 @@
+using System.Threading.Tasks;
+
+using Microsoft.Extensions.Logging;
+
+namespace Tgstation.Server.Tests.Live
+{
+	sealed class HardFailLoggerProvider : ILoggerProvider
+	{
+		public static bool BlockFails { get; set; }
+
+		public static Task FailureSource => failureSink.Task;
+
+		static readonly TaskCompletionSource failureSink = new TaskCompletionSource();
+
+		public ILogger CreateLogger(string categoryName) => new HardFailLogger(ex =>
+		{
+			if (!BlockFails)
+				failureSink.TrySetException(ex);
+		});
+
+		public void Dispose() { }
+	}
+}

tests/Tgstation.Server.Tests/Live/TestLiveServer.cs

@@ -698,6 +698,26 @@ namespace Tgstation.Server.Tests.Live
 		[TestMethod]
 		public async Task TestStandardTgsOperation()
 		{
+			const int MaximumTestMinutes = 30;
+			using var hardCancellationTokenSource = new CancellationTokenSource(TimeSpan.FromMinutes(MaximumTestMinutes));
+			var hardCancellationToken = hardCancellationTokenSource.Token;
+
+			ServiceCollectionExtensions.UseAdditionalLoggerProvider<HardFailLoggerProvider>();
+
+			var internalTask = TestTgsInternal(hardCancellationToken);
+			await Task.WhenAny(
+				internalTask,
+				HardFailLoggerProvider.FailureSource);
+
+			if (!internalTask.IsCompleted)
+			{
+				hardCancellationTokenSource.Cancel();
+				await Task.WhenAll(internalTask, HardFailLoggerProvider.FailureSource);
+			}
+		}
+
+		async Task TestTgsInternal(CancellationToken hardCancellationToken)
+		{ 
 			var discordConnectionString = Environment.GetEnvironmentVariable("TGS_TEST_DISCORD_TOKEN");
 			var ircConnectionString = Environment.GetEnvironmentVariable("TGS_TEST_IRC_CONNECTION_STRING");
 			var missingChatVarsCount = Convert.ToInt32(String.IsNullOrWhiteSpace(discordConnectionString))
@@ -734,9 +754,6 @@ namespace Tgstation.Server.Tests.Live
 
 			using var server = new LiveTestingServer(null, true);
 
-			const int MaximumTestMinutes = 180;
-			using var hardTimeoutCancellationTokenSource = new CancellationTokenSource(TimeSpan.FromMinutes(MaximumTestMinutes));
-			var hardCancellationToken = hardTimeoutCancellationTokenSource.Token;
 			using var serverCts = CancellationTokenSource.CreateLinkedTokenSource(hardCancellationToken);
 			var cancellationToken = serverCts.Token;
 
@@ -816,12 +833,13 @@ namespace Tgstation.Server.Tests.Live
 				// http bind test https://github.com/tgstation/tgstation-server/issues/1065
 				if (new PlatformIdentifier().IsWindows)
 				{
-					using var blockingSocket = new Socket(SocketType.Stream, ProtocolType.Tcp);
-					blockingSocket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ExclusiveAddressUse, true);
-					blockingSocket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, false);
-					blockingSocket.Bind(new IPEndPoint(IPAddress.Any, server.Url.Port));
+					HardFailLoggerProvider.BlockFails = true;
 					try
 					{
+						using var blockingSocket = new Socket(SocketType.Stream, ProtocolType.Tcp);
+						blockingSocket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ExclusiveAddressUse, true);
+						blockingSocket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, false);
+						blockingSocket.Bind(new IPEndPoint(IPAddress.Any, server.Url.Port));
 						// bind test run
 						await server.Run(cancellationToken);
 						Assert.Fail("Expected server task to end with a SocketException");
@@ -830,6 +848,10 @@ namespace Tgstation.Server.Tests.Live
 					{
 						Assert.AreEqual(ex.SocketErrorCode, SocketError.AddressAlreadyInUse);
 					}
+					finally
+					{
+						HardFailLoggerProvider.BlockFails = false;
+					}
 				}
 
 				await Task.WhenAny(serverTask, Task.Delay(TimeSpan.FromMinutes(1), cancellationToken));