Elasticsearch log ingest support

build/Version.props

@@ -4,7 +4,7 @@
   <Import Project="ControlPanelVersion.props" />
   <PropertyGroup>
     <TgsCoreVersion>4.12.0</TgsCoreVersion>
-    <TgsConfigVersion>3.0.0</TgsConfigVersion>
+    <TgsConfigVersion>3.0.1</TgsConfigVersion>
     <TgsApiVersion>9.0.1</TgsApiVersion>
     <TgsApiLibraryVersion>9.0.0</TgsApiLibraryVersion>
     <TgsClientVersion>10.0.0</TgsClientVersion>

src/Tgstation.Server.Host/Configuration/ElasticsearchConfiguration.cs

@@ -0,0 +1,53 @@
+namespace Tgstation.Server.Host.Configuration
+	{
+	/// <summary>
+	/// Configuration options pertaining to elasticsearch log storage.
+	/// </summary>
+	sealed class ElasticsearchConfiguration
+	{
+		/// <summary>
+		/// The key for the <see cref="Microsoft.Extensions.Configuration.IConfigurationSection"/> the <see cref="ElasticsearchConfiguration"/> resides in.
+		/// </summary>
+		public const string Section = "Elasticsearch";
+
+		/// <summary>
+		/// Default value of <see cref="Enable"/>.
+		/// </summary>
+		const bool DefaultEnable = false;
+
+		/// <summary>
+		/// Default value of <see cref="Host"/>.
+		/// </summary>
+		const string DefaultHost = "http://127.0.0.1:9200"; // localhost
+
+		/// <summary>
+		/// Default value of <see cref="Username"/>.
+		/// </summary>
+		const string DefaultUsername = "my_username";
+
+		/// <summary>
+		/// Default value of <see cref="Password"/>.
+		/// </summary>
+		const string DefaultPassword = "my_password";
+
+		/// <summary>
+		/// Do we want to enable elasticsearch or not?.
+		/// </summary>
+		public bool Enable { get; set; } = DefaultEnable;
+
+		/// <summary>
+		/// The host of the elasticsearch endpoint.
+		/// </summary>
+		public string Host { get; set; } = DefaultHost;
+
+		/// <summary>
+		/// Username for elasticsearch.
+		/// </summary>
+		public string Username { get; set; } = DefaultUsername;
+
+		/// <summary>
+		/// Password for elasticsearch.
+		/// </summary>
+		public string Password { get; set; } = DefaultPassword;
+	}
+}

src/Tgstation.Server.Host/Core/Application.cs

@@ -151,6 +151,7 @@ namespace Tgstation.Server.Host.Core
 						config.MinimumLevel.Override("System.Net.Http.HttpClient", microsoftEventLevel.Value);
 					}
 				},
+				Configuration,
 				sinkConfig =>
 				{
 					if (postSetupServices.FileLoggingConfiguration.Disable)

src/Tgstation.Server.Host/Extensions/ServiceCollectionExtensions.cs

@@ -2,12 +2,14 @@
 using System.Diagnostics;
 using System.Globalization;
 
+using Elastic.CommonSchema.Serilog;
+
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using Serilog;
 using Serilog.Configuration;
-
+using Serilog.Sinks.Elasticsearch;
 using Tgstation.Server.Host.Configuration;
 
 namespace Tgstation.Server.Host.Extensions
@@ -57,11 +59,13 @@ namespace Tgstation.Server.Host.Extensions
 		/// </summary>
 		/// <param name="serviceCollection">The <see cref="IServiceCollection"/> to configure.</param>
 		/// <param name="configurationAction">Additional configuration for a given <see cref="LoggerConfiguration"/>.</param>
+		/// <param name="tgsConfiguration">The service configuration for a given <see cref="IConfiguration"/>.</param>
 		/// <param name="sinkConfigurationAction">Additional configuration for a given <see cref="LoggerSinkConfiguration"/>.</param>
 		/// <returns>The updated <paramref name="serviceCollection"/>.</returns>
 		public static IServiceCollection SetupLogging(
 			this IServiceCollection serviceCollection,
 			Action<LoggerConfiguration> configurationAction,
+			IConfiguration tgsConfiguration,
 			Action<LoggerSinkConfiguration> sinkConfigurationAction = null)
 			=> serviceCollection.AddLogging(builder =>
 			{
@@ -85,6 +89,30 @@ namespace Tgstation.Server.Host.Extensions
 						sinkConfigurationAction?.Invoke(sinkConfiguration);
 					});
 
+				string elasticsearchEndpoint = tgsConfiguration.GetSection("Elasticsearch").GetSection("Host").Value;
+				string elasticsearchUser = tgsConfiguration.GetSection("Elasticsearch").GetSection("Username").Value;
+				string elasticsearchPassword = tgsConfiguration.GetSection("Elasticsearch").GetSection("Password").Value;
+				string raw_bool = tgsConfiguration.GetSection("Elasticsearch").GetSection("Enable").Value;
+				bool elasticsearchEnabled = false;
+
+				if (!string.IsNullOrEmpty(raw_bool))
+					elasticsearchEnabled = bool.Parse(raw_bool);
+
+				if (elasticsearchEnabled)
+				{
+					if (elasticsearchEndpoint == null)
+						throw new InvalidOperationException("Elasticsearch endpoint is null!");
+
+					configuration.WriteTo.Elasticsearch(new ElasticsearchSinkOptions(new Uri(elasticsearchEndpoint))
+					{
+						ModifyConnectionSettings = x => (!string.IsNullOrEmpty(elasticsearchUser) && !string.IsNullOrEmpty(elasticsearchPassword)) ? x.BasicAuthentication(elasticsearchUser, elasticsearchPassword) : null,
+						CustomFormatter = new EcsTextFormatter(),
+						AutoRegisterTemplate = true,
+						AutoRegisterTemplateVersion = AutoRegisterTemplateVersion.ESv7,
+						IndexFormat = "tgs4-logs",
+					});
+				}
+
 				builder.AddSerilog(configuration.CreateLogger(), true);
 
 				if (Debugger.IsAttached)

src/Tgstation.Server.Host/Setup/SetupApplication.cs

@@ -52,7 +52,7 @@ namespace Tgstation.Server.Host.Setup
 			if (services == null)
 				throw new ArgumentNullException(nameof(services));
 
-			services.SetupLogging(config => config.MinimumLevel.Override("Microsoft", LogEventLevel.Warning));
+			services.SetupLogging(config => config.MinimumLevel.Override("Microsoft", LogEventLevel.Warning), Configuration);
 
 			services.AddSingleton(IOManager);
 			services.AddSingleton(AssemblyInformationProvider);
@@ -66,6 +66,7 @@ namespace Tgstation.Server.Host.Setup
 			services.UseStandardConfig<DatabaseConfiguration>(Configuration);
 			services.UseStandardConfig<SecurityConfiguration>(Configuration);
 			services.UseStandardConfig<FileLoggingConfiguration>(Configuration);
+			services.UseStandardConfig<ElasticsearchConfiguration>(Configuration);
 
 			ConfigureHostedService(services);
 		}

src/Tgstation.Server.Host/Tgstation.Server.Host.csproj

@@ -67,6 +67,7 @@
     <PackageReference Include="Cyberboss.AspNetCore.AsyncInitializer" Version="1.2.0" />
     <PackageReference Include="Cyberboss.SmartIrc4net.Standard" Version="0.4.6" />
     <PackageReference Include="Discord.Net.WebSocket" Version="2.3.1" />
+    <PackageReference Include="Elastic.CommonSchema.Serilog" Version="1.5.3" />
     <PackageReference Include="GitLabApiClient" Version="1.7.0" />
     <PackageReference Include="LibGit2Sharp" Version="0.27.0-preview-0034" />
     <PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="3.1.13" />
@@ -91,6 +92,7 @@
     <PackageReference Include="Serilog.Extensions.Logging" Version="3.0.1" />
     <PackageReference Include="Serilog.Sinks.Async" Version="1.4.0" />
     <PackageReference Include="Serilog.Sinks.Console" Version="3.1.1" />
+    <PackageReference Include="Serilog.Sinks.Elasticsearch" Version="8.4.1" />
     <PackageReference Include="Serilog.Sinks.File" Version="4.1.0" />
     <PackageReference Include="StyleCop.Analyzers" Version="1.1.118">
       <PrivateAssets>all</PrivateAssets>