diff --git a/README.md b/README.md index 680f2f2a6b..c0f5290a1b 100644 --- a/README.md +++ b/README.md @@ -256,7 +256,9 @@ Create an `appsettings.Production.yml` file next to `appsettings.yml`. This will - `General:ConfigVersion`: Suppresses warnings about out of date config versions. You should change this after updating TGS to one with a new config version. The current version can be found on the releases page for your server version. -- `General:MinimumPasswordLength`: Minimum password length requirement for database users +- `General:MinimumPasswordLength`: Minimum password length requirement for database users. + +- `General:PrometheusPort`: Port Prometheus metrics are published on under /metrics. This can be set to the same value as the `ApiPort`, just note that accessing it does not require authentication. - `General:ValidInstancePaths`: Array meant to limit the directories in which instances may be created. diff --git a/build/Dockerfile b/build/Dockerfile index c281147ddd..64c43f8fdc 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -58,11 +58,12 @@ RUN export TGS_TELEMETRY_KEY_FILE="../../${TGS_TELEMETRY_KEY_FILE}" \ FROM mcr.microsoft.com/dotnet/aspnet:8.0-bookworm-slim -#needed for byond +#needed for byond, curl for healthchecks RUN apt-get update \ && apt-get install -y \ gcc-multilib \ gdb \ + curl \ && rm -rf /var/lib/apt/lists/* EXPOSE 5000 @@ -78,4 +79,6 @@ COPY --from=build /repo/build/tgs.docker.sh tgs.sh VOLUME ["/config_data", "/tgs_logs", "/app/lib"] +HEALTHCHECK CMD --curl --fail http://localhost:5000/health || exit + ENTRYPOINT ["./tgs.sh"] diff --git a/build/TestCommon.props b/build/TestCommon.props index f0f8f067f2..7d6476170b 100644 --- a/build/TestCommon.props +++ b/build/TestCommon.props @@ -13,7 +13,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/build/Version.props b/build/Version.props index dcf5a154ad..b47df5002c 100644 --- a/build/Version.props +++ b/build/Version.props @@ -3,8 +3,8 @@ - 6.13.0 - 5.4.0 + 6.14.0 + 5.5.0 10.12.1 0.5.0 7.0.0 diff --git a/src/Tgstation.Server.Api/Tgstation.Server.Api.csproj b/src/Tgstation.Server.Api/Tgstation.Server.Api.csproj index 27bbd839ba..52412e6e78 100644 --- a/src/Tgstation.Server.Api/Tgstation.Server.Api.csproj +++ b/src/Tgstation.Server.Api/Tgstation.Server.Api.csproj @@ -28,10 +28,9 @@ - + - diff --git a/src/Tgstation.Server.Client.GraphQL/Tgstation.Server.Client.GraphQL.csproj b/src/Tgstation.Server.Client.GraphQL/Tgstation.Server.Client.GraphQL.csproj index e9308d8bee..19901867f0 100644 --- a/src/Tgstation.Server.Client.GraphQL/Tgstation.Server.Client.GraphQL.csproj +++ b/src/Tgstation.Server.Client.GraphQL/Tgstation.Server.Client.GraphQL.csproj @@ -9,7 +9,7 @@ - + diff --git a/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj b/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj index 0870f67e4a..898b10b2f2 100644 --- a/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj +++ b/src/Tgstation.Server.Client/Tgstation.Server.Client.csproj @@ -11,9 +11,9 @@ - + - + diff --git a/src/Tgstation.Server.Host.Console/Tgstation.Server.Host.Console.csproj b/src/Tgstation.Server.Host.Console/Tgstation.Server.Host.Console.csproj index 889a862be4..c6fd7418cb 100644 --- a/src/Tgstation.Server.Host.Console/Tgstation.Server.Host.Console.csproj +++ b/src/Tgstation.Server.Host.Console/Tgstation.Server.Host.Console.csproj @@ -13,9 +13,9 @@ - + - + diff --git a/src/Tgstation.Server.Host.Service/Tgstation.Server.Host.Service.csproj b/src/Tgstation.Server.Host.Service/Tgstation.Server.Host.Service.csproj index 4bfa95022d..6af556b1fc 100644 --- a/src/Tgstation.Server.Host.Service/Tgstation.Server.Host.Service.csproj +++ b/src/Tgstation.Server.Host.Service/Tgstation.Server.Host.Service.csproj @@ -21,21 +21,21 @@ - + - + - + - + diff --git a/src/Tgstation.Server.Host/.config/dotnet-tools.json b/src/Tgstation.Server.Host/.config/dotnet-tools.json index 02afa3f072..eda2e98c23 100644 --- a/src/Tgstation.Server.Host/.config/dotnet-tools.json +++ b/src/Tgstation.Server.Host/.config/dotnet-tools.json @@ -3,7 +3,7 @@ "isRoot": true, "tools": { "dotnet-ef": { - "version": "8.0.11", + "version": "8.0.13", "commands": [ "dotnet-ef" ] diff --git a/src/Tgstation.Server.Host/Components/Chat/Providers/IrcProvider.cs b/src/Tgstation.Server.Host/Components/Chat/Providers/IrcProvider.cs index 2aa9916bf7..d449320bc7 100644 --- a/src/Tgstation.Server.Host/Components/Chat/Providers/IrcProvider.cs +++ b/src/Tgstation.Server.Host/Components/Chat/Providers/IrcProvider.cs @@ -227,8 +227,6 @@ namespace Tgstation.Server.Host.Components.Chat.Providers { ArgumentNullException.ThrowIfNull(revisionInformation); ArgumentNullException.ThrowIfNull(engineVersion); - ArgumentNullException.ThrowIfNull(gitHubOwner); - ArgumentNullException.ThrowIfNull(gitHubRepo); var commitInsert = revisionInformation.CommitSha![..7]; string remoteCommitInsert; diff --git a/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs b/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs index eab48e8172..9ffab92802 100644 --- a/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs +++ b/src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs @@ -8,6 +8,8 @@ using System.Threading.Tasks; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; +using Prometheus; + using Tgstation.Server.Api.Models; using Tgstation.Server.Api.Models.Internal; using Tgstation.Server.Common.Extensions; @@ -105,6 +107,21 @@ namespace Tgstation.Server.Host.Components.Deployment /// readonly Api.Models.Instance metadata; + /// + /// The number of attempted deployments. + /// + readonly Counter attemptedDeployments; + + /// + /// The number of successful deployments. + /// + readonly Counter successfulDeployments; + + /// + /// The number of failed deployments. + /// + readonly Counter failedDeployments; + /// /// for . /// @@ -149,6 +166,7 @@ namespace Tgstation.Server.Host.Components.Deployment /// The value of . /// The value of . /// The value of . + /// The to use. /// The value of . /// The value of . /// The value of . @@ -164,6 +182,7 @@ namespace Tgstation.Server.Host.Components.Deployment IRepositoryManager repositoryManager, IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory, IAsyncDelayer asyncDelayer, + IMetricFactory metricFactory, ILogger logger, SessionConfiguration sessionConfiguration, Api.Models.Instance metadata) @@ -177,12 +196,17 @@ namespace Tgstation.Server.Host.Components.Deployment this.processExecutor = processExecutor ?? throw new ArgumentNullException(nameof(processExecutor)); this.compileJobConsumer = compileJobConsumer ?? throw new ArgumentNullException(nameof(compileJobConsumer)); this.repositoryManager = repositoryManager ?? throw new ArgumentNullException(nameof(repositoryManager)); - this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer)); this.remoteDeploymentManagerFactory = remoteDeploymentManagerFactory ?? throw new ArgumentNullException(nameof(remoteDeploymentManagerFactory)); + this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer)); + ArgumentNullException.ThrowIfNull(metricFactory); this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); this.sessionConfiguration = sessionConfiguration ?? throw new ArgumentNullException(nameof(sessionConfiguration)); this.metadata = metadata ?? throw new ArgumentNullException(nameof(metadata)); + successfulDeployments = metricFactory.CreateCounter("tgs_successful_deployments", "The number of deployments that have completed successfully"); + failedDeployments = metricFactory.CreateCounter("tgs_failed_deployments", "The number of deployments that have failed"); + attemptedDeployments = metricFactory.CreateCounter("tgs_total_deployments", "The number of deployments that have been attempted"); + deploymentLock = new object(); } @@ -205,9 +229,12 @@ namespace Tgstation.Server.Host.Components.Deployment deploying = true; } + attemptedDeployments.Inc(); + currentChatCallback = null; currentDreamMakerOutput = null; Models.CompileJob? compileJob = null; + bool success = false; try { string? repoOwner = null; @@ -351,6 +378,7 @@ namespace Tgstation.Server.Host.Components.Deployment { var chatNotificationAction = currentChatCallback!(null, compileJob.Output!); await compileJobConsumer.LoadCompileJob(compileJob, chatNotificationAction, cancellationToken); + success = true; } catch { @@ -406,6 +434,10 @@ namespace Tgstation.Server.Host.Components.Deployment finally { deploying = false; + if (success) + successfulDeployments.Inc(); + else + failedDeployments.Inc(); } } #pragma warning restore CA1506 diff --git a/src/Tgstation.Server.Host/Components/InstanceFactory.cs b/src/Tgstation.Server.Host/Components/InstanceFactory.cs index 70ef1dd028..b5c93bac69 100644 --- a/src/Tgstation.Server.Host/Components/InstanceFactory.cs +++ b/src/Tgstation.Server.Host/Components/InstanceFactory.cs @@ -1,10 +1,14 @@ using System; +using System.Collections.Generic; +using System.Globalization; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using Prometheus; + using Tgstation.Server.Host.Components.Chat; using Tgstation.Server.Host.Components.Chat.Commands; using Tgstation.Server.Host.Components.Deployment; @@ -140,6 +144,11 @@ namespace Tgstation.Server.Host.Components /// readonly IDotnetDumpService dotnetDumpService; + /// + /// The for the . + /// + readonly IMetricFactory metricFactory; + /// /// The for the . /// @@ -183,6 +192,7 @@ namespace Tgstation.Server.Host.Components /// The value of . /// The value of . /// The value of . + /// The value of . /// The containing the value of . /// The containing the value of . public InstanceFactory( @@ -208,6 +218,7 @@ namespace Tgstation.Server.Host.Components IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory, IAsyncDelayer asyncDelayer, IDotnetDumpService dotnetDumpService, + IMetricFactory metricFactory, IOptions generalConfigurationOptions, IOptions sessionConfigurationOptions) { @@ -233,6 +244,7 @@ namespace Tgstation.Server.Host.Components this.remoteDeploymentManagerFactory = remoteDeploymentManagerFactory ?? throw new ArgumentNullException(nameof(remoteDeploymentManagerFactory)); this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer)); this.dotnetDumpService = dotnetDumpService ?? throw new ArgumentNullException(nameof(dotnetDumpService)); + this.metricFactory = metricFactory ?? throw new ArgumentNullException(nameof(metricFactory)); generalConfiguration = generalConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(generalConfigurationOptions)); sessionConfiguration = sessionConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(sessionConfigurationOptions)); } @@ -264,6 +276,13 @@ namespace Tgstation.Server.Host.Components var diagnosticsIOManager = new ResolvingIOManager(instanceIoManager, "Diagnostics"); var configurationIoManager = new ResolvingIOManager(instanceIoManager, "Configuration"); + var metricFactory = this.metricFactory.WithLabels( + new Dictionary + { + { "instance_name", metadata.Name! }, + { "instance_id", metadata.Id!.Value.ToString(CultureInfo.InvariantCulture) }, + }); + var configuration = new StaticFiles.Configuration( configurationIoManager, synchronousIOManager, @@ -323,6 +342,7 @@ namespace Tgstation.Server.Host.Components eventConsumer, asyncDelayer, dotnetDumpService, + metricFactory, loggerFactory, loggerFactory.CreateLogger(), sessionConfiguration, @@ -337,6 +357,7 @@ namespace Tgstation.Server.Host.Components diagnosticsIOManager, configuration, // watchdog doesn't need itself as an event consumer remoteDeploymentManagerFactory, + metricFactory, metadata, metadata.DreamDaemonSettings!); try @@ -357,6 +378,7 @@ namespace Tgstation.Server.Host.Components repoManager, remoteDeploymentManagerFactory, asyncDelayer, + metricFactory, loggerFactory.CreateLogger(), sessionConfiguration, metadata); diff --git a/src/Tgstation.Server.Host/Components/InstanceManager.cs b/src/Tgstation.Server.Host/Components/InstanceManager.cs index d0f91c4c46..1ded8b3dad 100644 --- a/src/Tgstation.Server.Host/Components/InstanceManager.cs +++ b/src/Tgstation.Server.Host/Components/InstanceManager.cs @@ -9,6 +9,8 @@ using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using Prometheus; + using Tgstation.Server.Api.Models; using Tgstation.Server.Common; using Tgstation.Server.Common.Extensions; @@ -150,6 +152,11 @@ namespace Tgstation.Server.Host.Components /// readonly CancellationTokenSource shutdownCancellationTokenSource; + /// + /// The count of online instances. + /// + readonly Gauge onlineInstances; + /// /// The original of . /// @@ -180,6 +187,8 @@ namespace Tgstation.Server.Host.Components /// The value of . /// The value of . /// The value of . + /// The used to create metrics. + /// The to use. /// The containing the value of . /// The containing the value of . /// The containing the value of . @@ -197,6 +206,8 @@ namespace Tgstation.Server.Host.Components ISwarmServiceController swarmServiceController, IConsole console, IPlatformIdentifier platformIdentifier, + IMetricFactory metricFactory, + ICollectorRegistry collectorRegistry, IOptions generalConfigurationOptions, IOptions swarmConfigurationOptions, IOptions internalConfigurationOptions, @@ -214,6 +225,8 @@ namespace Tgstation.Server.Host.Components this.swarmServiceController = swarmServiceController ?? throw new ArgumentNullException(nameof(swarmServiceController)); this.console = console ?? throw new ArgumentNullException(nameof(console)); this.platformIdentifier = platformIdentifier ?? throw new ArgumentNullException(nameof(platformIdentifier)); + ArgumentNullException.ThrowIfNull(metricFactory); + ArgumentNullException.ThrowIfNull(collectorRegistry); generalConfiguration = generalConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(generalConfigurationOptions)); swarmConfiguration = swarmConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(swarmConfigurationOptions)); internalConfiguration = internalConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(internalConfigurationOptions)); @@ -221,12 +234,21 @@ namespace Tgstation.Server.Host.Components originalConsoleTitle = console.Title; + onlineInstances = metricFactory.CreateGauge("tgs_online_instances", "The total number of instances online"); + instances = new Dictionary>(); bridgeHandlers = new Dictionary(); readyTcs = new TaskCompletionSource(); instanceStateChangeSemaphore = new SemaphoreSlim(1); startupCancellationTokenSource = new CancellationTokenSource(); shutdownCancellationTokenSource = new CancellationTokenSource(); + + collectorRegistry.AddBeforeCollectCallback(async cancellationToken => + { + using (await SemaphoreSlimContext.Lock(instanceStateChangeSemaphore, cancellationToken)) + foreach (var container in instances.Values) + container.Instance.Watchdog.RunMetricsScrape(); + }); } /// @@ -396,6 +418,7 @@ namespace Tgstation.Server.Host.Components finally { await container.Instance.DisposeAsync(); + onlineInstances.Dec(); } } } @@ -426,6 +449,8 @@ namespace Tgstation.Server.Host.Components instances.Add( instanceId, new ReferenceCountingContainer(instance)); + + onlineInstances.Inc(); } catch (Exception ex) { diff --git a/src/Tgstation.Server.Host/Components/Repository/Repository.cs b/src/Tgstation.Server.Host/Components/Repository/Repository.cs index 1aa2fa0842..b62cfbaa50 100644 --- a/src/Tgstation.Server.Host/Components/Repository/Repository.cs +++ b/src/Tgstation.Server.Host/Components/Repository/Repository.cs @@ -199,7 +199,7 @@ namespace Tgstation.Server.Host.Components.Repository var commitMessage = String.Format( CultureInfo.InvariantCulture, - "TGS Test Merge (#{0}){1}{2}", + "TGS Test Merge (PR {0}){1}{2}", testMergeParameters.Number, testMergeParameters.Comment != null ? Environment.NewLine diff --git a/src/Tgstation.Server.Host/Components/Session/SessionController.cs b/src/Tgstation.Server.Host/Components/Session/SessionController.cs index ac6d3dd6a3..3992acc6ef 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionController.cs @@ -519,6 +519,10 @@ namespace Tgstation.Server.Host.Components.Session return process.CreateDump(outputFile, minidump, cancellationToken); } + /// + public double MeasureProcessorTimeDelta() + => process.MeasureProcessorTimeDelta(); + /// /// The for . /// diff --git a/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs b/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs index 0785051b64..5c602ca1e5 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionControllerFactory.cs @@ -9,6 +9,8 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using Prometheus; + using Tgstation.Server.Api.Models; using Tgstation.Server.Api.Models.Internal; using Tgstation.Server.Common.Extensions; @@ -127,6 +129,16 @@ namespace Tgstation.Server.Host.Components.Session /// readonly SessionConfiguration sessionConfiguration; + /// + /// The number of sessions launched. + /// + readonly Counter sessionsLaunched; + + /// + /// The time the current session was launched. + /// + readonly Gauge lastSessionLaunch; + /// /// The for the . /// @@ -185,6 +197,7 @@ namespace Tgstation.Server.Host.Components.Session /// The value of . /// The value of . /// The value of . + /// The used to create metrics. /// The value of . /// The value of . /// The value of . @@ -204,6 +217,7 @@ namespace Tgstation.Server.Host.Components.Session IEventConsumer eventConsumer, IAsyncDelayer asyncDelayer, IDotnetDumpService dotnetDumpService, + IMetricFactory metricFactory, ILoggerFactory loggerFactory, ILogger logger, SessionConfiguration sessionConfiguration, @@ -224,10 +238,14 @@ namespace Tgstation.Server.Host.Components.Session this.eventConsumer = eventConsumer ?? throw new ArgumentNullException(nameof(eventConsumer)); this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer)); this.dotnetDumpService = dotnetDumpService ?? throw new ArgumentNullException(nameof(dotnetDumpService)); + ArgumentNullException.ThrowIfNull(metricFactory); this.loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory)); this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); this.sessionConfiguration = sessionConfiguration ?? throw new ArgumentNullException(nameof(sessionConfiguration)); this.instance = instance ?? throw new ArgumentNullException(nameof(instance)); + + sessionsLaunched = metricFactory.CreateCounter("tgs_sessions_launched", "The number of game server processes created"); + lastSessionLaunch = metricFactory.CreateGauge("tgs_session_start_time", "The UTC unix timestamp the most recent session was started"); } /// @@ -368,6 +386,12 @@ namespace Tgstation.Server.Host.Components.Session false, apiValidate); + if (!apiValidate) + { + sessionsLaunched.Inc(); + lastSessionLaunch.SetToCurrentTimeUtc(); + } + return sessionController; } catch diff --git a/src/Tgstation.Server.Host/Components/Watchdog/AdvancedWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/AdvancedWatchdog.cs index 68e44eee5d..168a8d77eb 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/AdvancedWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/AdvancedWatchdog.cs @@ -6,6 +6,8 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using Prometheus; + using Tgstation.Server.Api.Models; using Tgstation.Server.Api.Models.Internal; using Tgstation.Server.Host.Components.Chat; @@ -63,6 +65,7 @@ namespace Tgstation.Server.Host.Components.Watchdog /// The 'Diagnostics' for the . /// The for the . /// The for the . + /// The for the . /// The 'Game' for the . /// The value of . /// The for the . @@ -80,6 +83,7 @@ namespace Tgstation.Server.Host.Components.Watchdog IIOManager diagnosticsIOManager, IEventConsumer eventConsumer, IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory, + IMetricFactory metricFactory, IIOManager gameIOManager, IFilesystemLinkFactory linkFactory, ILogger logger, @@ -97,6 +101,7 @@ namespace Tgstation.Server.Host.Components.Watchdog diagnosticsIOManager, eventConsumer, remoteDeploymentManagerFactory, + metricFactory, gameIOManager, logger, initialLaunchParameters, diff --git a/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs index 5bc8089824..6e9520bc7d 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs @@ -6,6 +6,8 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using Prometheus; + using Tgstation.Server.Api.Models.Internal; using Tgstation.Server.Host.Components.Chat; using Tgstation.Server.Host.Components.Deployment; @@ -53,6 +55,7 @@ namespace Tgstation.Server.Host.Components.Watchdog /// The 'Diagnostics' for the . /// The for the . /// The for the . + /// The for the . /// The 'Game' for the . /// The for the . /// The for the . @@ -69,6 +72,7 @@ namespace Tgstation.Server.Host.Components.Watchdog IIOManager diagnosticsIOManager, IEventConsumer eventConsumer, IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory, + IMetricFactory metricFactory, IIOManager gameIOManager, ILogger logger, DreamDaemonLaunchParameters initialLaunchParameters, @@ -85,6 +89,7 @@ namespace Tgstation.Server.Host.Components.Watchdog diagnosticsIOManager, eventConsumer, remoteDeploymentManagerFactory, + metricFactory, gameIOManager, logger, initialLaunchParameters, diff --git a/src/Tgstation.Server.Host/Components/Watchdog/IWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/IWatchdog.cs index c1dd923c00..bacb2cb280 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/IWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/IWatchdog.cs @@ -117,5 +117,10 @@ namespace Tgstation.Server.Host.Components.Watchdog /// The for the operation. /// A resulting in if the broadcast succeeded., otherwise. ValueTask Broadcast(string message, CancellationToken cancellationToken); + + /// + /// Callback to update transient metrics. + /// + void RunMetricsScrape(); } } diff --git a/src/Tgstation.Server.Host/Components/Watchdog/IWatchdogFactory.cs b/src/Tgstation.Server.Host/Components/Watchdog/IWatchdogFactory.cs index 308cd3db54..e6b313af55 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/IWatchdogFactory.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/IWatchdogFactory.cs @@ -1,4 +1,6 @@ -using Tgstation.Server.Api.Models.Internal; +using Prometheus; + +using Tgstation.Server.Api.Models.Internal; using Tgstation.Server.Host.Components.Chat; using Tgstation.Server.Host.Components.Deployment; using Tgstation.Server.Host.Components.Deployment.Remote; @@ -24,6 +26,7 @@ namespace Tgstation.Server.Host.Components.Watchdog /// The pointing to the Diagnostics directory for the . /// The for the . /// The for the . + /// The used to create metrics. /// The for the . /// The initial for the . /// A new . @@ -36,6 +39,7 @@ namespace Tgstation.Server.Host.Components.Watchdog IIOManager diagnosticsIOManager, IEventConsumer eventConsumer, IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory, + IMetricFactory metricFactory, Api.Models.Instance instance, DreamDaemonSettings settings); } diff --git a/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs index b4c7a74229..b97517d0f7 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs @@ -5,6 +5,8 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using Prometheus; + using Tgstation.Server.Api.Models.Internal; using Tgstation.Server.Host.Components.Chat; using Tgstation.Server.Host.Components.Deployment; @@ -43,6 +45,7 @@ namespace Tgstation.Server.Host.Components.Watchdog /// The for the . /// The for the . /// The for the . + /// The for the . /// The pointing to the game directory for the .. /// The for the . /// The for the . @@ -61,6 +64,7 @@ namespace Tgstation.Server.Host.Components.Watchdog IIOManager diagnosticsIOManager, IEventConsumer eventConsumer, IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory, + IMetricFactory metricFactory, IIOManager gameIOManager, IFilesystemLinkFactory linkFactory, ILogger logger, @@ -79,6 +83,7 @@ namespace Tgstation.Server.Host.Components.Watchdog diagnosticsIOManager, eventConsumer, remoteDeploymentManagerFactory, + metricFactory, gameIOManager, linkFactory, logger, diff --git a/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdogFactory.cs b/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdogFactory.cs index 1d0d140bc0..23539d34ac 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdogFactory.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdogFactory.cs @@ -4,6 +4,8 @@ using System.Runtime.Versioning; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using Prometheus; + using Tgstation.Server.Api.Models.Internal; using Tgstation.Server.Host.Components.Chat; using Tgstation.Server.Host.Components.Deployment; @@ -60,6 +62,7 @@ namespace Tgstation.Server.Host.Components.Watchdog IIOManager diagnosticsIOManager, IEventConsumer eventConsumer, IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory, + IMetricFactory metricFactory, Api.Models.Instance instance, DreamDaemonSettings settings) => new PosixWatchdog( @@ -73,6 +76,7 @@ namespace Tgstation.Server.Host.Components.Watchdog diagnosticsIOManager, eventConsumer, remoteDeploymentManagerFactory, + metricFactory, gameIOManager, LinkFactory, LoggerFactory.CreateLogger(), diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs index 7fa4fc4926..805957535e 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs @@ -7,6 +7,8 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using Prometheus; + using Serilog.Context; using Tgstation.Server.Api.Models; @@ -123,6 +125,21 @@ namespace Tgstation.Server.Host.Components.Watchdog /// readonly SemaphoreSlim controllerDisposeSemaphore; + /// + /// The as a metric. + /// + readonly Gauge watchdogStatusMetric; + + /// + /// Active session CPU usage as a metric. + /// + readonly Gauge cpuUsageMetric; + + /// + /// as a metric. + /// + readonly Gauge ramUsageMetric; + /// /// The that is not the . /// @@ -201,6 +218,7 @@ namespace Tgstation.Server.Host.Components.Watchdog /// The value of . /// The value of . /// The value of . + /// The used to create metrics. /// The value of . /// The value of . /// The initial value of . May be modified. @@ -217,6 +235,7 @@ namespace Tgstation.Server.Host.Components.Watchdog IIOManager diagnosticsIOManager, IEventConsumer eventConsumer, IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory, + IMetricFactory metricFactory, IIOManager gameIOManager, ILogger logger, DreamDaemonLaunchParameters initialLaunchParameters, @@ -232,6 +251,7 @@ namespace Tgstation.Server.Host.Components.Watchdog this.diagnosticsIOManager = diagnosticsIOManager ?? throw new ArgumentNullException(nameof(diagnosticsIOManager)); this.eventConsumer = eventConsumer ?? throw new ArgumentNullException(nameof(eventConsumer)); this.remoteDeploymentManagerFactory = remoteDeploymentManagerFactory ?? throw new ArgumentNullException(nameof(remoteDeploymentManagerFactory)); + ArgumentNullException.ThrowIfNull(metricFactory); GameIOManager = gameIOManager ?? throw new ArgumentNullException(nameof(gameIOManager)); Logger = logger ?? throw new ArgumentNullException(nameof(logger)); ActiveLaunchParameters = initialLaunchParameters ?? throw new ArgumentNullException(nameof(initialLaunchParameters)); @@ -240,6 +260,12 @@ namespace Tgstation.Server.Host.Components.Watchdog ArgumentNullException.ThrowIfNull(serverControl); + watchdogStatusMetric = metricFactory.CreateGauge( + "tgs_watchdog_status", + $"TGS Watchdog status: {(int)WatchdogStatus.Offline} = Offline, {(int)WatchdogStatus.Online} = Online, {(int)WatchdogStatus.Restoring} = Restoring, {(int)WatchdogStatus.DelayedRestart} = Delayed Restart"); + cpuUsageMetric = metricFactory.CreateGauge("tgs_game_cpu_usage", "Estimated total CPU usage time for the game process from 0-1"); + ramUsageMetric = metricFactory.CreateGauge("tgs_game_ram_usage", "Total used bytes of private memory for the game process"); + chat.RegisterCommandHandler(this); ActiveLaunchParameters = initialLaunchParameters; @@ -272,6 +298,7 @@ namespace Tgstation.Server.Host.Components.Watchdog await DisposeAndNullControllersImpl(); controllerDisposeSemaphore.Dispose(); monitorCts?.Dispose(); + disposed = true; } @@ -501,6 +528,15 @@ namespace Tgstation.Server.Host.Components.Watchdog return response != null && response.ErrorMessage == null; } + /// + public void RunMetricsScrape() + { + watchdogStatusMetric.Set((int)Status); + var controller = GetActiveController(); + ramUsageMetric.Set(controller?.MemoryUsage ?? 0); + cpuUsageMetric.Set(controller?.MeasureProcessorTimeDelta() ?? 0); + } + /// async ValueTask IEventConsumer.HandleEvent(EventType eventType, IEnumerable parameters, bool deploymentPipeline, CancellationToken cancellationToken) { diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogFactory.cs b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogFactory.cs index e9a37ecb2b..d052001900 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogFactory.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogFactory.cs @@ -3,6 +3,8 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using Prometheus; + using Tgstation.Server.Api.Models.Internal; using Tgstation.Server.Host.Components.Chat; using Tgstation.Server.Host.Components.Deployment; @@ -77,6 +79,7 @@ namespace Tgstation.Server.Host.Components.Watchdog IIOManager diagnosticsIOManager, IEventConsumer eventConsumer, IRemoteDeploymentManagerFactory remoteDeploymentManagerfactory, + IMetricFactory metricFactory, Api.Models.Instance instance, DreamDaemonSettings settings) => new BasicWatchdog( @@ -90,6 +93,7 @@ namespace Tgstation.Server.Host.Components.Watchdog diagnosticsIOManager, eventConsumer, remoteDeploymentManagerfactory, + metricFactory, gameIOManager, LoggerFactory.CreateLogger(), settings, diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs index b03cad9bae..fcf76c5abc 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs @@ -3,6 +3,8 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using Prometheus; + using Tgstation.Server.Api.Models; using Tgstation.Server.Api.Models.Internal; using Tgstation.Server.Host.Components.Chat; @@ -35,6 +37,7 @@ namespace Tgstation.Server.Host.Components.Watchdog /// The for the . /// The for the . /// The for the . + /// The for the . /// The pointing to the game directory for the .. /// The for the . /// The for the . @@ -52,6 +55,7 @@ namespace Tgstation.Server.Host.Components.Watchdog IIOManager diagnosticsIOManager, IEventConsumer eventConsumer, IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory, + IMetricFactory metricFactory, IIOManager gameIOManager, IFilesystemLinkFactory linkFactory, ILogger logger, @@ -69,6 +73,7 @@ namespace Tgstation.Server.Host.Components.Watchdog diagnosticsIOManager, eventConsumer, remoteDeploymentManagerFactory, + metricFactory, gameIOManager, linkFactory, logger, diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs index 89ed6b145d..5639219021 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs @@ -3,6 +3,8 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using Prometheus; + using Tgstation.Server.Api.Models.Internal; using Tgstation.Server.Host.Components.Chat; using Tgstation.Server.Host.Components.Deployment; @@ -63,6 +65,7 @@ namespace Tgstation.Server.Host.Components.Watchdog IIOManager diagnosticsIOManager, IEventConsumer eventConsumer, IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory, + IMetricFactory metricFactory, Api.Models.Instance instance, DreamDaemonSettings settings) => new WindowsWatchdog( @@ -76,6 +79,7 @@ namespace Tgstation.Server.Host.Components.Watchdog diagnosticsIOManager, eventConsumer, remoteDeploymentManagerFactory, + metricFactory, gameIOManager, LinkFactory, LoggerFactory.CreateLogger(), diff --git a/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs b/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs index 5ea16ae64e..3ba8b3e878 100644 --- a/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs +++ b/src/Tgstation.Server.Host/Configuration/GeneralConfiguration.cs @@ -91,6 +91,11 @@ namespace Tgstation.Server.Host.Configuration /// public ushort ApiPort { get; set; } + /// + /// The port Prometheus metrics are published on, if any. + /// + public ushort? PrometheusPort { get; set; } + /// /// A classic GitHub personal access token to use for bypassing rate limits on requests. Requires no scopes. /// diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index dd88ce5d80..08f39565a6 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -30,6 +30,8 @@ using Microsoft.Extensions.Options; using Newtonsoft.Json; +using Prometheus; + using Serilog; using Serilog.Events; using Serilog.Formatting.Display; @@ -294,6 +296,21 @@ namespace Tgstation.Server.Host.Core services.AddHttpClient(); services.AddSingleton(); + // configure metrics + var prometheusPort = postSetupServices.GeneralConfiguration.PrometheusPort; + + services.AddSingleton(_ => Metrics.DefaultFactory); + services.AddSingleton(_ => Metrics.DefaultRegistry); + + if (prometheusPort.HasValue && prometheusPort != postSetupServices.GeneralConfiguration.ApiPort) + services.AddMetricServer(options => options.Port = prometheusPort.Value); + + services.UseHttpClientMetrics(); + + var healthChecksBuilder = services + .AddHealthChecks() + .ForwardToPrometheus(); + // configure graphql services .AddScoped() @@ -362,6 +379,9 @@ namespace Tgstation.Server.Host.Core configureAction(builder, databaseConfig); }); services.AddScoped(x => x.GetRequiredService()); + + healthChecksBuilder + .AddDbContextCheck(); } // add the correct database context type @@ -571,6 +591,9 @@ namespace Tgstation.Server.Host.Core // Wrap exceptions in a 500 (ErrorMessage) response applicationBuilder.UseServerErrorHandling(); + // metrics capture + applicationBuilder.UseHttpMetrics(); + // Add the X-Powered-By response header applicationBuilder.UseServerBranding(assemblyInformationProvider); @@ -692,6 +715,19 @@ namespace Tgstation.Server.Host.Core .MapGraphQL(Routes.GraphQL) .WithOptions(gqlOptions); } + + if (generalConfiguration.PrometheusPort.HasValue) + if (generalConfiguration.PrometheusPort == generalConfiguration.ApiPort) + { + endpoints.MapMetrics(); + logger.LogDebug("Prometheus being hosted alongside server"); + } + else + logger.LogDebug("Prometheus being hosted on port {prometheusPort}", generalConfiguration.PrometheusPort); + else + logger.LogTrace("Prometheus disabled"); + + endpoints.MapHealthChecks("/health"); }); // 404 anything that gets this far diff --git a/src/Tgstation.Server.Host/Jobs/JobService.cs b/src/Tgstation.Server.Host/Jobs/JobService.cs index d78cc7a287..519d00c2d3 100644 --- a/src/Tgstation.Server.Host/Jobs/JobService.cs +++ b/src/Tgstation.Server.Host/Jobs/JobService.cs @@ -9,6 +9,8 @@ using Microsoft.AspNetCore.SignalR; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; +using Prometheus; + using Serilog.Context; using Tgstation.Server.Api.Extensions; @@ -67,6 +69,16 @@ namespace Tgstation.Server.Host.Jobs /// readonly TaskCompletionSource activationTcs; + /// + /// Total number of jobs processed. + /// + readonly Counter processedJobs; + + /// + /// Jobs currently running. + /// + readonly Gauge runningJobs; + /// /// for various operations. /// @@ -87,19 +99,25 @@ namespace Tgstation.Server.Host.Jobs /// /// The value of . /// The value of . + /// The to use. /// The value of . /// The value of . public JobService( IConnectionMappedHubContext hub, IDatabaseContextFactory databaseContextFactory, + IMetricFactory metricFactory, ILoggerFactory loggerFactory, ILogger logger) { this.hub = hub ?? throw new ArgumentNullException(nameof(hub)); this.databaseContextFactory = databaseContextFactory ?? throw new ArgumentNullException(nameof(databaseContextFactory)); this.loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory)); + ArgumentNullException.ThrowIfNull(metricFactory); this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + runningJobs = metricFactory.CreateGauge("tgs_jobs_running", "The number of TGS jobs running across all instances"); + processedJobs = metricFactory.CreateCounter("tgs_jobs_processed", "The number of TGS jobs that have run previously"); + jobs = new Dictionary(); hubUpdateActions = new Dictionary(); activationTcs = new TaskCompletionSource(); @@ -363,6 +381,7 @@ namespace Tgstation.Server.Host.Jobs { var jid = job.Require(x => x.Id); using (LogContext.PushProperty(SerilogContextHelper.JobIdContextProperty, jid)) + using (runningJobs.TrackInProgress()) try { void LogException(Exception ex) => logger.LogDebug(ex, "Job {jobId} exited with error!", jid); @@ -553,6 +572,8 @@ namespace Tgstation.Server.Host.Jobs jobs.Remove(jid); handler.Dispose(); } + + processedJobs.Inc(); } } } diff --git a/src/Tgstation.Server.Host/System/IProcessBase.cs b/src/Tgstation.Server.Host/System/IProcessBase.cs index b95b0bf928..64d11da92d 100644 --- a/src/Tgstation.Server.Host/System/IProcessBase.cs +++ b/src/Tgstation.Server.Host/System/IProcessBase.cs @@ -24,6 +24,12 @@ namespace Tgstation.Server.Host.System /// long? MemoryUsage { get; } + /// + /// Gets the estimated CPU usage fraction of the process based on the last time this was called. + /// + /// The CPU's estimated usage as a value between 0 and 1. + double MeasureProcessorTimeDelta(); + /// /// Set's the owned to a non-normal value. /// diff --git a/src/Tgstation.Server.Host/System/Process.cs b/src/Tgstation.Server.Host/System/Process.cs index 76f9ef56f3..09ad4aa3af 100644 --- a/src/Tgstation.Server.Host/System/Process.cs +++ b/src/Tgstation.Server.Host/System/Process.cs @@ -87,10 +87,30 @@ namespace Tgstation.Server.Host.System /// readonly Task? readTask; + /// + /// object for measuring processor time usage. + /// + readonly object processTimeMeasureLock; + + /// + /// The last time was called. + /// + DateTimeOffset lastProcessorMeasureTime; + + /// + /// The last value of . + /// + TimeSpan lastProcessorUsageTime; + + /// + /// The last valid return value of . + /// + double lastProcessorUsageEstimation; + /// /// If the was disposed. /// - volatile int disposed; + int disposed; /// /// Initializes a new instance of the class. @@ -123,6 +143,8 @@ namespace Tgstation.Server.Host.System this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + processTimeMeasureLock = new object(); + Lifetime = WrapLifetimeTask(); if (preExisting) @@ -147,6 +169,8 @@ namespace Tgstation.Server.Host.System DefaultIOManager.BlockingTaskCreationOptions, TaskScheduler.Current); + MeasureProcessorTimeDelta(); + logger.LogTrace("Created process ID: {pid}", Id); } @@ -267,6 +291,34 @@ namespace Tgstation.Server.Host.System return processFeatures.CreateDump(handle, outputFile, minidump, cancellationToken); } + /// + public double MeasureProcessorTimeDelta() + { + lock (processTimeMeasureLock) + { + try + { + var now = DateTimeOffset.UtcNow; + var newTime = handle.TotalProcessorTime; + + var timeDelta = now - lastProcessorMeasureTime; + var newUsage = newTime - lastProcessorUsageTime; + + lastProcessorMeasureTime = now; + lastProcessorUsageTime = newTime; + + if (timeDelta != TimeSpan.Zero) + lastProcessorUsageEstimation = newUsage / (Environment.ProcessorCount * timeDelta); + } + catch (Exception ex) + { + logger.LogWarning(ex, "Error measuring processor time delta!"); + } + + return lastProcessorUsageEstimation; + } + } + /// /// Attaches a log message to the process' exit event. /// diff --git a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj index dca5d19f22..2d84f38fdb 100644 --- a/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj +++ b/src/Tgstation.Server.Host/Tgstation.Server.Host.csproj @@ -1,4 +1,4 @@ - + @@ -110,21 +110,23 @@ - + - + - + - + - + runtime; build; native; contentfiles; analyzers; buildtransitive - + - + + + @@ -132,11 +134,13 @@ - + + + - + @@ -152,11 +156,11 @@ - + - + diff --git a/src/Tgstation.Server.Host/Utils/GitHub/GitHubClientFactory.cs b/src/Tgstation.Server.Host/Utils/GitHub/GitHubClientFactory.cs index a424e1b4d5..2a1554f0e9 100644 --- a/src/Tgstation.Server.Host/Utils/GitHub/GitHubClientFactory.cs +++ b/src/Tgstation.Server.Host/Utils/GitHub/GitHubClientFactory.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IdentityModel.Tokens.Jwt; using System.Linq; +using System.Net.Http; using System.Security.Cryptography; using System.Text; using System.Threading; @@ -12,6 +13,7 @@ using Microsoft.Extensions.Options; using Microsoft.IdentityModel.Tokens; using Octokit; +using Octokit.Internal; using Tgstation.Server.Api.Models; using Tgstation.Server.Host.Configuration; @@ -38,6 +40,11 @@ namespace Tgstation.Server.Host.Utils.GitHub /// readonly IAssemblyInformationProvider assemblyInformationProvider; + /// + /// The for the . + /// + readonly IHttpMessageHandlerFactory httpMessageHandlerFactory; + /// /// The for the . /// @@ -62,14 +69,17 @@ namespace Tgstation.Server.Host.Utils.GitHub /// Initializes a new instance of the class. /// /// The value of . + /// The value of . /// The value of . /// The containing the value of . public GitHubClientFactory( IAssemblyInformationProvider assemblyInformationProvider, + IHttpMessageHandlerFactory httpMessageHandlerFactory, ILogger logger, IOptions generalConfigurationOptions) { this.assemblyInformationProvider = assemblyInformationProvider ?? throw new ArgumentNullException(nameof(assemblyInformationProvider)); + this.httpMessageHandlerFactory = httpMessageHandlerFactory ?? throw new ArgumentNullException(nameof(httpMessageHandlerFactory)); this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); generalConfiguration = generalConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(generalConfigurationOptions)); @@ -315,10 +325,33 @@ namespace Tgstation.Server.Host.Utils.GitHub GitHubClient CreateUnauthenticatedClient() { var product = assemblyInformationProvider.ProductInfoHeaderValue.Product!; - return new GitHubClient( - new ProductHeaderValue( - product.Name, - product.Version)); +#pragma warning disable CA2000 // Dispose objects before losing scope + var handler = httpMessageHandlerFactory.CreateHandler(); + try + { + var clientAdapter = new HttpClientAdapter(() => handler); +#pragma warning restore CA2000 // Dispose objects before losing scope + handler = null; + try + { + return new GitHubClient( + new Connection( + new ProductHeaderValue( + product.Name, + product.Version), + clientAdapter)); + } + catch + { + clientAdapter.Dispose(); + throw; + } + } + catch + { + handler?.Dispose(); + throw; + } } } } diff --git a/src/Tgstation.Server.Host/appsettings.yml b/src/Tgstation.Server.Host/appsettings.yml index 2978befbf9..add5c212bf 100644 --- a/src/Tgstation.Server.Host/appsettings.yml +++ b/src/Tgstation.Server.Host/appsettings.yml @@ -8,6 +8,7 @@ General: ByondTopicTimeout: 5000 # Timeout for BYOND /world/Topic() calls in milliseconds RestartTimeoutMinutes: 1 # Timeout for server restarts after requested by SIGTERM or the HTTP API ApiPort: 5000 # Port the HTTP API is hosted on + PrometheusPort: null # Port Prometheus metrics are published on under /metrics. This can be set to the same value as the ApiPort, just note that accessing it does not require authentication. UseBasicWatchdog: false # The basic watchdog hard restarts DreamDaemon when /world/proc/TgsReboot() is called in the DMAPI if a new deployment is available UserLimit: 100 # Maximum number of allowed users UserGroupLimit: 25 # Maximum number of allowed groups diff --git a/src/Tgstation.Server.Shared/Tgstation.Server.Shared.csproj b/src/Tgstation.Server.Shared/Tgstation.Server.Shared.csproj index bd8bbc1219..29e3b26c77 100644 --- a/src/Tgstation.Server.Shared/Tgstation.Server.Shared.csproj +++ b/src/Tgstation.Server.Shared/Tgstation.Server.Shared.csproj @@ -10,7 +10,7 @@ - + diff --git a/tests/Tgstation.Server.Client.Tests/Tgstation.Server.Client.Tests.csproj b/tests/Tgstation.Server.Client.Tests/Tgstation.Server.Client.Tests.csproj index 0c961dac12..886899a85b 100644 --- a/tests/Tgstation.Server.Client.Tests/Tgstation.Server.Client.Tests.csproj +++ b/tests/Tgstation.Server.Client.Tests/Tgstation.Server.Client.Tests.csproj @@ -6,7 +6,7 @@ - + diff --git a/tests/Tgstation.Server.Host.Tests/BasicHttpMessageHandlerFactory.cs b/tests/Tgstation.Server.Host.Tests/BasicHttpMessageHandlerFactory.cs new file mode 100644 index 0000000000..94dd57d3ea --- /dev/null +++ b/tests/Tgstation.Server.Host.Tests/BasicHttpMessageHandlerFactory.cs @@ -0,0 +1,19 @@ +using System; +using System.Net.Http; + +namespace Tgstation.Server.Host.Tests +{ + /// + /// Basic implementation for testiong + /// + public sealed class BasicHttpMessageHandlerFactory : IHttpMessageHandlerFactory, IDisposable + { + readonly HttpClientHandler handler = new(); + + public HttpMessageHandler CreateHandler(string name) + => handler; + + public void Dispose() + => handler.Dispose(); + } +} diff --git a/tests/Tgstation.Server.Host.Tests/Tgstation.Server.Host.Tests.csproj b/tests/Tgstation.Server.Host.Tests/Tgstation.Server.Host.Tests.csproj index 69c414d1e2..0849c3118b 100644 --- a/tests/Tgstation.Server.Host.Tests/Tgstation.Server.Host.Tests.csproj +++ b/tests/Tgstation.Server.Host.Tests/Tgstation.Server.Host.Tests.csproj @@ -7,7 +7,7 @@ - + diff --git a/tests/Tgstation.Server.Host.Tests/Utils/GitHub/TestGitHubClientFactory.cs b/tests/Tgstation.Server.Host.Tests/Utils/GitHub/TestGitHubClientFactory.cs index 5132f8ab01..01bb7bac1f 100644 --- a/tests/Tgstation.Server.Host.Tests/Utils/GitHub/TestGitHubClientFactory.cs +++ b/tests/Tgstation.Server.Host.Tests/Utils/GitHub/TestGitHubClientFactory.cs @@ -1,4 +1,5 @@ using System; +using System.Net.Http; using System.Net.Http.Headers; using System.Text; using System.Threading; @@ -15,6 +16,7 @@ using Octokit; using Tgstation.Server.Api.Models; using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.System; +using Tgstation.Server.Host.Tests; namespace Tgstation.Server.Host.Utils.GitHub.Tests { @@ -42,9 +44,10 @@ namespace Tgstation.Server.Host.Utils.GitHub.Tests [TestMethod] public void TestContructionThrows() { - Assert.ThrowsException(() => new GitHubClientFactory(null, null, null)); - Assert.ThrowsException(() => new GitHubClientFactory(Mock.Of(), null, null)); - Assert.ThrowsException(() => new GitHubClientFactory(Mock.Of(), Mock.Of>(), null)); + Assert.ThrowsException(() => new GitHubClientFactory(null, null, null, null)); + Assert.ThrowsException(() => new GitHubClientFactory(Mock.Of(), null, null, null)); + Assert.ThrowsException(() => new GitHubClientFactory(Mock.Of(), Mock.Of(), null, null)); + Assert.ThrowsException(() => new GitHubClientFactory(Mock.Of(), Mock.Of(), Mock.Of>(), null)); } [TestMethod] @@ -58,7 +61,7 @@ namespace Tgstation.Server.Host.Utils.GitHub.Tests var gc = new GeneralConfiguration(); Assert.IsNull(gc.GitHubAccessToken); mockOptions.SetupGet(x => x.Value).Returns(gc); - var factory = new GitHubClientFactory(mockApp.Object, loggerFactory.CreateLogger(), mockOptions.Object); + var factory = new GitHubClientFactory(mockApp.Object, new BasicHttpMessageHandlerFactory(), loggerFactory.CreateLogger(), mockOptions.Object); var client = await factory.CreateClient(CancellationToken.None); Assert.IsNotNull(client); @@ -84,7 +87,7 @@ namespace Tgstation.Server.Host.Utils.GitHub.Tests var mockOptions = new Mock>(); mockOptions.SetupGet(x => x.Value).Returns(new GeneralConfiguration()); - var factory = new GitHubClientFactory(mockApp.Object, loggerFactory.CreateLogger(), mockOptions.Object); + var factory = new GitHubClientFactory(mockApp.Object, new BasicHttpMessageHandlerFactory(), loggerFactory.CreateLogger(), mockOptions.Object); await Assert.ThrowsExceptionAsync(() => factory.CreateClient(null, CancellationToken.None).AsTask()); @@ -106,7 +109,7 @@ namespace Tgstation.Server.Host.Utils.GitHub.Tests var mockOptions = new Mock>(); mockOptions.SetupGet(x => x.Value).Returns(new GeneralConfiguration()); - var factory = new GitHubClientFactory(mockApp.Object, loggerFactory.CreateLogger(), mockOptions.Object); + var factory = new GitHubClientFactory(mockApp.Object, new BasicHttpMessageHandlerFactory(), loggerFactory.CreateLogger(), mockOptions.Object); var appID = Environment.GetEnvironmentVariable("TGS_TEST_APP_ID"); var privateKey = Environment.GetEnvironmentVariable("TGS_TEST_APP_PRIVATE_KEY"); @@ -144,7 +147,7 @@ namespace Tgstation.Server.Host.Utils.GitHub.Tests var mockOptions = new Mock>(); mockOptions.SetupGet(x => x.Value).Returns(new GeneralConfiguration()); - var factory = new GitHubClientFactory(mockApp.Object, loggerFactory.CreateLogger(), mockOptions.Object); + var factory = new GitHubClientFactory(mockApp.Object, new BasicHttpMessageHandlerFactory(), loggerFactory.CreateLogger(), mockOptions.Object); await Assert.ThrowsExceptionAsync(() => factory.CreateClient(null, CancellationToken.None).AsTask()); @@ -192,7 +195,7 @@ vTdVAoGBAI/jjUMdjkY43zhe3w2piwT0fhGfqm9ikdAB9IcgcptuS0ML0ZaWV/eO var mockOptions = new Mock>(); mockOptions.SetupGet(x => x.Value).Returns(new GeneralConfiguration()); - var factory = new GitHubClientFactory(mockApp.Object, loggerFactory.CreateLogger(), mockOptions.Object); + var factory = new GitHubClientFactory(mockApp.Object, new BasicHttpMessageHandlerFactory(), loggerFactory.CreateLogger(), mockOptions.Object); var client1 = await factory.CreateClient(CancellationToken.None); var client2 = await factory.CreateClient("asdf", CancellationToken.None); diff --git a/tests/Tgstation.Server.Tests/Live/DummyChatProvider.cs b/tests/Tgstation.Server.Tests/Live/DummyChatProvider.cs index b82cf58226..04fe1b542c 100644 --- a/tests/Tgstation.Server.Tests/Live/DummyChatProvider.cs +++ b/tests/Tgstation.Server.Tests/Live/DummyChatProvider.cs @@ -115,8 +115,6 @@ namespace Tgstation.Server.Tests.Live { ArgumentNullException.ThrowIfNull(revisionInformation); ArgumentNullException.ThrowIfNull(engineVersion); - ArgumentNullException.ThrowIfNull(gitHubOwner); - ArgumentNullException.ThrowIfNull(gitHubRepo); Assert.IsTrue(knownChannels.ContainsKey(channelId)); diff --git a/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs b/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs index bbd6bb176c..ac4f519fa6 100644 --- a/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs +++ b/tests/Tgstation.Server.Tests/Live/LiveTestingServer.cs @@ -156,6 +156,7 @@ namespace Tgstation.Server.Tests.Live $"Security:TokenExpiryMinutes=120", // timeouts are useless for us $"General:OpenDreamSuppressInstallOutput={TestingUtils.RunningInGitHubActions}", "Telemetry:DisableVersionReporting=true", + $"General:PrometheusPort={port}" }; if (MultiServerClient.UseGraphQL) diff --git a/tests/Tgstation.Server.Tests/Live/RawRequestTests.cs b/tests/Tgstation.Server.Tests/Live/RawRequestTests.cs index f04ce92007..2e26ff9bc7 100644 --- a/tests/Tgstation.Server.Tests/Live/RawRequestTests.cs +++ b/tests/Tgstation.Server.Tests/Live/RawRequestTests.cs @@ -455,6 +455,40 @@ namespace Tgstation.Server.Tests.Live } } + static async Task TestHealthChecks(IRestServerClient serverClient, CancellationToken cancellationToken) + { + var url = serverClient.Url; + // check that 400s are returned appropriately + using var httpClient = new HttpClient(); + using (var request = new HttpRequestMessage(HttpMethod.Get, url.ToString() + "health")) + { + request.Headers.Accept.Clear(); + request.Headers.UserAgent.Add(new ProductInfoHeaderValue("RootTest", "1.0.0")); + using var response = await httpClient.SendAsync(request, cancellationToken); + response.EnsureSuccessStatusCode(); + + var text = await response.Content.ReadAsStringAsync(cancellationToken); + Assert.AreEqual("Healthy", text.Trim()); + } + } + + static async Task TestPrometheusMetrics(IRestServerClient serverClient, CancellationToken cancellationToken) + { + var url = serverClient.Url; + // check that 400s are returned appropriately + using var httpClient = new HttpClient(); + using (var request = new HttpRequestMessage(HttpMethod.Get, url.ToString() + "metrics")) + { + request.Headers.Accept.Clear(); + request.Headers.UserAgent.Add(new ProductInfoHeaderValue("RootTest", "1.0.0")); + using var response = await httpClient.SendAsync(request, cancellationToken); + response.EnsureSuccessStatusCode(); + + var text = await response.Content.ReadAsStringAsync(cancellationToken); + Assert.IsTrue(text.Contains("tgs_jobs_running")); + } + } + static async Task TestGraphQLLogin(IRestServerClientFactory clientFactory, IRestServerClient restClient, CancellationToken cancellationToken) { if (!MultiServerClient.UseGraphQL) @@ -483,6 +517,8 @@ namespace Tgstation.Server.Tests.Live TestServerInformation(clientFactory, serverClient, cancellationToken), TestInvalidTransfers(serverClient, cancellationToken), RegressionTestForLeakedPasswordHashesBug(serverClient, cancellationToken), - TestSignalRUsage(clientFactory, serverClient, cancellationToken)); + TestSignalRUsage(clientFactory, serverClient, cancellationToken), + TestHealthChecks(serverClient, cancellationToken), + TestPrometheusMetrics(serverClient, cancellationToken)); } } diff --git a/tests/Tgstation.Server.Tests/Live/TestingGitHubService.cs b/tests/Tgstation.Server.Tests/Live/TestingGitHubService.cs index 7bb3537ce9..731e653a4c 100644 --- a/tests/Tgstation.Server.Tests/Live/TestingGitHubService.cs +++ b/tests/Tgstation.Server.Tests/Live/TestingGitHubService.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Net.Http; using System.Threading; using System.Threading.Tasks; @@ -15,6 +16,7 @@ using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.Extensions; using Tgstation.Server.Host.Security; using Tgstation.Server.Host.System; +using Tgstation.Server.Host.Tests; using Tgstation.Server.Host.Utils.GitHub; namespace Tgstation.Server.Tests.Live @@ -38,7 +40,7 @@ namespace Tgstation.Server.Tests.Live GitHubAccessToken = Environment.GetEnvironmentVariable("TGS_TEST_GITHUB_TOKEN") }); - var gitHubClientFactory = new GitHubClientFactory(new AssemblyInformationProvider(), Mock.Of>(), mockOptions.Object); + var gitHubClientFactory = new GitHubClientFactory(new AssemblyInformationProvider(), new BasicHttpMessageHandlerFactory(), Mock.Of>(), mockOptions.Object); RealClient = gitHubClientFactory.CreateClient(CancellationToken.None).GetAwaiter().GetResult(); } diff --git a/tests/Tgstation.Server.Tests/Tgstation.Server.Tests.csproj b/tests/Tgstation.Server.Tests/Tgstation.Server.Tests.csproj index 32edf3de15..e4b0a8244b 100644 --- a/tests/Tgstation.Server.Tests/Tgstation.Server.Tests.csproj +++ b/tests/Tgstation.Server.Tests/Tgstation.Server.Tests.csproj @@ -6,13 +6,14 @@ - + +