Merge branch 'dev' into tm_spam_reduction

This commit is contained in:
Drathek
2025-02-17 22:55:19 -06:00
committed by GitHub
45 changed files with 459 additions and 55 deletions
+3 -1
View File
@@ -256,7 +256,9 @@ Create an `appsettings.Production.yml` file next to `appsettings.yml`. This will
- `General:ConfigVersion`: Suppresses warnings about out of date config versions. You should change this after updating TGS to one with a new config version. The current version can be found on the releases page for your server version.
- `General:MinimumPasswordLength`: Minimum password length requirement for database users
- `General:MinimumPasswordLength`: Minimum password length requirement for database users.
- `General:PrometheusPort`: Port Prometheus metrics are published on under /metrics. This can be set to the same value as the `ApiPort`, just note that accessing it does not require authentication.
- `General:ValidInstancePaths`: Array meant to limit the directories in which instances may be created.
+4 -1
View File
@@ -58,11 +58,12 @@ RUN export TGS_TELEMETRY_KEY_FILE="../../${TGS_TELEMETRY_KEY_FILE}" \
FROM mcr.microsoft.com/dotnet/aspnet:8.0-bookworm-slim
#needed for byond
#needed for byond, curl for healthchecks
RUN apt-get update \
&& apt-get install -y \
gcc-multilib \
gdb \
curl \
&& rm -rf /var/lib/apt/lists/*
EXPOSE 5000
@@ -78,4 +79,6 @@ COPY --from=build /repo/build/tgs.docker.sh tgs.sh
VOLUME ["/config_data", "/tgs_logs", "/app/lib"]
HEALTHCHECK CMD --curl --fail http://localhost:5000/health || exit
ENTRYPOINT ["./tgs.sh"]
+1 -1
View File
@@ -13,7 +13,7 @@
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<!-- Usage: Hard to say what exactly this is for, but not including it removes the test icon and breaks vstest.console.exe for some reason -->
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.12.0" Condition="'$(TgsTestNoSdk)' != 'true'" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.13.0" Condition="'$(TgsTestNoSdk)' != 'true'" />
<!-- Usage: Dependency mocking for tests -->
<!-- Pinned: Be VERY careful about updating https://github.com/moq/moq/issues/1372 -->
<PackageReference Include="Moq" Version="4.20.72" />
+2 -2
View File
@@ -3,8 +3,8 @@
<!-- Integration tests will ensure they match across the board -->
<Import Project="WebpanelVersion.props" />
<PropertyGroup>
<TgsCoreVersion>6.13.0</TgsCoreVersion>
<TgsConfigVersion>5.4.0</TgsConfigVersion>
<TgsCoreVersion>6.14.0</TgsCoreVersion>
<TgsConfigVersion>5.5.0</TgsConfigVersion>
<TgsRestVersion>10.12.1</TgsRestVersion>
<TgsGraphQLVersion>0.5.0</TgsGraphQLVersion>
<TgsCommonLibraryVersion>7.0.0</TgsCommonLibraryVersion>
@@ -28,10 +28,9 @@
<!-- Usage: HTTP constants reference -->
<PackageReference Include="Microsoft.AspNetCore.Http.Extensions" Version="2.3.0" />
<!-- Usage: Decoding the 'nbf' property of JWTs -->
<PackageReference Include="Microsoft.IdentityModel.JsonWebTokens" Version="8.4.0" />
<PackageReference Include="Microsoft.IdentityModel.JsonWebTokens" Version="8.5.0" />
<!-- Usage: Data model annotating -->
<PackageReference Include="System.ComponentModel.Annotations" Version="5.0.0" />
<PackageReference Include="System.Text.Encodings.Web" Version="9.0.1" />
</ItemGroup>
<ItemGroup>
@@ -9,7 +9,7 @@
<ItemGroup>
<!-- GraphQL connector and code generator -->
<PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="9.0.1" />
<PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="9.0.2" />
<PackageReference Include="StrawberryShake.Server" Version="15.0.3" />
</ItemGroup>
@@ -11,9 +11,9 @@
<ItemGroup>
<!-- Usage: Connecting to SignalR hubs in API -->
<PackageReference Include="Microsoft.AspNetCore.SignalR.Client" Version="9.0.1" />
<PackageReference Include="Microsoft.AspNetCore.SignalR.Client" Version="9.0.2" />
<!-- Usage: Using target JSON serializer for API -->
<PackageReference Include="Microsoft.AspNetCore.SignalR.Protocols.NewtonsoftJson" Version="9.0.0" />
<PackageReference Include="Microsoft.AspNetCore.SignalR.Protocols.NewtonsoftJson" Version="9.0.2" />
</ItemGroup>
<ItemGroup>
@@ -13,9 +13,9 @@
<ItemGroup>
<!-- Usage: Identifying if we're running under SystemD -->
<PackageReference Include="Microsoft.Extensions.Hosting.Systemd" Version="9.0.1" />
<PackageReference Include="Microsoft.Extensions.Hosting.Systemd" Version="9.0.2" />
<!-- Usage: Console logging plugin -->
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="9.0.1" />
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="9.0.2" />
</ItemGroup>
<ItemGroup>
@@ -21,21 +21,21 @@
<!-- Usage: Command line argument support -->
<PackageReference Include="McMaster.Extensions.CommandLineUtils" Version="4.1.1" />
<!-- Usage: Identifies when we are running in the context of the Windows SCM -->
<PackageReference Include="Microsoft.Extensions.Hosting.WindowsServices" Version="9.0.1" />
<PackageReference Include="Microsoft.Extensions.Hosting.WindowsServices" Version="9.0.2" />
<!-- Usage: Windows event log logging plugin -->
<PackageReference Include="Microsoft.Extensions.Logging.EventLog" Version="9.0.2" />
<!-- Usage: Console logging plugin -->
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="9.0.1" />
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="9.0.2" />
<!-- Usage: Updated transitive dependency of Core.System.ServiceProcess -->
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
<!-- Usage: Updated transitive dependency of Core.System.ServiceProcess -->
<PackageReference Include="System.Drawing.Common" Version="9.0.1" />
<PackageReference Include="System.Drawing.Common" Version="9.0.2" />
<!-- Usage: Updated transitive dependency, unable to tell what of -->
<PackageReference Include="System.Private.Uri" Version="4.3.2" />
<!-- Usage: OS identification -->
<PackageReference Include="System.Runtime.InteropServices" Version="4.3.0" />
<!-- Usage: Windows Service Manager intergration -->
<PackageReference Include="System.ServiceProcess.ServiceController" Version="9.0.1" />
<PackageReference Include="System.ServiceProcess.ServiceController" Version="9.0.2" />
</ItemGroup>
<ItemGroup>
@@ -3,7 +3,7 @@
"isRoot": true,
"tools": {
"dotnet-ef": {
"version": "8.0.11",
"version": "8.0.13",
"commands": [
"dotnet-ef"
]
@@ -227,8 +227,6 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
{
ArgumentNullException.ThrowIfNull(revisionInformation);
ArgumentNullException.ThrowIfNull(engineVersion);
ArgumentNullException.ThrowIfNull(gitHubOwner);
ArgumentNullException.ThrowIfNull(gitHubRepo);
var commitInsert = revisionInformation.CommitSha![..7];
string remoteCommitInsert;
@@ -8,6 +8,8 @@ using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using Prometheus;
using Tgstation.Server.Api.Models;
using Tgstation.Server.Api.Models.Internal;
using Tgstation.Server.Common.Extensions;
@@ -105,6 +107,21 @@ namespace Tgstation.Server.Host.Components.Deployment
/// </summary>
readonly Api.Models.Instance metadata;
/// <summary>
/// The number of attempted deployments.
/// </summary>
readonly Counter attemptedDeployments;
/// <summary>
/// The number of successful deployments.
/// </summary>
readonly Counter successfulDeployments;
/// <summary>
/// The number of failed deployments.
/// </summary>
readonly Counter failedDeployments;
/// <summary>
/// <see langword="lock"/> <see cref="object"/> for <see cref="deploying"/>.
/// </summary>
@@ -149,6 +166,7 @@ namespace Tgstation.Server.Host.Components.Deployment
/// <param name="repositoryManager">The value of <see cref="repositoryManager"/>.</param>
/// <param name="remoteDeploymentManagerFactory">The value of <see cref="remoteDeploymentManagerFactory"/>.</param>
/// <param name="asyncDelayer">The value of <see cref="asyncDelayer"/>.</param>
/// <param name="metricFactory">The <see cref="IMetricFactory"/> to use.</param>
/// <param name="logger">The value of <see cref="logger"/>.</param>
/// <param name="sessionConfiguration">The value of <see cref="sessionConfiguration"/>.</param>
/// <param name="metadata">The value of <see cref="metadata"/>.</param>
@@ -164,6 +182,7 @@ namespace Tgstation.Server.Host.Components.Deployment
IRepositoryManager repositoryManager,
IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory,
IAsyncDelayer asyncDelayer,
IMetricFactory metricFactory,
ILogger<DreamMaker> logger,
SessionConfiguration sessionConfiguration,
Api.Models.Instance metadata)
@@ -177,12 +196,17 @@ namespace Tgstation.Server.Host.Components.Deployment
this.processExecutor = processExecutor ?? throw new ArgumentNullException(nameof(processExecutor));
this.compileJobConsumer = compileJobConsumer ?? throw new ArgumentNullException(nameof(compileJobConsumer));
this.repositoryManager = repositoryManager ?? throw new ArgumentNullException(nameof(repositoryManager));
this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer));
this.remoteDeploymentManagerFactory = remoteDeploymentManagerFactory ?? throw new ArgumentNullException(nameof(remoteDeploymentManagerFactory));
this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer));
ArgumentNullException.ThrowIfNull(metricFactory);
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
this.sessionConfiguration = sessionConfiguration ?? throw new ArgumentNullException(nameof(sessionConfiguration));
this.metadata = metadata ?? throw new ArgumentNullException(nameof(metadata));
successfulDeployments = metricFactory.CreateCounter("tgs_successful_deployments", "The number of deployments that have completed successfully");
failedDeployments = metricFactory.CreateCounter("tgs_failed_deployments", "The number of deployments that have failed");
attemptedDeployments = metricFactory.CreateCounter("tgs_total_deployments", "The number of deployments that have been attempted");
deploymentLock = new object();
}
@@ -205,9 +229,12 @@ namespace Tgstation.Server.Host.Components.Deployment
deploying = true;
}
attemptedDeployments.Inc();
currentChatCallback = null;
currentDreamMakerOutput = null;
Models.CompileJob? compileJob = null;
bool success = false;
try
{
string? repoOwner = null;
@@ -351,6 +378,7 @@ namespace Tgstation.Server.Host.Components.Deployment
{
var chatNotificationAction = currentChatCallback!(null, compileJob.Output!);
await compileJobConsumer.LoadCompileJob(compileJob, chatNotificationAction, cancellationToken);
success = true;
}
catch
{
@@ -406,6 +434,10 @@ namespace Tgstation.Server.Host.Components.Deployment
finally
{
deploying = false;
if (success)
successfulDeployments.Inc();
else
failedDeployments.Inc();
}
}
#pragma warning restore CA1506
@@ -1,10 +1,14 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Prometheus;
using Tgstation.Server.Host.Components.Chat;
using Tgstation.Server.Host.Components.Chat.Commands;
using Tgstation.Server.Host.Components.Deployment;
@@ -140,6 +144,11 @@ namespace Tgstation.Server.Host.Components
/// </summary>
readonly IDotnetDumpService dotnetDumpService;
/// <summary>
/// The <see cref="IMetricFactory"/> for the <see cref="InstanceFactory"/>.
/// </summary>
readonly IMetricFactory metricFactory;
/// <summary>
/// The <see cref="GeneralConfiguration"/> for the <see cref="InstanceFactory"/>.
/// </summary>
@@ -183,6 +192,7 @@ namespace Tgstation.Server.Host.Components
/// <param name="remoteDeploymentManagerFactory">The value of <see cref="remoteDeploymentManagerFactory"/>.</param>
/// <param name="asyncDelayer">The value of <see cref="asyncDelayer"/>.</param>
/// <param name="dotnetDumpService">The value of <see cref="dotnetDumpService"/>.</param>
/// <param name="metricFactory">The value of <see cref="metricFactory"/>.</param>
/// <param name="generalConfigurationOptions">The <see cref="IOptions{TOptions}"/> containing the value of <see cref="generalConfiguration"/>.</param>
/// <param name="sessionConfigurationOptions">The <see cref="IOptions{TOptions}"/> containing the value of <see cref="sessionConfiguration"/>.</param>
public InstanceFactory(
@@ -208,6 +218,7 @@ namespace Tgstation.Server.Host.Components
IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory,
IAsyncDelayer asyncDelayer,
IDotnetDumpService dotnetDumpService,
IMetricFactory metricFactory,
IOptions<GeneralConfiguration> generalConfigurationOptions,
IOptions<SessionConfiguration> sessionConfigurationOptions)
{
@@ -233,6 +244,7 @@ namespace Tgstation.Server.Host.Components
this.remoteDeploymentManagerFactory = remoteDeploymentManagerFactory ?? throw new ArgumentNullException(nameof(remoteDeploymentManagerFactory));
this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer));
this.dotnetDumpService = dotnetDumpService ?? throw new ArgumentNullException(nameof(dotnetDumpService));
this.metricFactory = metricFactory ?? throw new ArgumentNullException(nameof(metricFactory));
generalConfiguration = generalConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(generalConfigurationOptions));
sessionConfiguration = sessionConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(sessionConfigurationOptions));
}
@@ -264,6 +276,13 @@ namespace Tgstation.Server.Host.Components
var diagnosticsIOManager = new ResolvingIOManager(instanceIoManager, "Diagnostics");
var configurationIoManager = new ResolvingIOManager(instanceIoManager, "Configuration");
var metricFactory = this.metricFactory.WithLabels(
new Dictionary<string, string>
{
{ "instance_name", metadata.Name! },
{ "instance_id", metadata.Id!.Value.ToString(CultureInfo.InvariantCulture) },
});
var configuration = new StaticFiles.Configuration(
configurationIoManager,
synchronousIOManager,
@@ -323,6 +342,7 @@ namespace Tgstation.Server.Host.Components
eventConsumer,
asyncDelayer,
dotnetDumpService,
metricFactory,
loggerFactory,
loggerFactory.CreateLogger<SessionControllerFactory>(),
sessionConfiguration,
@@ -337,6 +357,7 @@ namespace Tgstation.Server.Host.Components
diagnosticsIOManager,
configuration, // watchdog doesn't need itself as an event consumer
remoteDeploymentManagerFactory,
metricFactory,
metadata,
metadata.DreamDaemonSettings!);
try
@@ -357,6 +378,7 @@ namespace Tgstation.Server.Host.Components
repoManager,
remoteDeploymentManagerFactory,
asyncDelayer,
metricFactory,
loggerFactory.CreateLogger<DreamMaker>(),
sessionConfiguration,
metadata);
@@ -9,6 +9,8 @@ using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Prometheus;
using Tgstation.Server.Api.Models;
using Tgstation.Server.Common;
using Tgstation.Server.Common.Extensions;
@@ -150,6 +152,11 @@ namespace Tgstation.Server.Host.Components
/// </summary>
readonly CancellationTokenSource shutdownCancellationTokenSource;
/// <summary>
/// The count of online instances.
/// </summary>
readonly Gauge onlineInstances;
/// <summary>
/// The original <see cref="IConsole.Title"/> of <see cref="console"/>.
/// </summary>
@@ -180,6 +187,8 @@ namespace Tgstation.Server.Host.Components
/// <param name="swarmServiceController">The value of <see cref="swarmServiceController"/>.</param>
/// <param name="console">The value of <see cref="console"/>.</param>
/// <param name="platformIdentifier">The value of <see cref="platformIdentifier"/>.</param>
/// <param name="metricFactory">The <see cref="IMetricFactory"/> used to create metrics.</param>
/// <param name="collectorRegistry">The <see cref="ICollectorRegistry"/> to use.</param>
/// <param name="generalConfigurationOptions">The <see cref="IOptions{TOptions}"/> containing the value of <see cref="generalConfiguration"/>.</param>
/// <param name="swarmConfigurationOptions">The <see cref="IOptions{TOptions}"/> containing the value of <see cref="swarmConfiguration"/>.</param>
/// <param name="internalConfigurationOptions">The <see cref="IOptions{TOptions}"/> containing the value of <see cref="internalConfiguration"/>.</param>
@@ -197,6 +206,8 @@ namespace Tgstation.Server.Host.Components
ISwarmServiceController swarmServiceController,
IConsole console,
IPlatformIdentifier platformIdentifier,
IMetricFactory metricFactory,
ICollectorRegistry collectorRegistry,
IOptions<GeneralConfiguration> generalConfigurationOptions,
IOptions<SwarmConfiguration> swarmConfigurationOptions,
IOptions<InternalConfiguration> internalConfigurationOptions,
@@ -214,6 +225,8 @@ namespace Tgstation.Server.Host.Components
this.swarmServiceController = swarmServiceController ?? throw new ArgumentNullException(nameof(swarmServiceController));
this.console = console ?? throw new ArgumentNullException(nameof(console));
this.platformIdentifier = platformIdentifier ?? throw new ArgumentNullException(nameof(platformIdentifier));
ArgumentNullException.ThrowIfNull(metricFactory);
ArgumentNullException.ThrowIfNull(collectorRegistry);
generalConfiguration = generalConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(generalConfigurationOptions));
swarmConfiguration = swarmConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(swarmConfigurationOptions));
internalConfiguration = internalConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(internalConfigurationOptions));
@@ -221,12 +234,21 @@ namespace Tgstation.Server.Host.Components
originalConsoleTitle = console.Title;
onlineInstances = metricFactory.CreateGauge("tgs_online_instances", "The total number of instances online");
instances = new Dictionary<long, ReferenceCountingContainer<IInstance, InstanceWrapper>>();
bridgeHandlers = new Dictionary<string, IBridgeHandler>();
readyTcs = new TaskCompletionSource();
instanceStateChangeSemaphore = new SemaphoreSlim(1);
startupCancellationTokenSource = new CancellationTokenSource();
shutdownCancellationTokenSource = new CancellationTokenSource();
collectorRegistry.AddBeforeCollectCallback(async cancellationToken =>
{
using (await SemaphoreSlimContext.Lock(instanceStateChangeSemaphore, cancellationToken))
foreach (var container in instances.Values)
container.Instance.Watchdog.RunMetricsScrape();
});
}
/// <inheritdoc />
@@ -396,6 +418,7 @@ namespace Tgstation.Server.Host.Components
finally
{
await container.Instance.DisposeAsync();
onlineInstances.Dec();
}
}
}
@@ -426,6 +449,8 @@ namespace Tgstation.Server.Host.Components
instances.Add(
instanceId,
new ReferenceCountingContainer<IInstance, InstanceWrapper>(instance));
onlineInstances.Inc();
}
catch (Exception ex)
{
@@ -199,7 +199,7 @@ namespace Tgstation.Server.Host.Components.Repository
var commitMessage = String.Format(
CultureInfo.InvariantCulture,
"TGS Test Merge (#{0}){1}{2}",
"TGS Test Merge (PR {0}){1}{2}",
testMergeParameters.Number,
testMergeParameters.Comment != null
? Environment.NewLine
@@ -519,6 +519,10 @@ namespace Tgstation.Server.Host.Components.Session
return process.CreateDump(outputFile, minidump, cancellationToken);
}
/// <inheritdoc />
public double MeasureProcessorTimeDelta()
=> process.MeasureProcessorTimeDelta();
/// <summary>
/// The <see cref="Task{TResult}"/> for <see cref="LaunchResult"/>.
/// </summary>
@@ -9,6 +9,8 @@ using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Prometheus;
using Tgstation.Server.Api.Models;
using Tgstation.Server.Api.Models.Internal;
using Tgstation.Server.Common.Extensions;
@@ -127,6 +129,16 @@ namespace Tgstation.Server.Host.Components.Session
/// </summary>
readonly SessionConfiguration sessionConfiguration;
/// <summary>
/// The number of sessions launched.
/// </summary>
readonly Counter sessionsLaunched;
/// <summary>
/// The time the current session was launched.
/// </summary>
readonly Gauge lastSessionLaunch;
/// <summary>
/// The <see cref="Api.Models.Instance"/> for the <see cref="SessionControllerFactory"/>.
/// </summary>
@@ -185,6 +197,7 @@ namespace Tgstation.Server.Host.Components.Session
/// <param name="eventConsumer">The value of <see cref="eventConsumer"/>.</param>
/// <param name="asyncDelayer">The value of <see cref="asyncDelayer"/>.</param>
/// <param name="dotnetDumpService">The value of <see cref="dotnetDumpService"/>.</param>
/// <param name="metricFactory">The <see cref="IMetricFactory"/> used to create metrics.</param>
/// <param name="loggerFactory">The value of <see cref="loggerFactory"/>.</param>
/// <param name="logger">The value of <see cref="logger"/>.</param>
/// <param name="sessionConfiguration">The value of <see cref="sessionConfiguration"/>.</param>
@@ -204,6 +217,7 @@ namespace Tgstation.Server.Host.Components.Session
IEventConsumer eventConsumer,
IAsyncDelayer asyncDelayer,
IDotnetDumpService dotnetDumpService,
IMetricFactory metricFactory,
ILoggerFactory loggerFactory,
ILogger<SessionControllerFactory> logger,
SessionConfiguration sessionConfiguration,
@@ -224,10 +238,14 @@ namespace Tgstation.Server.Host.Components.Session
this.eventConsumer = eventConsumer ?? throw new ArgumentNullException(nameof(eventConsumer));
this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer));
this.dotnetDumpService = dotnetDumpService ?? throw new ArgumentNullException(nameof(dotnetDumpService));
ArgumentNullException.ThrowIfNull(metricFactory);
this.loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory));
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
this.sessionConfiguration = sessionConfiguration ?? throw new ArgumentNullException(nameof(sessionConfiguration));
this.instance = instance ?? throw new ArgumentNullException(nameof(instance));
sessionsLaunched = metricFactory.CreateCounter("tgs_sessions_launched", "The number of game server processes created");
lastSessionLaunch = metricFactory.CreateGauge("tgs_session_start_time", "The UTC unix timestamp the most recent session was started");
}
/// <inheritdoc />
@@ -368,6 +386,12 @@ namespace Tgstation.Server.Host.Components.Session
false,
apiValidate);
if (!apiValidate)
{
sessionsLaunched.Inc();
lastSessionLaunch.SetToCurrentTimeUtc();
}
return sessionController;
}
catch
@@ -6,6 +6,8 @@ using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Prometheus;
using Tgstation.Server.Api.Models;
using Tgstation.Server.Api.Models.Internal;
using Tgstation.Server.Host.Components.Chat;
@@ -63,6 +65,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
/// <param name="diagnosticsIOManager">The 'Diagnostics' <see cref="IIOManager"/> for the <see cref="WatchdogBase"/>.</param>
/// <param name="eventConsumer">The <see cref="IEventConsumer"/> for the <see cref="WatchdogBase"/>.</param>
/// <param name="remoteDeploymentManagerFactory">The <see cref="IRemoteDeploymentManagerFactory"/> for the <see cref="WatchdogBase"/>.</param>
/// <param name="metricFactory">The <see cref="IMetricFactory"/> for the <see cref="WatchdogBase"/>.</param>
/// <param name="gameIOManager">The 'Game' <see cref="IIOManager"/> for the <see cref="WatchdogBase"/>.</param>
/// <param name="linkFactory">The value of <see cref="LinkFactory"/>.</param>
/// <param name="logger">The <see cref="ILogger"/> for the <see cref="WatchdogBase"/>.</param>
@@ -80,6 +83,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
IIOManager diagnosticsIOManager,
IEventConsumer eventConsumer,
IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory,
IMetricFactory metricFactory,
IIOManager gameIOManager,
IFilesystemLinkFactory linkFactory,
ILogger<AdvancedWatchdog> logger,
@@ -97,6 +101,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
diagnosticsIOManager,
eventConsumer,
remoteDeploymentManagerFactory,
metricFactory,
gameIOManager,
logger,
initialLaunchParameters,
@@ -6,6 +6,8 @@ using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Prometheus;
using Tgstation.Server.Api.Models.Internal;
using Tgstation.Server.Host.Components.Chat;
using Tgstation.Server.Host.Components.Deployment;
@@ -53,6 +55,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
/// <param name="diagnosticsIOManager">The 'Diagnostics' <see cref="IIOManager"/> for the <see cref="WatchdogBase"/>.</param>
/// <param name="eventConsumer">The <see cref="IEventConsumer"/> for the <see cref="WatchdogBase"/>.</param>
/// <param name="remoteDeploymentManagerFactory">The <see cref="IRemoteDeploymentManagerFactory"/> for the <see cref="WatchdogBase"/>.</param>
/// <param name="metricFactory">The <see cref="IMetricFactory"/> for the <see cref="WatchdogBase"/>.</param>
/// <param name="gameIOManager">The 'Game' <see cref="IIOManager"/> for the <see cref="WatchdogBase"/>.</param>
/// <param name="logger">The <see cref="ILogger"/> for the <see cref="WatchdogBase"/>.</param>
/// <param name="initialLaunchParameters">The <see cref="DreamDaemonLaunchParameters"/> for the <see cref="WatchdogBase"/>.</param>
@@ -69,6 +72,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
IIOManager diagnosticsIOManager,
IEventConsumer eventConsumer,
IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory,
IMetricFactory metricFactory,
IIOManager gameIOManager,
ILogger<BasicWatchdog> logger,
DreamDaemonLaunchParameters initialLaunchParameters,
@@ -85,6 +89,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
diagnosticsIOManager,
eventConsumer,
remoteDeploymentManagerFactory,
metricFactory,
gameIOManager,
logger,
initialLaunchParameters,
@@ -117,5 +117,10 @@ namespace Tgstation.Server.Host.Components.Watchdog
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
/// <returns>A <see cref="ValueTask{TResult}"/> resulting in <see langword="true"/> if the broadcast succeeded., <see langword="false"/> otherwise.</returns>
ValueTask<bool> Broadcast(string message, CancellationToken cancellationToken);
/// <summary>
/// Callback to update transient metrics.
/// </summary>
void RunMetricsScrape();
}
}
@@ -1,4 +1,6 @@
using Tgstation.Server.Api.Models.Internal;
using Prometheus;
using Tgstation.Server.Api.Models.Internal;
using Tgstation.Server.Host.Components.Chat;
using Tgstation.Server.Host.Components.Deployment;
using Tgstation.Server.Host.Components.Deployment.Remote;
@@ -24,6 +26,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
/// <param name="diagnosticsIOManager">The <see cref="IIOManager"/> pointing to the Diagnostics directory for the <see cref="IWatchdog"/>.</param>
/// <param name="eventConsumer">The <see cref="IEventConsumer"/> for the <see cref="IWatchdog"/>.</param>
/// <param name="remoteDeploymentManagerFactory">The <see cref="IRemoteDeploymentManagerFactory"/> for the <see cref="IWatchdog"/>.</param>
/// <param name="metricFactory">The <see cref="IMetricFactory"/> used to create metrics.</param>
/// <param name="instance">The <see cref="Instance"/> for the <see cref="IWatchdog"/>.</param>
/// <param name="settings">The initial <see cref="DreamDaemonSettings"/> for the <see cref="IWatchdog"/>.</param>
/// <returns>A new <see cref="IWatchdog"/>.</returns>
@@ -36,6 +39,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
IIOManager diagnosticsIOManager,
IEventConsumer eventConsumer,
IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory,
IMetricFactory metricFactory,
Api.Models.Instance instance,
DreamDaemonSettings settings);
}
@@ -5,6 +5,8 @@ using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Prometheus;
using Tgstation.Server.Api.Models.Internal;
using Tgstation.Server.Host.Components.Chat;
using Tgstation.Server.Host.Components.Deployment;
@@ -43,6 +45,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
/// <param name="diagnosticsIOManager">The <see cref="IIOManager"/> for the <see cref="WatchdogBase"/>.</param>
/// <param name="eventConsumer">The <see cref="IEventConsumer"/> for the <see cref="WatchdogBase"/>.</param>
/// <param name="remoteDeploymentManagerFactory">The <see cref="IRemoteDeploymentManagerFactory"/> for the <see cref="WatchdogBase"/>.</param>
/// <param name="metricFactory">The <see cref="IMetricFactory"/> for the <see cref="WatchdogBase"/>.</param>
/// <param name="gameIOManager">The <see cref="IIOManager"/> pointing to the game directory for the <see cref="AdvancedWatchdog"/>..</param>
/// <param name="linkFactory">The <see cref="IFilesystemLinkFactory"/> for the <see cref="AdvancedWatchdog"/>.</param>
/// <param name="logger">The <see cref="ILogger"/> for the <see cref="WatchdogBase"/>.</param>
@@ -61,6 +64,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
IIOManager diagnosticsIOManager,
IEventConsumer eventConsumer,
IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory,
IMetricFactory metricFactory,
IIOManager gameIOManager,
IFilesystemLinkFactory linkFactory,
ILogger<PosixWatchdog> logger,
@@ -79,6 +83,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
diagnosticsIOManager,
eventConsumer,
remoteDeploymentManagerFactory,
metricFactory,
gameIOManager,
linkFactory,
logger,
@@ -4,6 +4,8 @@ using System.Runtime.Versioning;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Prometheus;
using Tgstation.Server.Api.Models.Internal;
using Tgstation.Server.Host.Components.Chat;
using Tgstation.Server.Host.Components.Deployment;
@@ -60,6 +62,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
IIOManager diagnosticsIOManager,
IEventConsumer eventConsumer,
IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory,
IMetricFactory metricFactory,
Api.Models.Instance instance,
DreamDaemonSettings settings)
=> new PosixWatchdog(
@@ -73,6 +76,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
diagnosticsIOManager,
eventConsumer,
remoteDeploymentManagerFactory,
metricFactory,
gameIOManager,
LinkFactory,
LoggerFactory.CreateLogger<PosixWatchdog>(),
@@ -7,6 +7,8 @@ using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Prometheus;
using Serilog.Context;
using Tgstation.Server.Api.Models;
@@ -123,6 +125,21 @@ namespace Tgstation.Server.Host.Components.Watchdog
/// </summary>
readonly SemaphoreSlim controllerDisposeSemaphore;
/// <summary>
/// The <see cref="Status"/> as a metric.
/// </summary>
readonly Gauge watchdogStatusMetric;
/// <summary>
/// Active session CPU usage as a metric.
/// </summary>
readonly Gauge cpuUsageMetric;
/// <summary>
/// <see cref="MemoryUsage"/> as a metric.
/// </summary>
readonly Gauge ramUsageMetric;
/// <summary>
/// The <see cref="IEventConsumer"/> that is not the <see cref="WatchdogBase"/>.
/// </summary>
@@ -201,6 +218,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
/// <param name="diagnosticsIOManager">The value of <see cref="diagnosticsIOManager"/>.</param>
/// <param name="eventConsumer">The value of <see cref="EventConsumer"/>.</param>
/// <param name="remoteDeploymentManagerFactory">The value of <see cref="remoteDeploymentManagerFactory"/>.</param>
/// <param name="metricFactory">The <see cref="IMetricFactory"/> used to create metrics.</param>
/// <param name="gameIOManager">The value of <see cref="GameIOManager"/>.</param>
/// <param name="logger">The value of <see cref="Logger"/>.</param>
/// <param name="initialLaunchParameters">The initial value of <see cref="ActiveLaunchParameters"/>. May be modified.</param>
@@ -217,6 +235,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
IIOManager diagnosticsIOManager,
IEventConsumer eventConsumer,
IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory,
IMetricFactory metricFactory,
IIOManager gameIOManager,
ILogger<WatchdogBase> logger,
DreamDaemonLaunchParameters initialLaunchParameters,
@@ -232,6 +251,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
this.diagnosticsIOManager = diagnosticsIOManager ?? throw new ArgumentNullException(nameof(diagnosticsIOManager));
this.eventConsumer = eventConsumer ?? throw new ArgumentNullException(nameof(eventConsumer));
this.remoteDeploymentManagerFactory = remoteDeploymentManagerFactory ?? throw new ArgumentNullException(nameof(remoteDeploymentManagerFactory));
ArgumentNullException.ThrowIfNull(metricFactory);
GameIOManager = gameIOManager ?? throw new ArgumentNullException(nameof(gameIOManager));
Logger = logger ?? throw new ArgumentNullException(nameof(logger));
ActiveLaunchParameters = initialLaunchParameters ?? throw new ArgumentNullException(nameof(initialLaunchParameters));
@@ -240,6 +260,12 @@ namespace Tgstation.Server.Host.Components.Watchdog
ArgumentNullException.ThrowIfNull(serverControl);
watchdogStatusMetric = metricFactory.CreateGauge(
"tgs_watchdog_status",
$"TGS Watchdog status: {(int)WatchdogStatus.Offline} = Offline, {(int)WatchdogStatus.Online} = Online, {(int)WatchdogStatus.Restoring} = Restoring, {(int)WatchdogStatus.DelayedRestart} = Delayed Restart");
cpuUsageMetric = metricFactory.CreateGauge("tgs_game_cpu_usage", "Estimated total CPU usage time for the game process from 0-1");
ramUsageMetric = metricFactory.CreateGauge("tgs_game_ram_usage", "Total used bytes of private memory for the game process");
chat.RegisterCommandHandler(this);
ActiveLaunchParameters = initialLaunchParameters;
@@ -272,6 +298,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
await DisposeAndNullControllersImpl();
controllerDisposeSemaphore.Dispose();
monitorCts?.Dispose();
disposed = true;
}
@@ -501,6 +528,15 @@ namespace Tgstation.Server.Host.Components.Watchdog
return response != null && response.ErrorMessage == null;
}
/// <inheritdoc />
public void RunMetricsScrape()
{
watchdogStatusMetric.Set((int)Status);
var controller = GetActiveController();
ramUsageMetric.Set(controller?.MemoryUsage ?? 0);
cpuUsageMetric.Set(controller?.MeasureProcessorTimeDelta() ?? 0);
}
/// <inheritdoc />
async ValueTask IEventConsumer.HandleEvent(EventType eventType, IEnumerable<string?> parameters, bool deploymentPipeline, CancellationToken cancellationToken)
{
@@ -3,6 +3,8 @@
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Prometheus;
using Tgstation.Server.Api.Models.Internal;
using Tgstation.Server.Host.Components.Chat;
using Tgstation.Server.Host.Components.Deployment;
@@ -77,6 +79,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
IIOManager diagnosticsIOManager,
IEventConsumer eventConsumer,
IRemoteDeploymentManagerFactory remoteDeploymentManagerfactory,
IMetricFactory metricFactory,
Api.Models.Instance instance,
DreamDaemonSettings settings)
=> new BasicWatchdog(
@@ -90,6 +93,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
diagnosticsIOManager,
eventConsumer,
remoteDeploymentManagerfactory,
metricFactory,
gameIOManager,
LoggerFactory.CreateLogger<BasicWatchdog>(),
settings,
@@ -3,6 +3,8 @@ using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Prometheus;
using Tgstation.Server.Api.Models;
using Tgstation.Server.Api.Models.Internal;
using Tgstation.Server.Host.Components.Chat;
@@ -35,6 +37,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
/// <param name="diagnosticsIOManager">The <see cref="IIOManager"/> for the <see cref="WatchdogBase"/>.</param>
/// <param name="eventConsumer">The <see cref="IEventConsumer"/> for the <see cref="WatchdogBase"/>.</param>
/// <param name="remoteDeploymentManagerFactory">The <see cref="IRemoteDeploymentManagerFactory"/> for the <see cref="WatchdogBase"/>.</param>
/// <param name="metricFactory">The <see cref="IMetricFactory"/> for the <see cref="WatchdogBase"/>.</param>
/// <param name="gameIOManager">The <see cref="IIOManager"/> pointing to the game directory for the <see cref="AdvancedWatchdog"/>..</param>
/// <param name="linkFactory">The <see cref="IFilesystemLinkFactory"/> for the <see cref="AdvancedWatchdog"/>.</param>
/// <param name="logger">The <see cref="ILogger"/> for the <see cref="WatchdogBase"/>.</param>
@@ -52,6 +55,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
IIOManager diagnosticsIOManager,
IEventConsumer eventConsumer,
IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory,
IMetricFactory metricFactory,
IIOManager gameIOManager,
IFilesystemLinkFactory linkFactory,
ILogger<WindowsWatchdog> logger,
@@ -69,6 +73,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
diagnosticsIOManager,
eventConsumer,
remoteDeploymentManagerFactory,
metricFactory,
gameIOManager,
linkFactory,
logger,
@@ -3,6 +3,8 @@
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Prometheus;
using Tgstation.Server.Api.Models.Internal;
using Tgstation.Server.Host.Components.Chat;
using Tgstation.Server.Host.Components.Deployment;
@@ -63,6 +65,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
IIOManager diagnosticsIOManager,
IEventConsumer eventConsumer,
IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory,
IMetricFactory metricFactory,
Api.Models.Instance instance,
DreamDaemonSettings settings)
=> new WindowsWatchdog(
@@ -76,6 +79,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
diagnosticsIOManager,
eventConsumer,
remoteDeploymentManagerFactory,
metricFactory,
gameIOManager,
LinkFactory,
LoggerFactory.CreateLogger<WindowsWatchdog>(),
@@ -91,6 +91,11 @@ namespace Tgstation.Server.Host.Configuration
/// </summary>
public ushort ApiPort { get; set; }
/// <summary>
/// The port Prometheus metrics are published on, if any.
/// </summary>
public ushort? PrometheusPort { get; set; }
/// <summary>
/// A classic GitHub personal access token to use for bypassing rate limits on requests. Requires no scopes.
/// </summary>
@@ -30,6 +30,8 @@ using Microsoft.Extensions.Options;
using Newtonsoft.Json;
using Prometheus;
using Serilog;
using Serilog.Events;
using Serilog.Formatting.Display;
@@ -294,6 +296,21 @@ namespace Tgstation.Server.Host.Core
services.AddHttpClient();
services.AddSingleton<IAbstractHttpClientFactory, AbstractHttpClientFactory>();
// configure metrics
var prometheusPort = postSetupServices.GeneralConfiguration.PrometheusPort;
services.AddSingleton<IMetricFactory>(_ => Metrics.DefaultFactory);
services.AddSingleton<ICollectorRegistry>(_ => Metrics.DefaultRegistry);
if (prometheusPort.HasValue && prometheusPort != postSetupServices.GeneralConfiguration.ApiPort)
services.AddMetricServer(options => options.Port = prometheusPort.Value);
services.UseHttpClientMetrics();
var healthChecksBuilder = services
.AddHealthChecks()
.ForwardToPrometheus();
// configure graphql
services
.AddScoped<GraphQL.Subscriptions.ITopicEventReceiver, ShutdownAwareTopicEventReceiver>()
@@ -362,6 +379,9 @@ namespace Tgstation.Server.Host.Core
configureAction(builder, databaseConfig);
});
services.AddScoped<IDatabaseContext>(x => x.GetRequiredService<TContext>());
healthChecksBuilder
.AddDbContextCheck<TContext>();
}
// add the correct database context type
@@ -571,6 +591,9 @@ namespace Tgstation.Server.Host.Core
// Wrap exceptions in a 500 (ErrorMessage) response
applicationBuilder.UseServerErrorHandling();
// metrics capture
applicationBuilder.UseHttpMetrics();
// Add the X-Powered-By response header
applicationBuilder.UseServerBranding(assemblyInformationProvider);
@@ -692,6 +715,19 @@ namespace Tgstation.Server.Host.Core
.MapGraphQL(Routes.GraphQL)
.WithOptions(gqlOptions);
}
if (generalConfiguration.PrometheusPort.HasValue)
if (generalConfiguration.PrometheusPort == generalConfiguration.ApiPort)
{
endpoints.MapMetrics();
logger.LogDebug("Prometheus being hosted alongside server");
}
else
logger.LogDebug("Prometheus being hosted on port {prometheusPort}", generalConfiguration.PrometheusPort);
else
logger.LogTrace("Prometheus disabled");
endpoints.MapHealthChecks("/health");
});
// 404 anything that gets this far
@@ -9,6 +9,8 @@ using Microsoft.AspNetCore.SignalR;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using Prometheus;
using Serilog.Context;
using Tgstation.Server.Api.Extensions;
@@ -67,6 +69,16 @@ namespace Tgstation.Server.Host.Jobs
/// </summary>
readonly TaskCompletionSource<IInstanceCoreProvider> activationTcs;
/// <summary>
/// Total number of jobs processed.
/// </summary>
readonly Counter processedJobs;
/// <summary>
/// Jobs currently running.
/// </summary>
readonly Gauge runningJobs;
/// <summary>
/// <see langword="lock"/> <see cref="object"/> for various operations.
/// </summary>
@@ -87,19 +99,25 @@ namespace Tgstation.Server.Host.Jobs
/// </summary>
/// <param name="hub">The value of <see cref="hub"/>.</param>
/// <param name="databaseContextFactory">The value of <see cref="databaseContextFactory"/>.</param>
/// <param name="metricFactory">The <see cref="IMetricFactory"/> to use.</param>
/// <param name="loggerFactory">The value of <see cref="loggerFactory"/>.</param>
/// <param name="logger">The value of <see cref="logger"/>.</param>
public JobService(
IConnectionMappedHubContext<JobsHub, IJobsHub> hub,
IDatabaseContextFactory databaseContextFactory,
IMetricFactory metricFactory,
ILoggerFactory loggerFactory,
ILogger<JobService> logger)
{
this.hub = hub ?? throw new ArgumentNullException(nameof(hub));
this.databaseContextFactory = databaseContextFactory ?? throw new ArgumentNullException(nameof(databaseContextFactory));
this.loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory));
ArgumentNullException.ThrowIfNull(metricFactory);
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
runningJobs = metricFactory.CreateGauge("tgs_jobs_running", "The number of TGS jobs running across all instances");
processedJobs = metricFactory.CreateCounter("tgs_jobs_processed", "The number of TGS jobs that have run previously");
jobs = new Dictionary<long, JobHandler>();
hubUpdateActions = new Dictionary<long, Action>();
activationTcs = new TaskCompletionSource<IInstanceCoreProvider>();
@@ -363,6 +381,7 @@ namespace Tgstation.Server.Host.Jobs
{
var jid = job.Require(x => x.Id);
using (LogContext.PushProperty(SerilogContextHelper.JobIdContextProperty, jid))
using (runningJobs.TrackInProgress())
try
{
void LogException(Exception ex) => logger.LogDebug(ex, "Job {jobId} exited with error!", jid);
@@ -553,6 +572,8 @@ namespace Tgstation.Server.Host.Jobs
jobs.Remove(jid);
handler.Dispose();
}
processedJobs.Inc();
}
}
}
@@ -24,6 +24,12 @@ namespace Tgstation.Server.Host.System
/// </summary>
long? MemoryUsage { get; }
/// <summary>
/// Gets the estimated CPU usage fraction of the process based on the last time this was called.
/// </summary>
/// <returns>The CPU's estimated usage as a value between 0 and 1.</returns>
double MeasureProcessorTimeDelta();
/// <summary>
/// Set's the owned <see cref="global::System.Diagnostics.Process.PriorityClass"/> to a non-normal value.
/// </summary>
+53 -1
View File
@@ -87,10 +87,30 @@ namespace Tgstation.Server.Host.System
/// </summary>
readonly Task<string?>? readTask;
/// <summary>
/// <see langword="lock"/> object for measuring processor time usage.
/// </summary>
readonly object processTimeMeasureLock;
/// <summary>
/// The last time <see cref="MeasureProcessorTimeDelta"/> was called.
/// </summary>
DateTimeOffset lastProcessorMeasureTime;
/// <summary>
/// The last value of <see cref="global::System.Diagnostics.Process.TotalProcessorTime"/>.
/// </summary>
TimeSpan lastProcessorUsageTime;
/// <summary>
/// The last valid return value of <see cref="MeasureProcessorTimeDelta"/>.
/// </summary>
double lastProcessorUsageEstimation;
/// <summary>
/// If the <see cref="Process"/> was disposed.
/// </summary>
volatile int disposed;
int disposed;
/// <summary>
/// Initializes a new instance of the <see cref="Process"/> class.
@@ -123,6 +143,8 @@ namespace Tgstation.Server.Host.System
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
processTimeMeasureLock = new object();
Lifetime = WrapLifetimeTask();
if (preExisting)
@@ -147,6 +169,8 @@ namespace Tgstation.Server.Host.System
DefaultIOManager.BlockingTaskCreationOptions,
TaskScheduler.Current);
MeasureProcessorTimeDelta();
logger.LogTrace("Created process ID: {pid}", Id);
}
@@ -267,6 +291,34 @@ namespace Tgstation.Server.Host.System
return processFeatures.CreateDump(handle, outputFile, minidump, cancellationToken);
}
/// <inheritdoc />
public double MeasureProcessorTimeDelta()
{
lock (processTimeMeasureLock)
{
try
{
var now = DateTimeOffset.UtcNow;
var newTime = handle.TotalProcessorTime;
var timeDelta = now - lastProcessorMeasureTime;
var newUsage = newTime - lastProcessorUsageTime;
lastProcessorMeasureTime = now;
lastProcessorUsageTime = newTime;
if (timeDelta != TimeSpan.Zero)
lastProcessorUsageEstimation = newUsage / (Environment.ProcessorCount * timeDelta);
}
catch (Exception ex)
{
logger.LogWarning(ex, "Error measuring processor time delta!");
}
return lastProcessorUsageEstimation;
}
}
/// <summary>
/// Attaches a log message to the process' exit event.
/// </summary>
@@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk.Web">
<Project Sdk="Microsoft.NET.Sdk.Web">
<Import Project="../../build/SrcCommon.props" />
<PropertyGroup>
@@ -110,21 +110,23 @@
<!-- Usage: git interop -->
<PackageReference Include="LibGit2Sharp" Version="0.31.0" />
<!-- Usage: Support ""legacy"" Newotonsoft.Json in HTTP pipeline. The rest of our codebase uses Newtonsoft. -->
<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="8.0.12" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="8.0.13" />
<!-- Usage: Using target JSON serializer for API -->
<PackageReference Include="Microsoft.AspNetCore.SignalR.Protocols.NewtonsoftJson" Version="9.0.1" />
<PackageReference Include="Microsoft.AspNetCore.SignalR.Protocols.NewtonsoftJson" Version="9.0.2" />
<!-- Usage: Generating dumps of dotnet engine processes -->
<PackageReference Include="Microsoft.Diagnostics.NETCore.Client" Version="0.2.553101" />
<PackageReference Include="Microsoft.Diagnostics.NETCore.Client" Version="0.2.607501" />
<!-- Usage: Database ORM -->
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="8.0.11" />
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="8.0.13" />
<!-- Usage: Automatic migration generation using command line -->
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="8.0.11">
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="8.0.13">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<!-- Usage: Sqlite ORM plugin -->
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="8.0.11" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="8.0.13" />
<!-- Usage: MSSQL ORM plugin -->
<PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer" Version="8.0.11" />
<PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer" Version="8.0.13" />
<!-- Usage: Database connectivity health check -->
<PackageReference Include="Microsoft.Extensions.Diagnostics.HealthChecks.EntityFrameworkCore" Version="8.0.13" />
<!-- Usage: POSIX support for syscalls, signals, and symlinks -->
<PackageReference Include="Mono.Posix.NETStandard" Version="1.0.0" />
<!-- Usage: Cron string parsing -->
@@ -132,11 +134,13 @@
<!-- Usage: YAML config plugin -->
<PackageReference Include="NetEscapades.Configuration.Yaml" Version="3.1.0" />
<!-- Usage: PostgresSQL ORM plugin -->
<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="8.0.10" />
<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="8.0.11" />
<!-- Usage: MYSQL/MariaDB ORM plugin -->
<PackageReference Include="Pomelo.EntityFrameworkCore.MySql" Version="8.0.2" />
<!-- Usage: Publishing Prometheus metrics -->
<PackageReference Include="prometheus-net.AspNetCore.HealthChecks" Version="8.2.1" />
<!-- Usage: Discord interop -->
<PackageReference Include="Remora.Discord" Version="2024.3.0" />
<PackageReference Include="Remora.Discord" Version="2025.1.0" />
<!-- Usage: Rich logger builder -->
<PackageReference Include="Serilog.Extensions.Logging" Version="9.0.0" />
<!-- Usage: Async logging plugin -->
@@ -152,11 +156,11 @@
<!-- Usage: Newtonsoft.Json plugin for OpenAPI spec generator -->
<PackageReference Include="Swashbuckle.AspNetCore.Newtonsoft" Version="7.2.0" />
<!-- Usage: Windows authentication plugin allowing searching for users by name -->
<PackageReference Include="System.DirectoryServices.AccountManagement" Version="9.0.1" />
<PackageReference Include="System.DirectoryServices.AccountManagement" Version="9.0.2" />
<!-- Usage: Identifying owning user of Windows Process objects -->
<PackageReference Include="System.Management" Version="9.0.2" />
<!-- Usage: Temporary resolution to compatibility issues with EFCore 7 and .NET 8 -->
<PackageReference Include="System.Security.Permissions" Version="9.0.1" />
<PackageReference Include="System.Security.Permissions" Version="9.0.2" />
</ItemGroup>
<ItemGroup>
@@ -2,6 +2,7 @@
using System.Collections.Generic;
using System.IdentityModel.Tokens.Jwt;
using System.Linq;
using System.Net.Http;
using System.Security.Cryptography;
using System.Text;
using System.Threading;
@@ -12,6 +13,7 @@ using Microsoft.Extensions.Options;
using Microsoft.IdentityModel.Tokens;
using Octokit;
using Octokit.Internal;
using Tgstation.Server.Api.Models;
using Tgstation.Server.Host.Configuration;
@@ -38,6 +40,11 @@ namespace Tgstation.Server.Host.Utils.GitHub
/// </summary>
readonly IAssemblyInformationProvider assemblyInformationProvider;
/// <summary>
/// The <see cref="IHttpMessageHandlerFactory"/> for the <see cref="GitHubClientFactory"/>.
/// </summary>
readonly IHttpMessageHandlerFactory httpMessageHandlerFactory;
/// <summary>
/// The <see cref="ILogger"/> for the <see cref="GitHubClientFactory"/>.
/// </summary>
@@ -62,14 +69,17 @@ namespace Tgstation.Server.Host.Utils.GitHub
/// Initializes a new instance of the <see cref="GitHubClientFactory"/> class.
/// </summary>
/// <param name="assemblyInformationProvider">The value of <see cref="assemblyInformationProvider"/>.</param>
/// <param name="httpMessageHandlerFactory">The value of <see cref="httpMessageHandlerFactory"/>.</param>
/// <param name="logger">The value of <see cref="logger"/>.</param>
/// <param name="generalConfigurationOptions">The <see cref="IOptions{TOptions}"/> containing the value of <see cref="generalConfiguration"/>.</param>
public GitHubClientFactory(
IAssemblyInformationProvider assemblyInformationProvider,
IHttpMessageHandlerFactory httpMessageHandlerFactory,
ILogger<GitHubClientFactory> logger,
IOptions<GeneralConfiguration> generalConfigurationOptions)
{
this.assemblyInformationProvider = assemblyInformationProvider ?? throw new ArgumentNullException(nameof(assemblyInformationProvider));
this.httpMessageHandlerFactory = httpMessageHandlerFactory ?? throw new ArgumentNullException(nameof(httpMessageHandlerFactory));
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
generalConfiguration = generalConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(generalConfigurationOptions));
@@ -315,10 +325,33 @@ namespace Tgstation.Server.Host.Utils.GitHub
GitHubClient CreateUnauthenticatedClient()
{
var product = assemblyInformationProvider.ProductInfoHeaderValue.Product!;
return new GitHubClient(
new ProductHeaderValue(
product.Name,
product.Version));
#pragma warning disable CA2000 // Dispose objects before losing scope
var handler = httpMessageHandlerFactory.CreateHandler();
try
{
var clientAdapter = new HttpClientAdapter(() => handler);
#pragma warning restore CA2000 // Dispose objects before losing scope
handler = null;
try
{
return new GitHubClient(
new Connection(
new ProductHeaderValue(
product.Name,
product.Version),
clientAdapter));
}
catch
{
clientAdapter.Dispose();
throw;
}
}
catch
{
handler?.Dispose();
throw;
}
}
}
}
@@ -8,6 +8,7 @@ General:
ByondTopicTimeout: 5000 # Timeout for BYOND /world/Topic() calls in milliseconds
RestartTimeoutMinutes: 1 # Timeout for server restarts after requested by SIGTERM or the HTTP API
ApiPort: 5000 # Port the HTTP API is hosted on
PrometheusPort: null # Port Prometheus metrics are published on under /metrics. This can be set to the same value as the ApiPort, just note that accessing it does not require authentication.
UseBasicWatchdog: false # The basic watchdog hard restarts DreamDaemon when /world/proc/TgsReboot() is called in the DMAPI if a new deployment is available
UserLimit: 100 # Maximum number of allowed users
UserGroupLimit: 25 # Maximum number of allowed groups
@@ -10,7 +10,7 @@
<ItemGroup>
<!-- Usage: JWT injection into HTTP pipeline -->
<PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="8.0.12" />
<PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="8.0.13" />
<!-- Usage: GitHub.com interop -->
<PackageReference Include="Octokit" Version="14.0.0" />
<!-- Usage: YAML conversion of Version objects -->
@@ -6,7 +6,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="9.0.1" />
<PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="9.0.2" />
</ItemGroup>
<ItemGroup>
@@ -0,0 +1,19 @@
using System;
using System.Net.Http;
namespace Tgstation.Server.Host.Tests
{
/// <summary>
/// Basic <see cref="IHttpMessageHandlerFactory"/> implementation for testiong
/// </summary>
public sealed class BasicHttpMessageHandlerFactory : IHttpMessageHandlerFactory, IDisposable
{
readonly HttpClientHandler handler = new();
public HttpMessageHandler CreateHandler(string name)
=> handler;
public void Dispose()
=> handler.Dispose();
}
}
@@ -7,7 +7,7 @@
<ItemGroup>
<!-- Usage: Creating mock database implementations -->
<PackageReference Include="Microsoft.EntityFrameworkCore.InMemory" Version="8.0.11" />
<PackageReference Include="Microsoft.EntityFrameworkCore.InMemory" Version="8.0.13" />
</ItemGroup>
<ItemGroup>
@@ -1,4 +1,5 @@
using System;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
using System.Threading;
@@ -15,6 +16,7 @@ using Octokit;
using Tgstation.Server.Api.Models;
using Tgstation.Server.Host.Configuration;
using Tgstation.Server.Host.System;
using Tgstation.Server.Host.Tests;
namespace Tgstation.Server.Host.Utils.GitHub.Tests
{
@@ -42,9 +44,10 @@ namespace Tgstation.Server.Host.Utils.GitHub.Tests
[TestMethod]
public void TestContructionThrows()
{
Assert.ThrowsException<ArgumentNullException>(() => new GitHubClientFactory(null, null, null));
Assert.ThrowsException<ArgumentNullException>(() => new GitHubClientFactory(Mock.Of<IAssemblyInformationProvider>(), null, null));
Assert.ThrowsException<ArgumentNullException>(() => new GitHubClientFactory(Mock.Of<IAssemblyInformationProvider>(), Mock.Of<ILogger<GitHubClientFactory>>(), null));
Assert.ThrowsException<ArgumentNullException>(() => new GitHubClientFactory(null, null, null, null));
Assert.ThrowsException<ArgumentNullException>(() => new GitHubClientFactory(Mock.Of<IAssemblyInformationProvider>(), null, null, null));
Assert.ThrowsException<ArgumentNullException>(() => new GitHubClientFactory(Mock.Of<IAssemblyInformationProvider>(), Mock.Of<IHttpMessageHandlerFactory>(), null, null));
Assert.ThrowsException<ArgumentNullException>(() => new GitHubClientFactory(Mock.Of<IAssemblyInformationProvider>(), Mock.Of<IHttpMessageHandlerFactory>(), Mock.Of<ILogger<GitHubClientFactory>>(), null));
}
[TestMethod]
@@ -58,7 +61,7 @@ namespace Tgstation.Server.Host.Utils.GitHub.Tests
var gc = new GeneralConfiguration();
Assert.IsNull(gc.GitHubAccessToken);
mockOptions.SetupGet(x => x.Value).Returns(gc);
var factory = new GitHubClientFactory(mockApp.Object, loggerFactory.CreateLogger<GitHubClientFactory>(), mockOptions.Object);
var factory = new GitHubClientFactory(mockApp.Object, new BasicHttpMessageHandlerFactory(), loggerFactory.CreateLogger<GitHubClientFactory>(), mockOptions.Object);
var client = await factory.CreateClient(CancellationToken.None);
Assert.IsNotNull(client);
@@ -84,7 +87,7 @@ namespace Tgstation.Server.Host.Utils.GitHub.Tests
var mockOptions = new Mock<IOptions<GeneralConfiguration>>();
mockOptions.SetupGet(x => x.Value).Returns(new GeneralConfiguration());
var factory = new GitHubClientFactory(mockApp.Object, loggerFactory.CreateLogger<GitHubClientFactory>(), mockOptions.Object);
var factory = new GitHubClientFactory(mockApp.Object, new BasicHttpMessageHandlerFactory(), loggerFactory.CreateLogger<GitHubClientFactory>(), mockOptions.Object);
await Assert.ThrowsExceptionAsync<ArgumentNullException>(() => factory.CreateClient(null, CancellationToken.None).AsTask());
@@ -106,7 +109,7 @@ namespace Tgstation.Server.Host.Utils.GitHub.Tests
var mockOptions = new Mock<IOptions<GeneralConfiguration>>();
mockOptions.SetupGet(x => x.Value).Returns(new GeneralConfiguration());
var factory = new GitHubClientFactory(mockApp.Object, loggerFactory.CreateLogger<GitHubClientFactory>(), mockOptions.Object);
var factory = new GitHubClientFactory(mockApp.Object, new BasicHttpMessageHandlerFactory(), loggerFactory.CreateLogger<GitHubClientFactory>(), mockOptions.Object);
var appID = Environment.GetEnvironmentVariable("TGS_TEST_APP_ID");
var privateKey = Environment.GetEnvironmentVariable("TGS_TEST_APP_PRIVATE_KEY");
@@ -144,7 +147,7 @@ namespace Tgstation.Server.Host.Utils.GitHub.Tests
var mockOptions = new Mock<IOptions<GeneralConfiguration>>();
mockOptions.SetupGet(x => x.Value).Returns(new GeneralConfiguration());
var factory = new GitHubClientFactory(mockApp.Object, loggerFactory.CreateLogger<GitHubClientFactory>(), mockOptions.Object);
var factory = new GitHubClientFactory(mockApp.Object, new BasicHttpMessageHandlerFactory(), loggerFactory.CreateLogger<GitHubClientFactory>(), mockOptions.Object);
await Assert.ThrowsExceptionAsync<ArgumentNullException>(() => factory.CreateClient(null, CancellationToken.None).AsTask());
@@ -192,7 +195,7 @@ vTdVAoGBAI/jjUMdjkY43zhe3w2piwT0fhGfqm9ikdAB9IcgcptuS0ML0ZaWV/eO
var mockOptions = new Mock<IOptions<GeneralConfiguration>>();
mockOptions.SetupGet(x => x.Value).Returns(new GeneralConfiguration());
var factory = new GitHubClientFactory(mockApp.Object, loggerFactory.CreateLogger<GitHubClientFactory>(), mockOptions.Object);
var factory = new GitHubClientFactory(mockApp.Object, new BasicHttpMessageHandlerFactory(), loggerFactory.CreateLogger<GitHubClientFactory>(), mockOptions.Object);
var client1 = await factory.CreateClient(CancellationToken.None);
var client2 = await factory.CreateClient("asdf", CancellationToken.None);
@@ -115,8 +115,6 @@ namespace Tgstation.Server.Tests.Live
{
ArgumentNullException.ThrowIfNull(revisionInformation);
ArgumentNullException.ThrowIfNull(engineVersion);
ArgumentNullException.ThrowIfNull(gitHubOwner);
ArgumentNullException.ThrowIfNull(gitHubRepo);
Assert.IsTrue(knownChannels.ContainsKey(channelId));
@@ -156,6 +156,7 @@ namespace Tgstation.Server.Tests.Live
$"Security:TokenExpiryMinutes=120", // timeouts are useless for us
$"General:OpenDreamSuppressInstallOutput={TestingUtils.RunningInGitHubActions}",
"Telemetry:DisableVersionReporting=true",
$"General:PrometheusPort={port}"
};
if (MultiServerClient.UseGraphQL)
@@ -455,6 +455,40 @@ namespace Tgstation.Server.Tests.Live
}
}
static async Task TestHealthChecks(IRestServerClient serverClient, CancellationToken cancellationToken)
{
var url = serverClient.Url;
// check that 400s are returned appropriately
using var httpClient = new HttpClient();
using (var request = new HttpRequestMessage(HttpMethod.Get, url.ToString() + "health"))
{
request.Headers.Accept.Clear();
request.Headers.UserAgent.Add(new ProductInfoHeaderValue("RootTest", "1.0.0"));
using var response = await httpClient.SendAsync(request, cancellationToken);
response.EnsureSuccessStatusCode();
var text = await response.Content.ReadAsStringAsync(cancellationToken);
Assert.AreEqual("Healthy", text.Trim());
}
}
static async Task TestPrometheusMetrics(IRestServerClient serverClient, CancellationToken cancellationToken)
{
var url = serverClient.Url;
// check that 400s are returned appropriately
using var httpClient = new HttpClient();
using (var request = new HttpRequestMessage(HttpMethod.Get, url.ToString() + "metrics"))
{
request.Headers.Accept.Clear();
request.Headers.UserAgent.Add(new ProductInfoHeaderValue("RootTest", "1.0.0"));
using var response = await httpClient.SendAsync(request, cancellationToken);
response.EnsureSuccessStatusCode();
var text = await response.Content.ReadAsStringAsync(cancellationToken);
Assert.IsTrue(text.Contains("tgs_jobs_running"));
}
}
static async Task TestGraphQLLogin(IRestServerClientFactory clientFactory, IRestServerClient restClient, CancellationToken cancellationToken)
{
if (!MultiServerClient.UseGraphQL)
@@ -483,6 +517,8 @@ namespace Tgstation.Server.Tests.Live
TestServerInformation(clientFactory, serverClient, cancellationToken),
TestInvalidTransfers(serverClient, cancellationToken),
RegressionTestForLeakedPasswordHashesBug(serverClient, cancellationToken),
TestSignalRUsage(clientFactory, serverClient, cancellationToken));
TestSignalRUsage(clientFactory, serverClient, cancellationToken),
TestHealthChecks(serverClient, cancellationToken),
TestPrometheusMetrics(serverClient, cancellationToken));
}
}
@@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
@@ -15,6 +16,7 @@ using Tgstation.Server.Host.Configuration;
using Tgstation.Server.Host.Extensions;
using Tgstation.Server.Host.Security;
using Tgstation.Server.Host.System;
using Tgstation.Server.Host.Tests;
using Tgstation.Server.Host.Utils.GitHub;
namespace Tgstation.Server.Tests.Live
@@ -38,7 +40,7 @@ namespace Tgstation.Server.Tests.Live
GitHubAccessToken = Environment.GetEnvironmentVariable("TGS_TEST_GITHUB_TOKEN")
});
var gitHubClientFactory = new GitHubClientFactory(new AssemblyInformationProvider(), Mock.Of<ILogger<GitHubClientFactory>>(), mockOptions.Object);
var gitHubClientFactory = new GitHubClientFactory(new AssemblyInformationProvider(), new BasicHttpMessageHandlerFactory(), Mock.Of<ILogger<GitHubClientFactory>>(), mockOptions.Object);
RealClient = gitHubClientFactory.CreateClient(CancellationToken.None).GetAwaiter().GetResult();
}
@@ -6,13 +6,14 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="9.0.1" />
<PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="9.0.2" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\src\Tgstation.Server.Client.GraphQL\Tgstation.Server.Client.GraphQL.csproj" />
<ProjectReference Include="..\..\src\Tgstation.Server.Host.Watchdog\Tgstation.Server.Host.Watchdog.csproj" />
<ProjectReference Include="..\..\src\Tgstation.Server.Host\Tgstation.Server.Host.csproj" />
<ProjectReference Include="..\Tgstation.Server.Host.Tests\Tgstation.Server.Host.Tests.csproj" />
</ItemGroup>
<ItemGroup>