mirror of
https://github.com/tgstation/tgstation-server.git
synced 2026-07-16 18:43:43 +01:00
Merge branch 'dev' into tm_spam_reduction
This commit is contained in:
@@ -256,7 +256,9 @@ Create an `appsettings.Production.yml` file next to `appsettings.yml`. This will
|
||||
|
||||
- `General:ConfigVersion`: Suppresses warnings about out of date config versions. You should change this after updating TGS to one with a new config version. The current version can be found on the releases page for your server version.
|
||||
|
||||
- `General:MinimumPasswordLength`: Minimum password length requirement for database users
|
||||
- `General:MinimumPasswordLength`: Minimum password length requirement for database users.
|
||||
|
||||
- `General:PrometheusPort`: Port Prometheus metrics are published on under /metrics. This can be set to the same value as the `ApiPort`, just note that accessing it does not require authentication.
|
||||
|
||||
- `General:ValidInstancePaths`: Array meant to limit the directories in which instances may be created.
|
||||
|
||||
|
||||
+4
-1
@@ -58,11 +58,12 @@ RUN export TGS_TELEMETRY_KEY_FILE="../../${TGS_TELEMETRY_KEY_FILE}" \
|
||||
|
||||
FROM mcr.microsoft.com/dotnet/aspnet:8.0-bookworm-slim
|
||||
|
||||
#needed for byond
|
||||
#needed for byond, curl for healthchecks
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y \
|
||||
gcc-multilib \
|
||||
gdb \
|
||||
curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
EXPOSE 5000
|
||||
@@ -78,4 +79,6 @@ COPY --from=build /repo/build/tgs.docker.sh tgs.sh
|
||||
|
||||
VOLUME ["/config_data", "/tgs_logs", "/app/lib"]
|
||||
|
||||
HEALTHCHECK CMD --curl --fail http://localhost:5000/health || exit
|
||||
|
||||
ENTRYPOINT ["./tgs.sh"]
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
</PackageReference>
|
||||
<!-- Usage: Hard to say what exactly this is for, but not including it removes the test icon and breaks vstest.console.exe for some reason -->
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.12.0" Condition="'$(TgsTestNoSdk)' != 'true'" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.13.0" Condition="'$(TgsTestNoSdk)' != 'true'" />
|
||||
<!-- Usage: Dependency mocking for tests -->
|
||||
<!-- Pinned: Be VERY careful about updating https://github.com/moq/moq/issues/1372 -->
|
||||
<PackageReference Include="Moq" Version="4.20.72" />
|
||||
|
||||
+2
-2
@@ -3,8 +3,8 @@
|
||||
<!-- Integration tests will ensure they match across the board -->
|
||||
<Import Project="WebpanelVersion.props" />
|
||||
<PropertyGroup>
|
||||
<TgsCoreVersion>6.13.0</TgsCoreVersion>
|
||||
<TgsConfigVersion>5.4.0</TgsConfigVersion>
|
||||
<TgsCoreVersion>6.14.0</TgsCoreVersion>
|
||||
<TgsConfigVersion>5.5.0</TgsConfigVersion>
|
||||
<TgsRestVersion>10.12.1</TgsRestVersion>
|
||||
<TgsGraphQLVersion>0.5.0</TgsGraphQLVersion>
|
||||
<TgsCommonLibraryVersion>7.0.0</TgsCommonLibraryVersion>
|
||||
|
||||
@@ -28,10 +28,9 @@
|
||||
<!-- Usage: HTTP constants reference -->
|
||||
<PackageReference Include="Microsoft.AspNetCore.Http.Extensions" Version="2.3.0" />
|
||||
<!-- Usage: Decoding the 'nbf' property of JWTs -->
|
||||
<PackageReference Include="Microsoft.IdentityModel.JsonWebTokens" Version="8.4.0" />
|
||||
<PackageReference Include="Microsoft.IdentityModel.JsonWebTokens" Version="8.5.0" />
|
||||
<!-- Usage: Data model annotating -->
|
||||
<PackageReference Include="System.ComponentModel.Annotations" Version="5.0.0" />
|
||||
<PackageReference Include="System.Text.Encodings.Web" Version="9.0.1" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
|
||||
<ItemGroup>
|
||||
<!-- GraphQL connector and code generator -->
|
||||
<PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="9.0.1" />
|
||||
<PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="9.0.2" />
|
||||
<PackageReference Include="StrawberryShake.Server" Version="15.0.3" />
|
||||
</ItemGroup>
|
||||
|
||||
|
||||
@@ -11,9 +11,9 @@
|
||||
|
||||
<ItemGroup>
|
||||
<!-- Usage: Connecting to SignalR hubs in API -->
|
||||
<PackageReference Include="Microsoft.AspNetCore.SignalR.Client" Version="9.0.1" />
|
||||
<PackageReference Include="Microsoft.AspNetCore.SignalR.Client" Version="9.0.2" />
|
||||
<!-- Usage: Using target JSON serializer for API -->
|
||||
<PackageReference Include="Microsoft.AspNetCore.SignalR.Protocols.NewtonsoftJson" Version="9.0.0" />
|
||||
<PackageReference Include="Microsoft.AspNetCore.SignalR.Protocols.NewtonsoftJson" Version="9.0.2" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
||||
@@ -13,9 +13,9 @@
|
||||
|
||||
<ItemGroup>
|
||||
<!-- Usage: Identifying if we're running under SystemD -->
|
||||
<PackageReference Include="Microsoft.Extensions.Hosting.Systemd" Version="9.0.1" />
|
||||
<PackageReference Include="Microsoft.Extensions.Hosting.Systemd" Version="9.0.2" />
|
||||
<!-- Usage: Console logging plugin -->
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="9.0.1" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="9.0.2" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
||||
@@ -21,21 +21,21 @@
|
||||
<!-- Usage: Command line argument support -->
|
||||
<PackageReference Include="McMaster.Extensions.CommandLineUtils" Version="4.1.1" />
|
||||
<!-- Usage: Identifies when we are running in the context of the Windows SCM -->
|
||||
<PackageReference Include="Microsoft.Extensions.Hosting.WindowsServices" Version="9.0.1" />
|
||||
<PackageReference Include="Microsoft.Extensions.Hosting.WindowsServices" Version="9.0.2" />
|
||||
<!-- Usage: Windows event log logging plugin -->
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.EventLog" Version="9.0.2" />
|
||||
<!-- Usage: Console logging plugin -->
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="9.0.1" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="9.0.2" />
|
||||
<!-- Usage: Updated transitive dependency of Core.System.ServiceProcess -->
|
||||
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
|
||||
<!-- Usage: Updated transitive dependency of Core.System.ServiceProcess -->
|
||||
<PackageReference Include="System.Drawing.Common" Version="9.0.1" />
|
||||
<PackageReference Include="System.Drawing.Common" Version="9.0.2" />
|
||||
<!-- Usage: Updated transitive dependency, unable to tell what of -->
|
||||
<PackageReference Include="System.Private.Uri" Version="4.3.2" />
|
||||
<!-- Usage: OS identification -->
|
||||
<PackageReference Include="System.Runtime.InteropServices" Version="4.3.0" />
|
||||
<!-- Usage: Windows Service Manager intergration -->
|
||||
<PackageReference Include="System.ServiceProcess.ServiceController" Version="9.0.1" />
|
||||
<PackageReference Include="System.ServiceProcess.ServiceController" Version="9.0.2" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
"isRoot": true,
|
||||
"tools": {
|
||||
"dotnet-ef": {
|
||||
"version": "8.0.11",
|
||||
"version": "8.0.13",
|
||||
"commands": [
|
||||
"dotnet-ef"
|
||||
]
|
||||
|
||||
@@ -227,8 +227,6 @@ namespace Tgstation.Server.Host.Components.Chat.Providers
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(revisionInformation);
|
||||
ArgumentNullException.ThrowIfNull(engineVersion);
|
||||
ArgumentNullException.ThrowIfNull(gitHubOwner);
|
||||
ArgumentNullException.ThrowIfNull(gitHubRepo);
|
||||
|
||||
var commitInsert = revisionInformation.CommitSha![..7];
|
||||
string remoteCommitInsert;
|
||||
|
||||
@@ -8,6 +8,8 @@ using System.Threading.Tasks;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
using Prometheus;
|
||||
|
||||
using Tgstation.Server.Api.Models;
|
||||
using Tgstation.Server.Api.Models.Internal;
|
||||
using Tgstation.Server.Common.Extensions;
|
||||
@@ -105,6 +107,21 @@ namespace Tgstation.Server.Host.Components.Deployment
|
||||
/// </summary>
|
||||
readonly Api.Models.Instance metadata;
|
||||
|
||||
/// <summary>
|
||||
/// The number of attempted deployments.
|
||||
/// </summary>
|
||||
readonly Counter attemptedDeployments;
|
||||
|
||||
/// <summary>
|
||||
/// The number of successful deployments.
|
||||
/// </summary>
|
||||
readonly Counter successfulDeployments;
|
||||
|
||||
/// <summary>
|
||||
/// The number of failed deployments.
|
||||
/// </summary>
|
||||
readonly Counter failedDeployments;
|
||||
|
||||
/// <summary>
|
||||
/// <see langword="lock"/> <see cref="object"/> for <see cref="deploying"/>.
|
||||
/// </summary>
|
||||
@@ -149,6 +166,7 @@ namespace Tgstation.Server.Host.Components.Deployment
|
||||
/// <param name="repositoryManager">The value of <see cref="repositoryManager"/>.</param>
|
||||
/// <param name="remoteDeploymentManagerFactory">The value of <see cref="remoteDeploymentManagerFactory"/>.</param>
|
||||
/// <param name="asyncDelayer">The value of <see cref="asyncDelayer"/>.</param>
|
||||
/// <param name="metricFactory">The <see cref="IMetricFactory"/> to use.</param>
|
||||
/// <param name="logger">The value of <see cref="logger"/>.</param>
|
||||
/// <param name="sessionConfiguration">The value of <see cref="sessionConfiguration"/>.</param>
|
||||
/// <param name="metadata">The value of <see cref="metadata"/>.</param>
|
||||
@@ -164,6 +182,7 @@ namespace Tgstation.Server.Host.Components.Deployment
|
||||
IRepositoryManager repositoryManager,
|
||||
IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory,
|
||||
IAsyncDelayer asyncDelayer,
|
||||
IMetricFactory metricFactory,
|
||||
ILogger<DreamMaker> logger,
|
||||
SessionConfiguration sessionConfiguration,
|
||||
Api.Models.Instance metadata)
|
||||
@@ -177,12 +196,17 @@ namespace Tgstation.Server.Host.Components.Deployment
|
||||
this.processExecutor = processExecutor ?? throw new ArgumentNullException(nameof(processExecutor));
|
||||
this.compileJobConsumer = compileJobConsumer ?? throw new ArgumentNullException(nameof(compileJobConsumer));
|
||||
this.repositoryManager = repositoryManager ?? throw new ArgumentNullException(nameof(repositoryManager));
|
||||
this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer));
|
||||
this.remoteDeploymentManagerFactory = remoteDeploymentManagerFactory ?? throw new ArgumentNullException(nameof(remoteDeploymentManagerFactory));
|
||||
this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer));
|
||||
ArgumentNullException.ThrowIfNull(metricFactory);
|
||||
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
this.sessionConfiguration = sessionConfiguration ?? throw new ArgumentNullException(nameof(sessionConfiguration));
|
||||
this.metadata = metadata ?? throw new ArgumentNullException(nameof(metadata));
|
||||
|
||||
successfulDeployments = metricFactory.CreateCounter("tgs_successful_deployments", "The number of deployments that have completed successfully");
|
||||
failedDeployments = metricFactory.CreateCounter("tgs_failed_deployments", "The number of deployments that have failed");
|
||||
attemptedDeployments = metricFactory.CreateCounter("tgs_total_deployments", "The number of deployments that have been attempted");
|
||||
|
||||
deploymentLock = new object();
|
||||
}
|
||||
|
||||
@@ -205,9 +229,12 @@ namespace Tgstation.Server.Host.Components.Deployment
|
||||
deploying = true;
|
||||
}
|
||||
|
||||
attemptedDeployments.Inc();
|
||||
|
||||
currentChatCallback = null;
|
||||
currentDreamMakerOutput = null;
|
||||
Models.CompileJob? compileJob = null;
|
||||
bool success = false;
|
||||
try
|
||||
{
|
||||
string? repoOwner = null;
|
||||
@@ -351,6 +378,7 @@ namespace Tgstation.Server.Host.Components.Deployment
|
||||
{
|
||||
var chatNotificationAction = currentChatCallback!(null, compileJob.Output!);
|
||||
await compileJobConsumer.LoadCompileJob(compileJob, chatNotificationAction, cancellationToken);
|
||||
success = true;
|
||||
}
|
||||
catch
|
||||
{
|
||||
@@ -406,6 +434,10 @@ namespace Tgstation.Server.Host.Components.Deployment
|
||||
finally
|
||||
{
|
||||
deploying = false;
|
||||
if (success)
|
||||
successfulDeployments.Inc();
|
||||
else
|
||||
failedDeployments.Inc();
|
||||
}
|
||||
}
|
||||
#pragma warning restore CA1506
|
||||
|
||||
@@ -1,10 +1,14 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
using Prometheus;
|
||||
|
||||
using Tgstation.Server.Host.Components.Chat;
|
||||
using Tgstation.Server.Host.Components.Chat.Commands;
|
||||
using Tgstation.Server.Host.Components.Deployment;
|
||||
@@ -140,6 +144,11 @@ namespace Tgstation.Server.Host.Components
|
||||
/// </summary>
|
||||
readonly IDotnetDumpService dotnetDumpService;
|
||||
|
||||
/// <summary>
|
||||
/// The <see cref="IMetricFactory"/> for the <see cref="InstanceFactory"/>.
|
||||
/// </summary>
|
||||
readonly IMetricFactory metricFactory;
|
||||
|
||||
/// <summary>
|
||||
/// The <see cref="GeneralConfiguration"/> for the <see cref="InstanceFactory"/>.
|
||||
/// </summary>
|
||||
@@ -183,6 +192,7 @@ namespace Tgstation.Server.Host.Components
|
||||
/// <param name="remoteDeploymentManagerFactory">The value of <see cref="remoteDeploymentManagerFactory"/>.</param>
|
||||
/// <param name="asyncDelayer">The value of <see cref="asyncDelayer"/>.</param>
|
||||
/// <param name="dotnetDumpService">The value of <see cref="dotnetDumpService"/>.</param>
|
||||
/// <param name="metricFactory">The value of <see cref="metricFactory"/>.</param>
|
||||
/// <param name="generalConfigurationOptions">The <see cref="IOptions{TOptions}"/> containing the value of <see cref="generalConfiguration"/>.</param>
|
||||
/// <param name="sessionConfigurationOptions">The <see cref="IOptions{TOptions}"/> containing the value of <see cref="sessionConfiguration"/>.</param>
|
||||
public InstanceFactory(
|
||||
@@ -208,6 +218,7 @@ namespace Tgstation.Server.Host.Components
|
||||
IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory,
|
||||
IAsyncDelayer asyncDelayer,
|
||||
IDotnetDumpService dotnetDumpService,
|
||||
IMetricFactory metricFactory,
|
||||
IOptions<GeneralConfiguration> generalConfigurationOptions,
|
||||
IOptions<SessionConfiguration> sessionConfigurationOptions)
|
||||
{
|
||||
@@ -233,6 +244,7 @@ namespace Tgstation.Server.Host.Components
|
||||
this.remoteDeploymentManagerFactory = remoteDeploymentManagerFactory ?? throw new ArgumentNullException(nameof(remoteDeploymentManagerFactory));
|
||||
this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer));
|
||||
this.dotnetDumpService = dotnetDumpService ?? throw new ArgumentNullException(nameof(dotnetDumpService));
|
||||
this.metricFactory = metricFactory ?? throw new ArgumentNullException(nameof(metricFactory));
|
||||
generalConfiguration = generalConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(generalConfigurationOptions));
|
||||
sessionConfiguration = sessionConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(sessionConfigurationOptions));
|
||||
}
|
||||
@@ -264,6 +276,13 @@ namespace Tgstation.Server.Host.Components
|
||||
var diagnosticsIOManager = new ResolvingIOManager(instanceIoManager, "Diagnostics");
|
||||
var configurationIoManager = new ResolvingIOManager(instanceIoManager, "Configuration");
|
||||
|
||||
var metricFactory = this.metricFactory.WithLabels(
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
{ "instance_name", metadata.Name! },
|
||||
{ "instance_id", metadata.Id!.Value.ToString(CultureInfo.InvariantCulture) },
|
||||
});
|
||||
|
||||
var configuration = new StaticFiles.Configuration(
|
||||
configurationIoManager,
|
||||
synchronousIOManager,
|
||||
@@ -323,6 +342,7 @@ namespace Tgstation.Server.Host.Components
|
||||
eventConsumer,
|
||||
asyncDelayer,
|
||||
dotnetDumpService,
|
||||
metricFactory,
|
||||
loggerFactory,
|
||||
loggerFactory.CreateLogger<SessionControllerFactory>(),
|
||||
sessionConfiguration,
|
||||
@@ -337,6 +357,7 @@ namespace Tgstation.Server.Host.Components
|
||||
diagnosticsIOManager,
|
||||
configuration, // watchdog doesn't need itself as an event consumer
|
||||
remoteDeploymentManagerFactory,
|
||||
metricFactory,
|
||||
metadata,
|
||||
metadata.DreamDaemonSettings!);
|
||||
try
|
||||
@@ -357,6 +378,7 @@ namespace Tgstation.Server.Host.Components
|
||||
repoManager,
|
||||
remoteDeploymentManagerFactory,
|
||||
asyncDelayer,
|
||||
metricFactory,
|
||||
loggerFactory.CreateLogger<DreamMaker>(),
|
||||
sessionConfiguration,
|
||||
metadata);
|
||||
|
||||
@@ -9,6 +9,8 @@ using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
using Prometheus;
|
||||
|
||||
using Tgstation.Server.Api.Models;
|
||||
using Tgstation.Server.Common;
|
||||
using Tgstation.Server.Common.Extensions;
|
||||
@@ -150,6 +152,11 @@ namespace Tgstation.Server.Host.Components
|
||||
/// </summary>
|
||||
readonly CancellationTokenSource shutdownCancellationTokenSource;
|
||||
|
||||
/// <summary>
|
||||
/// The count of online instances.
|
||||
/// </summary>
|
||||
readonly Gauge onlineInstances;
|
||||
|
||||
/// <summary>
|
||||
/// The original <see cref="IConsole.Title"/> of <see cref="console"/>.
|
||||
/// </summary>
|
||||
@@ -180,6 +187,8 @@ namespace Tgstation.Server.Host.Components
|
||||
/// <param name="swarmServiceController">The value of <see cref="swarmServiceController"/>.</param>
|
||||
/// <param name="console">The value of <see cref="console"/>.</param>
|
||||
/// <param name="platformIdentifier">The value of <see cref="platformIdentifier"/>.</param>
|
||||
/// <param name="metricFactory">The <see cref="IMetricFactory"/> used to create metrics.</param>
|
||||
/// <param name="collectorRegistry">The <see cref="ICollectorRegistry"/> to use.</param>
|
||||
/// <param name="generalConfigurationOptions">The <see cref="IOptions{TOptions}"/> containing the value of <see cref="generalConfiguration"/>.</param>
|
||||
/// <param name="swarmConfigurationOptions">The <see cref="IOptions{TOptions}"/> containing the value of <see cref="swarmConfiguration"/>.</param>
|
||||
/// <param name="internalConfigurationOptions">The <see cref="IOptions{TOptions}"/> containing the value of <see cref="internalConfiguration"/>.</param>
|
||||
@@ -197,6 +206,8 @@ namespace Tgstation.Server.Host.Components
|
||||
ISwarmServiceController swarmServiceController,
|
||||
IConsole console,
|
||||
IPlatformIdentifier platformIdentifier,
|
||||
IMetricFactory metricFactory,
|
||||
ICollectorRegistry collectorRegistry,
|
||||
IOptions<GeneralConfiguration> generalConfigurationOptions,
|
||||
IOptions<SwarmConfiguration> swarmConfigurationOptions,
|
||||
IOptions<InternalConfiguration> internalConfigurationOptions,
|
||||
@@ -214,6 +225,8 @@ namespace Tgstation.Server.Host.Components
|
||||
this.swarmServiceController = swarmServiceController ?? throw new ArgumentNullException(nameof(swarmServiceController));
|
||||
this.console = console ?? throw new ArgumentNullException(nameof(console));
|
||||
this.platformIdentifier = platformIdentifier ?? throw new ArgumentNullException(nameof(platformIdentifier));
|
||||
ArgumentNullException.ThrowIfNull(metricFactory);
|
||||
ArgumentNullException.ThrowIfNull(collectorRegistry);
|
||||
generalConfiguration = generalConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(generalConfigurationOptions));
|
||||
swarmConfiguration = swarmConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(swarmConfigurationOptions));
|
||||
internalConfiguration = internalConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(internalConfigurationOptions));
|
||||
@@ -221,12 +234,21 @@ namespace Tgstation.Server.Host.Components
|
||||
|
||||
originalConsoleTitle = console.Title;
|
||||
|
||||
onlineInstances = metricFactory.CreateGauge("tgs_online_instances", "The total number of instances online");
|
||||
|
||||
instances = new Dictionary<long, ReferenceCountingContainer<IInstance, InstanceWrapper>>();
|
||||
bridgeHandlers = new Dictionary<string, IBridgeHandler>();
|
||||
readyTcs = new TaskCompletionSource();
|
||||
instanceStateChangeSemaphore = new SemaphoreSlim(1);
|
||||
startupCancellationTokenSource = new CancellationTokenSource();
|
||||
shutdownCancellationTokenSource = new CancellationTokenSource();
|
||||
|
||||
collectorRegistry.AddBeforeCollectCallback(async cancellationToken =>
|
||||
{
|
||||
using (await SemaphoreSlimContext.Lock(instanceStateChangeSemaphore, cancellationToken))
|
||||
foreach (var container in instances.Values)
|
||||
container.Instance.Watchdog.RunMetricsScrape();
|
||||
});
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
@@ -396,6 +418,7 @@ namespace Tgstation.Server.Host.Components
|
||||
finally
|
||||
{
|
||||
await container.Instance.DisposeAsync();
|
||||
onlineInstances.Dec();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -426,6 +449,8 @@ namespace Tgstation.Server.Host.Components
|
||||
instances.Add(
|
||||
instanceId,
|
||||
new ReferenceCountingContainer<IInstance, InstanceWrapper>(instance));
|
||||
|
||||
onlineInstances.Inc();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
|
||||
@@ -199,7 +199,7 @@ namespace Tgstation.Server.Host.Components.Repository
|
||||
|
||||
var commitMessage = String.Format(
|
||||
CultureInfo.InvariantCulture,
|
||||
"TGS Test Merge (#{0}){1}{2}",
|
||||
"TGS Test Merge (PR {0}){1}{2}",
|
||||
testMergeParameters.Number,
|
||||
testMergeParameters.Comment != null
|
||||
? Environment.NewLine
|
||||
|
||||
@@ -519,6 +519,10 @@ namespace Tgstation.Server.Host.Components.Session
|
||||
return process.CreateDump(outputFile, minidump, cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public double MeasureProcessorTimeDelta()
|
||||
=> process.MeasureProcessorTimeDelta();
|
||||
|
||||
/// <summary>
|
||||
/// The <see cref="Task{TResult}"/> for <see cref="LaunchResult"/>.
|
||||
/// </summary>
|
||||
|
||||
@@ -9,6 +9,8 @@ using System.Threading.Tasks;
|
||||
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
using Prometheus;
|
||||
|
||||
using Tgstation.Server.Api.Models;
|
||||
using Tgstation.Server.Api.Models.Internal;
|
||||
using Tgstation.Server.Common.Extensions;
|
||||
@@ -127,6 +129,16 @@ namespace Tgstation.Server.Host.Components.Session
|
||||
/// </summary>
|
||||
readonly SessionConfiguration sessionConfiguration;
|
||||
|
||||
/// <summary>
|
||||
/// The number of sessions launched.
|
||||
/// </summary>
|
||||
readonly Counter sessionsLaunched;
|
||||
|
||||
/// <summary>
|
||||
/// The time the current session was launched.
|
||||
/// </summary>
|
||||
readonly Gauge lastSessionLaunch;
|
||||
|
||||
/// <summary>
|
||||
/// The <see cref="Api.Models.Instance"/> for the <see cref="SessionControllerFactory"/>.
|
||||
/// </summary>
|
||||
@@ -185,6 +197,7 @@ namespace Tgstation.Server.Host.Components.Session
|
||||
/// <param name="eventConsumer">The value of <see cref="eventConsumer"/>.</param>
|
||||
/// <param name="asyncDelayer">The value of <see cref="asyncDelayer"/>.</param>
|
||||
/// <param name="dotnetDumpService">The value of <see cref="dotnetDumpService"/>.</param>
|
||||
/// <param name="metricFactory">The <see cref="IMetricFactory"/> used to create metrics.</param>
|
||||
/// <param name="loggerFactory">The value of <see cref="loggerFactory"/>.</param>
|
||||
/// <param name="logger">The value of <see cref="logger"/>.</param>
|
||||
/// <param name="sessionConfiguration">The value of <see cref="sessionConfiguration"/>.</param>
|
||||
@@ -204,6 +217,7 @@ namespace Tgstation.Server.Host.Components.Session
|
||||
IEventConsumer eventConsumer,
|
||||
IAsyncDelayer asyncDelayer,
|
||||
IDotnetDumpService dotnetDumpService,
|
||||
IMetricFactory metricFactory,
|
||||
ILoggerFactory loggerFactory,
|
||||
ILogger<SessionControllerFactory> logger,
|
||||
SessionConfiguration sessionConfiguration,
|
||||
@@ -224,10 +238,14 @@ namespace Tgstation.Server.Host.Components.Session
|
||||
this.eventConsumer = eventConsumer ?? throw new ArgumentNullException(nameof(eventConsumer));
|
||||
this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer));
|
||||
this.dotnetDumpService = dotnetDumpService ?? throw new ArgumentNullException(nameof(dotnetDumpService));
|
||||
ArgumentNullException.ThrowIfNull(metricFactory);
|
||||
this.loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory));
|
||||
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
this.sessionConfiguration = sessionConfiguration ?? throw new ArgumentNullException(nameof(sessionConfiguration));
|
||||
this.instance = instance ?? throw new ArgumentNullException(nameof(instance));
|
||||
|
||||
sessionsLaunched = metricFactory.CreateCounter("tgs_sessions_launched", "The number of game server processes created");
|
||||
lastSessionLaunch = metricFactory.CreateGauge("tgs_session_start_time", "The UTC unix timestamp the most recent session was started");
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
@@ -368,6 +386,12 @@ namespace Tgstation.Server.Host.Components.Session
|
||||
false,
|
||||
apiValidate);
|
||||
|
||||
if (!apiValidate)
|
||||
{
|
||||
sessionsLaunched.Inc();
|
||||
lastSessionLaunch.SetToCurrentTimeUtc();
|
||||
}
|
||||
|
||||
return sessionController;
|
||||
}
|
||||
catch
|
||||
|
||||
@@ -6,6 +6,8 @@ using System.Threading.Tasks;
|
||||
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
using Prometheus;
|
||||
|
||||
using Tgstation.Server.Api.Models;
|
||||
using Tgstation.Server.Api.Models.Internal;
|
||||
using Tgstation.Server.Host.Components.Chat;
|
||||
@@ -63,6 +65,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
/// <param name="diagnosticsIOManager">The 'Diagnostics' <see cref="IIOManager"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
/// <param name="eventConsumer">The <see cref="IEventConsumer"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
/// <param name="remoteDeploymentManagerFactory">The <see cref="IRemoteDeploymentManagerFactory"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
/// <param name="metricFactory">The <see cref="IMetricFactory"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
/// <param name="gameIOManager">The 'Game' <see cref="IIOManager"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
/// <param name="linkFactory">The value of <see cref="LinkFactory"/>.</param>
|
||||
/// <param name="logger">The <see cref="ILogger"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
@@ -80,6 +83,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
IIOManager diagnosticsIOManager,
|
||||
IEventConsumer eventConsumer,
|
||||
IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory,
|
||||
IMetricFactory metricFactory,
|
||||
IIOManager gameIOManager,
|
||||
IFilesystemLinkFactory linkFactory,
|
||||
ILogger<AdvancedWatchdog> logger,
|
||||
@@ -97,6 +101,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
diagnosticsIOManager,
|
||||
eventConsumer,
|
||||
remoteDeploymentManagerFactory,
|
||||
metricFactory,
|
||||
gameIOManager,
|
||||
logger,
|
||||
initialLaunchParameters,
|
||||
|
||||
@@ -6,6 +6,8 @@ using System.Threading.Tasks;
|
||||
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
using Prometheus;
|
||||
|
||||
using Tgstation.Server.Api.Models.Internal;
|
||||
using Tgstation.Server.Host.Components.Chat;
|
||||
using Tgstation.Server.Host.Components.Deployment;
|
||||
@@ -53,6 +55,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
/// <param name="diagnosticsIOManager">The 'Diagnostics' <see cref="IIOManager"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
/// <param name="eventConsumer">The <see cref="IEventConsumer"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
/// <param name="remoteDeploymentManagerFactory">The <see cref="IRemoteDeploymentManagerFactory"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
/// <param name="metricFactory">The <see cref="IMetricFactory"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
/// <param name="gameIOManager">The 'Game' <see cref="IIOManager"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
/// <param name="logger">The <see cref="ILogger"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
/// <param name="initialLaunchParameters">The <see cref="DreamDaemonLaunchParameters"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
@@ -69,6 +72,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
IIOManager diagnosticsIOManager,
|
||||
IEventConsumer eventConsumer,
|
||||
IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory,
|
||||
IMetricFactory metricFactory,
|
||||
IIOManager gameIOManager,
|
||||
ILogger<BasicWatchdog> logger,
|
||||
DreamDaemonLaunchParameters initialLaunchParameters,
|
||||
@@ -85,6 +89,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
diagnosticsIOManager,
|
||||
eventConsumer,
|
||||
remoteDeploymentManagerFactory,
|
||||
metricFactory,
|
||||
gameIOManager,
|
||||
logger,
|
||||
initialLaunchParameters,
|
||||
|
||||
@@ -117,5 +117,10 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
|
||||
/// <returns>A <see cref="ValueTask{TResult}"/> resulting in <see langword="true"/> if the broadcast succeeded., <see langword="false"/> otherwise.</returns>
|
||||
ValueTask<bool> Broadcast(string message, CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Callback to update transient metrics.
|
||||
/// </summary>
|
||||
void RunMetricsScrape();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
using Tgstation.Server.Api.Models.Internal;
|
||||
using Prometheus;
|
||||
|
||||
using Tgstation.Server.Api.Models.Internal;
|
||||
using Tgstation.Server.Host.Components.Chat;
|
||||
using Tgstation.Server.Host.Components.Deployment;
|
||||
using Tgstation.Server.Host.Components.Deployment.Remote;
|
||||
@@ -24,6 +26,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
/// <param name="diagnosticsIOManager">The <see cref="IIOManager"/> pointing to the Diagnostics directory for the <see cref="IWatchdog"/>.</param>
|
||||
/// <param name="eventConsumer">The <see cref="IEventConsumer"/> for the <see cref="IWatchdog"/>.</param>
|
||||
/// <param name="remoteDeploymentManagerFactory">The <see cref="IRemoteDeploymentManagerFactory"/> for the <see cref="IWatchdog"/>.</param>
|
||||
/// <param name="metricFactory">The <see cref="IMetricFactory"/> used to create metrics.</param>
|
||||
/// <param name="instance">The <see cref="Instance"/> for the <see cref="IWatchdog"/>.</param>
|
||||
/// <param name="settings">The initial <see cref="DreamDaemonSettings"/> for the <see cref="IWatchdog"/>.</param>
|
||||
/// <returns>A new <see cref="IWatchdog"/>.</returns>
|
||||
@@ -36,6 +39,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
IIOManager diagnosticsIOManager,
|
||||
IEventConsumer eventConsumer,
|
||||
IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory,
|
||||
IMetricFactory metricFactory,
|
||||
Api.Models.Instance instance,
|
||||
DreamDaemonSettings settings);
|
||||
}
|
||||
|
||||
@@ -5,6 +5,8 @@ using System.Threading.Tasks;
|
||||
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
using Prometheus;
|
||||
|
||||
using Tgstation.Server.Api.Models.Internal;
|
||||
using Tgstation.Server.Host.Components.Chat;
|
||||
using Tgstation.Server.Host.Components.Deployment;
|
||||
@@ -43,6 +45,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
/// <param name="diagnosticsIOManager">The <see cref="IIOManager"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
/// <param name="eventConsumer">The <see cref="IEventConsumer"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
/// <param name="remoteDeploymentManagerFactory">The <see cref="IRemoteDeploymentManagerFactory"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
/// <param name="metricFactory">The <see cref="IMetricFactory"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
/// <param name="gameIOManager">The <see cref="IIOManager"/> pointing to the game directory for the <see cref="AdvancedWatchdog"/>..</param>
|
||||
/// <param name="linkFactory">The <see cref="IFilesystemLinkFactory"/> for the <see cref="AdvancedWatchdog"/>.</param>
|
||||
/// <param name="logger">The <see cref="ILogger"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
@@ -61,6 +64,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
IIOManager diagnosticsIOManager,
|
||||
IEventConsumer eventConsumer,
|
||||
IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory,
|
||||
IMetricFactory metricFactory,
|
||||
IIOManager gameIOManager,
|
||||
IFilesystemLinkFactory linkFactory,
|
||||
ILogger<PosixWatchdog> logger,
|
||||
@@ -79,6 +83,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
diagnosticsIOManager,
|
||||
eventConsumer,
|
||||
remoteDeploymentManagerFactory,
|
||||
metricFactory,
|
||||
gameIOManager,
|
||||
linkFactory,
|
||||
logger,
|
||||
|
||||
@@ -4,6 +4,8 @@ using System.Runtime.Versioning;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
using Prometheus;
|
||||
|
||||
using Tgstation.Server.Api.Models.Internal;
|
||||
using Tgstation.Server.Host.Components.Chat;
|
||||
using Tgstation.Server.Host.Components.Deployment;
|
||||
@@ -60,6 +62,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
IIOManager diagnosticsIOManager,
|
||||
IEventConsumer eventConsumer,
|
||||
IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory,
|
||||
IMetricFactory metricFactory,
|
||||
Api.Models.Instance instance,
|
||||
DreamDaemonSettings settings)
|
||||
=> new PosixWatchdog(
|
||||
@@ -73,6 +76,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
diagnosticsIOManager,
|
||||
eventConsumer,
|
||||
remoteDeploymentManagerFactory,
|
||||
metricFactory,
|
||||
gameIOManager,
|
||||
LinkFactory,
|
||||
LoggerFactory.CreateLogger<PosixWatchdog>(),
|
||||
|
||||
@@ -7,6 +7,8 @@ using System.Threading.Tasks;
|
||||
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
using Prometheus;
|
||||
|
||||
using Serilog.Context;
|
||||
|
||||
using Tgstation.Server.Api.Models;
|
||||
@@ -123,6 +125,21 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
/// </summary>
|
||||
readonly SemaphoreSlim controllerDisposeSemaphore;
|
||||
|
||||
/// <summary>
|
||||
/// The <see cref="Status"/> as a metric.
|
||||
/// </summary>
|
||||
readonly Gauge watchdogStatusMetric;
|
||||
|
||||
/// <summary>
|
||||
/// Active session CPU usage as a metric.
|
||||
/// </summary>
|
||||
readonly Gauge cpuUsageMetric;
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="MemoryUsage"/> as a metric.
|
||||
/// </summary>
|
||||
readonly Gauge ramUsageMetric;
|
||||
|
||||
/// <summary>
|
||||
/// The <see cref="IEventConsumer"/> that is not the <see cref="WatchdogBase"/>.
|
||||
/// </summary>
|
||||
@@ -201,6 +218,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
/// <param name="diagnosticsIOManager">The value of <see cref="diagnosticsIOManager"/>.</param>
|
||||
/// <param name="eventConsumer">The value of <see cref="EventConsumer"/>.</param>
|
||||
/// <param name="remoteDeploymentManagerFactory">The value of <see cref="remoteDeploymentManagerFactory"/>.</param>
|
||||
/// <param name="metricFactory">The <see cref="IMetricFactory"/> used to create metrics.</param>
|
||||
/// <param name="gameIOManager">The value of <see cref="GameIOManager"/>.</param>
|
||||
/// <param name="logger">The value of <see cref="Logger"/>.</param>
|
||||
/// <param name="initialLaunchParameters">The initial value of <see cref="ActiveLaunchParameters"/>. May be modified.</param>
|
||||
@@ -217,6 +235,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
IIOManager diagnosticsIOManager,
|
||||
IEventConsumer eventConsumer,
|
||||
IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory,
|
||||
IMetricFactory metricFactory,
|
||||
IIOManager gameIOManager,
|
||||
ILogger<WatchdogBase> logger,
|
||||
DreamDaemonLaunchParameters initialLaunchParameters,
|
||||
@@ -232,6 +251,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
this.diagnosticsIOManager = diagnosticsIOManager ?? throw new ArgumentNullException(nameof(diagnosticsIOManager));
|
||||
this.eventConsumer = eventConsumer ?? throw new ArgumentNullException(nameof(eventConsumer));
|
||||
this.remoteDeploymentManagerFactory = remoteDeploymentManagerFactory ?? throw new ArgumentNullException(nameof(remoteDeploymentManagerFactory));
|
||||
ArgumentNullException.ThrowIfNull(metricFactory);
|
||||
GameIOManager = gameIOManager ?? throw new ArgumentNullException(nameof(gameIOManager));
|
||||
Logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
ActiveLaunchParameters = initialLaunchParameters ?? throw new ArgumentNullException(nameof(initialLaunchParameters));
|
||||
@@ -240,6 +260,12 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
|
||||
ArgumentNullException.ThrowIfNull(serverControl);
|
||||
|
||||
watchdogStatusMetric = metricFactory.CreateGauge(
|
||||
"tgs_watchdog_status",
|
||||
$"TGS Watchdog status: {(int)WatchdogStatus.Offline} = Offline, {(int)WatchdogStatus.Online} = Online, {(int)WatchdogStatus.Restoring} = Restoring, {(int)WatchdogStatus.DelayedRestart} = Delayed Restart");
|
||||
cpuUsageMetric = metricFactory.CreateGauge("tgs_game_cpu_usage", "Estimated total CPU usage time for the game process from 0-1");
|
||||
ramUsageMetric = metricFactory.CreateGauge("tgs_game_ram_usage", "Total used bytes of private memory for the game process");
|
||||
|
||||
chat.RegisterCommandHandler(this);
|
||||
|
||||
ActiveLaunchParameters = initialLaunchParameters;
|
||||
@@ -272,6 +298,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
await DisposeAndNullControllersImpl();
|
||||
controllerDisposeSemaphore.Dispose();
|
||||
monitorCts?.Dispose();
|
||||
|
||||
disposed = true;
|
||||
}
|
||||
|
||||
@@ -501,6 +528,15 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
return response != null && response.ErrorMessage == null;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void RunMetricsScrape()
|
||||
{
|
||||
watchdogStatusMetric.Set((int)Status);
|
||||
var controller = GetActiveController();
|
||||
ramUsageMetric.Set(controller?.MemoryUsage ?? 0);
|
||||
cpuUsageMetric.Set(controller?.MeasureProcessorTimeDelta() ?? 0);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
async ValueTask IEventConsumer.HandleEvent(EventType eventType, IEnumerable<string?> parameters, bool deploymentPipeline, CancellationToken cancellationToken)
|
||||
{
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
using Prometheus;
|
||||
|
||||
using Tgstation.Server.Api.Models.Internal;
|
||||
using Tgstation.Server.Host.Components.Chat;
|
||||
using Tgstation.Server.Host.Components.Deployment;
|
||||
@@ -77,6 +79,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
IIOManager diagnosticsIOManager,
|
||||
IEventConsumer eventConsumer,
|
||||
IRemoteDeploymentManagerFactory remoteDeploymentManagerfactory,
|
||||
IMetricFactory metricFactory,
|
||||
Api.Models.Instance instance,
|
||||
DreamDaemonSettings settings)
|
||||
=> new BasicWatchdog(
|
||||
@@ -90,6 +93,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
diagnosticsIOManager,
|
||||
eventConsumer,
|
||||
remoteDeploymentManagerfactory,
|
||||
metricFactory,
|
||||
gameIOManager,
|
||||
LoggerFactory.CreateLogger<BasicWatchdog>(),
|
||||
settings,
|
||||
|
||||
@@ -3,6 +3,8 @@ using System.Threading.Tasks;
|
||||
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
using Prometheus;
|
||||
|
||||
using Tgstation.Server.Api.Models;
|
||||
using Tgstation.Server.Api.Models.Internal;
|
||||
using Tgstation.Server.Host.Components.Chat;
|
||||
@@ -35,6 +37,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
/// <param name="diagnosticsIOManager">The <see cref="IIOManager"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
/// <param name="eventConsumer">The <see cref="IEventConsumer"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
/// <param name="remoteDeploymentManagerFactory">The <see cref="IRemoteDeploymentManagerFactory"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
/// <param name="metricFactory">The <see cref="IMetricFactory"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
/// <param name="gameIOManager">The <see cref="IIOManager"/> pointing to the game directory for the <see cref="AdvancedWatchdog"/>..</param>
|
||||
/// <param name="linkFactory">The <see cref="IFilesystemLinkFactory"/> for the <see cref="AdvancedWatchdog"/>.</param>
|
||||
/// <param name="logger">The <see cref="ILogger"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
@@ -52,6 +55,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
IIOManager diagnosticsIOManager,
|
||||
IEventConsumer eventConsumer,
|
||||
IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory,
|
||||
IMetricFactory metricFactory,
|
||||
IIOManager gameIOManager,
|
||||
IFilesystemLinkFactory linkFactory,
|
||||
ILogger<WindowsWatchdog> logger,
|
||||
@@ -69,6 +73,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
diagnosticsIOManager,
|
||||
eventConsumer,
|
||||
remoteDeploymentManagerFactory,
|
||||
metricFactory,
|
||||
gameIOManager,
|
||||
linkFactory,
|
||||
logger,
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
using Prometheus;
|
||||
|
||||
using Tgstation.Server.Api.Models.Internal;
|
||||
using Tgstation.Server.Host.Components.Chat;
|
||||
using Tgstation.Server.Host.Components.Deployment;
|
||||
@@ -63,6 +65,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
IIOManager diagnosticsIOManager,
|
||||
IEventConsumer eventConsumer,
|
||||
IRemoteDeploymentManagerFactory remoteDeploymentManagerFactory,
|
||||
IMetricFactory metricFactory,
|
||||
Api.Models.Instance instance,
|
||||
DreamDaemonSettings settings)
|
||||
=> new WindowsWatchdog(
|
||||
@@ -76,6 +79,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
diagnosticsIOManager,
|
||||
eventConsumer,
|
||||
remoteDeploymentManagerFactory,
|
||||
metricFactory,
|
||||
gameIOManager,
|
||||
LinkFactory,
|
||||
LoggerFactory.CreateLogger<WindowsWatchdog>(),
|
||||
|
||||
@@ -91,6 +91,11 @@ namespace Tgstation.Server.Host.Configuration
|
||||
/// </summary>
|
||||
public ushort ApiPort { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The port Prometheus metrics are published on, if any.
|
||||
/// </summary>
|
||||
public ushort? PrometheusPort { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// A classic GitHub personal access token to use for bypassing rate limits on requests. Requires no scopes.
|
||||
/// </summary>
|
||||
|
||||
@@ -30,6 +30,8 @@ using Microsoft.Extensions.Options;
|
||||
|
||||
using Newtonsoft.Json;
|
||||
|
||||
using Prometheus;
|
||||
|
||||
using Serilog;
|
||||
using Serilog.Events;
|
||||
using Serilog.Formatting.Display;
|
||||
@@ -294,6 +296,21 @@ namespace Tgstation.Server.Host.Core
|
||||
services.AddHttpClient();
|
||||
services.AddSingleton<IAbstractHttpClientFactory, AbstractHttpClientFactory>();
|
||||
|
||||
// configure metrics
|
||||
var prometheusPort = postSetupServices.GeneralConfiguration.PrometheusPort;
|
||||
|
||||
services.AddSingleton<IMetricFactory>(_ => Metrics.DefaultFactory);
|
||||
services.AddSingleton<ICollectorRegistry>(_ => Metrics.DefaultRegistry);
|
||||
|
||||
if (prometheusPort.HasValue && prometheusPort != postSetupServices.GeneralConfiguration.ApiPort)
|
||||
services.AddMetricServer(options => options.Port = prometheusPort.Value);
|
||||
|
||||
services.UseHttpClientMetrics();
|
||||
|
||||
var healthChecksBuilder = services
|
||||
.AddHealthChecks()
|
||||
.ForwardToPrometheus();
|
||||
|
||||
// configure graphql
|
||||
services
|
||||
.AddScoped<GraphQL.Subscriptions.ITopicEventReceiver, ShutdownAwareTopicEventReceiver>()
|
||||
@@ -362,6 +379,9 @@ namespace Tgstation.Server.Host.Core
|
||||
configureAction(builder, databaseConfig);
|
||||
});
|
||||
services.AddScoped<IDatabaseContext>(x => x.GetRequiredService<TContext>());
|
||||
|
||||
healthChecksBuilder
|
||||
.AddDbContextCheck<TContext>();
|
||||
}
|
||||
|
||||
// add the correct database context type
|
||||
@@ -571,6 +591,9 @@ namespace Tgstation.Server.Host.Core
|
||||
// Wrap exceptions in a 500 (ErrorMessage) response
|
||||
applicationBuilder.UseServerErrorHandling();
|
||||
|
||||
// metrics capture
|
||||
applicationBuilder.UseHttpMetrics();
|
||||
|
||||
// Add the X-Powered-By response header
|
||||
applicationBuilder.UseServerBranding(assemblyInformationProvider);
|
||||
|
||||
@@ -692,6 +715,19 @@ namespace Tgstation.Server.Host.Core
|
||||
.MapGraphQL(Routes.GraphQL)
|
||||
.WithOptions(gqlOptions);
|
||||
}
|
||||
|
||||
if (generalConfiguration.PrometheusPort.HasValue)
|
||||
if (generalConfiguration.PrometheusPort == generalConfiguration.ApiPort)
|
||||
{
|
||||
endpoints.MapMetrics();
|
||||
logger.LogDebug("Prometheus being hosted alongside server");
|
||||
}
|
||||
else
|
||||
logger.LogDebug("Prometheus being hosted on port {prometheusPort}", generalConfiguration.PrometheusPort);
|
||||
else
|
||||
logger.LogTrace("Prometheus disabled");
|
||||
|
||||
endpoints.MapHealthChecks("/health");
|
||||
});
|
||||
|
||||
// 404 anything that gets this far
|
||||
|
||||
@@ -9,6 +9,8 @@ using Microsoft.AspNetCore.SignalR;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
using Prometheus;
|
||||
|
||||
using Serilog.Context;
|
||||
|
||||
using Tgstation.Server.Api.Extensions;
|
||||
@@ -67,6 +69,16 @@ namespace Tgstation.Server.Host.Jobs
|
||||
/// </summary>
|
||||
readonly TaskCompletionSource<IInstanceCoreProvider> activationTcs;
|
||||
|
||||
/// <summary>
|
||||
/// Total number of jobs processed.
|
||||
/// </summary>
|
||||
readonly Counter processedJobs;
|
||||
|
||||
/// <summary>
|
||||
/// Jobs currently running.
|
||||
/// </summary>
|
||||
readonly Gauge runningJobs;
|
||||
|
||||
/// <summary>
|
||||
/// <see langword="lock"/> <see cref="object"/> for various operations.
|
||||
/// </summary>
|
||||
@@ -87,19 +99,25 @@ namespace Tgstation.Server.Host.Jobs
|
||||
/// </summary>
|
||||
/// <param name="hub">The value of <see cref="hub"/>.</param>
|
||||
/// <param name="databaseContextFactory">The value of <see cref="databaseContextFactory"/>.</param>
|
||||
/// <param name="metricFactory">The <see cref="IMetricFactory"/> to use.</param>
|
||||
/// <param name="loggerFactory">The value of <see cref="loggerFactory"/>.</param>
|
||||
/// <param name="logger">The value of <see cref="logger"/>.</param>
|
||||
public JobService(
|
||||
IConnectionMappedHubContext<JobsHub, IJobsHub> hub,
|
||||
IDatabaseContextFactory databaseContextFactory,
|
||||
IMetricFactory metricFactory,
|
||||
ILoggerFactory loggerFactory,
|
||||
ILogger<JobService> logger)
|
||||
{
|
||||
this.hub = hub ?? throw new ArgumentNullException(nameof(hub));
|
||||
this.databaseContextFactory = databaseContextFactory ?? throw new ArgumentNullException(nameof(databaseContextFactory));
|
||||
this.loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory));
|
||||
ArgumentNullException.ThrowIfNull(metricFactory);
|
||||
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
|
||||
runningJobs = metricFactory.CreateGauge("tgs_jobs_running", "The number of TGS jobs running across all instances");
|
||||
processedJobs = metricFactory.CreateCounter("tgs_jobs_processed", "The number of TGS jobs that have run previously");
|
||||
|
||||
jobs = new Dictionary<long, JobHandler>();
|
||||
hubUpdateActions = new Dictionary<long, Action>();
|
||||
activationTcs = new TaskCompletionSource<IInstanceCoreProvider>();
|
||||
@@ -363,6 +381,7 @@ namespace Tgstation.Server.Host.Jobs
|
||||
{
|
||||
var jid = job.Require(x => x.Id);
|
||||
using (LogContext.PushProperty(SerilogContextHelper.JobIdContextProperty, jid))
|
||||
using (runningJobs.TrackInProgress())
|
||||
try
|
||||
{
|
||||
void LogException(Exception ex) => logger.LogDebug(ex, "Job {jobId} exited with error!", jid);
|
||||
@@ -553,6 +572,8 @@ namespace Tgstation.Server.Host.Jobs
|
||||
jobs.Remove(jid);
|
||||
handler.Dispose();
|
||||
}
|
||||
|
||||
processedJobs.Inc();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,6 +24,12 @@ namespace Tgstation.Server.Host.System
|
||||
/// </summary>
|
||||
long? MemoryUsage { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the estimated CPU usage fraction of the process based on the last time this was called.
|
||||
/// </summary>
|
||||
/// <returns>The CPU's estimated usage as a value between 0 and 1.</returns>
|
||||
double MeasureProcessorTimeDelta();
|
||||
|
||||
/// <summary>
|
||||
/// Set's the owned <see cref="global::System.Diagnostics.Process.PriorityClass"/> to a non-normal value.
|
||||
/// </summary>
|
||||
|
||||
@@ -87,10 +87,30 @@ namespace Tgstation.Server.Host.System
|
||||
/// </summary>
|
||||
readonly Task<string?>? readTask;
|
||||
|
||||
/// <summary>
|
||||
/// <see langword="lock"/> object for measuring processor time usage.
|
||||
/// </summary>
|
||||
readonly object processTimeMeasureLock;
|
||||
|
||||
/// <summary>
|
||||
/// The last time <see cref="MeasureProcessorTimeDelta"/> was called.
|
||||
/// </summary>
|
||||
DateTimeOffset lastProcessorMeasureTime;
|
||||
|
||||
/// <summary>
|
||||
/// The last value of <see cref="global::System.Diagnostics.Process.TotalProcessorTime"/>.
|
||||
/// </summary>
|
||||
TimeSpan lastProcessorUsageTime;
|
||||
|
||||
/// <summary>
|
||||
/// The last valid return value of <see cref="MeasureProcessorTimeDelta"/>.
|
||||
/// </summary>
|
||||
double lastProcessorUsageEstimation;
|
||||
|
||||
/// <summary>
|
||||
/// If the <see cref="Process"/> was disposed.
|
||||
/// </summary>
|
||||
volatile int disposed;
|
||||
int disposed;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="Process"/> class.
|
||||
@@ -123,6 +143,8 @@ namespace Tgstation.Server.Host.System
|
||||
|
||||
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
|
||||
processTimeMeasureLock = new object();
|
||||
|
||||
Lifetime = WrapLifetimeTask();
|
||||
|
||||
if (preExisting)
|
||||
@@ -147,6 +169,8 @@ namespace Tgstation.Server.Host.System
|
||||
DefaultIOManager.BlockingTaskCreationOptions,
|
||||
TaskScheduler.Current);
|
||||
|
||||
MeasureProcessorTimeDelta();
|
||||
|
||||
logger.LogTrace("Created process ID: {pid}", Id);
|
||||
}
|
||||
|
||||
@@ -267,6 +291,34 @@ namespace Tgstation.Server.Host.System
|
||||
return processFeatures.CreateDump(handle, outputFile, minidump, cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public double MeasureProcessorTimeDelta()
|
||||
{
|
||||
lock (processTimeMeasureLock)
|
||||
{
|
||||
try
|
||||
{
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var newTime = handle.TotalProcessorTime;
|
||||
|
||||
var timeDelta = now - lastProcessorMeasureTime;
|
||||
var newUsage = newTime - lastProcessorUsageTime;
|
||||
|
||||
lastProcessorMeasureTime = now;
|
||||
lastProcessorUsageTime = newTime;
|
||||
|
||||
if (timeDelta != TimeSpan.Zero)
|
||||
lastProcessorUsageEstimation = newUsage / (Environment.ProcessorCount * timeDelta);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.LogWarning(ex, "Error measuring processor time delta!");
|
||||
}
|
||||
|
||||
return lastProcessorUsageEstimation;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Attaches a log message to the process' exit event.
|
||||
/// </summary>
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk.Web">
|
||||
<Project Sdk="Microsoft.NET.Sdk.Web">
|
||||
<Import Project="../../build/SrcCommon.props" />
|
||||
|
||||
<PropertyGroup>
|
||||
@@ -110,21 +110,23 @@
|
||||
<!-- Usage: git interop -->
|
||||
<PackageReference Include="LibGit2Sharp" Version="0.31.0" />
|
||||
<!-- Usage: Support ""legacy"" Newotonsoft.Json in HTTP pipeline. The rest of our codebase uses Newtonsoft. -->
|
||||
<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="8.0.12" />
|
||||
<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="8.0.13" />
|
||||
<!-- Usage: Using target JSON serializer for API -->
|
||||
<PackageReference Include="Microsoft.AspNetCore.SignalR.Protocols.NewtonsoftJson" Version="9.0.1" />
|
||||
<PackageReference Include="Microsoft.AspNetCore.SignalR.Protocols.NewtonsoftJson" Version="9.0.2" />
|
||||
<!-- Usage: Generating dumps of dotnet engine processes -->
|
||||
<PackageReference Include="Microsoft.Diagnostics.NETCore.Client" Version="0.2.553101" />
|
||||
<PackageReference Include="Microsoft.Diagnostics.NETCore.Client" Version="0.2.607501" />
|
||||
<!-- Usage: Database ORM -->
|
||||
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="8.0.11" />
|
||||
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="8.0.13" />
|
||||
<!-- Usage: Automatic migration generation using command line -->
|
||||
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="8.0.11">
|
||||
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="8.0.13">
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
</PackageReference>
|
||||
<!-- Usage: Sqlite ORM plugin -->
|
||||
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="8.0.11" />
|
||||
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="8.0.13" />
|
||||
<!-- Usage: MSSQL ORM plugin -->
|
||||
<PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer" Version="8.0.11" />
|
||||
<PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer" Version="8.0.13" />
|
||||
<!-- Usage: Database connectivity health check -->
|
||||
<PackageReference Include="Microsoft.Extensions.Diagnostics.HealthChecks.EntityFrameworkCore" Version="8.0.13" />
|
||||
<!-- Usage: POSIX support for syscalls, signals, and symlinks -->
|
||||
<PackageReference Include="Mono.Posix.NETStandard" Version="1.0.0" />
|
||||
<!-- Usage: Cron string parsing -->
|
||||
@@ -132,11 +134,13 @@
|
||||
<!-- Usage: YAML config plugin -->
|
||||
<PackageReference Include="NetEscapades.Configuration.Yaml" Version="3.1.0" />
|
||||
<!-- Usage: PostgresSQL ORM plugin -->
|
||||
<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="8.0.10" />
|
||||
<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="8.0.11" />
|
||||
<!-- Usage: MYSQL/MariaDB ORM plugin -->
|
||||
<PackageReference Include="Pomelo.EntityFrameworkCore.MySql" Version="8.0.2" />
|
||||
<!-- Usage: Publishing Prometheus metrics -->
|
||||
<PackageReference Include="prometheus-net.AspNetCore.HealthChecks" Version="8.2.1" />
|
||||
<!-- Usage: Discord interop -->
|
||||
<PackageReference Include="Remora.Discord" Version="2024.3.0" />
|
||||
<PackageReference Include="Remora.Discord" Version="2025.1.0" />
|
||||
<!-- Usage: Rich logger builder -->
|
||||
<PackageReference Include="Serilog.Extensions.Logging" Version="9.0.0" />
|
||||
<!-- Usage: Async logging plugin -->
|
||||
@@ -152,11 +156,11 @@
|
||||
<!-- Usage: Newtonsoft.Json plugin for OpenAPI spec generator -->
|
||||
<PackageReference Include="Swashbuckle.AspNetCore.Newtonsoft" Version="7.2.0" />
|
||||
<!-- Usage: Windows authentication plugin allowing searching for users by name -->
|
||||
<PackageReference Include="System.DirectoryServices.AccountManagement" Version="9.0.1" />
|
||||
<PackageReference Include="System.DirectoryServices.AccountManagement" Version="9.0.2" />
|
||||
<!-- Usage: Identifying owning user of Windows Process objects -->
|
||||
<PackageReference Include="System.Management" Version="9.0.2" />
|
||||
<!-- Usage: Temporary resolution to compatibility issues with EFCore 7 and .NET 8 -->
|
||||
<PackageReference Include="System.Security.Permissions" Version="9.0.1" />
|
||||
<PackageReference Include="System.Security.Permissions" Version="9.0.2" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
using System.Collections.Generic;
|
||||
using System.IdentityModel.Tokens.Jwt;
|
||||
using System.Linq;
|
||||
using System.Net.Http;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
@@ -12,6 +13,7 @@ using Microsoft.Extensions.Options;
|
||||
using Microsoft.IdentityModel.Tokens;
|
||||
|
||||
using Octokit;
|
||||
using Octokit.Internal;
|
||||
|
||||
using Tgstation.Server.Api.Models;
|
||||
using Tgstation.Server.Host.Configuration;
|
||||
@@ -38,6 +40,11 @@ namespace Tgstation.Server.Host.Utils.GitHub
|
||||
/// </summary>
|
||||
readonly IAssemblyInformationProvider assemblyInformationProvider;
|
||||
|
||||
/// <summary>
|
||||
/// The <see cref="IHttpMessageHandlerFactory"/> for the <see cref="GitHubClientFactory"/>.
|
||||
/// </summary>
|
||||
readonly IHttpMessageHandlerFactory httpMessageHandlerFactory;
|
||||
|
||||
/// <summary>
|
||||
/// The <see cref="ILogger"/> for the <see cref="GitHubClientFactory"/>.
|
||||
/// </summary>
|
||||
@@ -62,14 +69,17 @@ namespace Tgstation.Server.Host.Utils.GitHub
|
||||
/// Initializes a new instance of the <see cref="GitHubClientFactory"/> class.
|
||||
/// </summary>
|
||||
/// <param name="assemblyInformationProvider">The value of <see cref="assemblyInformationProvider"/>.</param>
|
||||
/// <param name="httpMessageHandlerFactory">The value of <see cref="httpMessageHandlerFactory"/>.</param>
|
||||
/// <param name="logger">The value of <see cref="logger"/>.</param>
|
||||
/// <param name="generalConfigurationOptions">The <see cref="IOptions{TOptions}"/> containing the value of <see cref="generalConfiguration"/>.</param>
|
||||
public GitHubClientFactory(
|
||||
IAssemblyInformationProvider assemblyInformationProvider,
|
||||
IHttpMessageHandlerFactory httpMessageHandlerFactory,
|
||||
ILogger<GitHubClientFactory> logger,
|
||||
IOptions<GeneralConfiguration> generalConfigurationOptions)
|
||||
{
|
||||
this.assemblyInformationProvider = assemblyInformationProvider ?? throw new ArgumentNullException(nameof(assemblyInformationProvider));
|
||||
this.httpMessageHandlerFactory = httpMessageHandlerFactory ?? throw new ArgumentNullException(nameof(httpMessageHandlerFactory));
|
||||
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
generalConfiguration = generalConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(generalConfigurationOptions));
|
||||
|
||||
@@ -315,10 +325,33 @@ namespace Tgstation.Server.Host.Utils.GitHub
|
||||
GitHubClient CreateUnauthenticatedClient()
|
||||
{
|
||||
var product = assemblyInformationProvider.ProductInfoHeaderValue.Product!;
|
||||
return new GitHubClient(
|
||||
new ProductHeaderValue(
|
||||
product.Name,
|
||||
product.Version));
|
||||
#pragma warning disable CA2000 // Dispose objects before losing scope
|
||||
var handler = httpMessageHandlerFactory.CreateHandler();
|
||||
try
|
||||
{
|
||||
var clientAdapter = new HttpClientAdapter(() => handler);
|
||||
#pragma warning restore CA2000 // Dispose objects before losing scope
|
||||
handler = null;
|
||||
try
|
||||
{
|
||||
return new GitHubClient(
|
||||
new Connection(
|
||||
new ProductHeaderValue(
|
||||
product.Name,
|
||||
product.Version),
|
||||
clientAdapter));
|
||||
}
|
||||
catch
|
||||
{
|
||||
clientAdapter.Dispose();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
handler?.Dispose();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ General:
|
||||
ByondTopicTimeout: 5000 # Timeout for BYOND /world/Topic() calls in milliseconds
|
||||
RestartTimeoutMinutes: 1 # Timeout for server restarts after requested by SIGTERM or the HTTP API
|
||||
ApiPort: 5000 # Port the HTTP API is hosted on
|
||||
PrometheusPort: null # Port Prometheus metrics are published on under /metrics. This can be set to the same value as the ApiPort, just note that accessing it does not require authentication.
|
||||
UseBasicWatchdog: false # The basic watchdog hard restarts DreamDaemon when /world/proc/TgsReboot() is called in the DMAPI if a new deployment is available
|
||||
UserLimit: 100 # Maximum number of allowed users
|
||||
UserGroupLimit: 25 # Maximum number of allowed groups
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
|
||||
<ItemGroup>
|
||||
<!-- Usage: JWT injection into HTTP pipeline -->
|
||||
<PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="8.0.12" />
|
||||
<PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="8.0.13" />
|
||||
<!-- Usage: GitHub.com interop -->
|
||||
<PackageReference Include="Octokit" Version="14.0.0" />
|
||||
<!-- Usage: YAML conversion of Version objects -->
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="9.0.1" />
|
||||
<PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="9.0.2" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
using System;
|
||||
using System.Net.Http;
|
||||
|
||||
namespace Tgstation.Server.Host.Tests
|
||||
{
|
||||
/// <summary>
|
||||
/// Basic <see cref="IHttpMessageHandlerFactory"/> implementation for testiong
|
||||
/// </summary>
|
||||
public sealed class BasicHttpMessageHandlerFactory : IHttpMessageHandlerFactory, IDisposable
|
||||
{
|
||||
readonly HttpClientHandler handler = new();
|
||||
|
||||
public HttpMessageHandler CreateHandler(string name)
|
||||
=> handler;
|
||||
|
||||
public void Dispose()
|
||||
=> handler.Dispose();
|
||||
}
|
||||
}
|
||||
@@ -7,7 +7,7 @@
|
||||
|
||||
<ItemGroup>
|
||||
<!-- Usage: Creating mock database implementations -->
|
||||
<PackageReference Include="Microsoft.EntityFrameworkCore.InMemory" Version="8.0.11" />
|
||||
<PackageReference Include="Microsoft.EntityFrameworkCore.InMemory" Version="8.0.13" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
using System;
|
||||
using System.Net.Http;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
@@ -15,6 +16,7 @@ using Octokit;
|
||||
using Tgstation.Server.Api.Models;
|
||||
using Tgstation.Server.Host.Configuration;
|
||||
using Tgstation.Server.Host.System;
|
||||
using Tgstation.Server.Host.Tests;
|
||||
|
||||
namespace Tgstation.Server.Host.Utils.GitHub.Tests
|
||||
{
|
||||
@@ -42,9 +44,10 @@ namespace Tgstation.Server.Host.Utils.GitHub.Tests
|
||||
[TestMethod]
|
||||
public void TestContructionThrows()
|
||||
{
|
||||
Assert.ThrowsException<ArgumentNullException>(() => new GitHubClientFactory(null, null, null));
|
||||
Assert.ThrowsException<ArgumentNullException>(() => new GitHubClientFactory(Mock.Of<IAssemblyInformationProvider>(), null, null));
|
||||
Assert.ThrowsException<ArgumentNullException>(() => new GitHubClientFactory(Mock.Of<IAssemblyInformationProvider>(), Mock.Of<ILogger<GitHubClientFactory>>(), null));
|
||||
Assert.ThrowsException<ArgumentNullException>(() => new GitHubClientFactory(null, null, null, null));
|
||||
Assert.ThrowsException<ArgumentNullException>(() => new GitHubClientFactory(Mock.Of<IAssemblyInformationProvider>(), null, null, null));
|
||||
Assert.ThrowsException<ArgumentNullException>(() => new GitHubClientFactory(Mock.Of<IAssemblyInformationProvider>(), Mock.Of<IHttpMessageHandlerFactory>(), null, null));
|
||||
Assert.ThrowsException<ArgumentNullException>(() => new GitHubClientFactory(Mock.Of<IAssemblyInformationProvider>(), Mock.Of<IHttpMessageHandlerFactory>(), Mock.Of<ILogger<GitHubClientFactory>>(), null));
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
@@ -58,7 +61,7 @@ namespace Tgstation.Server.Host.Utils.GitHub.Tests
|
||||
var gc = new GeneralConfiguration();
|
||||
Assert.IsNull(gc.GitHubAccessToken);
|
||||
mockOptions.SetupGet(x => x.Value).Returns(gc);
|
||||
var factory = new GitHubClientFactory(mockApp.Object, loggerFactory.CreateLogger<GitHubClientFactory>(), mockOptions.Object);
|
||||
var factory = new GitHubClientFactory(mockApp.Object, new BasicHttpMessageHandlerFactory(), loggerFactory.CreateLogger<GitHubClientFactory>(), mockOptions.Object);
|
||||
|
||||
var client = await factory.CreateClient(CancellationToken.None);
|
||||
Assert.IsNotNull(client);
|
||||
@@ -84,7 +87,7 @@ namespace Tgstation.Server.Host.Utils.GitHub.Tests
|
||||
|
||||
var mockOptions = new Mock<IOptions<GeneralConfiguration>>();
|
||||
mockOptions.SetupGet(x => x.Value).Returns(new GeneralConfiguration());
|
||||
var factory = new GitHubClientFactory(mockApp.Object, loggerFactory.CreateLogger<GitHubClientFactory>(), mockOptions.Object);
|
||||
var factory = new GitHubClientFactory(mockApp.Object, new BasicHttpMessageHandlerFactory(), loggerFactory.CreateLogger<GitHubClientFactory>(), mockOptions.Object);
|
||||
|
||||
await Assert.ThrowsExceptionAsync<ArgumentNullException>(() => factory.CreateClient(null, CancellationToken.None).AsTask());
|
||||
|
||||
@@ -106,7 +109,7 @@ namespace Tgstation.Server.Host.Utils.GitHub.Tests
|
||||
|
||||
var mockOptions = new Mock<IOptions<GeneralConfiguration>>();
|
||||
mockOptions.SetupGet(x => x.Value).Returns(new GeneralConfiguration());
|
||||
var factory = new GitHubClientFactory(mockApp.Object, loggerFactory.CreateLogger<GitHubClientFactory>(), mockOptions.Object);
|
||||
var factory = new GitHubClientFactory(mockApp.Object, new BasicHttpMessageHandlerFactory(), loggerFactory.CreateLogger<GitHubClientFactory>(), mockOptions.Object);
|
||||
|
||||
var appID = Environment.GetEnvironmentVariable("TGS_TEST_APP_ID");
|
||||
var privateKey = Environment.GetEnvironmentVariable("TGS_TEST_APP_PRIVATE_KEY");
|
||||
@@ -144,7 +147,7 @@ namespace Tgstation.Server.Host.Utils.GitHub.Tests
|
||||
|
||||
var mockOptions = new Mock<IOptions<GeneralConfiguration>>();
|
||||
mockOptions.SetupGet(x => x.Value).Returns(new GeneralConfiguration());
|
||||
var factory = new GitHubClientFactory(mockApp.Object, loggerFactory.CreateLogger<GitHubClientFactory>(), mockOptions.Object);
|
||||
var factory = new GitHubClientFactory(mockApp.Object, new BasicHttpMessageHandlerFactory(), loggerFactory.CreateLogger<GitHubClientFactory>(), mockOptions.Object);
|
||||
|
||||
await Assert.ThrowsExceptionAsync<ArgumentNullException>(() => factory.CreateClient(null, CancellationToken.None).AsTask());
|
||||
|
||||
@@ -192,7 +195,7 @@ vTdVAoGBAI/jjUMdjkY43zhe3w2piwT0fhGfqm9ikdAB9IcgcptuS0ML0ZaWV/eO
|
||||
|
||||
var mockOptions = new Mock<IOptions<GeneralConfiguration>>();
|
||||
mockOptions.SetupGet(x => x.Value).Returns(new GeneralConfiguration());
|
||||
var factory = new GitHubClientFactory(mockApp.Object, loggerFactory.CreateLogger<GitHubClientFactory>(), mockOptions.Object);
|
||||
var factory = new GitHubClientFactory(mockApp.Object, new BasicHttpMessageHandlerFactory(), loggerFactory.CreateLogger<GitHubClientFactory>(), mockOptions.Object);
|
||||
|
||||
var client1 = await factory.CreateClient(CancellationToken.None);
|
||||
var client2 = await factory.CreateClient("asdf", CancellationToken.None);
|
||||
|
||||
@@ -115,8 +115,6 @@ namespace Tgstation.Server.Tests.Live
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(revisionInformation);
|
||||
ArgumentNullException.ThrowIfNull(engineVersion);
|
||||
ArgumentNullException.ThrowIfNull(gitHubOwner);
|
||||
ArgumentNullException.ThrowIfNull(gitHubRepo);
|
||||
|
||||
Assert.IsTrue(knownChannels.ContainsKey(channelId));
|
||||
|
||||
|
||||
@@ -156,6 +156,7 @@ namespace Tgstation.Server.Tests.Live
|
||||
$"Security:TokenExpiryMinutes=120", // timeouts are useless for us
|
||||
$"General:OpenDreamSuppressInstallOutput={TestingUtils.RunningInGitHubActions}",
|
||||
"Telemetry:DisableVersionReporting=true",
|
||||
$"General:PrometheusPort={port}"
|
||||
};
|
||||
|
||||
if (MultiServerClient.UseGraphQL)
|
||||
|
||||
@@ -455,6 +455,40 @@ namespace Tgstation.Server.Tests.Live
|
||||
}
|
||||
}
|
||||
|
||||
static async Task TestHealthChecks(IRestServerClient serverClient, CancellationToken cancellationToken)
|
||||
{
|
||||
var url = serverClient.Url;
|
||||
// check that 400s are returned appropriately
|
||||
using var httpClient = new HttpClient();
|
||||
using (var request = new HttpRequestMessage(HttpMethod.Get, url.ToString() + "health"))
|
||||
{
|
||||
request.Headers.Accept.Clear();
|
||||
request.Headers.UserAgent.Add(new ProductInfoHeaderValue("RootTest", "1.0.0"));
|
||||
using var response = await httpClient.SendAsync(request, cancellationToken);
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
var text = await response.Content.ReadAsStringAsync(cancellationToken);
|
||||
Assert.AreEqual("Healthy", text.Trim());
|
||||
}
|
||||
}
|
||||
|
||||
static async Task TestPrometheusMetrics(IRestServerClient serverClient, CancellationToken cancellationToken)
|
||||
{
|
||||
var url = serverClient.Url;
|
||||
// check that 400s are returned appropriately
|
||||
using var httpClient = new HttpClient();
|
||||
using (var request = new HttpRequestMessage(HttpMethod.Get, url.ToString() + "metrics"))
|
||||
{
|
||||
request.Headers.Accept.Clear();
|
||||
request.Headers.UserAgent.Add(new ProductInfoHeaderValue("RootTest", "1.0.0"));
|
||||
using var response = await httpClient.SendAsync(request, cancellationToken);
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
var text = await response.Content.ReadAsStringAsync(cancellationToken);
|
||||
Assert.IsTrue(text.Contains("tgs_jobs_running"));
|
||||
}
|
||||
}
|
||||
|
||||
static async Task TestGraphQLLogin(IRestServerClientFactory clientFactory, IRestServerClient restClient, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!MultiServerClient.UseGraphQL)
|
||||
@@ -483,6 +517,8 @@ namespace Tgstation.Server.Tests.Live
|
||||
TestServerInformation(clientFactory, serverClient, cancellationToken),
|
||||
TestInvalidTransfers(serverClient, cancellationToken),
|
||||
RegressionTestForLeakedPasswordHashesBug(serverClient, cancellationToken),
|
||||
TestSignalRUsage(clientFactory, serverClient, cancellationToken));
|
||||
TestSignalRUsage(clientFactory, serverClient, cancellationToken),
|
||||
TestHealthChecks(serverClient, cancellationToken),
|
||||
TestPrometheusMetrics(serverClient, cancellationToken));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Net.Http;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
@@ -15,6 +16,7 @@ using Tgstation.Server.Host.Configuration;
|
||||
using Tgstation.Server.Host.Extensions;
|
||||
using Tgstation.Server.Host.Security;
|
||||
using Tgstation.Server.Host.System;
|
||||
using Tgstation.Server.Host.Tests;
|
||||
using Tgstation.Server.Host.Utils.GitHub;
|
||||
|
||||
namespace Tgstation.Server.Tests.Live
|
||||
@@ -38,7 +40,7 @@ namespace Tgstation.Server.Tests.Live
|
||||
GitHubAccessToken = Environment.GetEnvironmentVariable("TGS_TEST_GITHUB_TOKEN")
|
||||
});
|
||||
|
||||
var gitHubClientFactory = new GitHubClientFactory(new AssemblyInformationProvider(), Mock.Of<ILogger<GitHubClientFactory>>(), mockOptions.Object);
|
||||
var gitHubClientFactory = new GitHubClientFactory(new AssemblyInformationProvider(), new BasicHttpMessageHandlerFactory(), Mock.Of<ILogger<GitHubClientFactory>>(), mockOptions.Object);
|
||||
RealClient = gitHubClientFactory.CreateClient(CancellationToken.None).GetAwaiter().GetResult();
|
||||
}
|
||||
|
||||
|
||||
@@ -6,13 +6,14 @@
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="9.0.1" />
|
||||
<PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="9.0.2" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\src\Tgstation.Server.Client.GraphQL\Tgstation.Server.Client.GraphQL.csproj" />
|
||||
<ProjectReference Include="..\..\src\Tgstation.Server.Host.Watchdog\Tgstation.Server.Host.Watchdog.csproj" />
|
||||
<ProjectReference Include="..\..\src\Tgstation.Server.Host\Tgstation.Server.Host.csproj" />
|
||||
<ProjectReference Include="..\Tgstation.Server.Host.Tests\Tgstation.Server.Host.Tests.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
||||
Reference in New Issue
Block a user