Merge pull request #1859 from tgstation/DeploymentsTelemetry

Version Telemetry, Update Webpanel
This commit is contained in:
Jordan Dominion
2024-08-16 17:29:19 -04:00
committed by GitHub
29 changed files with 776 additions and 87 deletions
+2
View File
@@ -36,6 +36,8 @@ You can of course, as always, ask for help at [#coderbus](irc://irc.rizon.net/co
You need the .NET 8.0 SDK, node>=v20, and npm>=v5.7 (in your PATH) to compile the server. On Linux, you also need the `libgdiplus` package installed to generate icons.
You need to run `corepack enable` to configure node to correctly build the webpanel.
The recommended IDE is Visual Studio 2022 or VSCode.
In order to build the service version and/or the Windows installer you need a to run on Windows.
+156 -16
View File
@@ -36,6 +36,7 @@ env:
OD_MIN_COMPAT_DOTNET_VERSION: 7
OD_DOTNET_VERSION: 8
TGS_DOTNET_QUALITY: ga
TGS_WEBPANEL_NODE_VERSION: 20.x
TGS_TEST_GITHUB_TOKEN: ${{ secrets.LIVE_TESTS_TOKEN }}
TGS_RELEASE_NOTES_TOKEN: ${{ secrets.DEV_PUSH_TOKEN }}
PACKAGING_PRIVATE_KEY_PASSPHRASE: ${{ secrets.PACKAGING_PRIVATE_KEY_PASSPHRASE }}
@@ -52,6 +53,8 @@ jobs:
permissions:
security-events: write
actions: read
env:
TGS_TELEMETRY_KEY_FILE: /tmp/tgs_telemetry_key.txt
steps:
- name: Setup dotnet
uses: actions/setup-dotnet@v4
@@ -74,9 +77,16 @@ jobs:
with:
languages: csharp
- name: Setup Telemetry Key File
run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }}
- name: Build
run: dotnet build -c ReleaseNoWindows -p:TGS_HOST_NO_WEBPANEL=true
- name: Delete Telemetry Key File
if: always()
run: rm ${{ env.TGS_TELEMETRY_KEY_FILE }}
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v3
with:
@@ -356,6 +366,8 @@ jobs:
docker-build:
name: Build Docker Image
runs-on: ubuntu-latest
env:
TGS_TELEMETRY_KEY_FILE: tgs_telemetry_key.txt
steps:
- name: Checkout (Branch)
uses: actions/checkout@v4
@@ -367,8 +379,16 @@ jobs:
with:
ref: "refs/pull/${{ github.event.inputs.pull_request_number }}/merge"
- name: Setup Telemetry Key File
shell: bash
run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }}
- name: Build Docker Image
run: docker build . -f build/Dockerfile
run: docker build . -f build/Dockerfile --build-arg TGS_TELEMETRY_KEY_FILE=${{ env.TGS_TELEMETRY_KEY_FILE }}
- name: Delete Telemetry Key File
if: always()
run: rm ${{ env.TGS_TELEMETRY_KEY_FILE }}
linux-unit-tests:
name: Linux Tests
@@ -379,6 +399,7 @@ jobs:
env:
TGS_TEST_DISCORD_TOKEN: ${{ secrets.DISCORD_TOKEN }}
TGS_TEST_IRC_CONNECTION_STRING: ${{ secrets.IRC_CONNECTION_STRING }}
TGS_TELEMETRY_KEY_FILE: /tmp/tgs_telemetry_key.txt
runs-on: ubuntu-latest
steps:
- name: Install x86 libc Dependencies
@@ -393,6 +414,11 @@ jobs:
dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x'
dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }}
- name: Setup Node.JS
uses: actions/setup-node@v4
with:
node-version: ${{ env.TGS_WEBPANEL_NODE_VERSION }}
- name: Checkout (Branch)
uses: actions/checkout@v4
if: github.event_name == 'push' || github.event_name == 'schedule'
@@ -406,9 +432,19 @@ jobs:
- name: Restore
run: dotnet restore
- name: Enable Corepack
run: corepack enable
- name: Setup Telemetry Key File
run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }}
- name: Build
run: dotnet build -c ${{ matrix.configuration }}NoWindows
- name: Delete Telemetry Key File
if: always()
run: rm ${{ env.TGS_TELEMETRY_KEY_FILE }}
- name: Cache BYOND .zips
uses: actions/cache@v4
id: cache-byond
@@ -434,6 +470,7 @@ jobs:
env:
TGS_TEST_DISCORD_TOKEN: ${{ secrets.DISCORD_TOKEN }}
TGS_TEST_IRC_CONNECTION_STRING: ${{ secrets.IRC_CONNECTION_STRING }}
TGS_TELEMETRY_KEY_FILE: C:/tgs_telemetry_key.txt
runs-on: windows-latest
steps:
- name: Setup dotnet
@@ -442,6 +479,11 @@ jobs:
dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x'
dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }}
- name: Setup Node.JS
uses: actions/setup-node@v4
with:
node-version: ${{ env.TGS_WEBPANEL_NODE_VERSION }}
- name: Checkout (Branch)
uses: actions/checkout@v4
if: github.event_name == 'push' || github.event_name == 'schedule'
@@ -455,9 +497,21 @@ jobs:
- name: Restore
run: dotnet restore
- name: Enable Corepack
run: corepack enable
- name: Setup Telemetry Key File
shell: bash
run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }}
- name: Build
run: dotnet build -c ${{ matrix.configuration }}NoWix
- name: Delete Telemetry Key File
shell: bash
if: always()
run: rm ${{ env.TGS_TELEMETRY_KEY_FILE }}
- name: Cache BYOND .zips
uses: actions/cache@v4
id: cache-byond
@@ -483,6 +537,8 @@ jobs:
database-type: [ 'SqlServer', 'Sqlite', 'PostgresSql', 'MariaDB', 'MySql' ]
watchdog-type: [ 'Basic', 'Advanced' ]
configuration: [ 'Debug', 'Release' ]
env:
TGS_TELEMETRY_KEY_FILE: C:/tgs_telemetry_key.txt
runs-on: windows-latest
steps:
- name: Wait for LocalDB Connection # Do this first because we don't want to find out it's failing later
@@ -501,6 +557,11 @@ jobs:
${{ env.OD_MIN_COMPAT_DOTNET_VERSION }}.0.x
dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }}
- name: Setup Node.JS
uses: actions/setup-node@v4
with:
node-version: ${{ env.TGS_WEBPANEL_NODE_VERSION }}
- name: Set TGS_TEST_DUMP_API_SPEC
if: ${{ matrix.configuration == 'Release' && matrix.watchdog-type == 'Advanced' && matrix.database-type == 'SqlServer' }}
run: echo "TGS_TEST_DUMP_API_SPEC=yes" >> $Env:GITHUB_ENV
@@ -570,9 +631,21 @@ jobs:
- name: Restore
run: dotnet restore
- name: Enable Corepack
run: corepack enable
- name: Setup Telemetry Key File
shell: bash
run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }}
- name: Build
run: dotnet build -c ${{ matrix.configuration }} tests/Tgstation.Server.Tests/Tgstation.Server.Tests.csproj
- name: Delete Telemetry Key File
shell: bash
if: always()
run: rm ${{ env.TGS_TELEMETRY_KEY_FILE }}
- name: Cache BYOND .zips
uses: actions/cache@v4
id: cache-byond
@@ -713,6 +786,8 @@ jobs:
database-type: [ 'Sqlite', 'PostgresSql', 'MariaDB', 'MySql' ]
watchdog-type: [ 'Basic', 'Advanced' ]
configuration: [ 'Debug', 'Release' ]
env:
TGS_TELEMETRY_KEY_FILE: /tmp/tgs_telemetry_key.txt
runs-on: ubuntu-latest
steps:
- name: Disable ptrace_scope
@@ -733,6 +808,11 @@ jobs:
${{ env.OD_MIN_COMPAT_DOTNET_VERSION }}.0.x
dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }}
- name: Setup Node.JS
uses: actions/setup-node@v4
with:
node-version: ${{ env.TGS_WEBPANEL_NODE_VERSION }}
- name: Set Sqlite Connection Info
if: ${{ matrix.database-type == 'Sqlite' }}
run: |
@@ -775,9 +855,19 @@ jobs:
- name: Restore
run: dotnet restore
- name: Enable Corepack
run: corepack enable
- name: Setup Telemetry Key File
run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }}
- name: Build
run: dotnet build -c ${{ matrix.configuration }}NoWindows tests/Tgstation.Server.Tests/Tgstation.Server.Tests.csproj
- name: Delete Telemetry Key File
if: always()
run: rm ${{ env.TGS_TELEMETRY_KEY_FILE }}
- name: Cache BYOND .zips
uses: actions/cache@v4
id: cache-byond
@@ -1117,12 +1207,14 @@ jobs:
build-deb:
name: Build .deb Package # Can't do i386 due to https://github.com/dotnet/core/issues/4595
runs-on: ubuntu-latest
env:
TGS_TELEMETRY_KEY_FILE: /tmp/tgs_telemetry_key.txt
steps:
- name: Install Native Dependencies
run: |
sudo dpkg --add-architecture i386
sudo apt-get update
sudo apt-get install -y -o APT::Immediate-Configure=0 libstdc++6:i386 libgcc-s1:i386 gnupg2 xmlstarlet libgdiplus
sudo apt-get install -y -o APT::Immediate-Configure=0 libstdc++6:i386 libgcc-s1:i386
- name: Import GPG Key
if: (github.event_name == 'push' && contains(github.event.head_commit.message, '[TGSDeploy]') && (github.event.ref == 'refs/heads/master' || github.event.ref == 'refs/heads/dev'))
@@ -1160,27 +1252,37 @@ jobs:
with:
ref: "refs/pull/${{ github.event.inputs.pull_request_number }}/merge"
- name: Grab Most Recent Changelog
run: curl -L https://raw.githubusercontent.com/tgstation/tgstation-server/gh-pages/changelog.yml -o changelog.yml
- name: Setup Telemetry Key File
run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }}
- name: Execute Build Script (Unsigned)
if: (!(github.event_name == 'push' && contains(github.event.head_commit.message, '[TGSDeploy]') && github.event.ref == 'refs/heads/master'))
run: sudo -E build/package/deb/build_package.sh
- name: Execute Build Script (Signed)
if: (github.event_name == 'push' && contains(github.event.head_commit.message, '[TGSDeploy]') && github.event.ref == 'refs/heads/master')
env:
PACKAGING_KEYGRIP: ${{ vars.PACKAGING_KEYGRIP }}
run: sudo -E build/package/deb/build_package.sh
- name: Parse TGS version
run: |
echo "TGS_VERSION=$(xmlstarlet sel -N X="http://schemas.microsoft.com/developer/msbuild/2003" --template --value-of /X:Project/X:PropertyGroup/X:TgsCoreVersion build/Version.props)" >> $GITHUB_ENV
- name: Grab Most Recent Changelog
run: curl -L https://raw.githubusercontent.com/tgstation/tgstation-server/gh-pages/changelog.yml -o changelog.yml
- name: Execute Build Script (Unsigned)
if: (!(github.event_name == 'push' && contains(github.event.head_commit.message, '[TGSDeploy]') && (github.event.ref == 'refs/heads/master' || github.event.ref == 'refs/heads/dev')))
run: sudo -E build/package/deb/build_package.sh
- name: Execute Build Script (Signed)
if: (github.event_name == 'push' && contains(github.event.head_commit.message, '[TGSDeploy]') && (github.event.ref == 'refs/heads/master' || github.event.ref == 'refs/heads/dev'))
env:
PACKAGING_KEYGRIP: ${{ vars.PACKAGING_KEYGRIP }}
run: |
sudo -E build/package/deb/build_package.sh
- name: Verify Package Files are Signed
if: (github.event_name == 'push' && contains(github.event.head_commit.message, '[TGSDeploy]') && github.event.ref == 'refs/heads/master')
run:
gpg --verify tgstation-server_${{ env.TGS_VERSION }}-1.dsc
gpg --verify tgstation-server_${{ env.TGS_VERSION }}-1_amd64.changes
gpg --verify tgstation-server_${{ env.TGS_VERSION }}-1_amd64.buildinfo
- name: Delete Telemetry Key File
if: always()
run: rm ${{ env.TGS_TELEMETRY_KEY_FILE }}
- name: Test Install
run: |
sudo mkdir /etc/tgstation-server
@@ -1219,6 +1321,8 @@ jobs:
build-msi:
name: Build Windows Installer .exe
runs-on: windows-latest
env:
TGS_TELEMETRY_KEY_FILE: C:/tgs_telemetry_key.txt
steps:
- name: Install winget
uses: Cyberboss/install-winget@v1
@@ -1231,6 +1335,11 @@ jobs:
dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x'
dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }}
- name: Setup Node.JS
uses: actions/setup-node@v4
with:
node-version: ${{ env.TGS_WEBPANEL_NODE_VERSION }}
- name: Checkout (Branch)
uses: actions/checkout@v4
if: github.event_name == 'push' || github.event_name == 'schedule'
@@ -1252,9 +1361,21 @@ jobs:
- name: Restore
run: dotnet restore
- name: Enable Corepack
run: corepack enable
- name: Setup Telemetry Key File
shell: bash
run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }}
- name: Build Host
run: dotnet build -c Release src/Tgstation.Server.Host/Tgstation.Server.Host.csproj
- name: Delete Telemetry Key File
shell: bash
if: always()
run: rm ${{ env.TGS_TELEMETRY_KEY_FILE }}
- name: Build Service
run: dotnet build -c Release src/Tgstation.Server.Host.Service/Tgstation.Server.Host.Service.csproj
@@ -1620,6 +1741,8 @@ jobs:
needs: [deploy-dm, deploy-http, deployment-gate]
runs-on: windows-latest
if: github.event.ref == 'refs/heads/master' && contains(github.event.head_commit.message, '[TGSDeploy]')
env:
TGS_TELEMETRY_KEY_FILE: C:/tgs_telemetry_key.txt
steps:
- name: Setup dotnet
uses: actions/setup-dotnet@v4
@@ -1627,6 +1750,11 @@ jobs:
dotnet-version: '${{ env.TGS_DOTNET_VERSION }}.0.x'
dotnet-quality: ${{ env.TGS_DOTNET_QUALITY }}
- name: Setup Node.JS
uses: actions/setup-node@v4
with:
node-version: ${{ env.TGS_WEBPANEL_NODE_VERSION }}
- name: Checkout
uses: actions/checkout@v4
@@ -1638,9 +1766,21 @@ jobs:
cd build/package/winget
dotnet tool restore
- name: Build Host # We need to rebuild the installer.exe so it can be properly signed
- name: Enable Corepack
run: corepack enable
- name: Setup Telemetry Key File
shell: bash
run: echo "${{ secrets.TGS_TELEMETRY_KEY }}" > ${{ env.TGS_TELEMETRY_KEY_FILE }}
- name: Build Host # We need to rebuild the installer.exe so it can be properly signed
run: dotnet build -c Release src/Tgstation.Server.Host/Tgstation.Server.Host.csproj
- name: Delete Telemetry Key File
shell: bash
if: always()
run: rm ${{ env.TGS_TELEMETRY_KEY_FILE }}
- name: Build Service
run: dotnet build -c Release src/Tgstation.Server.Host.Service/Tgstation.Server.Host.Service.csproj
+6
View File
@@ -313,6 +313,12 @@ The following providers use the `ServerUrl` setting:
- Keycloak
- InvisionCommunity
- `Telemetry:DisableVersionReporting`: Prevents you installation and the version you're using from being reported on the source repository's deployments list
- `Telemetry:ServerFriendlyName`: Prevents anonymous TGS version usage statistics from being sent to be displayed on the repository.
- `Telemetry:VersionReportingRepositoryId`: The repository telemetry is sent to. For security reasons, this is not the main TGS repo. See the [tgstation-server-deployments](https://github.com/tgstation/tgstation-server-deployments) repository for more information.
### Database Configuration
If using a MariaDB/MySQL server, our client library [recommends you set 'utf8mb4' as your default charset](https://github.com/PomeloFoundation/Pomelo.EntityFrameworkCore.MySql#1-recommended-server-charset) disregard at your own risk.
+8 -2
View File
@@ -1,5 +1,8 @@
FROM mcr.microsoft.com/dotnet/sdk:8.0-bookworm-slim AS build
# Set in CI
ARG TGS_TELEMETRY_KEY_FILE=
# install node and npm
# replace shell with bash so we can source files
RUN curl --silent -o- https://raw.githubusercontent.com/creationix/nvm/v0.39.1/install.sh | sh
@@ -17,7 +20,8 @@ RUN . $NVM_DIR/nvm.sh \
&& apt-get install -y \
dos2unix \
libgdiplus \
&& rm -rf /var/lib/apt/lists/*
&& rm -rf /var/lib/apt/lists/* \
&& corepack enable
# Build web control panel
WORKDIR /repo/build
@@ -45,7 +49,9 @@ RUN dotnet publish -c Release -o /app \
&& build/RemoveUnsupportedRuntimes.sh /app
WORKDIR /repo/src/Tgstation.Server.Host
RUN dotnet publish -c Release -o /app/lib/Default \
RUN export TGS_TELEMETRY_KEY_FILE="../../${TGS_TELEMETRY_KEY_FILE}" \
&& dotnet publish -c Release -o /app/lib/Default \
&& cd ../.. \
&& build/RemoveUnsupportedRuntimes.sh /app/lib/Default \
&& mv /app/lib/Default/appsettings* /app
-1
View File
@@ -21,6 +21,5 @@
<TgsMariaDBRedistVersion>11.4.2</TgsMariaDBRedistVersion>
<!-- Only have this uncommented if the mariadb servers are shitting the bed, update if the version updates -->
<!--<TgsMariaDBFallbackRedist>https://mirror.its.dal.ca/mariadb//mariadb-10.11.8/winx64-packages/mariadb-10.11.8-winx64.msi</TgsMariaDBFallbackRedist>-->
<TgsYarnVersion>1.22.21</TgsYarnVersion>
</PropertyGroup>
</Project>
+1 -1
View File
@@ -1,6 +1,6 @@
<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<!-- This is in it's own file to help incremental building, changing it causes a complete rebuild of the web panel -->
<TgsWebpanelVersion>5.9.0</TgsWebpanelVersion>
<TgsWebpanelVersion>6.1.0</TgsWebpanelVersion>
</PropertyGroup>
</Project>
+3 -2
View File
@@ -18,7 +18,7 @@ apt-get install -y \
devscripts \
ca-certificates \
curl \
gnupg \
gnupg2 \
xmlstarlet \
libgdiplus
@@ -35,6 +35,8 @@ echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.co
apt-get update
apt-get install nodejs dotnet-sdk-8.0 -y
corepack enable
CURRENT_COMMIT=$(git rev-parse HEAD)
rm -rf packaging
@@ -67,7 +69,6 @@ cp build/tgstation-server.service debian/
SIGN_COMMAND="$SCRIPT_DIR/wrap_gpg.sh"
rm -f /tmp/tgs_wrap_gpg_output.log
set +e
if [[ -z "$PACKAGING_KEYGRIP" ]]; then
@@ -79,13 +79,13 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote
IGitHubService gitHubService;
if (instanceAuthenticated)
{
authenticatedGitHubService = gitHubServiceFactory.CreateService(repositorySettings.AccessToken!);
authenticatedGitHubService = await gitHubServiceFactory.CreateService(repositorySettings.AccessToken!, cancellationToken);
gitHubService = authenticatedGitHubService;
}
else
{
authenticatedGitHubService = null;
gitHubService = gitHubServiceFactory.CreateService();
gitHubService = await gitHubServiceFactory.CreateService(cancellationToken);
}
var repoOwner = remoteInformation.RemoteRepositoryOwner!;
@@ -175,8 +175,8 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote
}
var gitHubService = repositorySettings.AccessToken != null
? gitHubServiceFactory.CreateService(repositorySettings.AccessToken)
: gitHubServiceFactory.CreateService();
? await gitHubServiceFactory.CreateService(repositorySettings.AccessToken, cancellationToken)
: await gitHubServiceFactory.CreateService(cancellationToken);
var tasks = revisionInformation
.ActiveTestMerges
@@ -255,7 +255,7 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote
int testMergeNumber,
CancellationToken cancellationToken)
{
var gitHubService = gitHubServiceFactory.CreateService(repositorySettings.AccessToken!);
var gitHubService = await gitHubServiceFactory.CreateService(repositorySettings.AccessToken!, cancellationToken);
try
{
@@ -343,7 +343,7 @@ namespace Tgstation.Server.Host.Components.Deployment.Remote
return;
}
var gitHubService = gitHubServiceFactory.CreateService(gitHubAccessToken);
var gitHubService = await gitHubServiceFactory.CreateService(gitHubAccessToken, cancellationToken);
try
{
@@ -49,8 +49,8 @@ namespace Tgstation.Server.Host.Components.Repository
CancellationToken cancellationToken)
{
var gitHubService = repositorySettings.AccessToken != null
? gitHubServiceFactory.CreateService(repositorySettings.AccessToken)
: gitHubServiceFactory.CreateService();
? await gitHubServiceFactory.CreateService(repositorySettings.AccessToken, cancellationToken)
: await gitHubServiceFactory.CreateService(cancellationToken);
PullRequest? pr = null;
ApiException? exception = null;
@@ -0,0 +1,33 @@
namespace Tgstation.Server.Host.Configuration
{
/// <summary>
/// Configuration options for telemetry.
/// </summary>
public sealed class TelemetryConfiguration
{
/// <summary>
/// The key for the <see cref="Microsoft.Extensions.Configuration.IConfigurationSection"/> the <see cref="TelemetryConfiguration"/> resides in.
/// </summary>
public const string Section = "Telemetry";
/// <summary>
/// The default value of <see cref="VersionReportingRepositoryId"/>.
/// </summary>
private const long DefaultVersionReportingRepositoryId = 841149827; // https://github.com/tgstation/tgstation-server-deployments
/// <summary>
/// If version reporting telemetry is disabled.
/// </summary>
public bool DisableVersionReporting { get; set; }
/// <summary>
/// The friendly name used on GitHub deployments for version reporting. If <see langword="null"/> only the server <see cref="global::System.Guid"/> will be shown.
/// </summary>
public string? ServerFriendlyName { get; set; }
/// <summary>
/// The GitHub repository ID used for version reporting.
/// </summary>
public long? VersionReportingRepositoryId { get; set; } = DefaultVersionReportingRepositoryId;
}
}
@@ -165,7 +165,7 @@ namespace Tgstation.Server.Host.Controllers
Uri? repoUrl = null;
try
{
var gitHubService = gitHubServiceFactory.CreateService();
var gitHubService = await gitHubServiceFactory.CreateService(cancellationToken);
var repositoryUrlTask = gitHubService.GetUpdatesRepositoryUrl(cancellationToken);
var releases = await gitHubService.GetTgsReleases(cancellationToken);
@@ -143,6 +143,7 @@ namespace Tgstation.Server.Host.Core
services.UseStandardConfig<ControlPanelConfiguration>(Configuration);
services.UseStandardConfig<SwarmConfiguration>(Configuration);
services.UseStandardConfig<SessionConfiguration>(Configuration);
services.UseStandardConfig<TelemetryConfiguration>(Configuration);
// enable options which give us config reloading
services.AddOptions();
@@ -423,6 +424,7 @@ namespace Tgstation.Server.Host.Core
services.AddSingleton<IServerPortProvider, ServerPortProivder>();
services.AddSingleton<ITopicClientFactory, TopicClientFactory>();
services.AddHostedService<CommandPipeManager>();
services.AddHostedService<VersionReportingService>();
services.AddFileDownloader();
services.AddGitHub();
@@ -288,7 +288,7 @@ namespace Tgstation.Server.Host.Core
{
logger.LogDebug("Looking for GitHub releases version {version}...", newVersion);
var gitHubService = gitHubServiceFactory.CreateService();
var gitHubService = await gitHubServiceFactory.CreateService(cancellationToken);
var releases = await gitHubService.GetTgsReleases(cancellationToken);
foreach (var kvp in releases)
{
@@ -0,0 +1,269 @@
using System;
using System.Collections.Generic;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Octokit;
using Tgstation.Server.Common.Extensions;
using Tgstation.Server.Host.Configuration;
using Tgstation.Server.Host.Extensions;
using Tgstation.Server.Host.IO;
using Tgstation.Server.Host.Properties;
using Tgstation.Server.Host.System;
using Tgstation.Server.Host.Utils;
using Tgstation.Server.Host.Utils.GitHub;
namespace Tgstation.Server.Host.Core
{
/// <summary>
/// Handles TGS version reporting, if enabled.
/// </summary>
sealed class VersionReportingService : BackgroundService
{
/// <summary>
/// The <see cref="IGitHubClientFactory"/> for the <see cref="VersionReportingService"/>.
/// </summary>
readonly IGitHubClientFactory gitHubClientFactory;
/// <summary>
/// The <see cref="IIOManager"/> for the <see cref="VersionReportingService"/>.
/// </summary>
readonly IIOManager ioManager;
/// <summary>
/// The <see cref="IAsyncDelayer"/> for the <see cref="VersionReportingService"/>.
/// </summary>
readonly IAsyncDelayer asyncDelayer;
/// <summary>
/// The <see cref="IAssemblyInformationProvider"/> for the <see cref="VersionReportingService"/>.
/// </summary>
readonly IAssemblyInformationProvider assemblyInformationProvider;
/// <summary>
/// The <see cref="ILogger"/> for the <see cref="VersionReportingService"/>.
/// </summary>
readonly ILogger<VersionReportingService> logger;
/// <summary>
/// The <see cref="TelemetryConfiguration"/> for the <see cref="VersionReportingService"/>.
/// </summary>
readonly TelemetryConfiguration telemetryConfiguration;
/// <summary>
/// The <see cref="CancellationToken"/> passed to <see cref="StopAsync(CancellationToken)"/>.
/// </summary>
CancellationToken shutdownCancellationToken;
/// <summary>
/// Initializes a new instance of the <see cref="VersionReportingService"/> class.
/// </summary>
/// <param name="gitHubClientFactory">The value of <see cref="gitHubClientFactory"/>.</param>
/// <param name="ioManager">The value of <see cref="ioManager"/>.</param>
/// <param name="asyncDelayer">The value of <see cref="asyncDelayer"/>.</param>
/// <param name="assemblyInformationProvider">The value of <see cref="assemblyInformationProvider"/>.</param>
/// <param name="telemetryConfigurationOptions">The <see cref="IOptions{TOptions}"/> containing the value of <see cref="telemetryConfiguration"/>.</param>
/// <param name="logger">The value of <see cref="logger"/>.</param>
public VersionReportingService(
IGitHubClientFactory gitHubClientFactory,
IIOManager ioManager,
IAsyncDelayer asyncDelayer,
IAssemblyInformationProvider assemblyInformationProvider,
IOptions<TelemetryConfiguration> telemetryConfigurationOptions,
ILogger<VersionReportingService> logger)
{
this.gitHubClientFactory = gitHubClientFactory ?? throw new ArgumentNullException(nameof(gitHubClientFactory));
this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager));
this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer));
this.assemblyInformationProvider = assemblyInformationProvider ?? throw new ArgumentNullException(nameof(assemblyInformationProvider));
telemetryConfiguration = telemetryConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(telemetryConfigurationOptions));
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
/// <inheritdoc />
public override Task StopAsync(CancellationToken cancellationToken)
{
shutdownCancellationToken = cancellationToken;
return base.StopAsync(cancellationToken);
}
/// <inheritdoc />
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
if (telemetryConfiguration.DisableVersionReporting)
{
logger.LogDebug("Version telemetry disabled");
return;
}
if (!telemetryConfiguration.VersionReportingRepositoryId.HasValue)
{
logger.LogError("Version reporting repository is misconfigured. Telemetry cannot be sent!");
return;
}
var attribute = TelemetryAppSerializedKeyAttribute.Instance;
if (attribute == null)
{
logger.LogDebug("TGS build configuration does not allow for version telemetry");
return;
}
logger.LogDebug("Starting...");
try
{
var telemetryIdFile = ioManager.ResolvePath(
ioManager.ConcatPath(
ioManager.GetPathInLocalDirectory(assemblyInformationProvider),
"telemetry.id"));
Guid telemetryId;
if (!await ioManager.FileExists(telemetryIdFile, stoppingToken))
{
telemetryId = Guid.NewGuid();
await ioManager.WriteAllBytes(telemetryIdFile, Encoding.UTF8.GetBytes(telemetryId.ToString()), stoppingToken);
logger.LogInformation("Generated telemetry ID {telemetryId} and wrote to {file}", telemetryId, telemetryIdFile);
}
else
{
var contents = await ioManager.ReadAllBytes(telemetryIdFile, stoppingToken);
string guidStr;
try
{
guidStr = Encoding.UTF8.GetString(contents);
}
catch (Exception ex)
{
logger.LogError(ex, "Cannot decode telemetry ID from installation file ({path}). Telemetry will not be sent!", telemetryIdFile);
return;
}
if (!Guid.TryParse(guidStr, out telemetryId))
{
logger.LogError("Cannot parse telemetry ID from installation file ({path}). Telemetry will not be sent!", telemetryIdFile);
return;
}
}
try
{
while (!stoppingToken.IsCancellationRequested)
{
var nextDelayHours = await TryReportVersion(
telemetryId,
attribute.SerializedKey,
telemetryConfiguration.VersionReportingRepositoryId.Value,
false,
stoppingToken)
? 24
: 1;
logger.LogDebug("Next version report in {hours} hours", nextDelayHours);
await asyncDelayer.Delay(TimeSpan.FromHours(nextDelayHours), stoppingToken);
}
}
catch (OperationCanceledException ex)
{
logger.LogTrace(ex, "Inner cancellation");
}
shutdownCancellationToken.ThrowIfCancellationRequested();
logger.LogDebug("Sending shutdown telemetry");
await TryReportVersion(
telemetryId,
attribute.SerializedKey,
telemetryConfiguration.VersionReportingRepositoryId.Value,
true,
shutdownCancellationToken);
}
catch (OperationCanceledException ex)
{
logger.LogTrace(ex, "Exiting due to outer cancellation...");
}
catch (Exception ex)
{
logger.LogError(ex, "Crashed!");
}
}
/// <summary>
/// Make an attempt to report the current <see cref="IAssemblyInformationProvider.Version"/> to the configured GitHub repository.
/// </summary>
/// <param name="telemetryId">The telemetry <see cref="Guid"/> for the installation.</param>
/// <param name="serializedPem">The serialized authentication <see cref="string"/> for the <see cref="gitHubClientFactory"/>.</param>
/// <param name="repositoryId">The ID of the repository to send telemetry to.</param>
/// <param name="shutdown">If this is shutdown telemetry.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
/// <returns>A <see cref="ValueTask{TResult}"/> resulting in <see langword="true"/> if telemetry was reported successfully, <see langword="false"/> otherwise.</returns>
async ValueTask<bool> TryReportVersion(Guid telemetryId, string serializedPem, long repositoryId, bool shutdown, CancellationToken cancellationToken)
{
logger.LogDebug("Sending version telemetry...");
var serverFriendlyName = telemetryConfiguration.ServerFriendlyName;
if (String.IsNullOrWhiteSpace(serverFriendlyName))
serverFriendlyName = null;
logger.LogTrace(
"Repository ID: {repoId}, Server friendly name: {friendlyName}",
repositoryId,
serverFriendlyName == null
? "(null)"
: $"\"{serverFriendlyName}\"");
try
{
var gitHubClient = await gitHubClientFactory.CreateInstallationClient(
serializedPem,
repositoryId,
cancellationToken);
if (gitHubClient == null)
{
logger.LogWarning("Could not create GitHub client to connect to repository ID {repoId}!", repositoryId);
return false;
}
// remove this lookup once https://github.com/octokit/octokit.net/pull/2960 is merged and released
var repository = await gitHubClient.Repository.Get(repositoryId);
logger.LogTrace("Repository ID {id} resolved to {owner}/{name}", repositoryId, repository.Owner.Name, repository.Name);
var inputs = new Dictionary<string, object>
{
{ "telemetry_id", telemetryId.ToString() },
{ "tgs_semver", assemblyInformationProvider.Version.Semver().ToString() },
{ "shutdown", shutdown ? "true" : "false" },
};
if (serverFriendlyName != null)
inputs.Add("server_friendly_name", serverFriendlyName);
await gitHubClient.Actions.Workflows.CreateDispatch(
repository.Owner.Login,
repository.Name,
".github/workflows/tgs_deployments_telemetry.yml",
new CreateWorkflowDispatch("main")
{
Inputs = inputs,
});
logger.LogTrace("Telemetry sent successfully");
return true;
}
catch (Exception ex)
{
logger.LogWarning(ex, "Failed to report version!");
return false;
}
}
}
}
@@ -0,0 +1,33 @@
using System;
using System.Reflection;
namespace Tgstation.Server.Host.Properties
{
/// <summary>
/// Attribute for bundling the GitHub App serialized private key used for version telemetry.
/// </summary>
[AttributeUsage(AttributeTargets.Assembly)]
sealed class TelemetryAppSerializedKeyAttribute : Attribute
{
/// <summary>
/// Return the <see cref="Assembly"/>'s instance of the <see cref="TelemetryAppSerializedKeyAttribute"/>.
/// </summary>
public static TelemetryAppSerializedKeyAttribute? Instance => Assembly
.GetExecutingAssembly()
.GetCustomAttribute<TelemetryAppSerializedKeyAttribute>();
/// <summary>
/// The serialized GitHub App Client ID and private key.
/// </summary>
public string SerializedKey { get; }
/// <summary>
/// Initializes a new instance of the <see cref="TelemetryAppSerializedKeyAttribute"/> class.
/// </summary>
/// <param name="serializedKey">The value of <see cref="SerializedKey"/>.</param>
public TelemetryAppSerializedKeyAttribute(string serializedKey)
{
SerializedKey = serializedKey ?? throw new ArgumentNullException(nameof(serializedKey));
}
}
}
@@ -60,12 +60,12 @@ namespace Tgstation.Server.Host.Security.OAuth
{
logger.LogTrace("Validating response code...");
var gitHubService = gitHubServiceFactory.CreateService();
var gitHubService = await gitHubServiceFactory.CreateService(cancellationToken);
var token = await gitHubService.CreateOAuthAccessToken(oAuthConfiguration, code, cancellationToken);
if (token == null)
return null;
var authenticatedClient = gitHubServiceFactory.CreateService(token);
var authenticatedClient = await gitHubServiceFactory.CreateService(token, cancellationToken);
logger.LogTrace("Getting user details...");
var userId = await authenticatedClient.GetCurrentUserId(cancellationToken);
@@ -950,6 +950,31 @@ namespace Tgstation.Server.Host.Setup
};
}
/// <summary>
/// Prompts the user to create a <see cref="TelemetryConfiguration"/>.
/// </summary>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
/// <returns>A <see cref="ValueTask{TResult}"/> resulting in the new <see cref="TelemetryConfiguration"/>.</returns>
async ValueTask<TelemetryConfiguration> ConfigureTelemetry(CancellationToken cancellationToken)
{
bool enableReporting = await PromptYesNo("Enable version telemetry? This anonymously reports the TGS version in use.", true, cancellationToken);
string? serverFriendlyName = null;
if (enableReporting)
{
await console.WriteAsync("(Optional) Publically associate your reported version with a friendly name:", false, cancellationToken);
serverFriendlyName = await console.ReadLineAsync(false, cancellationToken);
if (String.IsNullOrWhiteSpace(serverFriendlyName))
serverFriendlyName = null;
}
return new TelemetryConfiguration
{
DisableVersionReporting = !enableReporting,
ServerFriendlyName = serverFriendlyName,
};
}
/// <summary>
/// Saves a given <see cref="Configuration"/> set to <paramref name="userConfigFileName"/>.
/// </summary>
@@ -961,6 +986,7 @@ namespace Tgstation.Server.Host.Setup
/// <param name="elasticsearchConfiguration">The <see cref="ElasticsearchConfiguration"/> to save.</param>
/// <param name="controlPanelConfiguration">The <see cref="ControlPanelConfiguration"/> to save.</param>
/// <param name="swarmConfiguration">The <see cref="SwarmConfiguration"/> to save.</param>
/// <param name="telemetryConfiguration">The <see cref="TelemetryConfiguration"/> to save.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
/// <returns>A <see cref="ValueTask"/> representing the running operation.</returns>
async ValueTask SaveConfiguration(
@@ -972,6 +998,7 @@ namespace Tgstation.Server.Host.Setup
ElasticsearchConfiguration? elasticsearchConfiguration,
ControlPanelConfiguration controlPanelConfiguration,
SwarmConfiguration? swarmConfiguration,
TelemetryConfiguration? telemetryConfiguration,
CancellationToken cancellationToken)
{
newGeneralConfiguration.ApiPort = hostingPort ?? GeneralConfiguration.DefaultApiPort;
@@ -984,6 +1011,7 @@ namespace Tgstation.Server.Host.Setup
{ ElasticsearchConfiguration.Section, elasticsearchConfiguration },
{ ControlPanelConfiguration.Section, controlPanelConfiguration },
{ SwarmConfiguration.Section, swarmConfiguration },
{ TelemetryConfiguration.Section, telemetryConfiguration },
};
var versionConverter = new VersionConverter();
@@ -1055,6 +1083,8 @@ namespace Tgstation.Server.Host.Setup
var swarmConfiguration = await ConfigureSwarm(cancellationToken);
var telemetryConfiguration = await ConfigureTelemetry(cancellationToken);
await console.WriteAsync(null, true, cancellationToken);
await console.WriteAsync(String.Format(CultureInfo.InvariantCulture, "Configuration complete! Saving to {0}", userConfigFileName), true, cancellationToken);
@@ -1067,6 +1097,7 @@ namespace Tgstation.Server.Host.Setup
elasticSearchConfiguration,
controlPanelConfiguration,
swarmConfiguration,
telemetryConfiguration,
cancellationToken);
}
@@ -1184,6 +1215,7 @@ namespace Tgstation.Server.Host.Setup
AllowAnyOrigin = true,
},
null,
null,
cancellationToken);
}
else
@@ -29,13 +29,13 @@
<Exec Command="git clone https://github.com/tgstation/tgstation-server-webpanel --branch v$(TgsWebpanelVersion) --depth 1 ClientApp" />
<RemoveDir Directories="ClientApp/.git" /> <!-- Allows git clean to clean it out -->
<Message Text="Restoring yarn packages..." Importance="high" />
<Exec WorkingDirectory="ClientApp" Command="npx --yes yarn@$(TgsYarnVersion) install --immutable" />
<Exec WorkingDirectory="ClientApp" Command="yarn install --immutable" />
<Touch Files="$(NpmInstallStampFile)" AlwaysCreate="true" />
</Target>
<Target Condition="'$(TGS_HOST_NO_WEBPANEL)' != 'true'" Name="NpmBuild" BeforeTargets="BeforeBuild" DependsOnTargets="ClientInstall" Inputs="../../build/WebpanelVersion.props" Outputs="wwwroot\index.html">
<Message Text="Building web control panel..." Importance="high" />
<Exec WorkingDirectory="ClientApp" Command="npx --yes yarn@$(TgsYarnVersion) run msbuild" />
<Exec WorkingDirectory="ClientApp" Command="yarn run msbuild" />
</Target>
<Target Name="NpmClean" AfterTargets="Clean">
@@ -45,21 +45,43 @@
<Target Name="ApplyMasterVersionsAttribute" BeforeTargets="CoreCompile">
<ItemGroup>
<AssemblyAttributes Include="Tgstation.Server.Host.Properties.MasterVersionsAttribute">
<MasterVersionAssemblyAttributes Include="Tgstation.Server.Host.Properties.MasterVersionsAttribute">
<_Parameter1>$(TgsConfigVersion)</_Parameter1>
<_Parameter2>$(TgsInteropVersion)</_Parameter2>
<_Parameter3>$(TgsWebpanelVersion)</_Parameter3>
<_Parameter4>$(TgsHostWatchdogVersion)</_Parameter4>
<_Parameter5>$(TgsMariaDBRedistVersion)</_Parameter5>
</AssemblyAttributes>
</MasterVersionAssemblyAttributes>
</ItemGroup>
<WriteCodeFragment AssemblyAttributes="@(AssemblyAttributes)" Language="C#" OutputDirectory="$(IntermediateOutputPath)" OutputFile="MasterVersionsAssemblyInfo.cs">
<WriteCodeFragment AssemblyAttributes="@(MasterVersionAssemblyAttributes)" Language="C#" OutputDirectory="$(IntermediateOutputPath)" OutputFile="MasterVersionsAssemblyInfo.cs">
<Output TaskParameter="OutputFile" ItemName="Compile" />
<Output TaskParameter="OutputFile" ItemName="FileWrites" />
</WriteCodeFragment>
</Target>
<Target Condition="'$(TGS_TELEMETRY_KEY_FILE)' != ''" Name="ApplyTelemetryAppSerializedKey" BeforeTargets="CoreCompile">
<Error Condition="!Exists('$(TGS_TELEMETRY_KEY_FILE)')" Text="TGS_TELEMETRY_KEY_FILE set but does not exist!" />
<ReadLinesFromFile
File="$(TGS_TELEMETRY_KEY_FILE)" >
<Output
TaskParameter="Lines"
ItemName="SerializedTelemetryKey"/>
</ReadLinesFromFile>
<ItemGroup>
<TelemetryAppSerializedKeyAssemblyAttributes Include="Tgstation.Server.Host.Properties.TelemetryAppSerializedKeyAttribute">
<_Parameter1>@(SerializedTelemetryKey)</_Parameter1>
</TelemetryAppSerializedKeyAssemblyAttributes>
</ItemGroup>
<WriteCodeFragment AssemblyAttributes="@(TelemetryAppSerializedKeyAssemblyAttributes)" Language="C#" OutputDirectory="$(IntermediateOutputPath)" OutputFile="TelemetryAppSerializedKeyAssemblyInfo.cs">
<Output TaskParameter="OutputFile" ItemName="Compile" />
<Output TaskParameter="OutputFile" ItemName="FileWrites" />
</WriteCodeFragment>
</Target>
<Target Condition="'$(TGS_TELEMETRY_KEY_FILE)' == '' And '$(CI)' != ''" Name="FailBuildInCIWithoutTelemetryKey" BeforeTargets="CoreCompile">
<Error Text="The TGS_TELEMETRY_KEY_FILE environment variable should be set in CI!"/>
</Target>
<ItemGroup>
<!-- Usage: Concise throw statements for native Win32 errors -->
<PackageReference Include="BetterWin32Errors" Version="0.2.0" />
@@ -1,9 +1,16 @@
using System;
using System.Collections.Generic;
using System.IdentityModel.Tokens.Jwt;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Microsoft.IdentityModel.Tokens;
using Octokit;
using Tgstation.Server.Host.Configuration;
@@ -12,7 +19,7 @@ using Tgstation.Server.Host.System;
namespace Tgstation.Server.Host.Utils.GitHub
{
/// <inheritdoc />
sealed class GitHubClientFactory : IGitHubClientFactory
sealed class GitHubClientFactory : IGitHubClientFactory, IDisposable
{
/// <summary>
/// Limit to the amount of days a <see cref="GitHubClient"/> can live in the <see cref="clientCache"/>.
@@ -45,6 +52,11 @@ namespace Tgstation.Server.Host.Utils.GitHub
/// </summary>
readonly Dictionary<string, (GitHubClient Client, DateTimeOffset LastUsed)> clientCache;
/// <summary>
/// The <see cref="SemaphoreSlim"/> used to guard access to <see cref="clientCache"/>.
/// </summary>
readonly SemaphoreSlim clientCacheSemaphore;
/// <summary>
/// Initializes a new instance of the <see cref="GitHubClientFactory"/> class.
/// </summary>
@@ -61,50 +73,139 @@ namespace Tgstation.Server.Host.Utils.GitHub
generalConfiguration = generalConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(generalConfigurationOptions));
clientCache = new Dictionary<string, (GitHubClient, DateTimeOffset)>();
clientCacheSemaphore = new SemaphoreSlim(1, 1);
}
/// <inheritdoc />
public IGitHubClient CreateClient() => GetOrCreateClient(generalConfiguration.GitHubAccessToken);
public void Dispose() => clientCacheSemaphore.Dispose();
/// <inheritdoc />
public IGitHubClient CreateClient(string accessToken)
=> GetOrCreateClient(
accessToken ?? throw new ArgumentNullException(nameof(accessToken)));
public async ValueTask<IGitHubClient> CreateClient(CancellationToken cancellationToken)
=> (await GetOrCreateClient(
generalConfiguration.GitHubAccessToken,
null,
cancellationToken))!;
/// <inheritdoc />
public async ValueTask<IGitHubClient> CreateClient(string accessToken, CancellationToken cancellationToken)
=> (await GetOrCreateClient(
accessToken ?? throw new ArgumentNullException(nameof(accessToken)),
null,
cancellationToken))!;
/// <inheritdoc />
public ValueTask<IGitHubClient?> CreateInstallationClient(string serializedPem, long repositoryId, CancellationToken cancellationToken)
=> GetOrCreateClient(serializedPem, repositoryId, cancellationToken);
/// <summary>
/// Retrieve a <see cref="GitHubClient"/> from the <see cref="clientCache"/> or add a new one based on a given <paramref name="accessToken"/>.
/// Retrieve a <see cref="GitHubClient"/> from the <see cref="clientCache"/> or add a new one based on a given <paramref name="accessTokenOrSerializedPem"/>.
/// </summary>
/// <param name="accessToken">Optional access token to use as credentials.</param>
/// <returns>The <see cref="GitHubClient"/> for the given <paramref name="accessToken"/>.</returns>
GitHubClient GetOrCreateClient(string? accessToken)
/// <param name="accessTokenOrSerializedPem">Optional access token to use as credentials or GitHub App private key. If using a private key, <paramref name="installationRepositoryId"/> must be set.</param>
/// <param name="installationRepositoryId">Setting this specifies <paramref name="accessTokenOrSerializedPem"/> is a private key and a GitHub App installation authenticated client will be returned.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
/// <returns>A <see cref="ValueTask{TResult}"/> resulting in the <see cref="GitHubClient"/> for the given <paramref name="accessTokenOrSerializedPem"/> or <see langword="null"/> if authentication failed.</returns>
#pragma warning disable CA1506 // TODO: Decomplexify
async ValueTask<IGitHubClient?> GetOrCreateClient(string? accessTokenOrSerializedPem, long? installationRepositoryId, CancellationToken cancellationToken)
#pragma warning restore CA1506
{
GitHubClient client;
bool cacheHit;
DateTimeOffset? lastUsed;
lock (clientCache)
using (await SemaphoreSlimContext.Lock(clientCacheSemaphore, cancellationToken))
{
string cacheKey;
if (String.IsNullOrWhiteSpace(accessToken))
if (String.IsNullOrWhiteSpace(accessTokenOrSerializedPem))
{
accessToken = null;
accessTokenOrSerializedPem = null;
cacheKey = DefaultCacheKey;
}
else
cacheKey = accessToken;
cacheKey = accessTokenOrSerializedPem;
cacheHit = clientCache.TryGetValue(cacheKey, out var tuple);
var now = DateTimeOffset.UtcNow;
if (!cacheHit)
{
logger.LogTrace("Creating new GitHubClient...");
var product = assemblyInformationProvider.ProductInfoHeaderValue.Product!;
client = new GitHubClient(
new ProductHeaderValue(
product.Name,
product.Version));
if (accessToken != null)
client.Credentials = new Credentials(accessToken);
if (accessTokenOrSerializedPem != null)
{
if (installationRepositoryId.HasValue)
{
logger.LogTrace("Performing GitHub App authentication for installation on repository {installationRepositoryId}", installationRepositoryId.Value);
var splits = accessTokenOrSerializedPem.Split(':');
if (splits.Length != 2)
{
logger.LogError("Failed to parse serialized Client ID & PEM! Expected 2 chunks, got {chunkCount}", splits.Length);
return null;
}
byte[] pemBytes;
try
{
pemBytes = Convert.FromBase64String(splits[1]);
}
catch (Exception ex)
{
logger.LogError(ex, "Failed to parse supposed base64 PEM!");
return null;
}
var pem = Encoding.UTF8.GetString(pemBytes);
using var rsa = RSA.Create();
rsa.ImportFromPem(pem);
var signingCredentials = new SigningCredentials(new RsaSecurityKey(rsa), SecurityAlgorithms.RsaSha256);
var jwtSecurityTokenHandler = new JwtSecurityTokenHandler { SetDefaultTimesOnTokenCreation = false };
var nowDateTime = DateTime.UtcNow;
var jwt = jwtSecurityTokenHandler.CreateToken(new SecurityTokenDescriptor
{
Issuer = splits[0],
Expires = nowDateTime.AddMinutes(10),
IssuedAt = nowDateTime,
SigningCredentials = signingCredentials,
});
var jwtStr = jwtSecurityTokenHandler.WriteToken(jwt);
client.Credentials = new Credentials(jwtStr, AuthenticationType.Bearer);
Installation installation;
try
{
installation = await client.GitHubApps.GetRepositoryInstallationForCurrent(installationRepositoryId.Value);
}
catch (Exception ex)
{
logger.LogError(ex, "Failed to perform app authentication!");
return null;
}
cancellationToken.ThrowIfCancellationRequested();
try
{
var installToken = await client.GitHubApps.CreateInstallationToken(installation.Id);
client.Credentials = new Credentials(installToken.Token);
}
catch (Exception ex)
{
logger.LogError(ex, "Failed to perform installation authentication!");
return null;
}
}
else
client.Credentials = new Credentials(accessTokenOrSerializedPem);
}
clientCache.Add(cacheKey, (Client: client, LastUsed: now));
lastUsed = null;
@@ -1,4 +1,6 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
@@ -44,13 +46,16 @@ namespace Tgstation.Server.Host.Utils.GitHub
}
/// <inheritdoc />
public IGitHubService CreateService() => CreateServiceImpl(gitHubClientFactory.CreateClient());
public async ValueTask<IGitHubService> CreateService(CancellationToken cancellationToken)
=> CreateServiceImpl(
await gitHubClientFactory.CreateClient(cancellationToken));
/// <inheritdoc />
public IAuthenticatedGitHubService CreateService(string accessToken)
public async ValueTask<IAuthenticatedGitHubService> CreateService(string accessToken, CancellationToken cancellationToken)
=> CreateServiceImpl(
gitHubClientFactory.CreateClient(
accessToken ?? throw new ArgumentNullException(nameof(accessToken))));
await gitHubClientFactory.CreateClient(
accessToken ?? throw new ArgumentNullException(nameof(accessToken)),
cancellationToken));
/// <summary>
/// Create a <see cref="GitHubService"/>.
@@ -1,4 +1,7 @@
using Octokit;
using System.Threading;
using System.Threading.Tasks;
using Octokit;
namespace Tgstation.Server.Host.Utils.GitHub
{
@@ -10,14 +13,25 @@ namespace Tgstation.Server.Host.Utils.GitHub
/// <summary>
/// Create a <see cref="IGitHubClient"/> client. Low rate limit unless the server's GitHubAccessToken is set to bypass it.
/// </summary>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
/// <returns>A new <see cref="IGitHubClient"/>.</returns>
IGitHubClient CreateClient();
ValueTask<IGitHubClient> CreateClient(CancellationToken cancellationToken);
/// <summary>
/// Create a client with authentication using a personal access token.
/// </summary>
/// <param name="accessToken">The GitHub personal access token.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
/// <returns>A new <see cref="IGitHubClient"/>.</returns>
IGitHubClient CreateClient(string accessToken);
ValueTask<IGitHubClient> CreateClient(string accessToken, CancellationToken cancellationToken);
/// <summary>
/// Creates a GitHub App client for an installation.
/// </summary>
/// <param name="pem">The private key <see cref="string"/>.</param>
/// <param name="repositoryId">The GitHub repository ID.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
/// <returns>A <see cref="ValueTask{TResult}"/> resulting in a new <see cref="IGitHubClient"/> for the given <paramref name="repositoryId"/> or <see langword="null"/> if authentication failed.</returns>
ValueTask<IGitHubClient?> CreateInstallationClient(string pem, long repositoryId, CancellationToken cancellationToken);
}
}
@@ -1,4 +1,7 @@
namespace Tgstation.Server.Host.Utils.GitHub
using System.Threading;
using System.Threading.Tasks;
namespace Tgstation.Server.Host.Utils.GitHub
{
/// <summary>
/// Factory for <see cref="IGitHubService"/>s.
@@ -8,14 +11,16 @@
/// <summary>
/// Create a <see cref="IGitHubService"/>.
/// </summary>
/// <returns>A new <see cref="IGitHubService"/>.</returns>
public IGitHubService CreateService();
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
/// <returns>A <see cref="ValueTask{TResult}"/> resulting in a new <see cref="IGitHubService"/>.</returns>
public ValueTask<IGitHubService> CreateService(CancellationToken cancellationToken);
/// <summary>
/// Create an <see cref="IAuthenticatedGitHubService"/>.
/// </summary>
/// <param name="accessToken">The access token to use for communication with GitHub.</param>
/// <returns>A new <see cref="IAuthenticatedGitHubService"/>.</returns>
public IAuthenticatedGitHubService CreateService(string accessToken);
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
/// <returns>A <see cref="ValueTask{TResult}"/> resulting in a new <see cref="IAuthenticatedGitHubService"/>.</returns>
public ValueTask<IAuthenticatedGitHubService> CreateService(string accessToken, CancellationToken cancellationToken);
}
}
@@ -78,3 +78,7 @@ Swarm: # Should be left empty if using swarm mode is not desired
# PublicAddress: # The public address of the swarm node
# ControllerAddress: # Required on non-controller nodes. The internal address of the swarm controller's API'. Should be left empty on the controller itself
# UpdateRequiredNodeCount: # The number of nodes expected to be in the swarm before initiating an update. This should count every server irrespective of whether or not they are the controller MINUS 1
Telemetry:
DisableVersionReporting: false # Prevents you installation and the version you're using from being reported on the source repository's deployments list
ServerFriendlyName: null # Sets a friendly name for your server in reported telemetry. Must be unique. First come first serve
VersionReportingRepositoryId: 841149827 # GitHub repostiory ID where the tgs_version_telemetry workflow can be found
@@ -191,6 +191,8 @@ namespace Tgstation.Server.Host.Setup.Tests
"y",
// swarm config
"n",
// telemetry config
"n",
//saved, now for second run
//this time use defaults amap
String.Empty,
@@ -230,6 +232,9 @@ namespace Tgstation.Server.Host.Setup.Tests
"privatekey",
"n",
"http://controller.com",
// telemetry config
"y",
"telemetry name",
//third run, we already hit all the code coverage so just get through it
String.Empty,
nameof(DatabaseType.MariaDB),
@@ -267,7 +272,9 @@ namespace Tgstation.Server.Host.Setup.Tests
"https://controllerinternal.com",
"https://controllerpublic.com",
"privatekey",
"y"
"y",
// telemetry config
"n",
};
var inputPos = 0;
@@ -1,5 +1,6 @@
using System;
using System.Net.Http.Headers;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
@@ -57,14 +58,14 @@ namespace Tgstation.Server.Host.Utils.GitHub.Tests
mockOptions.SetupGet(x => x.Value).Returns(gc);
var factory = new GitHubClientFactory(mockApp.Object, loggerFactory.CreateLogger<GitHubClientFactory>(), mockOptions.Object);
var client = factory.CreateClient();
var client = await factory.CreateClient(CancellationToken.None);
Assert.IsNotNull(client);
var credentials = await client.Connection.CredentialStore.GetCredentials();
Assert.AreEqual(AuthenticationType.Anonymous, credentials.AuthenticationType);
gc.GitHubAccessToken = "asdfasdfasdfasdfasdfasdf";
client = factory.CreateClient();
client = await factory.CreateClient(CancellationToken.None);
Assert.IsNotNull(client);
credentials = await client.Connection.CredentialStore.GetCredentials();
@@ -83,9 +84,9 @@ namespace Tgstation.Server.Host.Utils.GitHub.Tests
mockOptions.SetupGet(x => x.Value).Returns(new GeneralConfiguration());
var factory = new GitHubClientFactory(mockApp.Object, loggerFactory.CreateLogger<GitHubClientFactory>(), mockOptions.Object);
Assert.ThrowsException<ArgumentNullException>(() => factory.CreateClient(null));
await Assert.ThrowsExceptionAsync<ArgumentNullException>(() => factory.CreateClient(null, CancellationToken.None).AsTask());
var client = factory.CreateClient("asdf");
var client = await factory.CreateClient("asdf", CancellationToken.None);
Assert.IsNotNull(client);
var credentials = await client.Connection.CredentialStore.GetCredentials();
@@ -96,7 +97,7 @@ namespace Tgstation.Server.Host.Utils.GitHub.Tests
}
[TestMethod]
public void TestClientCaching()
public async Task TestClientCaching()
{
var mockApp = new Mock<IAssemblyInformationProvider>();
mockApp.SetupGet(x => x.ProductInfoHeaderValue).Returns(new ProductInfoHeaderValue("TGSTests", "1.2.3")).Verifiable();
@@ -105,10 +106,10 @@ namespace Tgstation.Server.Host.Utils.GitHub.Tests
mockOptions.SetupGet(x => x.Value).Returns(new GeneralConfiguration());
var factory = new GitHubClientFactory(mockApp.Object, loggerFactory.CreateLogger<GitHubClientFactory>(), mockOptions.Object);
var client1 = factory.CreateClient();
var client2 = factory.CreateClient("asdf");
var client3 = factory.CreateClient();
var client4 = factory.CreateClient("asdf");
var client1 = await factory.CreateClient(CancellationToken.None);
var client2 = await factory.CreateClient("asdf", CancellationToken.None);
var client3 = await factory.CreateClient(CancellationToken.None);
var client4 = await factory.CreateClient("asdf", CancellationToken.None);
Assert.ReferenceEquals(client1, client3);
Assert.ReferenceEquals(client2, client4);
}
@@ -1,4 +1,6 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
@@ -28,27 +30,29 @@ namespace Tgstation.Server.Host.Utils.GitHub.Tests
}
[TestMethod]
public void TestCreateService()
public async Task TestCreateService()
{
var mockFactory = new Mock<IGitHubClientFactory>();
mockFactory.Setup(x => x.CreateClient()).Returns(Mock.Of<IGitHubClient>()).Verifiable();
#pragma warning disable CA2012 // Use ValueTasks correctly
mockFactory.Setup(x => x.CreateClient(It.IsAny<CancellationToken>())).Returns(ValueTask.FromResult(Mock.Of<IGitHubClient>())).Verifiable();
var mockToken = "asdf";
mockFactory.Setup(x => x.CreateClient(mockToken)).Returns(Mock.Of<IGitHubClient>()).Verifiable();
mockFactory.Setup(x => x.CreateClient(mockToken, It.IsAny<CancellationToken>())).Returns(ValueTask.FromResult(Mock.Of<IGitHubClient>())).Verifiable();
#pragma warning restore CA2012 // Use ValueTasks correctly
var mockOptions = new Mock<IOptions<UpdatesConfiguration>>();
mockOptions.SetupGet(x => x.Value).Returns(new UpdatesConfiguration());
var factory = new GitHubServiceFactory(mockFactory.Object, Mock.Of<ILoggerFactory>(), mockOptions.Object);
Assert.ThrowsException<ArgumentNullException>(() => factory.CreateService(null));
await Assert.ThrowsExceptionAsync<ArgumentNullException>(() => factory.CreateService(null, CancellationToken.None).AsTask());
Assert.AreEqual(0, mockFactory.Invocations.Count);
var result1 = factory.CreateService();
var result1 = await factory.CreateService(CancellationToken.None);
Assert.IsNotNull(result1);
var result2 = factory.CreateService(mockToken);
var result2 = factory.CreateService(mockToken, CancellationToken.None);
Assert.IsNotNull(result2);
mockFactory.VerifyAll();
@@ -1,4 +1,6 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
@@ -18,13 +20,13 @@ namespace Tgstation.Server.Tests.Live
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public IGitHubService CreateService() => CreateDummyService();
public ValueTask<IGitHubService> CreateService(CancellationToken cancellationToken) => ValueTask.FromResult<IGitHubService>(CreateDummyService());
public IAuthenticatedGitHubService CreateService(string accessToken)
public ValueTask<IAuthenticatedGitHubService> CreateService(string accessToken, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(accessToken);
return CreateDummyService();
return ValueTask.FromResult<IAuthenticatedGitHubService>(CreateDummyService());
}
TestingGitHubService CreateDummyService() => new TestingGitHubService(cryptographySuite, logger);
@@ -155,6 +155,7 @@ namespace Tgstation.Server.Tests.Live
$"General:OpenDreamGitUrl={OpenDreamUrl}",
$"Security:TokenExpiryMinutes=120", // timeouts are useless for us
$"General:OpenDreamSuppressInstallOutput={TestingUtils.RunningInGitHubActions}",
"Telemetry:DisableVersionReporting=true",
};
swarmArgs = new List<string>();
@@ -39,7 +39,7 @@ namespace Tgstation.Server.Tests.Live
});
var gitHubClientFactory = new GitHubClientFactory(new AssemblyInformationProvider(), Mock.Of<ILogger<GitHubClientFactory>>(), mockOptions.Object);
RealClient = gitHubClientFactory.CreateClient();
RealClient = gitHubClientFactory.CreateClient(CancellationToken.None).GetAwaiter().GetResult();
}
public static async Task InitializeAndInject(CancellationToken cancellationToken)