- Fixed issues with performing git operations on private repositories using GitHub App authentication.
- Fixed RepoFetch event scripts being called with the TGS internal representation of GitHub private keys instead of a valid access token when app authentication was in use.
This commit is contained in:
Jordan Dominion
2026-01-26 23:09:46 -05:00
parent 599be3abd5
commit e32e84aa53
15 changed files with 196 additions and 88 deletions
+1 -1
View File
@@ -3,7 +3,7 @@
<!-- Integration tests will ensure they match across the board -->
<Import Project="WebpanelVersion.props" />
<PropertyGroup>
<TgsCoreVersion>6.19.0</TgsCoreVersion>
<TgsCoreVersion>6.19.1</TgsCoreVersion>
<TgsConfigVersion>5.9.0</TgsConfigVersion>
<TgsRestVersion>10.14.0</TgsRestVersion>
<TgsGraphQLVersion>0.7.0</TgsGraphQLVersion>
@@ -31,5 +31,9 @@ namespace Tgstation.Server.Host.Components.Repository
TestMergeParameters parameters,
RepositorySettings repositorySettings,
CancellationToken cancellationToken) => throw new NotSupportedException();
/// <inheritdoc />
public ValueTask<string?> TransformRepositoryPassword(string? rawPassword, CancellationToken cancellationToken)
=> ValueTask.FromResult(rawPassword);
}
}
@@ -3,6 +3,7 @@ using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Octokit;
using Tgstation.Server.Api.Models;
@@ -41,6 +42,19 @@ namespace Tgstation.Server.Host.Components.Repository
this.gitHubServiceFactory = gitHubServiceFactory ?? throw new ArgumentNullException(nameof(gitHubServiceFactory));
}
/// <inheritdoc />
public override async ValueTask<string?> TransformRepositoryPassword(string? rawPassword, CancellationToken cancellationToken)
{
if (rawPassword == null)
return null;
var gitHubService = await gitHubServiceFactory.CreateService(
rawPassword,
new RepositoryIdentifier(this),
cancellationToken);
return gitHubService?.GetGitPassword() ?? rawPassword;
}
/// <inheritdoc />
protected override async ValueTask<Models.TestMerge> GetTestMergeImpl(
TestMergeParameters parameters,
@@ -42,6 +42,10 @@ namespace Tgstation.Server.Host.Components.Repository
{
}
/// <inheritdoc />
public override ValueTask<string?> TransformRepositoryPassword(string? rawPassword, CancellationToken cancellationToken)
=> ValueTask.FromResult(rawPassword);
/// <inheritdoc />
protected override async ValueTask<Models.TestMerge> GetTestMergeImpl(
TestMergeParameters parameters,
@@ -83,6 +83,9 @@ namespace Tgstation.Server.Host.Components.Repository
return result;
}
/// <inheritdoc />
public abstract ValueTask<string?> TransformRepositoryPassword(string? rawPassword, CancellationToken cancellationToken);
/// <summary>
/// Implementation of <see cref="GetTestMerge(TestMergeParameters, RepositorySettings, CancellationToken)"/>.
/// </summary>
@@ -1,4 +1,7 @@
using LibGit2Sharp;
using System.Threading;
using System.Threading.Tasks;
using LibGit2Sharp;
using LibGit2Sharp.Handlers;
using Tgstation.Server.Host.Jobs;
@@ -13,10 +16,12 @@ namespace Tgstation.Server.Host.Components.Repository
/// <summary>
/// Generate a <see cref="CredentialsHandler"/> from a given <paramref name="username"/> and <paramref name="password"/>.
/// </summary>
/// <param name="remoteFeatures">The <see cref="IGitRemoteFeatures"/> for the repository in question.</param>
/// <param name="username">The optional username to use in the <see cref="CredentialsHandler"/>.</param>
/// <param name="password">The optional password to use in the <see cref="CredentialsHandler"/>.</param>
/// <returns>A new <see cref="CredentialsHandler"/>.</returns>
CredentialsHandler GenerateCredentialsHandler(string? username, string? password);
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
/// <returns>A <see cref="ValueTask{TResult}"/> resulting in a new <see cref="CredentialsHandler"/>.</returns>
ValueTask<CredentialsHandler> GenerateCredentialsHandler(IGitRemoteFeatures remoteFeatures, string? username, string? password, CancellationToken cancellationToken);
/// <summary>
/// Rethrow the authentication failure message as a <see cref="JobException"/> if it is one.
@@ -1,4 +1,7 @@
namespace Tgstation.Server.Host.Components.Repository
using System.Threading;
using System.Threading.Tasks;
namespace Tgstation.Server.Host.Components.Repository
{
/// <summary>
/// Provides features for remote git services.
@@ -14,5 +17,13 @@
/// Get.
/// </summary>
string TestMergeLocalBranchNameFormatter { get; }
/// <summary>
/// Transform a service's <paramref name="rawPassword"/> into a password usable by git.
/// </summary>
/// <param name="rawPassword">The raw password to transform.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
/// <returns>A <see cref="ValueTask{TResult}"/> resulting in the transformed password.</returns>
public ValueTask<string?> TransformRepositoryPassword(string? rawPassword, CancellationToken cancellationToken);
}
}
@@ -4,6 +4,7 @@ using System.Threading.Tasks;
using LibGit2Sharp;
using LibGit2Sharp.Handlers;
using Microsoft.Extensions.Logging;
using Tgstation.Server.Api.Models;
@@ -80,32 +81,38 @@ namespace Tgstation.Server.Host.Components.Repository
TaskScheduler.Current);
/// <inheritdoc />
public CredentialsHandler GenerateCredentialsHandler(string? username, string? password) => (a, b, supportedCredentialTypes) =>
public async ValueTask<CredentialsHandler> GenerateCredentialsHandler(IGitRemoteFeatures gitRemoteFeatures, string? username, string? password, CancellationToken cancellationToken)
{
var hasCreds = username != null;
var supportsUserPass = supportedCredentialTypes.HasFlag(SupportedCredentialTypes.UsernamePassword);
var supportsAnonymous = supportedCredentialTypes.HasFlag(SupportedCredentialTypes.Default);
ArgumentNullException.ThrowIfNull(gitRemoteFeatures);
logger.LogTrace(
"Credentials requested. Present: {credentialsPresent}. Supports anonymous: {credentialsSupportAnon}. Supports user/pass: {credentialsSupportUserPass}",
hasCreds,
supportsAnonymous,
supportsUserPass);
if (supportsUserPass && hasCreds)
return new UsernamePasswordCredentials
{
Username = username,
Password = password,
};
var transformedPassword = await gitRemoteFeatures.TransformRepositoryPassword(password, cancellationToken);
return (a, b, supportedCredentialTypes) =>
{
var hasCreds = username != null;
var supportsUserPass = supportedCredentialTypes.HasFlag(SupportedCredentialTypes.UsernamePassword);
var supportsAnonymous = supportedCredentialTypes.HasFlag(SupportedCredentialTypes.Default);
if (supportsAnonymous)
return new DefaultCredentials();
logger.LogTrace(
"Credentials requested. Present: {credentialsPresent}. Supports anonymous: {credentialsSupportAnon}. Supports user/pass: {credentialsSupportUserPass}",
hasCreds,
supportsAnonymous,
supportsUserPass);
if (supportsUserPass && hasCreds)
return new UsernamePasswordCredentials
{
Username = username,
Password = transformedPassword,
};
if (supportsUserPass)
throw new JobException(ErrorCode.RepoCredentialsRequired);
if (supportsAnonymous)
return new DefaultCredentials();
throw new JobException(ErrorCode.RepoCannotAuthenticate);
};
if (supportsUserPass)
throw new JobException(ErrorCode.RepoCredentialsRequired);
throw new JobException(ErrorCode.RepoCannotAuthenticate);
};
}
/// <inheritdoc />
public void CheckBadCredentialsException(LibGit2SharpException exception)
@@ -126,6 +126,47 @@ namespace Tgstation.Server.Host.Components.Repository
/// </summary>
readonly ILogger<Repository> logger;
/// <summary>
/// Generate a standard set of <see cref="PushOptions"/>.
/// </summary>
/// <param name="progressReporter"><see cref="JobProgressReporter"/> of the operation.</param>
/// <param name="credentialsHandler">The <see cref="CredentialsHandler"/> for the operation.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
/// <returns>A <see cref="ValueTask{TResult}"/> resulting in a new set of <see cref="PushOptions"/> and the associated <see cref="JobProgressReporter"/>s based off <paramref name="progressReporter"/>.</returns>
static (PushOptions PushOptions, IEnumerable<JobProgressReporter> SubProgressReporters) GeneratePushOptions(JobProgressReporter progressReporter, CredentialsHandler credentialsHandler, CancellationToken cancellationToken)
{
var packFileCountingReporter = progressReporter.CreateSection(null, 0.25);
var packFileDeltafyingReporter = progressReporter.CreateSection(null, 0.25);
var transferProgressReporter = progressReporter.CreateSection(null, 0.5);
return (
PushOptions: new PushOptions
{
OnPackBuilderProgress = (stage, current, total) =>
{
if (total < current)
total = current;
var percentage = ((double)current) / total;
(stage == PackBuilderStage.Counting ? packFileCountingReporter : packFileDeltafyingReporter).ReportProgress(percentage);
return !cancellationToken.IsCancellationRequested;
},
OnNegotiationCompletedBeforePush = (a) => !cancellationToken.IsCancellationRequested,
OnPushTransferProgress = (a, sentBytes, totalBytes) =>
{
packFileCountingReporter.ReportProgress((double)sentBytes / totalBytes);
return !cancellationToken.IsCancellationRequested;
},
CredentialsProvider = credentialsHandler,
},
SubProgressReporters: new List<JobProgressReporter>
{
packFileCountingReporter,
packFileDeltafyingReporter,
transferProgressReporter,
});
}
/// <summary>
/// Initializes a new instance of the <see cref="Repository"/> class.
/// </summary>
@@ -220,6 +261,8 @@ namespace Tgstation.Server.Host.Components.Repository
var sig = new Signature(new Identity(committerName, committerEmail), DateTimeOffset.UtcNow);
List<string>? conflictedPaths = null;
var credentialsHandler = await GenerateCredentialsHandler(username, password, cancellationToken);
await Task.Factory.StartNew(
() =>
{
@@ -238,7 +281,7 @@ namespace Tgstation.Server.Host.Components.Repository
new FetchOptions().Hydrate(
logger,
fetchReporter,
credentialsProvider.GenerateCredentialsHandler(username, password),
credentialsHandler,
cancellationToken),
logMessage);
}
@@ -425,14 +468,20 @@ namespace Tgstation.Server.Host.Components.Repository
logger.LogDebug("Fetch origin...");
var parameters = new List<string>();
var credentialsHandlerTask = GenerateCredentialsHandler(username, password, cancellationToken);
if (username != null)
{
parameters.Add(username);
if (password != null)
parameters.Add(password);
{
var transformedPassword = await gitRemoteFeatures.TransformRepositoryPassword(password, cancellationToken);
if (transformedPassword != null)
parameters.Add(transformedPassword);
}
}
await eventConsumer.HandleEvent(EventType.RepoFetch, parameters, true, deploymentPipeline, cancellationToken);
var credentialsHandler = await credentialsHandlerTask;
await Task.Factory.StartNew(
() =>
{
@@ -447,7 +496,7 @@ namespace Tgstation.Server.Host.Components.Repository
}.Hydrate(
logger,
subReporter,
credentialsProvider.GenerateCredentialsHandler(username, password),
credentialsHandler,
cancellationToken);
commands.Fetch(
@@ -725,9 +774,9 @@ namespace Tgstation.Server.Host.Components.Repository
if (!synchronizeTrackedBranch)
{
using var progressReporter2 = progressReporter.CreateSection("Push to temporary branch", remainingProgressFactor);
var credentialsHandler = await GenerateCredentialsHandler(username, password, cancellationToken);
await PushHeadToTemporaryBranch(
username,
password,
credentialsHandler,
progressReporter2,
cancellationToken);
return false;
@@ -741,6 +790,12 @@ namespace Tgstation.Server.Host.Components.Repository
}
logger.LogInformation("Synchronizing with origin...");
using var pushReporter = progressReporter.CreateSection("Push to origin", remainingProgressFactor);
var (pushOptions, progressReporters) = await GeneratePushOptions(
pushReporter,
username,
password,
cancellationToken);
return await Task.Factory.StartNew(
() =>
@@ -748,12 +803,6 @@ namespace Tgstation.Server.Host.Components.Repository
var remote = libGitRepo.Network.Remotes.First();
try
{
using var pushReporter = progressReporter.CreateSection("Push to origin", remainingProgressFactor);
var (pushOptions, progressReporters) = GeneratePushOptions(
pushReporter,
username,
password,
cancellationToken);
try
{
libGitRepo.Network.Push(
@@ -993,12 +1042,11 @@ namespace Tgstation.Server.Host.Components.Repository
/// <summary>
/// Force push the current repository HEAD to <see cref="RemoteTemporaryBranchName"/>;.
/// </summary>
/// <param name="username">The username to fetch from the origin repository.</param>
/// <param name="password">The password to fetch from the origin repository.</param>
/// <param name="credentialsHandler">The <see cref="CredentialsHandler"/> for the operation.</param>
/// <param name="progressReporter"><see cref="JobProgressReporter"/> of the operation.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
/// <returns>A <see cref="Task"/> representing the running operation.</returns>
Task PushHeadToTemporaryBranch(string username, string password, JobProgressReporter progressReporter, CancellationToken cancellationToken) => Task.Factory.StartNew(
Task PushHeadToTemporaryBranch(CredentialsHandler credentialsHandler, JobProgressReporter progressReporter, CancellationToken cancellationToken) => Task.Factory.StartNew(
() =>
{
logger.LogInformation("Pushing changes to temporary remote branch...");
@@ -1015,8 +1063,7 @@ namespace Tgstation.Server.Host.Components.Repository
{
var (pushOptions, progressReporters) = GeneratePushOptions(
mainPushReporter,
username,
password,
credentialsHandler,
cancellationToken);
try
@@ -1032,7 +1079,7 @@ namespace Tgstation.Server.Host.Components.Repository
var removalString = String.Format(CultureInfo.InvariantCulture, ":{0}", branch.CanonicalName);
using var forcePushReporter = progressReporter.CreateSection(null, 0.1);
var (forcePushOptions, forcePushReporters) = GeneratePushOptions(forcePushReporter, username, password, cancellationToken);
var (forcePushOptions, forcePushReporters) = GeneratePushOptions(forcePushReporter, credentialsHandler, cancellationToken);
try
{
libGitRepo.Network.Push(remote, removalString, forcePushOptions);
@@ -1068,40 +1115,12 @@ namespace Tgstation.Server.Host.Components.Repository
/// <param name="username">The username for the <see cref="credentialsProvider"/>.</param>
/// <param name="password">The password for the <see cref="credentialsProvider"/>.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
/// <returns>A new set of <see cref="PushOptions"/> and the associated <see cref="JobProgressReporter"/>s based off <paramref name="progressReporter"/>.</returns>
(PushOptions PushOptions, IEnumerable<JobProgressReporter> SubProgressReporters) GeneratePushOptions(JobProgressReporter progressReporter, string username, string password, CancellationToken cancellationToken)
{
var packFileCountingReporter = progressReporter.CreateSection(null, 0.25);
var packFileDeltafyingReporter = progressReporter.CreateSection(null, 0.25);
var transferProgressReporter = progressReporter.CreateSection(null, 0.5);
return (
PushOptions: new PushOptions
{
OnPackBuilderProgress = (stage, current, total) =>
{
if (total < current)
total = current;
var percentage = ((double)current) / total;
(stage == PackBuilderStage.Counting ? packFileCountingReporter : packFileDeltafyingReporter).ReportProgress(percentage);
return !cancellationToken.IsCancellationRequested;
},
OnNegotiationCompletedBeforePush = (a) => !cancellationToken.IsCancellationRequested,
OnPushTransferProgress = (a, sentBytes, totalBytes) =>
{
packFileCountingReporter.ReportProgress((double)sentBytes / totalBytes);
return !cancellationToken.IsCancellationRequested;
},
CredentialsProvider = credentialsProvider.GenerateCredentialsHandler(username, password),
},
SubProgressReporters: new List<JobProgressReporter>
{
packFileCountingReporter,
packFileDeltafyingReporter,
transferProgressReporter,
});
}
/// <returns>A <see cref="ValueTask{TResult}"/> resulting in a new set of <see cref="PushOptions"/> and the associated <see cref="JobProgressReporter"/>s based off <paramref name="progressReporter"/>.</returns>
async ValueTask<(PushOptions PushOptions, IEnumerable<JobProgressReporter> SubProgressReporters)> GeneratePushOptions(JobProgressReporter progressReporter, string username, string password, CancellationToken cancellationToken)
=> GeneratePushOptions(
progressReporter,
await GenerateCredentialsHandler(username, password, cancellationToken),
cancellationToken);
/// <summary>
/// Gets the path of <see cref="libGitRepo"/>.
@@ -1153,10 +1172,12 @@ namespace Tgstation.Server.Host.Components.Repository
using var fetchReporter = currentProgressReporter.CreateSection($"Fetch submodule {submodule.Name}", factor);
var credentialsHandler = await GenerateCredentialsHandler(username, password, cancellationToken);
submoduleUpdateOptions.FetchOptions.Hydrate(
logger,
fetchReporter,
credentialsProvider.GenerateCredentialsHandler(username, password),
credentialsHandler,
cancellationToken);
using var checkoutReporter = currentProgressReporter.CreateSection($"Checkout submodule {submodule.Name}", factor);
@@ -1229,6 +1250,16 @@ namespace Tgstation.Server.Host.Components.Repository
return RecursiveUpdateSubmodules(libGitRepo, progressReporter, GetRepositoryPath());
}
/// <summary>
/// Generate a <see cref="CredentialsHandler"/> from a given <paramref name="username"/> and <paramref name="password"/>.
/// </summary>
/// <param name="username">The optional username to use in the <see cref="CredentialsHandler"/>.</param>
/// <param name="password">The optional password to use in the <see cref="CredentialsHandler"/>.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
/// <returns>A new <see cref="CredentialsHandler"/>.</returns>
ValueTask<CredentialsHandler> GenerateCredentialsHandler(string? username, string? password, CancellationToken cancellationToken)
=> credentialsProvider.GenerateCredentialsHandler(gitRemoteFeatures, username, password, cancellationToken);
/// <summary>
/// Converts a given <paramref name="progressReporter"/> to a <see cref="LibGit2Sharp.Handlers.CheckoutProgressHandler"/>.
/// </summary>
@@ -166,7 +166,11 @@ namespace Tgstation.Server.Host.Components.Repository
cloneOptions.FetchOptions.Hydrate(
logger,
cloneProgressReporter,
repositoryFactory.GenerateCredentialsHandler(username, password),
await repositoryFactory.GenerateCredentialsHandler(
gitRemoteFeaturesFactory.CreateGitRemoteFeatures(url),
username,
password,
cancellationToken),
cancellationToken);
await repositoryFactory.Clone(
@@ -346,5 +346,9 @@ namespace Tgstation.Server.Host.Utils.GitHub
committish)
.WaitAsync(cancellationToken);
}
/// <inheritdoc />
public string GetGitPassword()
=> gitHubClient.Connection.Credentials.Password;
}
}
@@ -10,6 +10,12 @@ namespace Tgstation.Server.Host.Utils.GitHub
/// </summary>
public interface IAuthenticatedGitHubService : IGitHubService
{
/// <summary>
/// Gets the password to use for accessing a given repository.
/// </summary>
/// <returns>The password to use for accessing a GitHub repository with git.</returns>
string GetGitPassword();
/// <summary>
/// Create a comment on a given <paramref name="issueNumber"/>.
/// </summary>
@@ -1,10 +1,14 @@
using LibGit2Sharp;
using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using LibGit2Sharp;
using Microsoft.Extensions.Logging;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Moq;
using System;
using System.IO;
using System.Threading.Tasks;
namespace Tgstation.Server.Host.Components.Repository.Tests
{
@@ -14,7 +18,7 @@ namespace Tgstation.Server.Host.Components.Repository.Tests
[TestClass]
public sealed class TestRepositoryFactory
{
static LibGit2RepositoryFactory CreateFactory() => new (Mock.Of<ILogger<LibGit2RepositoryFactory>>());
static LibGit2RepositoryFactory CreateFactory() => new(Mock.Of<ILogger<LibGit2RepositoryFactory>>());
static async Task<LibGit2Sharp.IRepository> TestRepoLoading(
string path,
@@ -40,7 +44,7 @@ namespace Tgstation.Server.Host.Components.Repository.Tests
{
var factory = CreateFactory();
var cloneOpts = new CloneOptions();
cloneOpts.FetchOptions.CredentialsProvider = factory.GenerateCredentialsHandler(null, null);
cloneOpts.FetchOptions.CredentialsProvider = await factory.GenerateCredentialsHandler(Mock.Of<IGitRemoteFeatures>(), null, null, CancellationToken.None);
await factory.Clone(
new Uri("https://github.com/Cyberboss/Test"),
cloneOpts,
@@ -1,7 +1,6 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
@@ -23,6 +22,8 @@ namespace Tgstation.Server.Tests.Live
{
sealed class TestingGitHubService : IAuthenticatedGitHubService
{
static string TestAccessToken => Environment.GetEnvironmentVariable("TGS_TEST_GITHUB_TOKEN");
static Dictionary<Version, Release> releasesDictionary;
static PullRequest testPr;
static GitHubCommit testCommit;
@@ -37,7 +38,7 @@ namespace Tgstation.Server.Tests.Live
var mockOptions = new Mock<IOptionsMonitor<GeneralConfiguration>>();
mockOptions.SetupGet(x => x.CurrentValue).Returns(new GeneralConfiguration
{
GitHubAccessToken = Environment.GetEnvironmentVariable("TGS_TEST_GITHUB_TOKEN")
GitHubAccessToken = TestAccessToken,
});
var gitHubClientFactory = new GitHubClientFactory(new AssemblyInformationProvider(), new BasicHttpMessageHandlerFactory(), mockOptions.Object, Mock.Of<ILogger<GitHubClientFactory>>());
@@ -162,5 +163,11 @@ namespace Tgstation.Server.Tests.Live
logger.LogTrace("GetTgsReleases");
return ValueTask.FromResult(releasesDictionary);
}
public string GetGitPassword()
{
logger.LogTrace("GetGitPassword");
return TestAccessToken;
}
}
}
@@ -18,6 +18,7 @@ using Tgstation.Server.Host.Components.Repository;
using Tgstation.Server.Host.Configuration;
using Tgstation.Server.Host.IO;
using Tgstation.Server.Host.Jobs;
using Tgstation.Server.Host.Utils.GitHub;
namespace Tgstation.Server.Tests
{
@@ -81,7 +82,10 @@ namespace Tgstation.Server.Tests
tempPath),
Mock.Of<IEventConsumer>(),
new WindowsPostWriteHandler(),
Mock.Of<IGitRemoteFeaturesFactory>(),
new GitRemoteFeaturesFactory(
Mock.Of<IGitHubServiceFactory>(),
Mock.Of<ILoggerFactory>(),
Mock.Of<ILogger<GitRemoteFeaturesFactory>>()),
Mock.Of<IOptionsMonitor<GeneralConfiguration>>(),
Mock.Of<ILogger<Host.Components.Repository.Repository>>(),
Mock.Of<ILogger<RepositoryManager>>());