From f5429c1c21681b5c1de08b7a3fdabac21f789bbe Mon Sep 17 00:00:00 2001 From: Cyberboss Date: Wed, 19 Sep 2018 09:42:50 -0400 Subject: [PATCH 1/4] Remove untracked files after PreSynchronize script --- .../Components/Repository/Repository.cs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Repository/Repository.cs b/src/Tgstation.Server.Host/Components/Repository/Repository.cs index 30cb9c1d94..36a69e8484 100644 --- a/src/Tgstation.Server.Host/Components/Repository/Repository.cs +++ b/src/Tgstation.Server.Host/Components/Repository/Repository.cs @@ -397,8 +397,15 @@ namespace Tgstation.Server.Host.Components.Repository repository.Config.Set("user.email", committerEmail); }, cancellationToken, TaskCreationOptions.LongRunning, TaskScheduler.Current).ConfigureAwait(false); - if (!await eventConsumer.HandleEvent(EventType.RepoPreSynchronize, new List { ioMananger.ResolvePath(".") }, cancellationToken).ConfigureAwait(false)) - return; + try + { + if (!await eventConsumer.HandleEvent(EventType.RepoPreSynchronize, new List { ioMananger.ResolvePath(".") }, cancellationToken).ConfigureAwait(false)) + return; + } + finally + { + await Task.Factory.StartNew(repository.RemoveUntrackedFiles, cancellationToken, TaskCreationOptions.LongRunning, TaskScheduler.Current).ConfigureAwait(false); + } if (!synchronizeTrackedBranch) { From 9f935c292865982e9c86b0e2aeca9f819262050d Mon Sep 17 00:00:00 2001 From: Cyberboss Date: Wed, 19 Sep 2018 09:47:06 -0400 Subject: [PATCH 2/4] Inject loggers into RepositoryManager and Repository --- .../Components/InstanceFactory.cs | 2 +- .../Components/Repository/Repository.cs | 10 +++++++++- .../Components/Repository/RepositoryManager.cs | 17 +++++++++++++++-- 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/InstanceFactory.cs b/src/Tgstation.Server.Host/Components/InstanceFactory.cs index 35c0fd23df..a703c61c02 100644 --- a/src/Tgstation.Server.Host/Components/InstanceFactory.cs +++ b/src/Tgstation.Server.Host/Components/InstanceFactory.cs @@ -148,7 +148,7 @@ namespace Tgstation.Server.Host.Components var dmbFactory = new DmbFactory(databaseContextFactory, gameIoManager, loggerFactory.CreateLogger(), metadata.CloneMetadata()); try { - var repoManager = new RepositoryManager(metadata.RepositorySettings, repoIoManager, eventConsumer); + var repoManager = new RepositoryManager(metadata.RepositorySettings, repoIoManager, eventConsumer, loggerFactory.CreateLogger(), loggerFactory.CreateLogger()); try { var byond = new ByondManager(byondIOManager, byondInstaller, loggerFactory.CreateLogger()); diff --git a/src/Tgstation.Server.Host/Components/Repository/Repository.cs b/src/Tgstation.Server.Host/Components/Repository/Repository.cs index 36a69e8484..0259b42935 100644 --- a/src/Tgstation.Server.Host/Components/Repository/Repository.cs +++ b/src/Tgstation.Server.Host/Components/Repository/Repository.cs @@ -1,4 +1,5 @@ using LibGit2Sharp; +using Microsoft.Extensions.Logging; using System; using System.Collections.Generic; using System.Globalization; @@ -66,6 +67,11 @@ namespace Tgstation.Server.Host.Components.Repository /// readonly IEventConsumer eventConsumer; + /// + /// The for the + /// + readonly ILogger logger; + /// /// to be taken when is called /// @@ -90,12 +96,14 @@ namespace Tgstation.Server.Host.Components.Repository /// The value of /// The value of /// The value of + /// The value of /// The value if - public Repository(LibGit2Sharp.IRepository repository, IIOManager ioMananger, IEventConsumer eventConsumer, Action onDispose) + public Repository(LibGit2Sharp.IRepository repository, IIOManager ioMananger, IEventConsumer eventConsumer, ILogger logger, Action onDispose) { this.repository = repository ?? throw new ArgumentNullException(nameof(repository)); this.ioMananger = ioMananger ?? throw new ArgumentNullException(nameof(ioMananger)); this.eventConsumer = eventConsumer ?? throw new ArgumentNullException(nameof(eventConsumer)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); this.onDispose = onDispose ?? throw new ArgumentNullException(nameof(onDispose)); IsGitHubRepository = Origin.Contains(GitHubUrl, StringComparison.InvariantCultureIgnoreCase); if (IsGitHubRepository) diff --git a/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs b/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs index 9b7ef5e95b..a34c461d70 100644 --- a/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs +++ b/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs @@ -1,4 +1,5 @@ using LibGit2Sharp; +using Microsoft.Extensions.Logging; using System; using System.Threading; using System.Threading.Tasks; @@ -27,6 +28,16 @@ namespace Tgstation.Server.Host.Components.Repository /// readonly IEventConsumer eventConsumer; + /// + /// The created s + /// + readonly ILogger repositoryLogger; + + /// + /// The for the + /// + readonly ILogger logger; + /// /// The for the /// @@ -43,7 +54,9 @@ namespace Tgstation.Server.Host.Components.Repository /// The value of /// The value of /// The value of - public RepositoryManager(RepositorySettings repositorySettings, IIOManager ioManager, IEventConsumer eventConsumer) + /// The value of + /// The value of + public RepositoryManager(RepositorySettings repositorySettings, IIOManager ioManager, IEventConsumer eventConsumer, ILogger repositoryLogger, ILogger logger) { this.repositorySettings = repositorySettings ?? throw new ArgumentNullException(nameof(repositorySettings)); this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); @@ -139,7 +152,7 @@ namespace Tgstation.Server.Host.Components.Repository return null; } var localSemaphore = semaphore; - return new Repository(repo, ioManager, eventConsumer, () => + return new Repository(repo, ioManager, eventConsumer, repositoryLogger, () => { localSemaphore?.Release(); localSemaphore = null; From 305d076b29b673fba9ea3f1666d3106f79ae9c6d Mon Sep 17 00:00:00 2001 From: Cyberboss Date: Wed, 19 Sep 2018 10:04:29 -0400 Subject: [PATCH 3/4] Add logging to RepositoryManager --- .../Repository/RepositoryManager.cs | 77 +++++++++++++++---- 1 file changed, 64 insertions(+), 13 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs b/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs index a34c461d70..cc50ff085e 100644 --- a/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs +++ b/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs @@ -65,11 +65,21 @@ namespace Tgstation.Server.Host.Components.Repository } /// - public void Dispose() => semaphore.Dispose(); + public void Dispose() + { + logger.LogTrace("Disposed"); + semaphore.Dispose(); + } /// public async Task CloneRepository(Uri url, string initialBranch, string username, string password, Action progressReporter, CancellationToken cancellationToken) { + if (url == null) + throw new ArgumentNullException(nameof(url)); + if (progressReporter == null) + throw new ArgumentNullException(nameof(progressReporter)); + + logger.LogInformation("Begin clone {0} (Branch: {1})", url, initialBranch); lock (this) { if (CloneInProgress) @@ -79,6 +89,8 @@ namespace Tgstation.Server.Host.Components.Repository try { using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken).ConfigureAwait(false)) + { + logger.LogTrace("Semaphore acquired"); if (!await ioManager.DirectoryExists(".", cancellationToken).ConfigureAwait(false)) try { @@ -100,11 +112,31 @@ namespace Tgstation.Server.Host.Components.Repository OnUpdateTips = (a, b, c) => !cancellationToken.IsCancellationRequested, RepositoryOperationStarting = (a) => !cancellationToken.IsCancellationRequested, BranchName = initialBranch, - CredentialsProvider = (a, b, c) => username != null ? (Credentials)new UsernamePasswordCredentials + CredentialsProvider = (a, b, supportedCredentialTypes) => { - Username = username, - Password = password - } : new DefaultCredentials() + var hasCreds = username != null; + var supportsUserPass = supportedCredentialTypes.HasFlag(SupportedCredentialTypes.UsernamePassword); + var supportsAnonymous = supportedCredentialTypes.HasFlag(SupportedCredentialTypes.Default); + + logger.LogTrace("Credentials requested. Present: {0}. Supports anonymous: {1}. Supports user/pass: {2}", hasCreds, supportsAnonymous, supportsUserPass); + if (supportsUserPass) + { + if (hasCreds) + return new UsernamePasswordCredentials + { + Username = username, + Password = password + }; + } + + if (supportsAnonymous) + return new DefaultCredentials(); + + if (hasCreds) + throw new JobException("Remote does not support anonymous authentication!"); + + throw new JobException("Server does not support anonymous or username/password authentication!"); + } }); } catch (UserCancelledException) { } @@ -115,13 +147,22 @@ namespace Tgstation.Server.Host.Components.Repository { try { + logger.LogTrace("Deleting partially cloned repository..."); await ioManager.DeleteDirectory(".", default).ConfigureAwait(false); } - catch { } + catch (Exception e) + { + logger.LogDebug("Error deleting partially cloned repository! Exception: {0}", e); + } throw; } else + { + logger.LogDebug("Repository exists, clone aborted!"); return null; + } + } + logger.LogInformation("Clone complete!"); } finally { @@ -133,6 +174,7 @@ namespace Tgstation.Server.Host.Components.Repository /// public async Task LoadRepository(CancellationToken cancellationToken) { + logger.LogTrace("Begin LoadRepository..."); lock (this) if (CloneInProgress) throw new InvalidOperationException("The repository is being cloned!"); @@ -142,28 +184,37 @@ namespace Tgstation.Server.Host.Components.Repository { try { + logger.LogTrace("Creating LibGit2Sharp.Repository..."); repo = new LibGit2Sharp.Repository(ioManager.ResolvePath(".")); } - catch (RepositoryNotFoundException) { } + catch (RepositoryNotFoundException e) + { + logger.LogDebug("Repository not found!"); + logger.LogTrace("Exception: {0}", e); + } + catch + { + semaphore.Release(); + throw; + } }, cancellationToken, TaskCreationOptions.LongRunning, TaskScheduler.Current).ConfigureAwait(false); if (repo == null) { semaphore.Release(); return null; } - var localSemaphore = semaphore; - return new Repository(repo, ioManager, eventConsumer, repositoryLogger, () => - { - localSemaphore?.Release(); - localSemaphore = null; - }); + return new Repository(repo, ioManager, eventConsumer, repositoryLogger, () => semaphore.Release()); } /// public async Task DeleteRepository(CancellationToken cancellationToken) { + logger.LogInformation("Deleting repository..."); using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken).ConfigureAwait(false)) + { + logger.LogTrace("Semaphore acquired, deleting Repository directory..."); await ioManager.DeleteDirectory(".", cancellationToken).ConfigureAwait(false); + } } } } From 3f821d22f84d6b97bda63c375d3fe7774ebc1e3d Mon Sep 17 00:00:00 2001 From: Cyberboss Date: Wed, 19 Sep 2018 11:46:13 -0400 Subject: [PATCH 4/4] Massive Repository Cleanup: - Added (I)CredentialsProvider for easier formatting - Refactored out several portions of duplicated Repository code - Many more repository functions now take progress reporters via OnCheckoutProgress and PushOptions progress handlers - Added much needed logging to Repository functions - Converted most Repository InvalidOperationExceptions to JobExceptions - Added needed calls to RemoveUntrackedFiles() in several places - Progress reporters are now required arguments --- .../Components/Instance.cs | 31 ++- .../Components/InstanceFactory.cs | 11 +- .../Repository/CredentialsProvider.cs | 52 ++++ .../Repository/ICredentialsProvider.cs | 18 ++ .../Components/Repository/IRepository.cs | 19 +- .../Components/Repository/Repository.cs | 260 +++++++++++++----- .../Repository/RepositoryManager.cs | 45 ++- .../Controllers/RepositoryController.cs | 53 ++-- src/Tgstation.Server.Host/Core/Application.cs | 2 + 9 files changed, 346 insertions(+), 145 deletions(-) create mode 100644 src/Tgstation.Server.Host/Components/Repository/CredentialsProvider.cs create mode 100644 src/Tgstation.Server.Host/Components/Repository/ICredentialsProvider.cs diff --git a/src/Tgstation.Server.Host/Components/Instance.cs b/src/Tgstation.Server.Host/Components/Instance.cs index 1c36c2e917..91302f8fdc 100644 --- a/src/Tgstation.Server.Host/Components/Instance.cs +++ b/src/Tgstation.Server.Host/Components/Instance.cs @@ -207,9 +207,19 @@ namespace Tgstation.Server.Host.Components var repositorySettingsTask = databaseContext.RepositorySettings.Where(x => x.InstanceId == metadata.Id).FirstAsync(jobCancellationToken); //assume 5 steps with synchronize - const int ProgressSections = 5; + const int ProgressSections = 7; const int ProgressStep = 100 / ProgressSections; - progressReporter(0 * ProgressStep); + + + const int NumSteps = 3; + var doneSteps = 0; + + Action NextProgressReporter() + { + var tmpDoneSteps = doneSteps; + ++doneSteps; + return progress => progressReporter((progress + 100 * tmpDoneSteps) / NumSteps); + }; using (var repo = await RepositoryManager.LoadRepository(jobCancellationToken).ConfigureAwait(false)) { @@ -219,17 +229,11 @@ namespace Tgstation.Server.Host.Components noRepo = true; return; } - progressReporter(1 * ProgressStep); var repositorySettings = await repositorySettingsTask.ConfigureAwait(false); - - const int SecondStepProgress = 2 * ProgressStep; - progressReporter(SecondStepProgress); - + //the main point of auto update is to pull the remote - await repo.FetchOrigin(repositorySettings.AccessUser, repositorySettings.AccessToken, x => progressReporter(SecondStepProgress + (x / ProgressSections)), jobCancellationToken).ConfigureAwait(false); - - progressReporter(3 * ProgressStep); + await repo.FetchOrigin(repositorySettings.AccessUser, repositorySettings.AccessToken, NextProgressReporter(), jobCancellationToken).ConfigureAwait(false); var startSha = repo.Head; @@ -237,21 +241,20 @@ namespace Tgstation.Server.Host.Components bool shouldSyncTracked; if (repositorySettings.AutoUpdatesKeepTestMerges.Value) { - var result = await repo.MergeOrigin(repositorySettings.CommitterName, repositorySettings.CommitterEmail, jobCancellationToken).ConfigureAwait(false); + var result = await repo.MergeOrigin(repositorySettings.CommitterName, repositorySettings.CommitterEmail, NextProgressReporter(), jobCancellationToken).ConfigureAwait(false); if (!result.HasValue) return; shouldSyncTracked = result.Value; } else { - await repo.ResetToOrigin(jobCancellationToken).ConfigureAwait(false); + await repo.ResetToOrigin(NextProgressReporter(), jobCancellationToken).ConfigureAwait(false); shouldSyncTracked = true; } - progressReporter(4 * ProgressStep); //synch if necessary if (repositorySettings.AutoUpdatesSynchronize.Value && startSha != repo.Head) - await repo.Sychronize(repositorySettings.AccessUser, repositorySettings.AccessToken, repositorySettings.CommitterName, repositorySettings.CommitterEmail, shouldSyncTracked, jobCancellationToken).ConfigureAwait(false); + await repo.Sychronize(repositorySettings.AccessUser, repositorySettings.AccessToken, repositorySettings.CommitterName, repositorySettings.CommitterEmail, NextProgressReporter(), shouldSyncTracked, jobCancellationToken).ConfigureAwait(false); progressReporter(5 * ProgressStep); } diff --git a/src/Tgstation.Server.Host/Components/InstanceFactory.cs b/src/Tgstation.Server.Host/Components/InstanceFactory.cs index a703c61c02..c9494cbd0c 100644 --- a/src/Tgstation.Server.Host/Components/InstanceFactory.cs +++ b/src/Tgstation.Server.Host/Components/InstanceFactory.cs @@ -93,6 +93,11 @@ namespace Tgstation.Server.Host.Components /// readonly IJobManager jobManager; + /// + /// The for the + /// + readonly ICredentialsProvider credentialsProvider; + /// /// Construct an /// @@ -111,7 +116,8 @@ namespace Tgstation.Server.Host.Components /// The value of /// The value of /// The value of - public InstanceFactory(IIOManager ioManager, IDatabaseContextFactory databaseContextFactory, IApplication application, ILoggerFactory loggerFactory, IByondTopicSender byondTopicSender, IServerControl serverUpdater, ICryptographySuite cryptographySuite, ISynchronousIOManager synchronousIOManager, ISymlinkFactory symlinkFactory, IByondInstaller byondInstaller, IProviderFactory providerFactory, IProcessExecutor processExecutor, IPostWriteHandler postWriteHandler, IWatchdogFactory watchdogFactory, IJobManager jobManager) + /// The value of + public InstanceFactory(IIOManager ioManager, IDatabaseContextFactory databaseContextFactory, IApplication application, ILoggerFactory loggerFactory, IByondTopicSender byondTopicSender, IServerControl serverUpdater, ICryptographySuite cryptographySuite, ISynchronousIOManager synchronousIOManager, ISymlinkFactory symlinkFactory, IByondInstaller byondInstaller, IProviderFactory providerFactory, IProcessExecutor processExecutor, IPostWriteHandler postWriteHandler, IWatchdogFactory watchdogFactory, IJobManager jobManager, ICredentialsProvider credentialsProvider) { this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); this.databaseContextFactory = databaseContextFactory ?? throw new ArgumentNullException(nameof(databaseContextFactory)); @@ -128,6 +134,7 @@ namespace Tgstation.Server.Host.Components this.postWriteHandler = postWriteHandler ?? throw new ArgumentNullException(nameof(postWriteHandler)); this.watchdogFactory = watchdogFactory ?? throw new ArgumentNullException(nameof(watchdogFactory)); this.jobManager = jobManager ?? throw new ArgumentNullException(nameof(jobManager)); + this.credentialsProvider = credentialsProvider ?? throw new ArgumentNullException(nameof(credentialsProvider)); } /// @@ -148,7 +155,7 @@ namespace Tgstation.Server.Host.Components var dmbFactory = new DmbFactory(databaseContextFactory, gameIoManager, loggerFactory.CreateLogger(), metadata.CloneMetadata()); try { - var repoManager = new RepositoryManager(metadata.RepositorySettings, repoIoManager, eventConsumer, loggerFactory.CreateLogger(), loggerFactory.CreateLogger()); + var repoManager = new RepositoryManager(metadata.RepositorySettings, repoIoManager, eventConsumer, credentialsProvider, loggerFactory.CreateLogger(), loggerFactory.CreateLogger()); try { var byond = new ByondManager(byondIOManager, byondInstaller, loggerFactory.CreateLogger()); diff --git a/src/Tgstation.Server.Host/Components/Repository/CredentialsProvider.cs b/src/Tgstation.Server.Host/Components/Repository/CredentialsProvider.cs new file mode 100644 index 0000000000..38dd416d11 --- /dev/null +++ b/src/Tgstation.Server.Host/Components/Repository/CredentialsProvider.cs @@ -0,0 +1,52 @@ +using LibGit2Sharp; +using LibGit2Sharp.Handlers; +using Microsoft.Extensions.Logging; +using System; + +namespace Tgstation.Server.Host.Components.Repository +{ + /// + sealed class CredentialsProvider : ICredentialsProvider + { + /// + /// The for the + /// + readonly ILogger logger; + + /// + /// Construct a + /// + /// The value of + public CredentialsProvider(ILogger logger) + { + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + /// + public CredentialsHandler GenerateHandler(string username, string password) => (a, b, supportedCredentialTypes) => + { + var hasCreds = username != null; + var supportsUserPass = supportedCredentialTypes.HasFlag(SupportedCredentialTypes.UsernamePassword); + var supportsAnonymous = supportedCredentialTypes.HasFlag(SupportedCredentialTypes.Default); + + logger.LogTrace("Credentials requested. Present: {0}. Supports anonymous: {1}. Supports user/pass: {2}", hasCreds, supportsAnonymous, supportsUserPass); + if (supportsUserPass) + { + if (hasCreds) + return new UsernamePasswordCredentials + { + Username = username, + Password = password + }; + } + + if (supportsAnonymous) + return new DefaultCredentials(); + + if (hasCreds) + throw new JobException("Remote does not support anonymous authentication!"); + + throw new JobException("Server does not support anonymous or username/password authentication!"); + }; + } +} diff --git a/src/Tgstation.Server.Host/Components/Repository/ICredentialsProvider.cs b/src/Tgstation.Server.Host/Components/Repository/ICredentialsProvider.cs new file mode 100644 index 0000000000..c130b02a78 --- /dev/null +++ b/src/Tgstation.Server.Host/Components/Repository/ICredentialsProvider.cs @@ -0,0 +1,18 @@ +using LibGit2Sharp.Handlers; + +namespace Tgstation.Server.Host.Components.Repository +{ + /// + /// For generating s + /// + interface ICredentialsProvider + { + /// + /// Generate a from a given and + /// + /// The optional username to use in the + /// The optional password to use in the + /// A new + CredentialsHandler GenerateHandler(string username, string password); + } +} \ No newline at end of file diff --git a/src/Tgstation.Server.Host/Components/Repository/IRepository.cs b/src/Tgstation.Server.Host/Components/Repository/IRepository.cs index 5cdd2e08ea..594469d138 100644 --- a/src/Tgstation.Server.Host/Components/Repository/IRepository.cs +++ b/src/Tgstation.Server.Host/Components/Repository/IRepository.cs @@ -57,9 +57,10 @@ namespace Tgstation.Server.Host.Components.Repository /// Checks out a given /// /// The sha or reference to checkout + /// to report 0-100 progress of the operation /// The for the operation /// A representing the running operation - Task CheckoutObject(string committish, CancellationToken cancellationToken); + Task CheckoutObject(string committish, Action progressReporter, CancellationToken cancellationToken); /// /// Attempt to merge a GitHub pull request into HEAD @@ -70,7 +71,7 @@ namespace Tgstation.Server.Host.Components.Repository /// The username to fetch from the origin repository /// The password to fetch from the origin repository /// The for the operation - /// Optional function to report 0-100 progress of the clone + /// to report 0-100 progress of the operation /// A resulting in a representing the merge result that is after a fast forward or up to date, on a merge, on a conflict Task AddTestMerge(TestMergeParameters testMergeParameters, string committerName, string committerEmail, string username, string password, Action progressReporter, CancellationToken cancellationToken); @@ -79,7 +80,7 @@ namespace Tgstation.Server.Host.Components.Repository /// /// The username to fetch from the origin repository /// The password to fetch from the origin repository - /// Optional function to report 0-100 progress of the clone + /// to report 0-100 progress of the operation /// The for the operation /// A representing the running operation Task FetchOrigin(string username, string password, Action progressReporter, CancellationToken cancellationToken); @@ -87,26 +88,29 @@ namespace Tgstation.Server.Host.Components.Repository /// /// Requires the current HEAD to be a tracked reference. Hard resets the reference to what it tracks on the origin repository /// + /// to report 0-100 progress of the operation /// The for the operation /// A resulting in the SHA of the new HEAD - Task ResetToOrigin(CancellationToken cancellationToken); + Task ResetToOrigin(Action progressReporter, CancellationToken cancellationToken); /// /// Requires the current HEAD to be a reference. Hard resets the reference to the given sha /// /// The sha hash to reset to + /// to report 0-100 progress of the operation /// The for the operation /// A resulting in the SHA of the new HEAD - Task ResetToSha(string sha, CancellationToken cancellationToken); + Task ResetToSha(string sha, Action progressReporter, CancellationToken cancellationToken); /// /// Requires the current HEAD to be a tracked reference. Merges the reference to what it tracks on the origin repository /// /// The name of the merge committer /// The e-mail of the merge committer + /// to report 0-100 progress of the operation /// The for the operation /// A resulting in a representing the merge result that is after a fast forward, on a merge or up to date, on a conflict - Task MergeOrigin(string committerName, string committerEmail, CancellationToken cancellationToken); + Task MergeOrigin(string committerName, string committerEmail, Action progressReporter, CancellationToken cancellationToken); /// /// Runs the synchronize event script and attempts to push any changes made to the if on a tracked branch @@ -116,9 +120,10 @@ namespace Tgstation.Server.Host.Components.Repository /// The name of the potential committer /// The e-mail of the potential committer /// If the synchronizations should be made to the tracked reference as opposed to a temporary branch + /// to report 0-100 progress of the operation /// The for the operation /// A representing the running operation - Task Sychronize(string username, string password, string committerName, string committerEmail, bool synchronizeTrackedBranch, CancellationToken cancellationToken); + Task Sychronize(string username, string password, string committerName, string committerEmail, Action progressReporter, bool synchronizeTrackedBranch, CancellationToken cancellationToken); /// /// Copies the current working directory to a given diff --git a/src/Tgstation.Server.Host/Components/Repository/Repository.cs b/src/Tgstation.Server.Host/Components/Repository/Repository.cs index 0259b42935..7eb8890da0 100644 --- a/src/Tgstation.Server.Host/Components/Repository/Repository.cs +++ b/src/Tgstation.Server.Host/Components/Repository/Repository.cs @@ -1,4 +1,5 @@ using LibGit2Sharp; +using LibGit2Sharp.Handlers; using Microsoft.Extensions.Logging; using System; using System.Collections.Generic; @@ -67,6 +68,11 @@ namespace Tgstation.Server.Host.Components.Repository /// readonly IEventConsumer eventConsumer; + /// + /// The for the + /// + readonly ICredentialsProvider credentialsProvider; + /// /// The for the /// @@ -77,7 +83,7 @@ namespace Tgstation.Server.Host.Components.Repository /// readonly Action onDispose; - static void GetRepositoryOwnerName(string remote, out string owner, out string name) + void GetRepositoryOwnerName(string remote, out string owner, out string name) { //Assume standard gh format: [(git)|(https)]://github.com/owner/repo(.git)[0-1] //Yes use .git twice in case it was weird @@ -88,21 +94,32 @@ namespace Tgstation.Server.Host.Components.Repository var splits = remote.Split('/'); name = splits[splits.Length - 1]; owner = splits[splits.Length - 2].Split('.')[0]; + + logger.LogTrace("GetRepositoryOwnerName({0}) => {1} / {2}", remote, owner, name); } + /// + /// Converts a given to a + /// + /// to report 0-100 progress of the operation + /// A based on + static CheckoutProgressHandler CheckoutProgressHandler(Action progressReporter) => (a, completedSteps, totalSteps) => progressReporter((int)((((float)completedSteps) / totalSteps) * 100)); + /// /// Construct a /// /// The value of /// The value of /// The value of + /// The value of /// The value of /// The value if - public Repository(LibGit2Sharp.IRepository repository, IIOManager ioMananger, IEventConsumer eventConsumer, ILogger logger, Action onDispose) + public Repository(LibGit2Sharp.IRepository repository, IIOManager ioMananger, IEventConsumer eventConsumer, ICredentialsProvider credentialsProvider, ILogger logger, Action onDispose) { this.repository = repository ?? throw new ArgumentNullException(nameof(repository)); this.ioMananger = ioMananger ?? throw new ArgumentNullException(nameof(ioMananger)); this.eventConsumer = eventConsumer ?? throw new ArgumentNullException(nameof(eventConsumer)); + this.credentialsProvider = credentialsProvider ?? throw new ArgumentNullException(nameof(credentialsProvider)); this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); this.onDispose = onDispose ?? throw new ArgumentNullException(nameof(onDispose)); IsGitHubRepository = Origin.Contains(GitHubUrl, StringComparison.InvariantCultureIgnoreCase); @@ -117,20 +134,57 @@ namespace Tgstation.Server.Host.Components.Repository /// public void Dispose() { + logger.LogTrace("Disposing..."); repository.Dispose(); onDispose.Invoke(); } + /// + /// Generate a standard set of + /// + /// to report 0-100 progress of the operation + /// The username for the + /// The password for the + /// The for the operation + /// A new set of + PushOptions GeneratePushOptions(Action progressReporter, string username, string password, CancellationToken cancellationToken) => new PushOptions + { + OnPackBuilderProgress = (stage, current, total) => + { + var baseProgress = stage == PackBuilderStage.Counting ? 0 : 25; + progressReporter(baseProgress + ((int)(25 * ((float)current) / total))); + return !cancellationToken.IsCancellationRequested; + }, + OnNegotiationCompletedBeforePush = (a) => !cancellationToken.IsCancellationRequested, + OnPushTransferProgress = (a, sentBytes, totalBytes) => + { + progressReporter(50 + ((int)(50 * ((float)sentBytes) / totalBytes))); + return !cancellationToken.IsCancellationRequested; + }, + CredentialsProvider = credentialsProvider.GenerateHandler(username, password) + }; + /// /// Runs a blocking force checkout to /// /// The committish to checkout - void RawCheckout(string committish) + /// Progress reporter + /// The for the operation + void RawCheckout(string committish, Action progressReporter, CancellationToken cancellationToken) { + logger.LogTrace("Checkout: {0}", committish); + + progressReporter(0); + cancellationToken.ThrowIfCancellationRequested(); + Commands.Checkout(repository, committish, new CheckoutOptions { - CheckoutModifiers = CheckoutModifiers.Force + CheckoutModifiers = CheckoutModifiers.Force, + OnCheckoutProgress = CheckoutProgressHandler(progressReporter) }); + + cancellationToken.ThrowIfCancellationRequested(); + repository.RemoveUntrackedFiles(); } @@ -139,22 +193,25 @@ namespace Tgstation.Server.Host.Components.Repository { if (testMergeParameters == null) throw new ArgumentNullException(nameof(testMergeParameters)); - if (committerName == null) throw new ArgumentNullException(nameof(committerName)); if (committerEmail == null) throw new ArgumentNullException(nameof(committerEmail)); + if (progressReporter == null) + throw new ArgumentNullException(nameof(progressReporter)); + + logger.LogDebug("Begin AddTestMerge: #{0} at {1} ({4}) by <{2} ({3})>", testMergeParameters.Number, testMergeParameters.PullRequestRevision?.Substring(0, 7), committerName, committerEmail, testMergeParameters.Comment); if (!IsGitHubRepository) - throw new InvalidOperationException("Test merging is only available on GitHub hosted origin repositories!"); + throw new JobException("Test merging is only available on GitHub hosted origin repositories!"); var commitMessage = String.Format(CultureInfo.InvariantCulture, "Test merge of pull request #{0}{1}{2}", testMergeParameters.Number.Value, testMergeParameters.Comment != null ? Environment.NewLine : String.Empty, testMergeParameters.Comment ?? String.Empty); - var prBranchName = String.Format(CultureInfo.InvariantCulture, "pr-{0}", testMergeParameters.Number); var localBranchName = String.Format(CultureInfo.InvariantCulture, "pull/{0}/headrefs/heads/{1}", testMergeParameters.Number, prBranchName); - var Refspec = new List { String.Format(CultureInfo.InvariantCulture, "pull/{0}/head:{1}", testMergeParameters.Number, prBranchName) }; + var refSpec = String.Format(CultureInfo.InvariantCulture, "pull/{0}/head:{1}", testMergeParameters.Number, prBranchName); + var refSpecList = new List { refSpec }; var logMessage = String.Format(CultureInfo.InvariantCulture, "Merge remote pull request #{0}", testMergeParameters.Number); var originalCommit = repository.Head; @@ -168,39 +225,45 @@ namespace Tgstation.Server.Host.Components.Repository { try { + logger.LogTrace("Fetching refspec {0}...", refSpec); + var remote = repository.Network.Remotes.First(); - Commands.Fetch((LibGit2Sharp.Repository)repository, remote.Name, Refspec, new FetchOptions + progressReporter(0); + Commands.Fetch((LibGit2Sharp.Repository)repository, remote.Name, refSpecList, new FetchOptions { Prune = true, OnProgress = (a) => !cancellationToken.IsCancellationRequested, OnTransferProgress = (a) => { - var percentage = 100 * (((float)a.IndexedObjects + a.ReceivedObjects) / (a.TotalObjects * 2)); - progressReporter?.Invoke((int)percentage); + var percentage = 50 * (((float)a.IndexedObjects + a.ReceivedObjects) / (a.TotalObjects * 2)); + progressReporter((int)percentage); return !cancellationToken.IsCancellationRequested; }, OnUpdateTips = (a, b, c) => !cancellationToken.IsCancellationRequested, - CredentialsProvider = (a, b, c) => username != null ? (Credentials)new UsernamePasswordCredentials - { - Username = username, - Password = password - } : new DefaultCredentials() + CredentialsProvider = credentialsProvider.GenerateHandler(username, password) }, logMessage); } catch (UserCancelledException) { } cancellationToken.ThrowIfCancellationRequested(); + repository.RemoveUntrackedFiles(); + + cancellationToken.ThrowIfCancellationRequested(); + testMergeParameters.PullRequestRevision = repository.Lookup(testMergeParameters.PullRequestRevision ?? localBranchName).Sha; cancellationToken.ThrowIfCancellationRequested(); + logger.LogTrace("Merging {0} into {1}...", testMergeParameters.PullRequestRevision.Substring(0, 7), Reference); + result = repository.Merge(testMergeParameters.PullRequestRevision, sig, new MergeOptions { CommitOnSuccess = commitMessage == null, FailOnConflict = true, FastForwardStrategy = FastForwardStrategy.NoFastForward, - SkipReuc = true + SkipReuc = true, + OnCheckoutProgress = (a, completedSteps, totalSteps) => progressReporter(50 + ((int)((((float)completedSteps) / totalSteps) * 50))) }); } finally @@ -212,7 +275,9 @@ namespace Tgstation.Server.Host.Components.Repository if (result.Status == MergeStatus.Conflicts) { - RawCheckout(originalCommit.CanonicalName ?? originalCommit.Tip.Sha); + var revertTo = originalCommit.CanonicalName ?? originalCommit.Tip.Sha; + logger.LogDebug("Merge conflict, aborting and reverting to {0}", revertTo); + RawCheckout(revertTo, progressReporter, cancellationToken); cancellationToken.ThrowIfCancellationRequested(); } @@ -226,27 +291,41 @@ namespace Tgstation.Server.Host.Components.Repository } if (commitMessage != null && result.Status != MergeStatus.UpToDate) - repository.Commit(commitMessage, sig, sig, new CommitOptions + { + logger.LogTrace("Committing merge: \"{0}\"...", commitMessage); + await Task.Factory.StartNew(() => repository.Commit(commitMessage, sig, sig, new CommitOptions { PrettifyMessage = true - }); + }), cancellationToken, TaskCreationOptions.LongRunning, TaskScheduler.Current).ConfigureAwait(false); + } return true; } /// - public async Task CheckoutObject(string committish, CancellationToken cancellationToken) + public async Task CheckoutObject(string committish, Action progressReporter, CancellationToken cancellationToken) { if (committish == null) throw new ArgumentNullException(nameof(committish)); + if (progressReporter == null) + throw new ArgumentNullException(nameof(progressReporter)); + logger.LogDebug("Checkout object: {0}...", committish); await eventConsumer.HandleEvent(EventType.RepoCheckout, new List { committish }, cancellationToken).ConfigureAwait(false); - await Task.Factory.StartNew(() => RawCheckout(committish), cancellationToken, TaskCreationOptions.LongRunning, TaskScheduler.Current).ConfigureAwait(false); + await Task.Factory.StartNew(() => + { + repository.RemoveUntrackedFiles(); + RawCheckout(committish, progressReporter, cancellationToken); + }, cancellationToken, TaskCreationOptions.LongRunning, TaskScheduler.Current).ConfigureAwait(false); } /// - public Task FetchOrigin(string username, string password, Action progressReporter, CancellationToken cancellationToken) => Task.WhenAll( - eventConsumer.HandleEvent(EventType.RepoFetch, Array.Empty(), cancellationToken), - Task.Factory.StartNew(() => + public async Task FetchOrigin(string username, string password, Action progressReporter, CancellationToken cancellationToken) + { + if (progressReporter == null) + throw new ArgumentNullException(nameof(progressReporter)); + logger.LogDebug("Fetch origin..."); + await eventConsumer.HandleEvent(EventType.RepoFetch, Array.Empty(), cancellationToken).ConfigureAwait(false); + await Task.Factory.StartNew(() => { var remote = repository.Network.Remotes.First(); try @@ -258,32 +337,31 @@ namespace Tgstation.Server.Host.Components.Repository OnTransferProgress = (a) => { var percentage = 100 * (((float)a.IndexedObjects + a.ReceivedObjects) / (a.TotalObjects * 2)); - progressReporter?.Invoke((int)percentage); + progressReporter((int)percentage); return !cancellationToken.IsCancellationRequested; }, OnUpdateTips = (a, b, c) => !cancellationToken.IsCancellationRequested, - CredentialsProvider = (a, b, c) => username != null ? (Credentials)new UsernamePasswordCredentials - { - Username = username, - Password = password - } : new DefaultCredentials() + CredentialsProvider = credentialsProvider.GenerateHandler(username, password) }, "Fetch origin commits"); } catch (UserCancelledException) { cancellationToken.ThrowIfCancellationRequested(); } - }, cancellationToken, TaskCreationOptions.LongRunning, TaskScheduler.Current)); + }, cancellationToken, TaskCreationOptions.LongRunning, TaskScheduler.Current).ConfigureAwait(false); + } /// - /// Force push the current repository HEAD to ; + /// Force push the current repository HEAD to ; /// /// The username to fetch from the origin repository /// The password to fetch from the origin repository + /// to report 0-100 progress of the operation /// The for the operation /// A representing the running operation - Task PushHeadToTemporaryBranch(string username, string password, CancellationToken cancellationToken) => Task.Factory.StartNew(() => + Task PushHeadToTemporaryBranch(string username, string password, Action progressReporter, CancellationToken cancellationToken) => Task.Factory.StartNew(() => { + logger.LogInformation("Pushing changes to temporary remote branch..."); var branch = repository.CreateBranch(RemoteTemporaryBranchName); try { @@ -291,17 +369,7 @@ namespace Tgstation.Server.Host.Components.Repository var remote = repository.Network.Remotes.First(); try { - repository.Network.Push(remote, String.Format(CultureInfo.InvariantCulture, "+{0}:{0}", branch.CanonicalName), new PushOptions - { - OnPackBuilderProgress = (a, b, c) => !cancellationToken.IsCancellationRequested, - OnNegotiationCompletedBeforePush = (a) => !cancellationToken.IsCancellationRequested, - OnPushTransferProgress = (a, b, c) => !cancellationToken.IsCancellationRequested, - CredentialsProvider = (a, b, c) => username != null ? (Credentials)new UsernamePasswordCredentials - { - Username = username, - Password = password - } : new DefaultCredentials() - }); + repository.Network.Push(remote, String.Format(CultureInfo.InvariantCulture, "+{0}:{0}", branch.CanonicalName), GeneratePushOptions(progressReporter, username, password, cancellationToken)); } catch (UserCancelledException) { @@ -315,21 +383,41 @@ namespace Tgstation.Server.Host.Components.Repository }, cancellationToken, TaskCreationOptions.LongRunning, TaskScheduler.Current); /// - public async Task ResetToOrigin(CancellationToken cancellationToken) + public async Task ResetToOrigin(Action progressReporter, CancellationToken cancellationToken) { - if (!repository.Head.IsTracking) - throw new InvalidOperationException("Cannot reset to origin while not on a tracked reference!"); + if (progressReporter == null) + throw new ArgumentNullException(nameof(progressReporter)); + if (!Tracking) + throw new JobException("Cannot reset to origin while not on a tracked reference!"); + logger.LogTrace("Reset to origin..."); var trackedBranch = repository.Head.TrackedBranch; await eventConsumer.HandleEvent(EventType.RepoResetOrigin, new List { trackedBranch.FriendlyName, trackedBranch.Tip.Sha }, cancellationToken).ConfigureAwait(false); - await ResetToSha(trackedBranch.Tip.Sha, cancellationToken).ConfigureAwait(false); + await ResetToSha(trackedBranch.Tip.Sha, progressReporter, cancellationToken).ConfigureAwait(false); } /// - public Task ResetToSha(string sha, CancellationToken cancellationToken) => Task.Factory.StartNew(() => + public Task ResetToSha(string sha, Action progressReporter, CancellationToken cancellationToken) => Task.Factory.StartNew(() => { - repository.Reset(ResetMode.Hard, sha); - cancellationToken.ThrowIfCancellationRequested(); + if (sha == null) + throw new ArgumentNullException(nameof(sha)); + if (progressReporter == null) + throw new ArgumentNullException(nameof(progressReporter)); + + logger.LogDebug("Reset to sha: {0}", sha.Substring(0, 7)); + repository.RemoveUntrackedFiles(); + cancellationToken.ThrowIfCancellationRequested(); + + var gitObject = repository.Lookup(sha, ObjectType.Commit); + cancellationToken.ThrowIfCancellationRequested(); + + if (gitObject == null) + throw new InvalidOperationException(String.Format(CultureInfo.InvariantCulture, "Cannot reset to non-existent SHA: {0}", sha)); + + repository.Reset(ResetMode.Hard, gitObject.Peel(), new CheckoutOptions + { + OnCheckoutProgress = CheckoutProgressHandler(progressReporter) + }); }, cancellationToken, TaskCreationOptions.LongRunning, TaskScheduler.Current); /// @@ -337,12 +425,16 @@ namespace Tgstation.Server.Host.Components.Repository { if (path == null) throw new ArgumentNullException(nameof(path)); + logger.LogTrace("Copying to {0}...", path); await ioMananger.CopyDirectory(".", path, new List { ".git" }, cancellationToken).ConfigureAwait(false); } /// - public async Task MergeOrigin(string committerName, string committerEmail, CancellationToken cancellationToken) + public async Task MergeOrigin(string committerName, string committerEmail, Action progressReporter, CancellationToken cancellationToken) { + if (progressReporter == null) + throw new ArgumentNullException(nameof(progressReporter)); + MergeResult result = null; Branch trackedBranch = null; @@ -350,23 +442,30 @@ namespace Tgstation.Server.Host.Components.Repository await Task.Factory.StartNew(() => { - if (!repository.Head.IsTracking) - throw new InvalidOperationException("Cannot reset to origin while not on a tracked reference!"); - trackedBranch = repository.Head.TrackedBranch; + if (!Tracking) + throw new JobException("Cannot reset to origin while not on a tracked reference!"); + repository.RemoveUntrackedFiles(); + + cancellationToken.ThrowIfCancellationRequested(); + + trackedBranch = repository.Head.TrackedBranch; + logger.LogDebug("Merge origin/{2}: <{0} ({1})>", committerName, committerEmail, trackedBranch.FriendlyName); result = repository.Merge(trackedBranch, new Signature(new Identity(committerName, committerEmail), DateTimeOffset.Now), new MergeOptions { CommitOnSuccess = true, FailOnConflict = true, FastForwardStrategy = FastForwardStrategy.Default, SkipReuc = true, + OnCheckoutProgress = CheckoutProgressHandler(progressReporter) }); cancellationToken.ThrowIfCancellationRequested(); if (result.Status == MergeStatus.Conflicts) { - RawCheckout(oldHead.CanonicalName); + logger.LogDebug("Merge conflict, aborting and reverting to {0}", oldHead.FriendlyName); + RawCheckout(oldHead.CanonicalName, progressReporter, cancellationToken); cancellationToken.ThrowIfCancellationRequested(); } @@ -383,64 +482,73 @@ namespace Tgstation.Server.Host.Components.Repository } /// - public async Task Sychronize(string username, string password, string committerName, string committerEmail, bool synchronizeTrackedBranch, CancellationToken cancellationToken) + public async Task Sychronize(string username, string password, string committerName, string committerEmail, Action progressReporter, bool synchronizeTrackedBranch, CancellationToken cancellationToken) { + if (committerName == null) + throw new ArgumentNullException(nameof(committerName)); + if (committerEmail == null) + throw new ArgumentNullException(nameof(committerEmail)); + if (progressReporter == null) + throw new ArgumentNullException(nameof(progressReporter)); + if (username == null && password == null) + { + logger.LogTrace("Not synchronizing due to lack of credentials!"); return; + } + logger.LogTrace("Begin Synchronize..."); if (username == null) throw new ArgumentNullException(nameof(username)); if (password == null) throw new ArgumentNullException(nameof(password)); - if (committerName == null) - throw new ArgumentNullException(nameof(committerName)); - if (committerEmail == null) - throw new ArgumentNullException(nameof(committerEmail)); var startHead = Head; + logger.LogTrace("Configuring <{0} ({1})> as author/committer", committerName, committerEmail); await Task.Factory.StartNew(() => { repository.Config.Set("user.name", committerName); + cancellationToken.ThrowIfCancellationRequested(); repository.Config.Set("user.email", committerEmail); }, cancellationToken, TaskCreationOptions.LongRunning, TaskScheduler.Current).ConfigureAwait(false); + cancellationToken.ThrowIfCancellationRequested(); try { if (!await eventConsumer.HandleEvent(EventType.RepoPreSynchronize, new List { ioMananger.ResolvePath(".") }, cancellationToken).ConfigureAwait(false)) + { + logger.LogDebug("Aborted synchronize due to event handler response!"); return; + } } finally { + logger.LogTrace("Cleaning untracked files..."); await Task.Factory.StartNew(repository.RemoveUntrackedFiles, cancellationToken, TaskCreationOptions.LongRunning, TaskScheduler.Current).ConfigureAwait(false); } if (!synchronizeTrackedBranch) { - await PushHeadToTemporaryBranch(username, password, cancellationToken).ConfigureAwait(false); + await PushHeadToTemporaryBranch(username, password, progressReporter, cancellationToken).ConfigureAwait(false); return; } - if (Head == startHead || !repository.Head.IsTracking) + var sameHead = Head == startHead; + if (sameHead || !Tracking) + { + logger.LogTrace("Aborted synchronize due to {0}!", sameHead ? "lack of changes" : "not being on tracked reference"); return; + } + + logger.LogInformation("Synchronizing with origin..."); await Task.Factory.StartNew(() => { - cancellationToken.ThrowIfCancellationRequested(); var remote = repository.Network.Remotes.First(); try { - repository.Network.Push(repository.Head, new PushOptions - { - OnPackBuilderProgress = (a, b, c) => !cancellationToken.IsCancellationRequested, - OnNegotiationCompletedBeforePush = (a) => !cancellationToken.IsCancellationRequested, - OnPushTransferProgress = (a, b, c) => !cancellationToken.IsCancellationRequested, - CredentialsProvider = (a, b, c) => username != null ? (Credentials)new UsernamePasswordCredentials - { - Username = username, - Password = password - } : new DefaultCredentials() - }); + repository.Network.Push(repository.Head, GeneratePushOptions(progressReporter, username, password, cancellationToken)); } catch (UserCancelledException) { diff --git a/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs b/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs index cc50ff085e..2ec1c57468 100644 --- a/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs +++ b/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs @@ -28,6 +28,11 @@ namespace Tgstation.Server.Host.Components.Repository /// readonly IEventConsumer eventConsumer; + /// + /// The for the + /// + readonly ICredentialsProvider credentialsProvider; + /// /// The created s /// @@ -54,20 +59,24 @@ namespace Tgstation.Server.Host.Components.Repository /// The value of /// The value of /// The value of + /// The value of /// The value of /// The value of - public RepositoryManager(RepositorySettings repositorySettings, IIOManager ioManager, IEventConsumer eventConsumer, ILogger repositoryLogger, ILogger logger) + public RepositoryManager(RepositorySettings repositorySettings, IIOManager ioManager, IEventConsumer eventConsumer, ICredentialsProvider credentialsProvider, ILogger repositoryLogger, ILogger logger) { this.repositorySettings = repositorySettings ?? throw new ArgumentNullException(nameof(repositorySettings)); this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); this.eventConsumer = eventConsumer ?? throw new ArgumentNullException(nameof(eventConsumer)); + this.credentialsProvider = credentialsProvider ?? throw new ArgumentNullException(nameof(credentialsProvider)); + this.repositoryLogger = repositoryLogger ?? throw new ArgumentNullException(nameof(repositoryLogger)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); semaphore = new SemaphoreSlim(1); } /// public void Dispose() { - logger.LogTrace("Disposed"); + logger.LogTrace("Disposing..."); semaphore.Dispose(); } @@ -112,31 +121,7 @@ namespace Tgstation.Server.Host.Components.Repository OnUpdateTips = (a, b, c) => !cancellationToken.IsCancellationRequested, RepositoryOperationStarting = (a) => !cancellationToken.IsCancellationRequested, BranchName = initialBranch, - CredentialsProvider = (a, b, supportedCredentialTypes) => - { - var hasCreds = username != null; - var supportsUserPass = supportedCredentialTypes.HasFlag(SupportedCredentialTypes.UsernamePassword); - var supportsAnonymous = supportedCredentialTypes.HasFlag(SupportedCredentialTypes.Default); - - logger.LogTrace("Credentials requested. Present: {0}. Supports anonymous: {1}. Supports user/pass: {2}", hasCreds, supportsAnonymous, supportsUserPass); - if (supportsUserPass) - { - if (hasCreds) - return new UsernamePasswordCredentials - { - Username = username, - Password = password - }; - } - - if (supportsAnonymous) - return new DefaultCredentials(); - - if (hasCreds) - throw new JobException("Remote does not support anonymous authentication!"); - - throw new JobException("Server does not support anonymous or username/password authentication!"); - } + CredentialsProvider = credentialsProvider.GenerateHandler(username, password) }); } catch (UserCancelledException) { } @@ -203,7 +188,11 @@ namespace Tgstation.Server.Host.Components.Repository semaphore.Release(); return null; } - return new Repository(repo, ioManager, eventConsumer, repositoryLogger, () => semaphore.Release()); + return new Repository(repo, ioManager, eventConsumer, credentialsProvider, repositoryLogger, () => + { + logger.LogTrace("Releasing semaphore due to Repository disposal..."); + semaphore.Release(); + }); } /// diff --git a/src/Tgstation.Server.Host/Controllers/RepositoryController.cs b/src/Tgstation.Server.Host/Controllers/RepositoryController.cs index d04b10b932..7c18d5ec06 100644 --- a/src/Tgstation.Server.Host/Controllers/RepositoryController.cs +++ b/src/Tgstation.Server.Host/Controllers/RepositoryController.cs @@ -434,10 +434,19 @@ namespace Tgstation.Server.Host.Controllers var committerName = currentModel.ShowTestMergeCommitters.Value ? AuthenticationContext.User.Name : currentModel.CommitterName; - var numFetches = (model.NewTestMerges?.Count ?? 0) + (model.UpdateFromOrigin == true ? 1 : 0); - var doneFetches = 0; - if (numFetches > 0) - progressReporter(0); + var hardResettingToOriginReference = model.UpdateFromOrigin == true && model.Reference != null; + + var numSteps = (model.NewTestMerges?.Count ?? 0) + (model.UpdateFromOrigin == true ? 1 : 0) + (!modelHasShaOrReference ? 2 : (hardResettingToOriginReference ? 3 : 1)); + var doneSteps = 0; + + Action NextProgressReporter() + { + var tmpDoneSteps = doneSteps; + ++doneSteps; + return progress => progressReporter((progress + 100 * tmpDoneSteps) / numSteps); + }; + + progressReporter(0); //get a base line for where we are Models.RevisionInformation lastRevisionInfo = null; @@ -465,19 +474,21 @@ namespace Tgstation.Server.Host.Controllers { if (!repo.Tracking) throw new JobException("Not on an updatable reference!"); - await repo.FetchOrigin(currentModel.AccessUser, currentModel.AccessToken, x => progressReporter(x / numFetches), ct).ConfigureAwait(false); - doneFetches = 1; + await repo.FetchOrigin(currentModel.AccessUser, currentModel.AccessToken, NextProgressReporter(), ct).ConfigureAwait(false); + doneSteps = 1; if (!modelHasShaOrReference) { - var fastForward = await repo.MergeOrigin(committerName, currentModel.CommitterEmail, ct).ConfigureAwait(false); + var fastForward = await repo.MergeOrigin(committerName, currentModel.CommitterEmail, NextProgressReporter(), ct).ConfigureAwait(false); if (!fastForward.HasValue) throw new JobException("Merge conflict occurred during origin update!"); await UpdateRevInfo().ConfigureAwait(false); if (fastForward.Value) { lastRevisionInfo.OriginCommitSha = repo.Head; - await repo.Sychronize(currentModel.AccessUser, currentModel.AccessToken, currentModel.CommitterName, currentModel.CommitterEmail, true, ct).ConfigureAwait(false); + await repo.Sychronize(currentModel.AccessUser, currentModel.AccessToken, currentModel.CommitterName, currentModel.CommitterEmail, NextProgressReporter(), true, ct).ConfigureAwait(false); } + else + NextProgressReporter()(100); } } @@ -493,16 +504,18 @@ namespace Tgstation.Server.Host.Controllers if ((isSha && model.Reference != null) || (!isSha && model.CheckoutSha != null)) throw new JobException("Attempted to checkout a SHA or reference that was actually the opposite!"); - await repo.CheckoutObject(committish, ct).ConfigureAwait(false); + await repo.CheckoutObject(committish, NextProgressReporter(), ct).ConfigureAwait(false); await LoadRevisionInformation(repo, databaseContext, attachedInstance, null, x => lastRevisionInfo = x, ct).ConfigureAwait(false); //we've either seen origin before or what we're checking out is on origin } + else + NextProgressReporter()(100); - if (model.UpdateFromOrigin == true && model.Reference != null) + if (hardResettingToOriginReference) { if (!repo.Tracking) throw new JobException("Checked out reference does not track a remote object!"); - await repo.ResetToOrigin(ct).ConfigureAwait(false); - await repo.Sychronize(currentModel.AccessUser, currentModel.AccessToken, currentModel.CommitterName, currentModel.CommitterEmail, true, ct).ConfigureAwait(false); + await repo.ResetToOrigin(NextProgressReporter(), ct).ConfigureAwait(false); + await repo.Sychronize(currentModel.AccessUser, currentModel.AccessToken, currentModel.CommitterName, currentModel.CommitterEmail, NextProgressReporter(), true, ct).ConfigureAwait(false); await LoadRevisionInformation(repo, databaseContext, attachedInstance, null, x => lastRevisionInfo = x, ct).ConfigureAwait(false); //repo head is on origin so force this //will update the db if necessary @@ -619,7 +632,7 @@ namespace Tgstation.Server.Host.Controllers if (revInfoWereLookingFor != null) { //goteem - await repo.ResetToSha(revInfoWereLookingFor.CommitSha, cancellationToken).ConfigureAwait(false); + await repo.ResetToSha(revInfoWereLookingFor.CommitSha, NextProgressReporter(), cancellationToken).ConfigureAwait(false); lastRevisionInfo = revInfoWereLookingFor; } @@ -660,12 +673,12 @@ namespace Tgstation.Server.Host.Controllers if (I.PullRequestRevision == null && pr != null) I.PullRequestRevision = pr.Head.Sha; - var mergeResult = await repo.AddTestMerge(I, committerName, currentModel.CommitterEmail, currentModel.AccessUser, currentModel.AccessToken, x => progressReporter((x + 100 * doneFetches) / numFetches), ct).ConfigureAwait(false); + var mergeResult = await repo.AddTestMerge(I, committerName, currentModel.CommitterEmail, currentModel.AccessUser, currentModel.AccessToken, NextProgressReporter(), ct).ConfigureAwait(false); if (!mergeResult.HasValue) //conflict, we don't care, dd already knows continue; - ++doneFetches; + ++doneSteps; var revInfoUpdateTask = UpdateRevInfo(); @@ -695,17 +708,21 @@ namespace Tgstation.Server.Host.Controllers if (startSha != repo.Head) { - await repo.Sychronize(currentModel.AccessUser, currentModel.AccessToken, currentModel.CommitterName, currentModel.CommitterEmail, false, ct).ConfigureAwait(false); + await repo.Sychronize(currentModel.AccessUser, currentModel.AccessToken, currentModel.CommitterName, currentModel.CommitterEmail, NextProgressReporter(), false, ct).ConfigureAwait(false); await UpdateRevInfo().ConfigureAwait(false); } await databaseContext.Save(ct).ConfigureAwait(false); } catch { + doneSteps = 0; + numSteps = 2; //the stuff didn't make it into the db, forget what we've done and abort - await repo.CheckoutObject(startReference ?? startSha, default).ConfigureAwait(false); + await repo.CheckoutObject(startReference ?? startSha, NextProgressReporter(), default).ConfigureAwait(false); if (startReference != null && repo.Head != startSha) - await repo.ResetToSha(startSha, default).ConfigureAwait(false); + await repo.ResetToSha(startSha, NextProgressReporter(), default).ConfigureAwait(false); + else + progressReporter(100); throw; } } diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index 6233129ca1..b024cde939 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -22,6 +22,7 @@ using System.Threading.Tasks; using Tgstation.Server.Host.Components; using Tgstation.Server.Host.Components.Byond; using Tgstation.Server.Host.Components.Chat; +using Tgstation.Server.Host.Components.Repository; using Tgstation.Server.Host.Components.Watchdog; using Tgstation.Server.Host.Configuration; using Tgstation.Server.Host.Controllers; @@ -183,6 +184,7 @@ namespace Tgstation.Server.Host.Core services.AddSingleton, PasswordHasher>(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddSingleton();