diff --git a/src/Tgstation.Server.Host/Components/Instance.cs b/src/Tgstation.Server.Host/Components/Instance.cs index 259da44cc1..cb16e111f4 100644 --- a/src/Tgstation.Server.Host/Components/Instance.cs +++ b/src/Tgstation.Server.Host/Components/Instance.cs @@ -1,5 +1,4 @@ using Microsoft.EntityFrameworkCore; -using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; using System; using System.Linq; @@ -11,6 +10,7 @@ using Tgstation.Server.Host.Components.Compiler; using Tgstation.Server.Host.Components.Repository; using Tgstation.Server.Host.Components.Watchdog; using Tgstation.Server.Host.Core; +using Tgstation.Server.Host.Models; namespace Tgstation.Server.Host.Components { @@ -122,14 +122,15 @@ namespace Tgstation.Server.Host.Components { await Task.Delay(new TimeSpan(0, minutes, 0), cancellationToken).ConfigureAwait(false); - string accessToken = null, projectName = null; + RepositorySettings repositorySettings = null; + string projectName = null; int timeout = 0; var dbTask = databaseContextFactory.UseContext(async (db) => { var instanceQuery = db.Instances.Where(x => x.Id == metadata.Id); var timeoutTask = instanceQuery.Select(x => x.DreamDaemonSettings.StartupTimeout).FirstAsync(cancellationToken); var projectNameTask = instanceQuery.Select(x => x.DreamMakerSettings.ProjectName).FirstOrDefaultAsync(cancellationToken); - accessToken = await instanceQuery.Select(x => x.RepositorySettings.AccessToken).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + repositorySettings = await instanceQuery.Select(x => x.RepositorySettings).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); projectName = await projectNameTask.ConfigureAwait(false); timeout = (await timeoutTask.ConfigureAwait(false)).Value; }); @@ -150,8 +151,27 @@ namespace Tgstation.Server.Host.Components } if (repo == null) continue; - await repo.FetchOrigin(accessToken, null, cancellationToken).ConfigureAwait(false); - await repo.ResetToOrigin(cancellationToken).ConfigureAwait(false); + + await repo.FetchOrigin(repositorySettings.AccessUser, repositorySettings.AccessToken, null, cancellationToken).ConfigureAwait(false); + + var startSha = repo.Head; + bool shouldSyncTracked; + if (repositorySettings.AutoUpdatesKeepTestMerges.Value) + { + var result = await repo.MergeOrigin(repositorySettings.CommitterName, repositorySettings.CommitterEmail, cancellationToken).ConfigureAwait(false); + if (!result.HasValue) + continue; + shouldSyncTracked = result.Value; + } + else + { + await repo.ResetToOrigin(cancellationToken).ConfigureAwait(false); + shouldSyncTracked = true; + } + + if (repositorySettings.AutoUpdatesSynchronize.Value && startSha != repo.Head) + await repo.Sychronize(repositorySettings.AccessUser, repositorySettings.AccessToken, shouldSyncTracked, cancellationToken).ConfigureAwait(false); + var job = await DreamMaker.Compile(projectName, timeout, repo, cancellationToken).ConfigureAwait(false); } } diff --git a/src/Tgstation.Server.Host/Components/Repository/IRepository.cs b/src/Tgstation.Server.Host/Components/Repository/IRepository.cs index 21e3cbcd56..16a64142f1 100644 --- a/src/Tgstation.Server.Host/Components/Repository/IRepository.cs +++ b/src/Tgstation.Server.Host/Components/Repository/IRepository.cs @@ -59,20 +59,23 @@ namespace Tgstation.Server.Host.Components.Repository /// The commit in the pull request to merge /// The name of the merge committer /// The e-mail of the merge committer - /// The access string to fetch from the origin repository + /// The commit message + /// The username to fetch from the origin repository + /// The password to fetch from the origin repository /// The for the operation /// Optional function to report 0-100 progress of the clone /// A resulting in a representing the merge result that is after a fast forward or up to date, on a merge, on a conflict - Task AddTestMerge(int pullRequestNumber, string targetCommit, string committerName, string committerEmail, string accessString, Action progressReporter, CancellationToken cancellationToken); + Task AddTestMerge(int pullRequestNumber, string targetCommit, string committerName, string committerEmail, string commitMessage, string username, string password, Action progressReporter, CancellationToken cancellationToken); /// /// Fetch commits from the origin repository /// - /// The access string to fetch from the origin repository + /// The username to fetch from the origin repository + /// The password to fetch from the origin repository /// Optional function to report 0-100 progress of the clone /// The for the operation /// A representing the running operation - Task FetchOrigin(string accessString, Action progressReporter, CancellationToken cancellationToken); + Task FetchOrigin(string username, string password, Action progressReporter, CancellationToken cancellationToken); /// /// Requires the current HEAD to be a tracked reference. Hard resets the reference to what it tracks on the origin repository @@ -81,6 +84,14 @@ namespace Tgstation.Server.Host.Components.Repository /// A resulting in the SHA of the new HEAD Task ResetToOrigin(CancellationToken cancellationToken); + /// + /// Requires the current HEAD to be a reference. Hard resets the reference to the given sha + /// + /// The sha hash to reset to + /// The for the operation + /// A resulting in the SHA of the new HEAD + Task ResetToSha(string sha, CancellationToken cancellationToken); + /// /// Requires the current HEAD to be a tracked reference. Merges the reference to what it tracks on the origin repository /// @@ -90,21 +101,15 @@ namespace Tgstation.Server.Host.Components.Repository /// A resulting in a representing the merge result that is after a fast forward or up to date, on a merge, on a conflict Task MergeOrigin(string committerName, string committerEmail, CancellationToken cancellationToken); - /// - /// Force push the current repository HEAD to ; - /// - /// The access string to fetch from the origin repository - /// The for the operation - /// A representing the running operation - Task PushHeadToTemporaryBranch(string accessString, CancellationToken cancellationToken); - /// /// Runs the synchronize event script and attempts to push any changes made to the if on a tracked branch /// - /// The access string to push to the origin repository + /// The username to fetch from the origin repository + /// The password to fetch from the origin repository + /// If the synchronizations should be made to the tracked reference as opposed to a temporary branch /// The for the operation /// A representing the running operation - Task Sychronize(string accessString, CancellationToken cancellationToken); + Task Sychronize(string username, string password, bool synchronizeTrackedBranch, CancellationToken cancellationToken); /// /// Copies the current working directory to a given diff --git a/src/Tgstation.Server.Host/Components/Repository/IRepositoryManager.cs b/src/Tgstation.Server.Host/Components/Repository/IRepositoryManager.cs index 08c6a8cd60..a8cb1fd123 100644 --- a/src/Tgstation.Server.Host/Components/Repository/IRepositoryManager.cs +++ b/src/Tgstation.Server.Host/Components/Repository/IRepositoryManager.cs @@ -21,11 +21,12 @@ namespace Tgstation.Server.Host.Components.Repository /// /// The of the remote repository to clone /// The branch to clone - /// The access string to clone from + /// The username to clone from + /// The password to clone from /// A function to report 0-100 progress of the clone /// The for the operation /// The newly cloned , if one already exists - Task CloneRepository(Uri url, string initialBranch, string accessString, Action progressReporter, CancellationToken cancellationToken); + Task CloneRepository(Uri url, string initialBranch, string username, string password, Action progressReporter, CancellationToken cancellationToken); /// /// Delete the current repository diff --git a/src/Tgstation.Server.Host/Components/Repository/Repository.cs b/src/Tgstation.Server.Host/Components/Repository/Repository.cs index b0f3a769dd..b04a0efa63 100644 --- a/src/Tgstation.Server.Host/Components/Repository/Repository.cs +++ b/src/Tgstation.Server.Host/Components/Repository/Repository.cs @@ -106,24 +106,6 @@ namespace Tgstation.Server.Host.Components.Repository repository.Dispose(); onDispose.Invoke(); } - /// - /// Convert to an "https://@{url} equivalent - /// - /// The URL to convert - /// The containing authentication info for the remote repository - /// An authenticated URL for accessing the remote repository - public static string GenerateAuthUrl(string url, string accessString) - { - if (url == null) - throw new ArgumentNullException(nameof(url)); - if (String.IsNullOrWhiteSpace(accessString)) - return url; - const string HttProtocolSecure = "HTTPS://"; - if (!url.ToUpperInvariant().StartsWith(HttProtocolSecure, StringComparison.InvariantCulture)) - throw new InvalidOperationException("Cannot use access string without HTTPS remote!"); - //ONLY support https urls - return url.ToUpperInvariant().Replace(HttProtocolSecure, String.Concat(HttProtocolSecure, accessString, '@'), StringComparison.Ordinal); - } /// /// Runs a blocking force checkout to @@ -139,7 +121,7 @@ namespace Tgstation.Server.Host.Components.Repository } /// - public async Task AddTestMerge(int pullRequestNumber, string targetCommit, string committerName, string committerEmail, string accessString, Action progressReporter, CancellationToken cancellationToken) + public async Task AddTestMerge(int pullRequestNumber, string targetCommit, string committerName, string committerEmail, string commitMessage, string username, string password, Action progressReporter, CancellationToken cancellationToken) { if (!IsGitHubRepository) @@ -154,12 +136,13 @@ namespace Tgstation.Server.Host.Components.Repository var originalCommit = repository.Head; MergeResult result = null; + + var sig = new Signature(new Identity(committerName, committerEmail), DateTimeOffset.Now); await Task.Factory.StartNew(() => { - var remote = repository.Network.Remotes.Add("temp_pr_fetch", GenerateAuthUrl(Origin, accessString)); try { - cancellationToken.ThrowIfCancellationRequested(); + var remote = repository.Network.Remotes.First(); Commands.Fetch((LibGit2Sharp.Repository)repository, remote.Name, Refspec, new FetchOptions { Prune = true, @@ -170,23 +153,24 @@ namespace Tgstation.Server.Host.Components.Repository progressReporter?.Invoke((int)percentage); return !cancellationToken.IsCancellationRequested; }, - OnUpdateTips = (a, b, c) => !cancellationToken.IsCancellationRequested + OnUpdateTips = (a, b, c) => !cancellationToken.IsCancellationRequested, + CredentialsProvider = (a, b, c) => username != null ? (Credentials)new UsernamePasswordCredentials + { + Username = username, + Password = password + } : new DefaultCredentials() }, logMessage); } - catch (UserCancelledException) { } - finally + catch (UserCancelledException) { - repository.Network.Remotes.Remove(remote.Name); - //commit is there and we never gc so - repository.Branches.Remove(localBranchName); - repository.Branches.Remove(prBranchName); + cancellationToken.ThrowIfCancellationRequested(); } cancellationToken.ThrowIfCancellationRequested(); - result = repository.Merge(targetCommit, new Signature(new Identity(committerName, committerEmail), DateTimeOffset.Now), new MergeOptions + result = repository.Merge(targetCommit, sig, new MergeOptions { - CommitOnSuccess = true, + CommitOnSuccess = commitMessage == null, FailOnConflict = true, FastForwardStrategy = FastForwardStrategy.NoFastForward, SkipReuc = true @@ -209,6 +193,12 @@ namespace Tgstation.Server.Host.Components.Repository return false; } + if (commitMessage != null) + repository.Commit(commitMessage, sig, sig, new CommitOptions + { + PrettifyMessage = true + }); + return true; } @@ -222,7 +212,7 @@ namespace Tgstation.Server.Host.Components.Repository } /// - public Task FetchOrigin(string accessString, Action progressReporter, CancellationToken cancellationToken) => Task.WhenAll( + public Task FetchOrigin(string username, string password, Action progressReporter, CancellationToken cancellationToken) => Task.WhenAll( eventConsumer.HandleEvent(EventType.RepoFetch, Array.Empty(), cancellationToken), Task.Factory.StartNew(() => { @@ -239,7 +229,12 @@ namespace Tgstation.Server.Host.Components.Repository progressReporter?.Invoke((int)percentage); return !cancellationToken.IsCancellationRequested; }, - OnUpdateTips = (a, b, c) => !cancellationToken.IsCancellationRequested + OnUpdateTips = (a, b, c) => !cancellationToken.IsCancellationRequested, + CredentialsProvider = (a, b, c) => username != null ? (Credentials)new UsernamePasswordCredentials + { + Username = username, + Password = password + } : new DefaultCredentials() }, "Fetch origin commits"); } catch (UserCancelledException) @@ -248,34 +243,37 @@ namespace Tgstation.Server.Host.Components.Repository } }, cancellationToken, TaskCreationOptions.LongRunning, TaskScheduler.Current)); - /// - public Task PushHeadToTemporaryBranch(string accessString, CancellationToken cancellationToken) => Task.Factory.StartNew(() => + /// + /// Force push the current repository HEAD to ; + /// + /// The username to fetch from the origin repository + /// The password to fetch from the origin repository + /// The for the operation + /// A representing the running operation + Task PushHeadToTemporaryBranch(string username, string password, CancellationToken cancellationToken) => Task.Factory.StartNew(() => { var branch = repository.CreateBranch(RemoteTemporaryBranchName); try { cancellationToken.ThrowIfCancellationRequested(); - var remote = repository.Network.Remotes.Add("temp_push", GenerateAuthUrl(Origin, accessString)); + var remote = repository.Network.Remotes.First(); try { - cancellationToken.ThrowIfCancellationRequested(); - try + repository.Network.Push(remote, String.Format(CultureInfo.InvariantCulture, "+{0}:{0}", branch.CanonicalName), new PushOptions { - repository.Network.Push(remote, String.Format(CultureInfo.InvariantCulture, "+{0}:{0}", branch.CanonicalName), new PushOptions + OnPackBuilderProgress = (a, b, c) => !cancellationToken.IsCancellationRequested, + OnNegotiationCompletedBeforePush = (a) => !cancellationToken.IsCancellationRequested, + OnPushTransferProgress = (a, b, c) => !cancellationToken.IsCancellationRequested, + CredentialsProvider = (a, b, c) => username != null ? (Credentials)new UsernamePasswordCredentials { - OnPackBuilderProgress = (a, b, c) => !cancellationToken.IsCancellationRequested, - OnNegotiationCompletedBeforePush = (a) => !cancellationToken.IsCancellationRequested, - OnPushTransferProgress = (a, b, c) => !cancellationToken.IsCancellationRequested - }); - } - catch (UserCancelledException) - { - cancellationToken.ThrowIfCancellationRequested(); - } + Username = username, + Password = password + } : new DefaultCredentials() + }); } - finally + catch (UserCancelledException) { - repository.Network.Remotes.Remove(remote.Name); + cancellationToken.ThrowIfCancellationRequested(); } } finally @@ -291,17 +289,17 @@ namespace Tgstation.Server.Host.Components.Repository throw new InvalidOperationException("Cannot reset to origin while not on a tracked reference!"); var trackedBranch = repository.Head.TrackedBranch; await eventConsumer.HandleEvent(EventType.RepoResetOrigin, new List { trackedBranch.FriendlyName, trackedBranch.Tip.Sha }, cancellationToken).ConfigureAwait(false); - await Task.Factory.StartNew(() => - { - Commands.Checkout((LibGit2Sharp.Repository)repository, repository.Head.TrackedBranch, new CheckoutOptions - { - CheckoutModifiers = CheckoutModifiers.Force - }); - cancellationToken.ThrowIfCancellationRequested(); - repository.RemoveUntrackedFiles(); - }, cancellationToken, TaskCreationOptions.LongRunning, TaskScheduler.Current).ConfigureAwait(false); + await ResetToSha(trackedBranch.Tip.Sha, cancellationToken).ConfigureAwait(false); } + /// + public Task ResetToSha(string sha, CancellationToken cancellationToken) => Task.Factory.StartNew(() => + { + repository.Reset(ResetMode.Hard, sha); + cancellationToken.ThrowIfCancellationRequested(); + repository.RemoveUntrackedFiles(); + }, cancellationToken, TaskCreationOptions.LongRunning, TaskScheduler.Current); + /// public async Task CopyTo(string path, CancellationToken cancellationToken) { @@ -353,40 +351,43 @@ namespace Tgstation.Server.Host.Components.Repository } /// - public async Task Sychronize(string accessString, CancellationToken cancellationToken) + public async Task Sychronize(string username, string password, bool synchronizeTrackedBranch, CancellationToken cancellationToken) { var startHead = Head; if (!await eventConsumer.HandleEvent(EventType.RepoPreSynchronize, new List { ioMananger.ResolvePath(".") }, cancellationToken).ConfigureAwait(false)) return; + if (!synchronizeTrackedBranch) + { + await PushHeadToTemporaryBranch(username, password, cancellationToken).ConfigureAwait(false); + return; + } + if (Head == startHead || !repository.Head.IsTracking) return; await Task.Factory.StartNew(() => { cancellationToken.ThrowIfCancellationRequested(); - var remote = repository.Network.Remotes.Add("temp_push", GenerateAuthUrl(Origin, accessString)); + var remote = repository.Network.Remotes.First(); try { - cancellationToken.ThrowIfCancellationRequested(); - try + repository.Network.Push(repository.Head, new PushOptions { - repository.Network.Push(repository.Head, new PushOptions + OnPackBuilderProgress = (a, b, c) => !cancellationToken.IsCancellationRequested, + OnNegotiationCompletedBeforePush = (a) => !cancellationToken.IsCancellationRequested, + OnPushTransferProgress = (a, b, c) => !cancellationToken.IsCancellationRequested, + CredentialsProvider = (a, b, c) => username != null ? (Credentials)new UsernamePasswordCredentials { - OnPackBuilderProgress = (a, b, c) => !cancellationToken.IsCancellationRequested, - OnNegotiationCompletedBeforePush = (a) => !cancellationToken.IsCancellationRequested, - OnPushTransferProgress = (a, b, c) => !cancellationToken.IsCancellationRequested - }); - } - catch (UserCancelledException) - { - cancellationToken.ThrowIfCancellationRequested(); - } + Username = username, + Password = password + } : new DefaultCredentials() + }); } - finally + catch (UserCancelledException) { - repository.Network.Remotes.Remove(remote.Name); + cancellationToken.ThrowIfCancellationRequested(); } }, cancellationToken, TaskCreationOptions.LongRunning, TaskScheduler.Current).ConfigureAwait(false); } diff --git a/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs b/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs index f594f32132..fb861acaa6 100644 --- a/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs +++ b/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs @@ -49,7 +49,7 @@ namespace Tgstation.Server.Host.Components.Repository public void Dispose() => semaphore.Dispose(); /// - public async Task CloneRepository(Uri url, string initialBranch, string accessString, Action progressReporter, CancellationToken cancellationToken) + public async Task CloneRepository(Uri url, string initialBranch, string username, string password, Action progressReporter, CancellationToken cancellationToken) { using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken).ConfigureAwait(false)) if (!await ioManager.DirectoryExists(".", cancellationToken).ConfigureAwait(false)) @@ -60,7 +60,7 @@ namespace Tgstation.Server.Host.Components.Repository string path = null; try { - path = LibGit2Sharp.Repository.Clone(Repository.GenerateAuthUrl(url.ToString(), accessString), ioManager.ResolvePath("."), new CloneOptions + path = LibGit2Sharp.Repository.Clone(url.ToString(), ioManager.ResolvePath("."), new CloneOptions { OnProgress = (a) => !cancellationToken.IsCancellationRequested, OnTransferProgress = (a) => @@ -72,7 +72,12 @@ namespace Tgstation.Server.Host.Components.Repository RecurseSubmodules = true, OnUpdateTips = (a, b, c) => !cancellationToken.IsCancellationRequested, RepositoryOperationStarting = (a) => !cancellationToken.IsCancellationRequested, - BranchName = initialBranch + BranchName = initialBranch, + CredentialsProvider = (a, b, c) => username != null ? (Credentials)new UsernamePasswordCredentials + { + Username = username, + Password = password + } : new DefaultCredentials() }); } catch (UserCancelledException) { } diff --git a/src/Tgstation.Server.Host/Controllers/ApiController.cs b/src/Tgstation.Server.Host/Controllers/ApiController.cs index c28a6620c0..58186e1891 100644 --- a/src/Tgstation.Server.Host/Controllers/ApiController.cs +++ b/src/Tgstation.Server.Host/Controllers/ApiController.cs @@ -158,7 +158,7 @@ namespace Tgstation.Server.Host.Controllers return; } - Logger.LogInformation("Request made by User ID {0}. Api version: {1}. User-Agent: {2}", AuthenticationContext.User?.Id.ToString(CultureInfo.InvariantCulture) ?? "NULL", ApiHeaders.ApiVersion, ApiHeaders.UserAgent); + Logger.LogInformation("Request made by User ID {0}. Api version: {1}. User-Agent: {2}", AuthenticationContext?.User.Id.ToString(CultureInfo.InvariantCulture) ?? "NULL", ApiHeaders.ApiVersion, ApiHeaders.UserAgent); await base.OnActionExecutionAsync(context, next).ConfigureAwait(false); } } diff --git a/src/Tgstation.Server.Host/Controllers/InstanceController.cs b/src/Tgstation.Server.Host/Controllers/InstanceController.cs index 9435a6786e..04530c31bc 100644 --- a/src/Tgstation.Server.Host/Controllers/InstanceController.cs +++ b/src/Tgstation.Server.Host/Controllers/InstanceController.cs @@ -124,7 +124,7 @@ namespace Tgstation.Server.Host.Controllers Path = model.Path, RepositorySettings = new RepositorySettings { - CommitterEmail = "tgstation-server@user.noreply.github.com", + CommitterEmail = "tgstation-server@users.noreply.github.com", CommitterName = application.VersionString, PushTestMergeCommits = false, ShowTestMergeCommitters = true, diff --git a/src/Tgstation.Server.Host/Controllers/RepositoryController.cs b/src/Tgstation.Server.Host/Controllers/RepositoryController.cs index 1d1ef12163..61904c5b99 100644 --- a/src/Tgstation.Server.Host/Controllers/RepositoryController.cs +++ b/src/Tgstation.Server.Host/Controllers/RepositoryController.cs @@ -1,5 +1,6 @@ using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using System; using System.Collections.Generic; @@ -56,36 +57,41 @@ namespace Tgstation.Server.Host.Controllers this.jobManager = jobManager ?? throw new ArgumentNullException(nameof(jobManager)); } - static string GetAccessString(Api.Models.Internal.RepositorySettings repositorySettings) => repositorySettings.AccessUser != null ? String.Concat(repositorySettings.AccessUser, '@', repositorySettings.AccessToken) : null; - - async Task LoadRevisionInformation(Components.Repository.IRepository repository, string lastOriginCommitSha, Action revInfoSink, CancellationToken cancellationToken) + static async Task LoadRevisionInformation(Components.Repository.IRepository repository, IDatabaseContext databaseContext, Models.Instance instance, string lastOriginCommitSha, Action revInfoSink, CancellationToken cancellationToken) { var repoSha = repository.Head; - var revisionInfo = await DatabaseContext.RevisionInformations.Where(x => x.CommitSha == repoSha) + + IQueryable queryTarget = databaseContext.RevisionInformations; + + var revisionInfo = await databaseContext.RevisionInformations.Where(x => x.CommitSha == repoSha && x.Instance.Id == instance.Id) .Include(x => x.CompileJobs) - .Include(x => x.ActiveTestMerges) //minimal info, they can query the rest if they're allowed + .Include(x => x.ActiveTestMerges).ThenInclude(x => x.TestMerge) //minimal info, they can query the rest if they're allowed .FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); //search every rev info because LOL SHA COLLISIONS + if (revisionInfo == default) + revisionInfo = databaseContext.RevisionInformations.Local.Where(x => x.CommitSha == repoSha).FirstOrDefault(); + var needsDbUpdate = revisionInfo == default; if (needsDbUpdate) { //needs insertion revisionInfo = new Models.RevisionInformation { + Instance = instance, CommitSha = repoSha, CompileJobs = new List(), ActiveTestMerges = new List() //non null vals for api returns }; - lock (DatabaseContext) //cleaner this way - DatabaseContext.RevisionInformations.Add(revisionInfo); + lock (databaseContext) //cleaner this way + databaseContext.RevisionInformations.Add(revisionInfo); } revisionInfo.OriginCommitSha = revisionInfo.OriginCommitSha ?? lastOriginCommitSha ?? repository.Head; revInfoSink?.Invoke(revisionInfo); return needsDbUpdate; } - async Task PopulateApi(Repository model, Components.Repository.IRepository repository, string lastOriginCommitSha, Action revInfoSink, CancellationToken cancellationToken) + static async Task PopulateApi(Repository model, Components.Repository.IRepository repository, IDatabaseContext databaseContext, Models.Instance instance, string lastOriginCommitSha, Action revInfoSink, CancellationToken cancellationToken) { model.IsGitHub = repository.IsGitHubRepository; model.Origin = repository.Origin; @@ -93,7 +99,7 @@ namespace Tgstation.Server.Host.Controllers //rev info stuff Models.RevisionInformation revisionInfo = null; - var needsDbUpdate = await LoadRevisionInformation(repository, lastOriginCommitSha, x => revisionInfo = x, cancellationToken).ConfigureAwait(false); + var needsDbUpdate = await LoadRevisionInformation(repository, databaseContext, instance, lastOriginCommitSha, x => revisionInfo = x, cancellationToken).ConfigureAwait(false); model.RevisionInformation = revisionInfo.ToApi(); revInfoSink?.Invoke(revisionInfo); return needsDbUpdate; @@ -149,11 +155,11 @@ namespace Tgstation.Server.Host.Controllers var api = currentModel.ToApi(); await jobManager.RegisterOperation(job, async (paramJob, serviceProvider, progressReporter, ct) => { - using (var repos = await repoManager.CloneRepository(new Uri(origin), cloneBranch, GetAccessString(currentModel), progressReporter, cancellationToken).ConfigureAwait(false)) + using (var repos = await repoManager.CloneRepository(new Uri(origin), cloneBranch, currentModel.AccessUser, currentModel.AccessToken, progressReporter, cancellationToken).ConfigureAwait(false)) { if (repos == null) throw new Exception("Filesystem conflict while cloning repository!"); - await PopulateApi(api, repo, null, null, cancellationToken).ConfigureAwait(false); + await PopulateApi(api, repo, DatabaseContext, Instance, null, null, cancellationToken).ConfigureAwait(false); } }, cancellationToken).ConfigureAwait(false); @@ -211,7 +217,7 @@ namespace Tgstation.Server.Host.Controllers using (var repo = await instanceManager.GetInstance(Instance).RepositoryManager.LoadRepository(cancellationToken).ConfigureAwait(false)) { - if (repo != null && await PopulateApi(api, repo, null, null, cancellationToken).ConfigureAwait(false)) + if (repo != null && await PopulateApi(api, repo, DatabaseContext, Instance, null, null, cancellationToken).ConfigureAwait(false)) //user may have fucked with the repo without telling us, do what we can await DatabaseContext.Save(cancellationToken).ConfigureAwait(false); return Json(api); @@ -273,8 +279,15 @@ namespace Tgstation.Server.Host.Controllers || (model.UpdateFromOrigin == true && !userRights.HasFlag(RepositoryRights.UpdateBranch))) return Forbid(); - //no, just one save changes at the end for sanity - //await DatabaseContext.Save(cancellationToken).ConfigureAwait(false); + if (currentModel.AccessToken.Length == 0 || currentModel.AccessUser.Length == 0) + { + //setting an empty string clears everything + currentModel.AccessUser = null; + currentModel.AccessToken = null; + } + + //this is just db stuf so stow it away + await DatabaseContext.Save(cancellationToken).ConfigureAwait(false); var job = new Models.Job { @@ -294,13 +307,13 @@ namespace Tgstation.Server.Host.Controllers var modelHasShaOrReference = model.CheckoutSha != null || model.Reference != null; + var startReference = repo.Reference; var startSha = repo.Head; if (newTestMerges && !repo.IsGitHubRepository) throw new InvalidOperationException("Cannot test merge on a non GitHub based repository!"); var committerName = currentModel.ShowTestMergeCommitters.Value ? AuthenticationContext.User.Name : currentModel.CommitterName; - var accessString = GetAccessString(currentModel); var numFetches = (model.NewTestMerges?.Count ?? 0) + (model.UpdateFromOrigin == true ? 1 : 0); var doneFetches = 0; @@ -309,13 +322,21 @@ namespace Tgstation.Server.Host.Controllers //get a base line for where we are Models.RevisionInformation lastRevisionInfo = null; - await LoadRevisionInformation(repo, null, x => lastRevisionInfo = x, cancellationToken).ConfigureAwait(false); + + var databaseContext = serviceProvider.GetRequiredService(); + var attachedInstance = new Models.Instance + { + Id = Instance.Id + }; + databaseContext.Instances.Attach(attachedInstance); + + await LoadRevisionInformation(repo, databaseContext, attachedInstance, null, x => lastRevisionInfo = x, cancellationToken).ConfigureAwait(false); //apply new rev info, tracking applied test merges async Task UpdateRevInfo() { var last = lastRevisionInfo; - await LoadRevisionInformation(repo, last.OriginCommitSha, x => lastRevisionInfo = x, cancellationToken).ConfigureAwait(false); + await LoadRevisionInformation(repo, databaseContext, attachedInstance, last.OriginCommitSha, x => lastRevisionInfo = x, cancellationToken).ConfigureAwait(false); lastRevisionInfo.ActiveTestMerges.AddRange(last.ActiveTestMerges); }; @@ -326,7 +347,7 @@ namespace Tgstation.Server.Host.Controllers { if (!repo.Tracking && model.Reference == null) throw new InvalidOperationException("Not on an updatable reference!"); - await repo.FetchOrigin(accessString, x => progressReporter(x / numFetches), cancellationToken).ConfigureAwait(false); + await repo.FetchOrigin(currentModel.AccessUser, currentModel.AccessToken, x => progressReporter(x / numFetches), cancellationToken).ConfigureAwait(false); doneFetches = 1; if (!modelHasShaOrReference) { @@ -338,7 +359,7 @@ namespace Tgstation.Server.Host.Controllers lastRevisionInfo.OriginCommitSha = repo.Head; } } - + //checkout/hard reset if (modelHasShaOrReference) { @@ -346,7 +367,7 @@ namespace Tgstation.Server.Host.Controllers || (model.Reference != null && repo.Reference != model.Reference)) { await repo.CheckoutObject(model.CheckoutSha ?? model.Reference, cancellationToken).ConfigureAwait(false); - await LoadRevisionInformation(repo, null, x => lastRevisionInfo = x, cancellationToken).ConfigureAwait(false); //we've either seen origin before or what we're checking out is on origin + await LoadRevisionInformation(repo, databaseContext, attachedInstance, null, x => lastRevisionInfo = x, cancellationToken).ConfigureAwait(false); //we've either seen origin before or what we're checking out is on origin } if (model.UpdateFromOrigin == true && model.Reference != null) @@ -354,7 +375,8 @@ namespace Tgstation.Server.Host.Controllers if (!repo.Tracking) throw new InvalidOperationException("Checked out reference does not track a remote object!"); await repo.ResetToOrigin(cancellationToken).ConfigureAwait(false); - await LoadRevisionInformation(repo, null, x => lastRevisionInfo = x, cancellationToken).ConfigureAwait(false); + await repo.Sychronize(currentModel.AccessUser, currentModel.AccessToken, true, cancellationToken).ConfigureAwait(false); + await LoadRevisionInformation(repo, databaseContext, attachedInstance, null, x => lastRevisionInfo = x, cancellationToken).ConfigureAwait(false); //repo head is on origin so force this //will update the db if necessary lastRevisionInfo.OriginCommitSha = repo.Head; @@ -364,26 +386,21 @@ namespace Tgstation.Server.Host.Controllers //test merging if (newTestMerges) { + var contextUser = new Models.User + { + Id = AuthenticationContext.User.Id + }; + databaseContext.Users.Attach(contextUser); + var repoOwner = repo.GitHubOwner; var repoName = repo.GitHubRepoName; foreach (var I in model.NewTestMerges) { - var prTask = gitHubClient.PullRequest.Get(repoOwner, repoName, I.Number); - - var mergeResult = await repo.AddTestMerge(I.Number, I.PullRequestRevision, committerName, currentModel.CommitterEmail, accessString, x => progressReporter((x + 100 * doneFetches) / numFetches), cancellationToken).ConfigureAwait(false); - - if (!mergeResult.HasValue) //conflict, we don't care, dd already knows - continue; - - ++doneFetches; - - var revInfoUpdateTask = UpdateRevInfo(); - Octokit.PullRequest pr = null; string errorMessage = null; try { - pr = await prTask.ConfigureAwait(false); + pr = await gitHubClient.PullRequest.Get(repoOwner, repoName, I.Number).ConfigureAwait(false); } catch (Octokit.RateLimitExceededException) { @@ -396,20 +413,27 @@ namespace Tgstation.Server.Host.Controllers errorMessage = "P.R.E. NOT FOUND"; } + var mergeResult = await repo.AddTestMerge(I.Number, I.PullRequestRevision, committerName, currentModel.CommitterEmail, String.Format(CultureInfo.InvariantCulture, "Test merge of pull request #{0}{1}{2}", I.Number, I.Comment != null ? Environment.NewLine : null, I.Comment), currentModel.AccessUser, currentModel.AccessToken, x => progressReporter((x + 100 * doneFetches) / numFetches), cancellationToken).ConfigureAwait(false); + + if (!mergeResult.HasValue) //conflict, we don't care, dd already knows + continue; + + ++doneFetches; + + var revInfoUpdateTask = UpdateRevInfo(); + var tm = new Models.TestMerge { Author = pr?.User.Login ?? errorMessage, - BodyAtMerge = pr?.Body, + BodyAtMerge = pr?.Body ?? errorMessage ?? String.Empty, MergedAt = DateTimeOffset.Now, - TitleAtMerge = pr?.Title ?? errorMessage, + TitleAtMerge = pr?.Title ?? errorMessage ?? String.Empty, Comment = I.Comment, Number = I.Number, - MergedBy = AuthenticationContext.User, + MergedBy = contextUser, PullRequestRevision = I.PullRequestRevision, Url = pr?.HtmlUrl ?? errorMessage }; - if (pr == null) - tm.BodyAtMerge = errorMessage; await revInfoUpdateTask.ConfigureAwait(false); @@ -423,15 +447,17 @@ namespace Tgstation.Server.Host.Controllers if (startSha != repo.Head) { - await repo.Sychronize(accessString, ct).ConfigureAwait(false); + await repo.Sychronize(currentModel.AccessUser, currentModel.AccessToken, false, ct).ConfigureAwait(false); await UpdateRevInfo().ConfigureAwait(false); } - await DatabaseContext.Save(cancellationToken).ConfigureAwait(false); + await databaseContext.Save(cancellationToken).ConfigureAwait(false); } catch { //the stuff didn't make it into the db, forget what we've done and abort - await repo.CheckoutObject(startSha, default).ConfigureAwait(false); + await repo.CheckoutObject(startReference ?? startSha, default).ConfigureAwait(false); + if (startReference != null && repo.Head != startSha) + await repo.ResetToSha(startSha, default).ConfigureAwait(false); throw; } } diff --git a/src/Tgstation.Server.Host/Models/RepositorySettings.cs b/src/Tgstation.Server.Host/Models/RepositorySettings.cs index 18fda50bfb..f4fa66bbf6 100644 --- a/src/Tgstation.Server.Host/Models/RepositorySettings.cs +++ b/src/Tgstation.Server.Host/Models/RepositorySettings.cs @@ -1,4 +1,5 @@ -using System.ComponentModel.DataAnnotations; +using System; +using System.ComponentModel.DataAnnotations; using Tgstation.Server.Api.Models; namespace Tgstation.Server.Host.Models @@ -21,19 +22,19 @@ namespace Tgstation.Server.Host.Models /// [Required] public Instance Instance { get; set; } - + /// - public Repository ToApi() => new Repository { + public Repository ToApi() => new Repository + { //AccessToken = AccessToken, //never show this AccessUser = AccessUser, AutoUpdatesKeepTestMerges = AutoUpdatesKeepTestMerges, AutoUpdatesSynchronize = AutoUpdatesSynchronize, CommitterEmail = CommitterEmail, CommitterName = CommitterName, - //intentionally don't populate origin just in case PushTestMergeCommits = PushTestMergeCommits, - //revision information and the rest retrieved by controller ShowTestMergeCommitters = ShowTestMergeCommitters + //revision information and the rest retrieved by controller }; } } diff --git a/src/Tgstation.Server.Host/Models/RevisionInformation.cs b/src/Tgstation.Server.Host/Models/RevisionInformation.cs index 4ff7c0bd79..99d606a6b1 100644 --- a/src/Tgstation.Server.Host/Models/RevisionInformation.cs +++ b/src/Tgstation.Server.Host/Models/RevisionInformation.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using System.ComponentModel.DataAnnotations; using System.Linq; namespace Tgstation.Server.Host.Models @@ -14,6 +15,7 @@ namespace Tgstation.Server.Host.Models /// /// The the belongs to /// + [Required] public Instance Instance { get; set; } /// diff --git a/src/Tgstation.Server.Host/Security/WindowsSystemIdentityFactory.cs b/src/Tgstation.Server.Host/Security/WindowsSystemIdentityFactory.cs index 7c4aa091f2..f89bdf48b2 100644 --- a/src/Tgstation.Server.Host/Security/WindowsSystemIdentityFactory.cs +++ b/src/Tgstation.Server.Host/Security/WindowsSystemIdentityFactory.cs @@ -1,8 +1,6 @@ using Microsoft.Win32.SafeHandles; using System; -using System.ComponentModel; using System.DirectoryServices.AccountManagement; -using System.Runtime.InteropServices; using System.Security.Principal; using System.Threading; using System.Threading.Tasks; diff --git a/tools/TGS.postman_collection.json b/tools/TGS.postman_collection.json index a4153fa1d9..70a76a745b 100644 --- a/tools/TGS.postman_collection.json +++ b/tools/TGS.postman_collection.json @@ -1658,7 +1658,93 @@ ], "body": { "mode": "raw", - "raw": "{\n\t\"updateReference\": true\n}" + "raw": "{\n\t\"updateFromOrigin\": true\n}" + }, + "url": { + "raw": "localhost:5000/Repository", + "host": [ + "localhost" + ], + "port": "5000", + "path": [ + "Repository" + ] + } + }, + "response": [] + }, + { + "name": "Set access info", + "request": { + "method": "POST", + "header": [ + { + "key": "Accept", + "value": "application/json" + }, + { + "key": "User-Agent", + "value": "Postman/1.0" + }, + { + "key": "User-Agent", + "value": "Tgstation.Server.Api/4.0.0.0" + }, + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "InstanceId", + "value": "1" + } + ], + "body": { + "mode": "raw", + "raw": "{\n\t\"accessUser\": \"Cyberboss\",\n\t\"accessToken\": \"this can be a password or personal access token, you should use the latter\"\n}" + }, + "url": { + "raw": "localhost:5000/Repository", + "host": [ + "localhost" + ], + "port": "5000", + "path": [ + "Repository" + ] + } + }, + "response": [] + }, + { + "name": "Reset to origin", + "request": { + "method": "POST", + "header": [ + { + "key": "Accept", + "value": "application/json" + }, + { + "key": "User-Agent", + "value": "Postman/1.0" + }, + { + "key": "User-Agent", + "value": "Tgstation.Server.Api/4.0.0.0" + }, + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "InstanceId", + "value": "1" + } + ], + "body": { + "mode": "raw", + "raw": "{\n\t\"updateFromOrigin\": true,\n\t\"reference\": \"master\"\n}" }, "url": { "raw": "localhost:5000/Repository", @@ -1701,7 +1787,7 @@ ], "body": { "mode": "raw", - "raw": "{\n\t\"updateReference\": true,\n\t\"reference\": \"master\",\n\t\"newTestMerges\": [\n\t\t{\n\t\t\t\"number\": 39476,\n\t\t\t\"pullRequestRevision\": \"11edfe5\"\n\t\t},\n\t\t{\n\t\t\t\"number\": 39469,\n\t\t\t\"pullRequestRevision\": \"ee4f00d\",\n\t\t\t\"comment\": \"babby's first pr\"\n\t\t}\n\t\t]\n}" + "raw": "{\n\t\"updateFromOrigin\": true,\n\t\"reference\": \"master\",\n\t\"newTestMerges\": [\n\t\t{\n\t\t\t\"number\": 39514,\n\t\t\t\"pullRequestRevision\": \"28c6762\",\n\t\t\t\"comment\": \"babby's first pr\"\n\t\t},\n\t\t{\n\t\t\t\"number\": 39512,\n\t\t\t\"pullRequestRevision\": \"09be343791fd233444fb3985ef54496e277397f7\",\n\t\t\t\"comment\": \"gud boi antur\"\n\t\t}\n\t\t]\n}" }, "url": { "raw": "localhost:5000/Repository", @@ -1805,6 +1891,56 @@ ], "_postman_isSubFolder": true }, + { + "name": "Compiler", + "description": "", + "item": [ + { + "name": "Read Info Copy", + "request": { + "method": "GET", + "header": [ + { + "key": "Accept", + "value": "application/json" + }, + { + "key": "User-Agent", + "value": "Postman/1.0" + }, + { + "key": "User-Agent", + "value": "Tgstation.Server.Api/4.0.0.0" + }, + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "InstanceId", + "value": "1" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": 1,\n \"online\": false\n}" + }, + "url": { + "raw": "localhost:5000/Repository", + "host": [ + "localhost" + ], + "port": "5000", + "path": [ + "Repository" + ] + } + }, + "response": [] + } + ], + "_postman_isSubFolder": true + }, { "name": "Online Instance ID 1", "request": {