From 0a585c8cb4051aa5f858f6532240e71f64830a5d Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Sun, 29 Jul 2018 22:02:04 -0400 Subject: [PATCH] Repo controller create --- .../Models/Internal/RepositorySettings.cs | 12 +++ src/Tgstation.Server.Api/Models/Repository.cs | 8 +- .../Models/RevisionInformation.cs | 7 +- .../Components/Instance.cs | 40 +++++++-- .../Components/InstanceFactory.cs | 4 +- .../Repository/IRepositoryManager.cs | 14 ++- .../Repository/RepositoryManager.cs | 69 ++++++++++----- .../Controllers/DreamMakerController.cs | 5 ++ .../Controllers/RepositoryController.cs | 87 ++++++++++++++++++- .../Models/RepositorySettings.cs | 17 +++- .../Models/RevisionInformation.cs | 3 +- 11 files changed, 218 insertions(+), 48 deletions(-) diff --git a/src/Tgstation.Server.Api/Models/Internal/RepositorySettings.cs b/src/Tgstation.Server.Api/Models/Internal/RepositorySettings.cs index 14dac247ab..4afc3d8394 100644 --- a/src/Tgstation.Server.Api/Models/Internal/RepositorySettings.cs +++ b/src/Tgstation.Server.Api/Models/Internal/RepositorySettings.cs @@ -9,6 +9,18 @@ namespace Tgstation.Server.Api.Models.Internal [Model(RightsType.Repository, ReadRight = RepositoryRights.Read, RequiresInstance = true)] public class RepositorySettings { + /// + /// The origin URL. If , the does not exist + /// + [Permissions(WriteRight = RepositoryRights.SetOrigin)] + public string Origin { get; set; } + + /// + /// The last commit recognized from + /// + [Permissions(DenyWrite = true)] + public string LastOriginCommitSha { get; set; } + /// /// The name of the committer /// diff --git a/src/Tgstation.Server.Api/Models/Repository.cs b/src/Tgstation.Server.Api/Models/Repository.cs index db3ed5b303..13a4b48814 100644 --- a/src/Tgstation.Server.Api/Models/Repository.cs +++ b/src/Tgstation.Server.Api/Models/Repository.cs @@ -8,17 +8,11 @@ namespace Tgstation.Server.Api.Models /// public sealed class Repository : Internal.RepositorySettings { - /// - /// The origin URL. If , the does not exist - /// - [Permissions(WriteRight = RepositoryRights.SetOrigin)] - public string Origin { get; set; } - /// /// The commit HEAD points to /// [Permissions(WriteRight = RepositoryRights.SetSha)] - public string NewRevision { get; set; } + public string Sha { get; set; } /// /// The current for the diff --git a/src/Tgstation.Server.Api/Models/RevisionInformation.cs b/src/Tgstation.Server.Api/Models/RevisionInformation.cs index 0c769f02e7..9a3de97f05 100644 --- a/src/Tgstation.Server.Api/Models/RevisionInformation.cs +++ b/src/Tgstation.Server.Api/Models/RevisionInformation.cs @@ -4,10 +4,15 @@ namespace Tgstation.Server.Api.Models { /// public sealed class RevisionInformation : Internal.RevisionInformation - { + { /// /// The s active in the /// public List TestMerges { get; set; } + + /// + /// The s made from the + /// + public List CompileJobs { get; set; } } } diff --git a/src/Tgstation.Server.Host/Components/Instance.cs b/src/Tgstation.Server.Host/Components/Instance.cs index 9a22a5ea9a..6d16695529 100644 --- a/src/Tgstation.Server.Host/Components/Instance.cs +++ b/src/Tgstation.Server.Host/Components/Instance.cs @@ -1,5 +1,6 @@ using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; using System; using System.Linq; using System.Threading; @@ -33,7 +34,7 @@ namespace Tgstation.Server.Host.Components /// public StaticFiles.IConfiguration Configuration { get; } - + /// /// The for the /// @@ -49,6 +50,11 @@ namespace Tgstation.Server.Host.Components /// readonly IDmbFactory dmbFactory; + /// + /// The for the + /// + readonly ILogger logger; + /// /// The for the /// @@ -76,7 +82,8 @@ namespace Tgstation.Server.Host.Components /// The value of /// The value of /// The value of - public Instance(Api.Models.Instance metadata, IRepositoryManager repositoryManager, IByondManager byondManager, IDreamMaker dreamMaker, IWatchdog watchdog, IChat chat, StaticFiles.IConfiguration configuration, ICompileJobConsumer compileJobConsumer, IDatabaseContextFactory databaseContextFactory, IDmbFactory dmbFactory) + /// The value of + public Instance(Api.Models.Instance metadata, IRepositoryManager repositoryManager, IByondManager byondManager, IDreamMaker dreamMaker, IWatchdog watchdog, IChat chat, StaticFiles.IConfiguration configuration, ICompileJobConsumer compileJobConsumer, IDatabaseContextFactory databaseContextFactory, IDmbFactory dmbFactory, ILogger logger) { this.metadata = metadata ?? throw new ArgumentNullException(nameof(metadata)); RepositoryManager = repositoryManager ?? throw new ArgumentNullException(nameof(repositoryManager)); @@ -88,6 +95,7 @@ namespace Tgstation.Server.Host.Components this.compileJobConsumer = compileJobConsumer ?? throw new ArgumentNullException(nameof(compileJobConsumer)); this.databaseContextFactory = databaseContextFactory ?? throw new ArgumentNullException(nameof(databaseContextFactory)); this.dmbFactory = dmbFactory ?? throw new ArgumentNullException(nameof(dmbFactory)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); } /// @@ -109,9 +117,8 @@ namespace Tgstation.Server.Host.Components /// A representing the running operation async Task TimerLoop(int minutes, CancellationToken cancellationToken) { - try - { - while (true) + while (true) + try { await Task.Delay(new TimeSpan(0, minutes, 0), cancellationToken).ConfigureAwait(false); @@ -128,14 +135,31 @@ namespace Tgstation.Server.Host.Components }); using (var repo = await RepositoryManager.LoadRepository(cancellationToken).ConfigureAwait(false)) { - await dbTask.ConfigureAwait(false); + try + { + await dbTask.ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception e) + { + logger.LogWarning("Database error in auto update loop! Exception: {0}", e); + continue; + } + if (repo == null) + continue; await repo.FetchOrigin(accessToken, cancellationToken).ConfigureAwait(false); await repo.ResetToOrigin(cancellationToken).ConfigureAwait(false); var job = await DreamMaker.Compile(projectName, timeout, repo, cancellationToken).ConfigureAwait(false); } } - } - catch (OperationCanceledException) { } + catch (OperationCanceledException) { } + catch (Exception e) + { + logger.LogError("Error in auto update loop! Exception: {0}", e); + } } /// diff --git a/src/Tgstation.Server.Host/Components/InstanceFactory.cs b/src/Tgstation.Server.Host/Components/InstanceFactory.cs index f6c402c1fa..9646712c84 100644 --- a/src/Tgstation.Server.Host/Components/InstanceFactory.cs +++ b/src/Tgstation.Server.Host/Components/InstanceFactory.cs @@ -143,7 +143,7 @@ namespace Tgstation.Server.Host.Components var commandFactory = new CommandFactory(application); var chatFactory = new ChatFactory(instanceIoManager, loggerFactory, commandFactory, providerFactory); - var repoManager = new RepositoryManager(metadata.RepositorySettings, repoIoManager); + var repoManager = new RepositoryManager(metadata.RepositorySettings, repoIoManager, eventConsumer); try { var byond = new ByondManager(byondIOManager, byondInstaller, loggerFactory.CreateLogger()); @@ -160,7 +160,7 @@ namespace Tgstation.Server.Host.Components { var dreamMaker = new DreamMaker(byond, ioManager, configuration, sessionControllerFactory, dmbFactory, application, eventConsumer, loggerFactory.CreateLogger()); - return new Instance(metadata.CloneMetadata(), repoManager, byond, dreamMaker, watchdog, chat, configuration, dmbFactory, databaseContextFactory, dmbFactory); + return new Instance(metadata.CloneMetadata(), repoManager, byond, dreamMaker, watchdog, chat, configuration, dmbFactory, databaseContextFactory, dmbFactory, loggerFactory.CreateLogger()); } catch { diff --git a/src/Tgstation.Server.Host/Components/Repository/IRepositoryManager.cs b/src/Tgstation.Server.Host/Components/Repository/IRepositoryManager.cs index 87544e2938..92b5ad0edd 100644 --- a/src/Tgstation.Server.Host/Components/Repository/IRepositoryManager.cs +++ b/src/Tgstation.Server.Host/Components/Repository/IRepositoryManager.cs @@ -13,16 +13,24 @@ namespace Tgstation.Server.Host.Components.Repository /// Attempt to load the from the default location /// /// The for the operation - /// The loaded + /// The loaded if it exists, otherwise Task LoadRepository(CancellationToken cancellationToken); /// - /// Delete the current and replaces it with a clone of the repository at + /// Clone the repository at /// /// The of the remote repository to clone + /// The branch to clone /// The access string to clone from /// The for the operation /// The newly cloned - Task CloneRepository(Uri url, string accessString, CancellationToken cancellationToken); + Task CloneRepository(Uri url, string initialBranch, string accessString, CancellationToken cancellationToken); + + /// + /// Delete the current repository + /// + /// The for the operation + /// A representing the running operation + Task DeleteRepository(CancellationToken cancellationToken); } } diff --git a/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs b/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs index 75c67c1e6f..ecaf3c5d98 100644 --- a/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs +++ b/src/Tgstation.Server.Host/Components/Repository/RepositoryManager.cs @@ -3,6 +3,7 @@ using System; using System.Threading; using System.Threading.Tasks; using Tgstation.Server.Api.Models.Internal; +using Tgstation.Server.Host.Core; using Tgstation.Server.Host.IO; namespace Tgstation.Server.Host.Components.Repository @@ -15,6 +16,11 @@ namespace Tgstation.Server.Host.Components.Repository /// readonly IIOManager ioManager; + /// + /// The for the + /// + readonly IEventConsumer eventConsumer; + /// /// The for the /// @@ -30,10 +36,12 @@ namespace Tgstation.Server.Host.Components.Repository /// /// The value of /// The value of - public RepositoryManager(RepositorySettings repositorySettings, IIOManager ioManager) + /// The value of + public RepositoryManager(RepositorySettings repositorySettings, IIOManager ioManager, IEventConsumer eventConsumer) { this.repositorySettings = repositorySettings ?? throw new ArgumentNullException(nameof(repositorySettings)); this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); + this.eventConsumer = eventConsumer ?? throw new ArgumentNullException(nameof(eventConsumer)); semaphore = new SemaphoreSlim(1); } @@ -41,28 +49,32 @@ namespace Tgstation.Server.Host.Components.Repository public void Dispose() => semaphore.Dispose(); /// - public async Task CloneRepository(Uri url, string accessString, CancellationToken cancellationToken) + public async Task CloneRepository(Uri url, string initialBranch, string accessString, CancellationToken cancellationToken) { - await ioManager.DeleteDirectory(".", cancellationToken).ConfigureAwait(false); - - await Task.Factory.StartNew(() => - { - string path = null; - try + using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken).ConfigureAwait(false)) + if (!await ioManager.DirectoryExists(".", cancellationToken).ConfigureAwait(false)) { - path = LibGit2Sharp.Repository.Clone(Repository.GenerateAuthUrl(url.ToString(), accessString), ioManager.ResolvePath("."), new CloneOptions - { - OnProgress = (a) => !cancellationToken.IsCancellationRequested, - OnTransferProgress = (a) => !cancellationToken.IsCancellationRequested, - RecurseSubmodules = true, - OnUpdateTips = (a, b, c) => !cancellationToken.IsCancellationRequested, - RepositoryOperationStarting = (a) => !cancellationToken.IsCancellationRequested - }); - } - catch (UserCancelledException) { } - cancellationToken.ThrowIfCancellationRequested(); - }, cancellationToken, TaskCreationOptions.LongRunning, TaskScheduler.Current).ConfigureAwait(false); + await DeleteRepository(cancellationToken).ConfigureAwait(false); + await Task.Factory.StartNew(() => + { + string path = null; + try + { + path = LibGit2Sharp.Repository.Clone(Repository.GenerateAuthUrl(url.ToString(), accessString), ioManager.ResolvePath("."), new CloneOptions + { + OnProgress = (a) => !cancellationToken.IsCancellationRequested, + OnTransferProgress = (a) => !cancellationToken.IsCancellationRequested, + RecurseSubmodules = true, + OnUpdateTips = (a, b, c) => !cancellationToken.IsCancellationRequested, + RepositoryOperationStarting = (a) => !cancellationToken.IsCancellationRequested, + BranchName = initialBranch + }); + } + catch (UserCancelledException) { } + cancellationToken.ThrowIfCancellationRequested(); + }, cancellationToken, TaskCreationOptions.LongRunning, TaskScheduler.Current).ConfigureAwait(false); + } return await LoadRepository(cancellationToken).ConfigureAwait(false); } @@ -73,14 +85,27 @@ namespace Tgstation.Server.Host.Components.Repository LibGit2Sharp.Repository repo = null; await Task.Factory.StartNew(() => { - repo = new LibGit2Sharp.Repository(ioManager.ResolvePath(".")); + try + { + repo = new LibGit2Sharp.Repository(ioManager.ResolvePath(".")); + } + catch (RepositoryNotFoundException) { } }, cancellationToken, TaskCreationOptions.LongRunning, TaskScheduler.Current).ConfigureAwait(false); + if (repo == null) + return null; var localSemaphore = semaphore; - return new Repository(repo, ioManager, () => + return new Repository(repo, ioManager, eventConsumer, () => { localSemaphore?.Release(); localSemaphore = null; }); } + + /// + public async Task DeleteRepository(CancellationToken cancellationToken) + { + using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken).ConfigureAwait(false)) + await ioManager.DeleteDirectory(".", cancellationToken).ConfigureAwait(false); + } } } diff --git a/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs b/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs index a29ac1dcf7..d25f914517 100644 --- a/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs +++ b/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs @@ -129,6 +129,11 @@ namespace Tgstation.Server.Host.Controllers Task revInfoTask; using (var repo = await instance.RepositoryManager.LoadRepository(cancellationToken).ConfigureAwait(false)) { + if (repo == null) + { + job.ExceptionDetails = "Missing repository!"; + return; + } revInfoTask = databaseContext.RevisionInformations.Where(x => x.CommitSha == repo.Head).Select(x => new RevisionInformation { Id = x.Id }).FirstAsync(); compileJob = await instance.DreamMaker.Compile(projectName, timeout.Value, repo, cancellationToken).ConfigureAwait(false); } diff --git a/src/Tgstation.Server.Host/Controllers/RepositoryController.cs b/src/Tgstation.Server.Host/Controllers/RepositoryController.cs index e662494c48..1c126a276b 100644 --- a/src/Tgstation.Server.Host/Controllers/RepositoryController.cs +++ b/src/Tgstation.Server.Host/Controllers/RepositoryController.cs @@ -1,5 +1,13 @@ using Microsoft.AspNetCore.Mvc; +using Microsoft.EntityFrameworkCore; using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using Tgstation.Server.Api.Models; +using Tgstation.Server.Api.Rights; using Tgstation.Server.Host.Components; using Tgstation.Server.Host.Models; using Tgstation.Server.Host.Security; @@ -9,8 +17,8 @@ namespace Tgstation.Server.Host.Controllers /// /// Controller for managing the s /// - [Route("/" + nameof(Api.Models.Repository))] - public sealed class RepositoryController : ModelController + [Route("/" + nameof(Repository))] + public sealed class RepositoryController : ModelController { /// /// The for the @@ -22,10 +30,83 @@ namespace Tgstation.Server.Host.Controllers /// /// The for the /// The for the - /// The value of + /// The value of public RepositoryController(IDatabaseContext databaseContext, IAuthenticationContextFactory authenticationContextFactory, IInstanceManager instanceManager) : base(databaseContext, authenticationContextFactory, true) { this.instanceManager = instanceManager ?? throw new ArgumentNullException(nameof(instanceManager)); } + + static string GetAccessString(Api.Models.Internal.RepositorySettings repositorySettings) => repositorySettings.AccessUser != null ? String.Concat(repositorySettings.AccessUser, '@', repositorySettings.AccessToken) : null; + + async Task PopulateApi(Repository model, Components.Repository.IRepository repository, string lastOriginCommitSha, CancellationToken cancellationToken) + { + model.IsGitHub = repository.IsGitHubRepository; + model.Origin = repository.Origin; + model.Reference = repository.Reference; + model.Sha = repository.Head; + + //rev info stuff + var revisionInfo = await DatabaseContext.RevisionInformations.Where(x => x.CommitSha == model.Sha) + .Include(x => x.CompileJobs) + .Include(x => x.TestMerges) //minimal info, they can query the rest if they're allowed + .FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); //search every rev info because LOL SHA COLLISIONS + + var needsDbUpdate = revisionInfo == default; + if (needsDbUpdate) + { + //needs insertion + revisionInfo = new Models.RevisionInformation + { + CommitSha = model.Sha, + CompileJobs = new List(), + TestMerges = new List(), //non null vals for api returns + OriginCommitSha = lastOriginCommitSha ?? model.Sha + }; + + DatabaseContext.RevisionInformations.Add(revisionInfo); + await DatabaseContext.Save(cancellationToken).ConfigureAwait(false); + } + + model.RevisionInformation = revisionInfo.ToApi(); + return needsDbUpdate; + } + + /// + [TgsAuthorize(RepositoryRights.SetOrigin)] + public override async Task Create([FromBody] Repository model, CancellationToken cancellationToken) + { + if (model == null) + return BadRequest(new { message = "Missing request model!" }); + + if (model.Origin == null) + return BadRequest(new { message = "Missing repo origin!" }); + + if (model.AccessUser == null ^ model.AccessToken == null) + return BadRequest(new { message = "Either both accessToken and accessUser must be present or neither!" }); + + var currentModel = await DatabaseContext.Instances.Where(x => x.Id == Instance.Id).Select(x => x.RepositorySettings).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + + if (currentModel == default) + return StatusCode((int)HttpStatusCode.Gone); + + + currentModel.AccessToken = model.AccessToken; + currentModel.AccessUser = model.AccessUser; + currentModel.Origin = model.Origin; //intentionally only these fields, user not allowed to change anything else atm + var cloneBranch = model.Reference; + + var repoManager = instanceManager.GetInstance(Instance).RepositoryManager; + using (var repo = await repoManager.CloneRepository(new Uri(currentModel.Origin), cloneBranch, GetAccessString(currentModel), cancellationToken).ConfigureAwait(false)) + { + if (repo == null) + //clone conflict + return Conflict(); + var api = currentModel.ToApi(); + await PopulateApi(api, repo, null, cancellationToken).ConfigureAwait(false); + currentModel.LastOriginCommitSha = repo.Head; + await DatabaseContext.Save(cancellationToken).ConfigureAwait(false); + return Json(api); + } + } } } diff --git a/src/Tgstation.Server.Host/Models/RepositorySettings.cs b/src/Tgstation.Server.Host/Models/RepositorySettings.cs index 3d42baf883..2a9eef753e 100644 --- a/src/Tgstation.Server.Host/Models/RepositorySettings.cs +++ b/src/Tgstation.Server.Host/Models/RepositorySettings.cs @@ -1,9 +1,10 @@ using System.ComponentModel.DataAnnotations; +using Tgstation.Server.Api.Models; namespace Tgstation.Server.Host.Models { /// - public sealed class RepositorySettings : Api.Models.Internal.RepositorySettings + public sealed class RepositorySettings : Api.Models.Internal.RepositorySettings, IApiConvertable { /// /// The row Id @@ -20,5 +21,19 @@ namespace Tgstation.Server.Host.Models /// [Required] public Instance Instance { get; set; } + + /// + public Repository ToApi() => new Repository { + AccessToken = AccessToken, + AccessUser = AccessUser, + AutoUpdatesKeepTestMerges = AutoUpdatesKeepTestMerges, + AutoUpdatesSynchronize = AutoUpdatesSynchronize, + CommitterEmail = CommitterEmail, + CommitterName = CommitterName, + //intentionally don't populate origin just in case + PushTestMergeCommits = PushTestMergeCommits, + //revision information and the rest retrieved by controller + ShowTestMergeCommitters = ShowTestMergeCommitters + }; } } diff --git a/src/Tgstation.Server.Host/Models/RevisionInformation.cs b/src/Tgstation.Server.Host/Models/RevisionInformation.cs index 69960d6356..23db58d0f8 100644 --- a/src/Tgstation.Server.Host/Models/RevisionInformation.cs +++ b/src/Tgstation.Server.Host/Models/RevisionInformation.cs @@ -31,7 +31,8 @@ namespace Tgstation.Server.Host.Models { CommitSha = CommitSha, OriginCommitSha = OriginCommitSha, - TestMerges = TestMerges.Select(x => x.ToApi()).ToList() + TestMerges = TestMerges.Select(x => x.ToApi()).ToList(), + CompileJobs = CompileJobs.Select(x => x.ToApi()).ToList() }; } }