diff --git a/docs/API.dox b/docs/API.dox index a74f211420..80b65f0099 100644 --- a/docs/API.dox +++ b/docs/API.dox @@ -9,7 +9,7 @@ The TGS4 API is designed to be a fully realized RESTful service. Once hosted, fo Routes and their usages are defined as follows -`[I (If Instance is required)] "" [Request Model] => ` +[I (If Instance is required)] <`Http Method`> "<`Route`>`" [Request Model] => <`Response Model`> @section api_lib Official Libraries @@ -29,7 +29,7 @@ This document will reference the canonical C# models in the @ref Tgstation.Serve @section api_header Headers -TGS4 expects this set of headers. Failure to provide them may result in +TGS4 expects this set of headers. Failure to provide them may result in 400 error responses - User-Agent: The user agent product header value of the calling program. Should be in the form Agent/Version (i.e. SomeTgsClient/1.2.4) - Accept: application/json @@ -284,14 +284,14 @@ git pull (Only if @ref Tgstation.Server.Api.Models.Repository.Reference is set): } @endcode -git checkout (Unsets @ref Tgstation.Server.Api.Models.Repository.Reference): +`git checkout (Unsets @ref Tgstation.Server.Api.Models.Repository.Reference):` @code{.json} { "checkoutSha": "" } @endcode -git checkout -f && git clean -fxd (Sets @ref Tgstation.Server.Api.Models.Repository.Reference): +`git checkout -f && git clean -fxd (Sets @ref Tgstation.Server.Api.Models.Repository.Reference):` @code{.json} { "reference": "" @@ -364,4 +364,14 @@ The job object returned represents the compile job {} @endcode +The compiler endpoint is also used to read compile jobs + +To list successful compile jobs ids use: + +I GET "/DreamMaker/List" => Array of @ref Tgstation.Server.Api.Models.CompileJob + +To get a specific compile job use: + +I GET "/DreamMaker/{CompileJobId}" => @ref Tgstation.Server.Api.Models.CompileJob + */ diff --git a/src/Tgstation.Server.Api/Models/DreamMaker.cs b/src/Tgstation.Server.Api/Models/DreamMaker.cs index de1d893846..f50fb7ac64 100644 --- a/src/Tgstation.Server.Api/Models/DreamMaker.cs +++ b/src/Tgstation.Server.Api/Models/DreamMaker.cs @@ -1,5 +1,4 @@ using Tgstation.Server.Api.Models.Internal; -using Tgstation.Server.Api.Rights; namespace Tgstation.Server.Api.Models { diff --git a/src/Tgstation.Server.Api/Models/Internal/DreamMakerSettings.cs b/src/Tgstation.Server.Api/Models/Internal/DreamMakerSettings.cs index 154167ec82..86ad593915 100644 --- a/src/Tgstation.Server.Api/Models/Internal/DreamMakerSettings.cs +++ b/src/Tgstation.Server.Api/Models/Internal/DreamMakerSettings.cs @@ -1,4 +1,5 @@ -using Tgstation.Server.Api.Rights; +using System.ComponentModel.DataAnnotations; +using Tgstation.Server.Api.Rights; namespace Tgstation.Server.Api.Models.Internal { @@ -13,5 +14,12 @@ namespace Tgstation.Server.Api.Models.Internal /// [Permissions(WriteRight = DreamMakerRights.SetDme)] public string ProjectName { get; set; } + + /// + /// The port used during compilation to validate the TGS API + /// + [Permissions(WriteRight = DreamMakerRights.SetApiValidationPort)] + [Required] + public ushort? ApiValidationPort { get; set; } } } diff --git a/src/Tgstation.Server.Api/Rights/DreamMakerRights.cs b/src/Tgstation.Server.Api/Rights/DreamMakerRights.cs index bce942c165..3ad8cf6078 100644 --- a/src/Tgstation.Server.Api/Rights/DreamMakerRights.cs +++ b/src/Tgstation.Server.Api/Rights/DreamMakerRights.cs @@ -27,6 +27,14 @@ namespace Tgstation.Server.Api.Rights /// /// User may modify /// - SetDme = 8 + SetDme = 8, + /// + /// User may modify + /// + SetApiValidationPort = 16, + /// + /// User may list and read all s + /// + List = 32 } } diff --git a/src/Tgstation.Server.Host/Components/Chat/Chat.cs b/src/Tgstation.Server.Host/Components/Chat/Chat.cs index 7178a1e2cd..15121806f3 100644 --- a/src/Tgstation.Server.Host/Components/Chat/Chat.cs +++ b/src/Tgstation.Server.Host/Components/Chat/Chat.cs @@ -164,8 +164,6 @@ namespace Tgstation.Server.Host.Components.Chat /// A representing the running operation async Task ProcessMessage(IProvider provider, Message message, CancellationToken cancellationToken) { - logger.LogTrace("Chat message: {0}. User (Note unconverted provider Id): {1}", message.Content, JsonConvert.SerializeObject(message.User)); - //map the channel if it's private and we haven't seen it if (message.User.Channel.IsPrivate) lock (providers) @@ -205,6 +203,8 @@ namespace Tgstation.Server.Host.Components.Chat //no mention return; + logger.LogTrace("Chat command: {0}. User (True provider Id): {1}", message.Content, JsonConvert.SerializeObject(message.User)); + if (addressed) splits.RemoveAt(0); diff --git a/src/Tgstation.Server.Host/Components/Compiler/DreamMaker.cs b/src/Tgstation.Server.Host/Components/Compiler/DreamMaker.cs index a269fcea9d..3adfbf0405 100644 --- a/src/Tgstation.Server.Host/Components/Compiler/DreamMaker.cs +++ b/src/Tgstation.Server.Host/Components/Compiler/DreamMaker.cs @@ -80,7 +80,7 @@ namespace Tgstation.Server.Host.Components.Compiler /// The for /// readonly ILogger logger; - + /// /// Construct /// @@ -115,15 +115,16 @@ namespace Tgstation.Server.Host.Components.Compiler /// The level to use to validate the API /// The for the operation /// The current + /// The port to use for API validation /// The for the operation /// A resulting in if the DMAPI was successfully validated, otherwise - async Task VerifyApi(uint timeout, DreamDaemonSecurity securityLevel, Models.CompileJob job, IByondExecutableLock byondLock, CancellationToken cancellationToken) + async Task VerifyApi(uint timeout, DreamDaemonSecurity securityLevel, Models.CompileJob job, IByondExecutableLock byondLock, ushort portToUse, CancellationToken cancellationToken) { - logger.LogTrace("Verifying DMAPI..."); + logger.LogTrace("Verifying DMAPI..."); var launchParameters = new DreamDaemonLaunchParameters { AllowWebClient = false, - PrimaryPort = 0, //pick any port + PrimaryPort = portToUse, SecurityLevel = securityLevel, //all it needs to read the file and exit StartupTimeout = timeout }; @@ -229,8 +230,14 @@ namespace Tgstation.Server.Host.Components.Compiler } /// - public async Task Compile(Models.RevisionInformation revisionInformation, string projectName, DreamDaemonSecurity securityLevel, uint apiValidateTimeout, IRepository repository, CancellationToken cancellationToken) + public async Task Compile(Models.RevisionInformation revisionInformation, DreamMakerSettings dreamMakerSettings, DreamDaemonSecurity securityLevel, uint apiValidateTimeout, IRepository repository, CancellationToken cancellationToken) { + if (revisionInformation == null) + throw new ArgumentNullException(nameof(revisionInformation)); + + if (dreamMakerSettings == null) + throw new ArgumentNullException(nameof(dreamMakerSettings)); + if (repository == null) throw new ArgumentNullException(nameof(repository)); @@ -242,7 +249,7 @@ namespace Tgstation.Server.Host.Components.Compiler var job = new Models.CompileJob { DirectoryName = Guid.NewGuid(), - DmeName = projectName, + DmeName = dreamMakerSettings.ProjectName, RevisionInformation = revisionInformation }; @@ -260,12 +267,15 @@ namespace Tgstation.Server.Host.Components.Compiler Status = CompilerStatus.Copying; } - var commitInsert = revisionInformation.CommitSha; - var remoteCommitInsert = String.Empty; - if (commitInsert == revisionInformation.OriginCommitSha) - commitInsert = String.Format(CultureInfo.InvariantCulture, "^{0}", commitInsert.Substring(0, 7)); + var commitInsert = revisionInformation.CommitSha.Substring(0, 7); + string remoteCommitInsert; + if (revisionInformation.CommitSha == revisionInformation.OriginCommitSha) + { + commitInsert = String.Format(CultureInfo.InvariantCulture, "^{0}", commitInsert); + remoteCommitInsert = String.Empty; + } else - remoteCommitInsert = String.Format(CultureInfo.InvariantCulture, ". Remote commit: ^{0}", revisionInformation.OriginCommitSha); + remoteCommitInsert = String.Format(CultureInfo.InvariantCulture, ". Remote commit: ^{0}", revisionInformation.OriginCommitSha.Substring(0, 7)); var testmergeInsert = revisionInformation.ActiveTestMerges.Count == 0 ? String.Empty : String.Format(CultureInfo.InvariantCulture, " (Test Merges: {0})", String.Join(", ", revisionInformation.ActiveTestMerges.Select(x => x.TestMerge).Select(x => @@ -286,11 +296,11 @@ namespace Tgstation.Server.Host.Components.Compiler var dirA = ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName); var dirB = ioManager.ConcatPath(job.DirectoryName.ToString(), BDirectoryName); - async Task CleanupFailedCompile() + async Task CleanupFailedCompile(bool announce) { logger.LogTrace("Cleaning compile directory..."); Status = CompilerStatus.Cleanup; - var chatTask = chat.SendUpdateMessage("DM: Deploy failed!", cancellationToken); + var chatTask = announce ? chat.SendUpdateMessage("Deploy failed!", cancellationToken) : Task.CompletedTask; try { await ioManager.DeleteDirectory(job.DirectoryName.ToString(), CancellationToken.None).ConfigureAwait(false); @@ -314,7 +324,8 @@ namespace Tgstation.Server.Host.Components.Compiler Status = CompilerStatus.PreCompile; - await eventConsumer.HandleEvent(EventType.CompileStart, new List { ioManager.ResolvePath(ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName)), repoOrigin }, cancellationToken).ConfigureAwait(false); + var resolvedGameDirectory = ioManager.ResolvePath(ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName)); + await eventConsumer.HandleEvent(EventType.CompileStart, new List { resolvedGameDirectory, repoOrigin }, cancellationToken).ConfigureAwait(false); Status = CompilerStatus.Modifying; @@ -339,53 +350,51 @@ namespace Tgstation.Server.Host.Components.Compiler Status = CompilerStatus.Compiling; //run compiler, verify api - bool ddVerified; job.ByondVersion = byondLock.Version.ToString(); await RunDreamMaker(byondLock.DreamMakerPath, job, cancellationToken).ConfigureAwait(false); - Status = CompilerStatus.Verifying; + if (job.ExitCode == 0) + { + Status = CompilerStatus.Verifying; - ddVerified = job.ExitCode == 0 && await VerifyApi(apiValidateTimeout, securityLevel, job, byondLock, cancellationToken).ConfigureAwait(false); + job.DMApiValidated = await VerifyApi(apiValidateTimeout, securityLevel, job, byondLock, dreamMakerSettings.ApiValidationPort.Value, cancellationToken).ConfigureAwait(false); + } - if (!ddVerified) + if (job.DMApiValidated != true) { //server never validated or compile failed - await CleanupFailedCompile().ConfigureAwait(false); - await eventConsumer.HandleEvent(EventType.CompileFailure, new List { job.ExitCode == 0 ? "1" : "0" }, cancellationToken).ConfigureAwait(false); + await eventConsumer.HandleEvent(EventType.CompileFailure, new List { resolvedGameDirectory, job.ExitCode == 0 ? "1" : "0" }, cancellationToken).ConfigureAwait(false); + throw new Exception(job.ExitCode == 0 ? "Validation of the TGS api failed!" : "DM exited with a non-zero code!"); } - else - { - job.DMApiValidated = true; - logger.LogTrace("Running post compile event..."); - Status = CompilerStatus.PostCompile; - await eventConsumer.HandleEvent(EventType.CompileComplete, new List { ioManager.ResolvePath(ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName)) }, cancellationToken).ConfigureAwait(false); + logger.LogTrace("Running post compile event..."); + Status = CompilerStatus.PostCompile; + await eventConsumer.HandleEvent(EventType.CompileComplete, new List { ioManager.ResolvePath(ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName)) }, cancellationToken).ConfigureAwait(false); - logger.LogTrace("Duplicating compiled game..."); - Status = CompilerStatus.Duplicating; + logger.LogTrace("Duplicating compiled game..."); + Status = CompilerStatus.Duplicating; - //duplicate the dmb et al - await ioManager.CopyDirectory(dirA, dirB, null, cancellationToken).ConfigureAwait(false); + //duplicate the dmb et al + await ioManager.CopyDirectory(dirA, dirB, null, cancellationToken).ConfigureAwait(false); - logger.LogTrace("Applying static game file symlinks..."); - Status = CompilerStatus.Symlinking; + logger.LogTrace("Applying static game file symlinks..."); + Status = CompilerStatus.Symlinking; - //symlink in the static data - var symATask = configuration.SymlinkStaticFilesTo(fullDirA, cancellationToken); - var symBTask = configuration.SymlinkStaticFilesTo(ioManager.ResolvePath(dirB), cancellationToken); + //symlink in the static data + var symATask = configuration.SymlinkStaticFilesTo(fullDirA, cancellationToken); + var symBTask = configuration.SymlinkStaticFilesTo(ioManager.ResolvePath(dirB), cancellationToken); - await Task.WhenAll(symATask, symBTask).ConfigureAwait(false); + await Task.WhenAll(symATask, symBTask).ConfigureAwait(false); - await chat.SendUpdateMessage("Deployment complete! Changes will be applied on next server reboot.", cancellationToken).ConfigureAwait(false); + await chat.SendUpdateMessage("Deployment complete! Changes will be applied on next server reboot.", cancellationToken).ConfigureAwait(false); - logger.LogDebug("Compile complete!"); - } + logger.LogDebug("Compile complete!"); return job; } - catch + catch (Exception e) { - await CleanupFailedCompile().ConfigureAwait(false); + await CleanupFailedCompile(!(e is OperationCanceledException)).ConfigureAwait(false); throw; } } diff --git a/src/Tgstation.Server.Host/Components/Compiler/IDreamMaker.cs b/src/Tgstation.Server.Host/Components/Compiler/IDreamMaker.cs index 9bbc9eda84..923de6c414 100644 --- a/src/Tgstation.Server.Host/Components/Compiler/IDreamMaker.cs +++ b/src/Tgstation.Server.Host/Components/Compiler/IDreamMaker.cs @@ -1,6 +1,7 @@ using System.Threading; using System.Threading.Tasks; using Tgstation.Server.Api.Models; +using Tgstation.Server.Api.Models.Internal; using Tgstation.Server.Host.Components.Repository; namespace Tgstation.Server.Host.Components.Compiler @@ -19,12 +20,12 @@ namespace Tgstation.Server.Host.Components.Compiler /// Starts a compile /// /// The being compiled from the - /// The optional name of the .dme to compile without the extension if not pre + /// The for the compile /// The level allowed for API validation /// The time in seconds to wait while validating the API /// The to copy from /// The for the operation /// A resulting in the partially populated for the operation. In particular, note the field will only have it's field populated - Task Compile(Models.RevisionInformation revisionInformation, string projectName, DreamDaemonSecurity securityLevel, uint apiValidateTimeout, IRepository repository, CancellationToken cancellationToken); + Task Compile(Models.RevisionInformation revisionInformation, DreamMakerSettings dreamMakerSettings, DreamDaemonSecurity securityLevel, uint apiValidateTimeout, IRepository repository, CancellationToken cancellationToken); } } \ No newline at end of file diff --git a/src/Tgstation.Server.Host/Components/EventType.cs b/src/Tgstation.Server.Host/Components/EventType.cs index 57de1c2bac..e60a1661a5 100644 --- a/src/Tgstation.Server.Host/Components/EventType.cs +++ b/src/Tgstation.Server.Host/Components/EventType.cs @@ -47,7 +47,7 @@ /// CompileCancelled = 9, /// - /// Parameters: "1" if compile succeeded and api validation failed, "0" otherwise + /// Parameters: Game directory path, "1" if compile succeeded and api validation failed, "0" otherwise /// CompileFailure = 10, /// diff --git a/src/Tgstation.Server.Host/Components/Instance.cs b/src/Tgstation.Server.Host/Components/Instance.cs index 59de36fb74..98d88ce16b 100644 --- a/src/Tgstation.Server.Host/Components/Instance.cs +++ b/src/Tgstation.Server.Host/Components/Instance.cs @@ -133,8 +133,8 @@ namespace Tgstation.Server.Host.Components StartupTimeout = x.StartupTimeout, SecurityLevel = x.SecurityLevel }).FirstAsync(cancellationToken); - var projectNameTask = instanceQuery.Select(x => x.DreamMakerSettings.ProjectName).FirstOrDefaultAsync(cancellationToken); - var repositorySettingsTask = instanceQuery.Select(x => x.RepositorySettings).FirstOrDefaultAsync(cancellationToken); + var dmSettingsTask = instanceQuery.Select(x => x.DreamMakerSettings).FirstAsync(cancellationToken); + var repositorySettingsTask = instanceQuery.Select(x => x.RepositorySettings).FirstAsync(cancellationToken); using (var repo = await RepositoryManager.LoadRepository(cancellationToken).ConfigureAwait(false)) { @@ -169,7 +169,7 @@ namespace Tgstation.Server.Host.Components await repo.Sychronize(repositorySettings.AccessUser, repositorySettings.AccessToken, shouldSyncTracked, cancellationToken).ConfigureAwait(false); //finish other queries - var projectName = await projectNameTask.ConfigureAwait(false); + var dmSettings = await dmSettingsTask.ConfigureAwait(false); var ddSettings = await ddSettingsTask.ConfigureAwait(false); var revInfo = await revInfoTask.ConfigureAwait(false); @@ -190,7 +190,7 @@ namespace Tgstation.Server.Host.Components } //finally start compile - job = await DreamMaker.Compile(revInfo, projectName, ddSettings.SecurityLevel.Value, ddSettings.StartupTimeout.Value, repo, cancellationToken).ConfigureAwait(false); + job = await DreamMaker.Compile(revInfo, dmSettings, ddSettings.SecurityLevel.Value, ddSettings.StartupTimeout.Value, repo, cancellationToken).ConfigureAwait(false); } db.CompileJobs.Add(job); diff --git a/src/Tgstation.Server.Host/Components/Interop/CommContext.cs b/src/Tgstation.Server.Host/Components/Interop/CommContext.cs index 19ee6d7777..515c30a882 100644 --- a/src/Tgstation.Server.Host/Components/Interop/CommContext.cs +++ b/src/Tgstation.Server.Host/Components/Interop/CommContext.cs @@ -1,6 +1,7 @@ using Microsoft.Extensions.Logging; using Newtonsoft.Json; using System; +using System.Collections.Generic; using System.IO; using System.Text; using System.Threading; @@ -96,7 +97,10 @@ namespace Tgstation.Server.Host.Components.Interop CommCommand command; try { - command = JsonConvert.DeserializeObject(file); + command = new CommCommand + { + Parameters = JsonConvert.DeserializeObject>(file) + }; } catch (JsonSerializationException ex) { diff --git a/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs b/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs index 8138100014..2b855a85a2 100644 --- a/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs +++ b/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs @@ -58,6 +58,29 @@ namespace Tgstation.Server.Host.Controllers }); } + /// + [TgsAuthorize(DreamMakerRights.List)] + public override async Task GetId(long id, CancellationToken cancellationToken) + { + var compileJob = await DatabaseContext.CompileJobs + .Where(x => x.Id == id && x.Job.Instance.Id == Instance.Id) + .Include(x => x.Job).ThenInclude(x => x.StartedBy) + .Include(x => x.RevisionInformation).ThenInclude(x => x.PrimaryTestMerge) + .Include(x => x.RevisionInformation).ThenInclude(x => x.ActiveTestMerges).ThenInclude(x => x.TestMerge) + .FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + if (compileJob == default) + return NotFound(); + return Json(compileJob.ToApi()); + } + + /// + [TgsAuthorize(DreamMakerRights.List)] + public override async Task List(CancellationToken cancellationToken) + { + var compileJobs = await DatabaseContext.CompileJobs.Where(x => x.Job.Instance.Id == Instance.Id).ToListAsync(cancellationToken).ConfigureAwait(false); + return Json(compileJobs.Select(x => x.ToApi())); + } + /// [TgsAuthorize(DreamMakerRights.Compile)] public override async Task Create([FromBody] Api.Models.DreamMaker model, CancellationToken cancellationToken) @@ -75,15 +98,33 @@ namespace Tgstation.Server.Host.Controllers } /// - [TgsAuthorize(DreamMakerRights.SetDme)] + [TgsAuthorize(DreamMakerRights.SetDme | DreamMakerRights.SetApiValidationPort)] public override async Task Update([FromBody] Api.Models.DreamMaker model, CancellationToken cancellationToken) { var hostModel = new DreamMakerSettings { InstanceId = Instance.Id }; + DatabaseContext.DreamMakerSettings.Attach(hostModel); - hostModel.ProjectName = model.ProjectName; + + if (model.ProjectName != null) + { + if (!AuthenticationContext.InstanceUser.DreamMakerRights.Value.HasFlag(DreamMakerRights.SetDme)) + return Forbid(); + if (model.ProjectName.Length == 0) + hostModel.ProjectName = null; + else + hostModel.ProjectName = model.ProjectName; + } + + if (model.ApiValidationPort.HasValue) + { + if (!AuthenticationContext.InstanceUser.DreamMakerRights.Value.HasFlag(DreamMakerRights.SetApiValidationPort)) + return Forbid(); + hostModel.ApiValidationPort = model.ApiValidationPort; + } + await DatabaseContext.Save(cancellationToken).ConfigureAwait(false); return await Read(cancellationToken).ConfigureAwait(false); } @@ -104,8 +145,8 @@ namespace Tgstation.Server.Host.Controllers var ddSettingsTask = databaseContext.DreamDaemonSettings.Where(x => x.InstanceId == instanceModel.Id).Select(x => new DreamDaemonSettings{ StartupTimeout = x.StartupTimeout, SecurityLevel = x.SecurityLevel - }).FirstOrDefaultAsync(cancellationToken); - var projectName = await databaseContext.DreamMakerSettings.Where(x => x.InstanceId == instanceModel.Id).Select(x => x.ProjectName).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + }).FirstAsync(cancellationToken); + var dreamMakerSettings = await databaseContext.DreamMakerSettings.Where(x => x.InstanceId == instanceModel.Id).FirstAsync(cancellationToken).ConfigureAwait(false); var ddSettings = await ddSettingsTask.ConfigureAwait(false); var instance = instanceManager.GetInstance(instanceModel); @@ -136,7 +177,7 @@ namespace Tgstation.Server.Host.Controllers databaseContext.Instances.Attach(revInfo.Instance); } - compileJob = await instance.DreamMaker.Compile(revInfo, projectName, ddSettings.SecurityLevel.Value, ddSettings.StartupTimeout.Value, repo, cancellationToken).ConfigureAwait(false); + compileJob = await instance.DreamMaker.Compile(revInfo, dreamMakerSettings, ddSettings.SecurityLevel.Value, ddSettings.StartupTimeout.Value, repo, cancellationToken).ConfigureAwait(false); } if (compileJob.DMApiValidated != true) diff --git a/src/Tgstation.Server.Host/Controllers/InstanceController.cs b/src/Tgstation.Server.Host/Controllers/InstanceController.cs index 503e6afb1f..e8482bd11d 100644 --- a/src/Tgstation.Server.Host/Controllers/InstanceController.cs +++ b/src/Tgstation.Server.Host/Controllers/InstanceController.cs @@ -122,7 +122,10 @@ namespace Tgstation.Server.Host.Controllers SoftShutdown = false, StartupTimeout = 20 }, - DreamMakerSettings = new DreamMakerSettings(), + DreamMakerSettings = new DreamMakerSettings + { + ApiValidationPort = 1339 + }, Name = model.Name, Online = false, Path = model.Path, diff --git a/src/Tgstation.Server.Host/Controllers/RepositoryController.cs b/src/Tgstation.Server.Host/Controllers/RepositoryController.cs index b170498ca5..686317f7fd 100644 --- a/src/Tgstation.Server.Host/Controllers/RepositoryController.cs +++ b/src/Tgstation.Server.Host/Controllers/RepositoryController.cs @@ -63,7 +63,7 @@ namespace Tgstation.Server.Host.Controllers IQueryable queryTarget = databaseContext.RevisionInformations; - var revisionInfo = await databaseContext.RevisionInformations.Where(x => x.CommitSha == repoSha && x.Instance.Id == instance.Id) + var revisionInfo = await databaseContext.RevisionInformations.Where(x => x.CommitSha == repoSha && x.Instance.Id == instance.Id) .Include(x => x.CompileJobs) .Include(x => x.ActiveTestMerges).ThenInclude(x => x.TestMerge) //minimal info, they can query the rest if they're allowed .FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); //search every rev info because LOL SHA COLLISIONS @@ -243,9 +243,12 @@ namespace Tgstation.Server.Host.Controllers if (model.Origin != null) return BadRequest(new ErrorMessage { Message = "origin cannot be modified without deleting the repository!" }); - if(model.NewTestMerges.Any(x => !x.Number.HasValue)) + if(model.NewTestMerges?.Any(x => !x.Number.HasValue) == true) return BadRequest(new ErrorMessage { Message = "All new test merges must provide a number!" }); + if(model.NewTestMerges?.Any(x => model.NewTestMerges.Any(y => x != y && x.Number == y.Number)) == true) + return BadRequest(new ErrorMessage { Message = "Cannot test merge the same PR twice in one job!" }); + var newTestMerges = model.NewTestMerges != null && model.NewTestMerges.Count > 0; var userRights = (RepositoryRights)AuthenticationContext.GetRight(RightsType.Repository); if (newTestMerges && !userRights.HasFlag(RepositoryRights.MergePullRequest)) @@ -282,7 +285,7 @@ namespace Tgstation.Server.Host.Controllers || (model.UpdateFromOrigin == true && !userRights.HasFlag(RepositoryRights.UpdateBranch))) return Forbid(); - if (currentModel.AccessToken.Length == 0 && currentModel.AccessUser.Length == 0) + if (currentModel.AccessToken?.Length == 0 && currentModel.AccessUser?.Length == 0) { //setting an empty string clears everything currentModel.AccessUser = null; @@ -385,50 +388,92 @@ namespace Tgstation.Server.Host.Controllers lastRevisionInfo.OriginCommitSha = repo.Head; } } - + + Dictionary prMap = null; //test merging if (newTestMerges) { - //optimization: if we've already merged these exact same commits in this fashion before, just find the rev info for it and check it out - Models.RevisionInformation revInfoWereLookingFor = null; - if(lastRevisionInfo.OriginCommitSha == lastRevisionInfo.CommitSha) - { - foreach (var I in model.NewTestMerges) - //normalize the shas to lowercase ala libgit2 -#pragma warning disable CA1308 // Normalize strings to uppercase - I.PullRequestRevision = I.PullRequestRevision?.ToLowerInvariant(); -#pragma warning restore CA1308 // Normalize strings to uppercase + //bit of sanitization + foreach (var I in model.NewTestMerges.Where(x => String.IsNullOrWhiteSpace(x.PullRequestRevision))) + I.PullRequestRevision = null; - revInfoWereLookingFor = await databaseContext.RevisionInformations - .Where(x => x.OriginCommitSha == lastRevisionInfo.OriginCommitSha && x.ActiveTestMerges.Count == model.NewTestMerges.Count) - //split here cause this bit probably has to be done locally - .Where(x => x.ActiveTestMerges.Select(y => y.TestMerge) - .All(y => model.NewTestMerges.Any(z => y.Number == z.Number && y.PullRequestRevision.StartsWith(z.PullRequestRevision, StringComparison.Ordinal)))) - .Include(x => x.ActiveTestMerges).ThenInclude(x => x.TestMerge) - .FirstOrDefaultAsync(ct).ConfigureAwait(false); + var gitHubClient = currentModel.AccessToken != null ? gitHubClientFactory.CreateClient(currentModel.AccessToken) : gitHubClientFactory.CreateClient(); + + var repoOwner = repo.GitHubOwner; + var repoName = repo.GitHubRepoName; + + Models.RevisionInformation revInfoWereLookingFor = null; + //optimization: if we've already merged these exact same commits in this fashion before, just find the rev info for it and check it out + if (lastRevisionInfo.OriginCommitSha == lastRevisionInfo.CommitSha) + { + //In order for this to work though we need the shas of all the commits + if (model.NewTestMerges.Any(x => x.PullRequestRevision == null)) + prMap = new Dictionary(); + + bool cantSearch = false; + foreach (var I in model.NewTestMerges) + { + if (I.PullRequestRevision != null) + //normalize the shas to lowercase ala libgit2 +#pragma warning disable CA1308 // Normalize strings to uppercase + I.PullRequestRevision = I.PullRequestRevision?.ToLowerInvariant(); +#pragma warning restore CA1308 // Normalize strings to uppercase + else + //retrieve the latest sha + try + { + var pr = await gitHubClient.PullRequest.Get(repoOwner, repoName, I.Number.Value).ConfigureAwait(false); + prMap.Add(I.Number.Value, pr); + I.PullRequestRevision = pr.Head.Sha; + } + catch + { + cantSearch = true; + break; + } + } + + if (!cantSearch) + { + var dbPull = await databaseContext.RevisionInformations + .Where(x => x.Instance.Id == Instance.Id && x.OriginCommitSha == lastRevisionInfo.OriginCommitSha && x.ActiveTestMerges.Count == model.NewTestMerges.Count) + .Include(x => x.ActiveTestMerges) + .ThenInclude(x => x.TestMerge) + .ToListAsync(cancellationToken).ConfigureAwait(false); + //split here cause this bit has to be done locally + revInfoWereLookingFor = dbPull + .Where(x => x.ActiveTestMerges.Select(y => y.TestMerge) + .All(y => model.NewTestMerges.Any(z => + y.Number == z.Number + && y.PullRequestRevision.StartsWith(z.PullRequestRevision, StringComparison.Ordinal) + && y.Comment == z.Comment || z.Comment == null))) + .FirstOrDefault(); + } } - if (revInfoWereLookingFor != null) + { + //goteem await repo.ResetToSha(revInfoWereLookingFor.CommitSha, cancellationToken).ConfigureAwait(false); + lastRevisionInfo = revInfoWereLookingFor; + } else { - var gitHubClient = currentModel.AccessToken != null ? gitHubClientFactory.CreateClient(currentModel.AccessToken) : gitHubClientFactory.CreateClient(); var contextUser = new Models.User { Id = AuthenticationContext.User.Id }; databaseContext.Users.Attach(contextUser); - var repoOwner = repo.GitHubOwner; - var repoName = repo.GitHubRepoName; foreach (var I in model.NewTestMerges) { Octokit.PullRequest pr = null; string errorMessage = null; try { - pr = await gitHubClient.PullRequest.Get(repoOwner, repoName, I.Number.Value).ConfigureAwait(false); + //load from cache if possible + if (prMap == null || !prMap.TryGetValue(I.Number.Value, out pr)) + pr = await gitHubClient.PullRequest.Get(repoOwner, repoName, I.Number.Value).ConfigureAwait(false); } catch (Octokit.RateLimitExceededException) { diff --git a/v4_prototype_TODO.txt b/v4_prototype_TODO.txt index 7333b2c540..934a941146 100644 --- a/v4_prototype_TODO.txt +++ b/v4_prototype_TODO.txt @@ -1,3 +1,5 @@ +Change permission bits to u64 + Verify the byond cache folder location on linux Test watchdog