From 9cfb039f3092542e5e5c0ca2bd9290def23050b1 Mon Sep 17 00:00:00 2001 From: Dominion Date: Sat, 1 Apr 2023 17:31:42 -0400 Subject: [PATCH 1/3] Documentation updates --- README.md | 1 + docs/Features.dox | 34 ++++++++++++++++++++++++++++------ 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 7df09e1651..c314551797 100644 --- a/README.md +++ b/README.md @@ -422,6 +422,7 @@ TGS supports creating infinite chat bots for notifying staff or players of thing - Internet Relay Chat (IRC) - Discord + - NOTE: Bot MUST have Message Content Intent enabled More can be added by providing a new implementation of the [IProvider](src/Tgstation.Server.Host/Components/Chat/Providers/IProvider.cs) interface diff --git a/docs/Features.dox b/docs/Features.dox index 8958e28807..9281f07f86 100644 --- a/docs/Features.dox +++ b/docs/Features.dox @@ -10,7 +10,7 @@ - Limitation: Users can be disabled but not deleted. - Proper Long Running Operation Support: Server actions take a long time, from a git pull to a DreamMaker compile. TGS now internally allows for them to be run in parallel with each other and provides an audit record via the database. This is an improvement over the old system where connections had to be held open for the duration of operations. - Database Backend: TGS requires an SQL database to operate. This allows for much better concurrency and is just overall much cleaner than the old single json file storage blob per instance. - - Limitation: There is a one-to-one relationship with a TGS server and a database. **DO NOT SHARE TGS DATABASES**. + - Limitation: There is a one-to-one relationship with a TGS server and a database. **DO NOT SHARE TGS DATABASES OUTSIDE OF SWARM MODE**. - Linux/Docker Support: TGS is Linux and docker compatible. (Note this does not mean that rust-g and BSQL work out of the box, they must be compiled using event scripts like PreCompile.sh). - Limitation: TGS has a dependency on the native library libgit2 which is known to cause issues on Linux. The binaries distributed with TGS are kept up to date with the upstream repository, but out of the box Linux support can't be assured in every environment. Docker is guaranteed to always work, however. See the repository for the distributed binary here: https://github.com/libgit2/libgit2sharp.nativebinaries. - Limitation: System based logins are not supported on Linux. https://github.com/tgstation/tgstation-server/issues/709 @@ -30,13 +30,12 @@ - Safe/Ultrasafe Security Support: Thanks to the new DMAPI, the ultrasafe and safe security levels may be used without running into BYOND's limitations. But no one really cares... - Private/Invisible visibility Support: Stored per instance. - Self Upgrading: To upgrade TGS3 you needed to download and run the installer. This was pretty seamless, but it's now even better in versions >=4 as the command to upgrade can be given straight to the API. At that point the server will handle downloading the update, detaching running DreamDaemon instances, restarting with the new version, and reattaching to them. Easier than ever patch delivery. +- Multi-server co-operation: TGS installed on multiple machines can share the same database by using Swarm Mode. This mode keeps TGS and account details in sync on all machines. +- OAuth 2.0 Support: Integrate with your favorite 3rd party authentication providers - *Gasp* TESTING: TGS currently has over 60% code coverage in automated unit and full stack integration tests. I aim to have that number ever increasing to prevent trivial mistakes. Big improvement over V3 which had... literally none... Along with these features, nearly every single V3 feature has been included and possibly improved in some fashion. This includes stuff like Windows accounts for logins, and using ACLs for static file handling. The following exceptions exist but are planned for future updates: - Process memory/CPU diagnostic data is not generated: https://github.com/tgstation/tgstation-server/issues/611 -- Process dumps may not be created: https://github.com/tgstation/tgstation-server/issues/612 -- The DMAPI is required in DreamMaker code: https://github.com/tgstation/tgstation-server/issues/934 -- The option to "Initialize Game Directories" is no longer present, but a variant is still performed every deployment for smoother error handling. - Direct server announcements are no longer present but may be readded upon request. @section features_list Comprehensive Feature List @@ -44,11 +43,26 @@ Along with these features, nearly every single V3 feature has been included and tgstation-server is a BYOND server managment suite. It includes all the following features - Standalone server with OpenAPI 3.0 defined HTTP REST API +- Web based client included - Secure user authentication, management, and permissions system - Support for using Windows accounts as logins -- Long running operation support via the jobs subsystem +- OAuth 2.0 support with several providers + - InvisionCommunity + - /tg/ Forums + - Discord + - GitHub + - Keycloak +- Long running operation support via the jobs subsystem with error handling and stage reporting - Logging system with configurable levels of verbosity + - Logs can be downloaded via the API + - Supports Elasticsearch ingesting - Able to self update with user input +- Swarm system to coordinate servers across systems +- Database backend supporting several providers + - Microsoft SQL Server + - MySQL/MariaDB + - PostgresSQL + - SQLite - Instances: Managing multiple sets of the below features seperately on the same machine - Git repository managment - Cloning @@ -63,10 +77,13 @@ tgstation-server is a BYOND server managment suite. It includes all the followin - Includes DirectX installation on Windows and proper handling of POSIX shared libraries - Supporting any amount of installed versions - Cleaning the BYOND cache + - Uploading a custom version .zip - Code Deployment - Compiles git code with DreamMaker and the active BYOND version - Validates interop with the compiled binary - Can be scheduled to pull the repository and build on a given interval + - Test merge commenting support for GitHub and GitLab + - GitHub Deployment Environments support - The Watchdog, a DreamDaemon uptime monitor - Automatically restarts the server when crashed until told to stop - "Heartbeat" system for checking DreamDaemon isn't hung @@ -75,11 +92,15 @@ tgstation-server is a BYOND server managment suite. It includes all the followin - Automatically handles Windows prompts related to trusted mode - TGS restarts/updates do not interrupt the DreamDaemon process - Raises process priority of DreamDaemon for increased performance + - Support for logging DreamDaemon output + - Support for the `-profile` command line option + - Safe session recovery from TGS crashes - Chat System - Support multiple chat bots per instance. Current providers: - Discord - - Supports rich embeds when deployments occur + - Supports rich embeds created from DM code and when deployments occur - IRC + - Supports several authentication types - Bots come out of the box with some basic commands like showing the server revision - Support different chat channel types & tags for use in DM code - Configuration system @@ -94,5 +115,6 @@ tgstation-server is a BYOND server managment suite. It includes all the followin - Provides API for implementing chat bot commands in DM code - Gives the ability to restart the DreamDaemon process from DM code - Functions with all 3 DreamDaemon security levels + - Provides notifications of TGS side events - Allows specifying the .dmb's minimum required security level */ From b6084292bd3b563977a4df3eb922ac4e4a3eb133 Mon Sep 17 00:00:00 2001 From: Dominion Date: Sat, 1 Apr 2023 17:55:03 -0400 Subject: [PATCH 2/3] Add support for showing conflicting files in repo update error --- .../Components/Events/EventType.cs | 2 +- .../Components/Repository/IRepository.cs | 4 +- .../Components/Repository/Repository.cs | 49 ++++++++++++------- .../Repository/RepositoryUpdateService.cs | 6 ++- .../Components/Repository/TestMergeResult.cs | 22 +++++++++ 5 files changed, 61 insertions(+), 22 deletions(-) create mode 100644 src/Tgstation.Server.Host/Components/Repository/TestMergeResult.cs diff --git a/src/Tgstation.Server.Host/Components/Events/EventType.cs b/src/Tgstation.Server.Host/Components/Events/EventType.cs index a518f37712..0a54518a24 100644 --- a/src/Tgstation.Server.Host/Components/Events/EventType.cs +++ b/src/Tgstation.Server.Host/Components/Events/EventType.cs @@ -85,7 +85,7 @@ InstanceAutoUpdateStart, /// - /// Parameters: Base sha, target sha, base reference, target reference + /// Parameters: Base sha, target sha, base reference, target reference, all conflicting files /// [EventScript("RepoMergeConflict")] RepoMergeConflict, diff --git a/src/Tgstation.Server.Host/Components/Repository/IRepository.cs b/src/Tgstation.Server.Host/Components/Repository/IRepository.cs index 73574da2af..ae54a9a8c7 100644 --- a/src/Tgstation.Server.Host/Components/Repository/IRepository.cs +++ b/src/Tgstation.Server.Host/Components/Repository/IRepository.cs @@ -69,8 +69,8 @@ namespace Tgstation.Server.Host.Components.Repository /// If a submodule update should be attempted after the merge. /// The to report progress of the operation. /// The for the operation. - /// A resulting in a representing the merge result that is after a fast forward or up to date, on a non-fast-forward, on a conflict. - Task AddTestMerge( + /// A resulting in the . + Task AddTestMerge( TestMergeParameters testMergeParameters, string committerName, string committerEmail, diff --git a/src/Tgstation.Server.Host/Components/Repository/Repository.cs b/src/Tgstation.Server.Host/Components/Repository/Repository.cs index 9e0bea04f3..db3d04bcc9 100644 --- a/src/Tgstation.Server.Host/Components/Repository/Repository.cs +++ b/src/Tgstation.Server.Host/Components/Repository/Repository.cs @@ -172,7 +172,7 @@ namespace Tgstation.Server.Host.Components.Repository /// #pragma warning disable CA1506 // TODO: Decomplexify - public async Task AddTestMerge( + public async Task AddTestMerge( TestMergeParameters testMergeParameters, string committerName, string committerEmail, @@ -225,6 +225,7 @@ namespace Tgstation.Server.Host.Components.Repository var progressFactor = 1.0 / (updateSubmodules ? 3 : 2); var sig = new Signature(new Identity(committerName, committerEmail), DateTimeOffset.UtcNow); + List conflictedPaths = null; await Task.Factory.StartNew( () => { @@ -279,7 +280,7 @@ namespace Tgstation.Server.Host.Components.Repository result = libGitRepo.Merge(testMergeParameters.TargetCommitSha, sig, new MergeOptions { CommitOnSuccess = commitMessage == null, - FailOnConflict = true, + FailOnConflict = false, // Needed to get conflicting files FastForwardStrategy = FastForwardStrategy.NoFastForward, SkipReuc = true, OnCheckoutProgress = CheckoutProgressHandler( @@ -295,6 +296,12 @@ namespace Tgstation.Server.Host.Components.Repository if (result.Status == MergeStatus.Conflicts) { + var repoStatus = libGitRepo.RetrieveStatus(); + conflictedPaths = new List(); + foreach (var file in repoStatus) + if (file.State == FileStatus.Conflicted) + conflictedPaths.Add(file.FilePath); + var revertTo = originalCommit.CanonicalName ?? originalCommit.Tip.Sha; logger.LogDebug("Merge conflict, aborting and reverting to {0}", revertTo); progressReporter.ReportProgress(0); @@ -306,23 +313,29 @@ namespace Tgstation.Server.Host.Components.Repository }, cancellationToken, DefaultIOManager.BlockingTaskCreationOptions, - TaskScheduler.Current) - ; + TaskScheduler.Current); if (result.Status == MergeStatus.Conflicts) { + var arguments = new List + { + originalCommit.Tip.Sha, + testMergeParameters.TargetCommitSha, + originalCommit.FriendlyName ?? UnknownReference, + testMergeBranchName, + }; + + arguments.AddRange(conflictedPaths); + await eventConsumer.HandleEvent( EventType.RepoMergeConflict, - new List - { - originalCommit.Tip.Sha, - testMergeParameters.TargetCommitSha, - originalCommit.FriendlyName ?? UnknownReference, - testMergeBranchName, - }, - cancellationToken) - ; - return null; + arguments, + cancellationToken); + return new TestMergeResult + { + Status = result.Status, + ConflictingFiles = conflictedPaths, + }; } if (result.Status != MergeStatus.UpToDate) @@ -356,10 +369,12 @@ namespace Tgstation.Server.Host.Components.Repository testMergeParameters.TargetCommitSha, testMergeParameters.Comment, }, - cancellationToken) - ; + cancellationToken); - return result.Status != MergeStatus.NonFastForward; + return new TestMergeResult + { + Status = result.Status, + }; } #pragma warning restore CA1506 diff --git a/src/Tgstation.Server.Host/Components/Repository/RepositoryUpdateService.cs b/src/Tgstation.Server.Host/Components/Repository/RepositoryUpdateService.cs index 5e6b9808cf..4f283079de 100644 --- a/src/Tgstation.Server.Host/Components/Repository/RepositoryUpdateService.cs +++ b/src/Tgstation.Server.Host/Components/Repository/RepositoryUpdateService.cs @@ -4,6 +4,8 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; +using LibGit2Sharp; + using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; @@ -499,11 +501,11 @@ namespace Tgstation.Server.Host.Components.Repository NextProgressReporter($"Test merge #{newTestMerge.Number}"), cancellationToken); - if (mergeResult == null) + if (mergeResult.Status == MergeStatus.Conflicts) throw new JobException( ErrorCode.RepoTestMergeConflict, new JobException( - $"Test Merge #{newTestMerge.Number} at {newTestMerge.TargetCommitSha[..7]} conflicted!")); + $"Test Merge #{newTestMerge.Number} at {newTestMerge.TargetCommitSha[..7]} conflicted! Conflicting files:{Environment.NewLine}{String.Join(Environment.NewLine, mergeResult.ConflictingFiles.Select(file => $"\t- /{file}"))}")); Models.TestMerge fullTestMerge; try diff --git a/src/Tgstation.Server.Host/Components/Repository/TestMergeResult.cs b/src/Tgstation.Server.Host/Components/Repository/TestMergeResult.cs new file mode 100644 index 0000000000..387a6baad1 --- /dev/null +++ b/src/Tgstation.Server.Host/Components/Repository/TestMergeResult.cs @@ -0,0 +1,22 @@ +using System.Collections.Generic; + +using LibGit2Sharp; + +namespace Tgstation.Server.Host.Components.Repository +{ + /// + /// Represents the result of a repository test merge attempt. + /// + public sealed class TestMergeResult + { + /// + /// The resulting . + /// + public MergeStatus Status { get; init; } + + /// + /// List of conflicting file paths relative to the repository root. Only present if is . + /// + public IReadOnlyList ConflictingFiles { get; init; } + } +} From 3a22a86e15b312ff917bb2c049b4cee3a6f5773a Mon Sep 17 00:00:00 2001 From: Dominion Date: Sat, 1 Apr 2023 18:20:37 -0400 Subject: [PATCH 3/3] Update to webpanel version 4.17.0 --- build/ControlPanelVersion.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/ControlPanelVersion.props b/build/ControlPanelVersion.props index 01d8243a3d..ff4985b645 100644 --- a/build/ControlPanelVersion.props +++ b/build/ControlPanelVersion.props @@ -1,6 +1,6 @@ - 4.15.0 + 4.17.0