diff --git a/.travis.yml b/.travis.yml index d5196bdd03..685ce06e27 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,6 +32,7 @@ jobs: packages: - libc6-i386 - libstdc++6:i386 + - gdb - env: - DoxGeneration=false - DockerBuild=false @@ -51,6 +52,7 @@ jobs: packages: - libc6-i386 - libstdc++6:i386 + - gdb - env: - DoxGeneration=false - DockerBuild=false @@ -74,6 +76,7 @@ jobs: - postgresql-12 - libc6-i386 - libstdc++6:i386 + - gdb - env: - DoxGeneration=false - DockerBuild=false diff --git a/README.md b/README.md index 16bf6e737d..ecf0d06e30 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,7 @@ The following dependencies are required to run tgstation-server on Linux alongsi - libc6-i386 - libstdc++6:i386 - libssl1.0.0 +- gdb (for using gcore to create core dumps) - gcc-multilib (Only on 64-bit systems) Note that tgstation-server has only ever been tested on Linux via it's [docker environment](build/Dockerfile#L22). If you are having trouble with something in a native installation, or figure out a required workaround, please contact project maintainers so this documentation may be better updated. diff --git a/build/Dockerfile b/build/Dockerfile index aa852c81e0..fa4e18131f 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -58,7 +58,8 @@ FROM mcr.microsoft.com/dotnet/core/aspnet:3.1-bionic #needed for byond RUN apt-get update \ && apt-get install -y \ - gcc-multilib \ + gcc-multilib \ + gdb \ && rm -rf /var/lib/apt/lists/* EXPOSE 5000 diff --git a/build/integration_test.sh b/build/integration_test.sh index 8492301359..33342223ac 100755 --- a/build/integration_test.sh +++ b/build/integration_test.sh @@ -1,6 +1,9 @@ #!/bin/bash set -e +# Needed so gcore can work +echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope + export TGS4_TEST_DISCORD_CHANNEL=493119635319947269 export TGS4_TEST_IRC_CHANNEL=\#botbus export TGS4_TEST_TEMP_DIRECTORY=~/tgs4_test diff --git a/src/Tgstation.Server.Api/Models/DreamDaemon.cs b/src/Tgstation.Server.Api/Models/DreamDaemon.cs index 38e29363e0..8f2545af4f 100644 --- a/src/Tgstation.Server.Api/Models/DreamDaemon.cs +++ b/src/Tgstation.Server.Api/Models/DreamDaemon.cs @@ -48,5 +48,10 @@ namespace Tgstation.Server.Api.Models /// If the server is undergoing a soft shutdown /// public bool? SoftShutdown { get; set; } + + /// + /// If a dump of the active DreamDaemon executable should be created. + /// + public bool? CreateDump { get; set; } } } diff --git a/src/Tgstation.Server.Api/Models/ErrorCode.cs b/src/Tgstation.Server.Api/Models/ErrorCode.cs index 1347fafed7..b6f3bd1e5b 100644 --- a/src/Tgstation.Server.Api/Models/ErrorCode.cs +++ b/src/Tgstation.Server.Api/Models/ErrorCode.cs @@ -519,5 +519,17 @@ namespace Tgstation.Server.Api.Models /// [Description("Cannot cancel the job as it is no longer running.")] JobStopped, + + /// + /// Missing GCore executable. + /// + [Description("Attempted to create a process dump but /usr/bin/gcore could not be located!")] + MissingGCore, + + /// + /// Non-zero gcore exit code. + /// + [Description("Could not create dump as gcore exited with a non-zero exit code!")] + GCoreFailure, } } \ No newline at end of file diff --git a/src/Tgstation.Server.Api/Rights/DreamDaemonRights.cs b/src/Tgstation.Server.Api/Rights/DreamDaemonRights.cs index ed341547cd..fd4c53f641 100644 --- a/src/Tgstation.Server.Api/Rights/DreamDaemonRights.cs +++ b/src/Tgstation.Server.Api/Rights/DreamDaemonRights.cs @@ -77,5 +77,10 @@ namespace Tgstation.Server.Api.Rights /// User can change /// SetHeartbeatInterval = 4096, + + /// + /// User can create DreamDaemon process dumps. + /// + CreateDump = 8192, } } diff --git a/src/Tgstation.Server.Api/Routes.cs b/src/Tgstation.Server.Api/Routes.cs index 6a2da6c8f0..2bb3f4e603 100644 --- a/src/Tgstation.Server.Api/Routes.cs +++ b/src/Tgstation.Server.Api/Routes.cs @@ -43,6 +43,11 @@ namespace Tgstation.Server.Api /// public const string DreamDaemon = Root + nameof(Models.DreamDaemon); + /// + /// For accessing DD diagnostics + /// + public const string Diagnostics = DreamDaemon + "/Diagnostics"; + /// /// The controller /// diff --git a/src/Tgstation.Server.Client/Components/DreamDaemonClient.cs b/src/Tgstation.Server.Client/Components/DreamDaemonClient.cs index 6c2d5869d2..566c8058fe 100644 --- a/src/Tgstation.Server.Client/Components/DreamDaemonClient.cs +++ b/src/Tgstation.Server.Client/Components/DreamDaemonClient.cs @@ -44,5 +44,8 @@ namespace Tgstation.Server.Client.Components /// public Task Update(DreamDaemon dreamDaemon, CancellationToken cancellationToken) => apiClient.Update(Routes.DreamDaemon, dreamDaemon ?? throw new ArgumentNullException(nameof(dreamDaemon)), instance.Id, cancellationToken); + + /// + public Task CreateDump(CancellationToken cancellationToken) => apiClient.Patch(Routes.Diagnostics, instance.Id, cancellationToken); } } \ No newline at end of file diff --git a/src/Tgstation.Server.Client/Components/IDreamDaemonClient.cs b/src/Tgstation.Server.Client/Components/IDreamDaemonClient.cs index 94fd0b5dbe..f5b78966bb 100644 --- a/src/Tgstation.Server.Client/Components/IDreamDaemonClient.cs +++ b/src/Tgstation.Server.Client/Components/IDreamDaemonClient.cs @@ -44,5 +44,12 @@ namespace Tgstation.Server.Client.Components /// The for the operation /// A resulting in the information Task Update(DreamDaemon dreamDaemon, CancellationToken cancellationToken); + + /// + /// Start a job to create a process dump of the active DreamDaemon executable. + /// + /// The for the operation. + /// A resulting in the of the running operation. + Task CreateDump(CancellationToken cancellationToken); } } diff --git a/src/Tgstation.Server.Host/Components/InstanceFactory.cs b/src/Tgstation.Server.Host/Components/InstanceFactory.cs index 844aa5c264..15876d33cd 100644 --- a/src/Tgstation.Server.Host/Components/InstanceFactory.cs +++ b/src/Tgstation.Server.Host/Components/InstanceFactory.cs @@ -202,6 +202,7 @@ namespace Tgstation.Server.Host.Components var repoIoManager = new ResolvingIOManager(instanceIoManager, "Repository"); var byondIOManager = new ResolvingIOManager(instanceIoManager, "Byond"); var gameIoManager = new ResolvingIOManager(instanceIoManager, "Game"); + var diagnosticsIOManager = new ResolvingIOManager(instanceIoManager, "Diagnostics"); var configurationIoManager = new ResolvingIOManager(instanceIoManager, "Configuration"); var configuration = new StaticFiles.Configuration(configurationIoManager, synchronousIOManager, symlinkFactory, processExecutor, postWriteHandler, platformIdentifier, loggerFactory.CreateLogger()); @@ -248,6 +249,7 @@ namespace Tgstation.Server.Host.Components reattachInfoHandler, sessionControllerFactory, gameIoManager, + diagnosticsIOManager, metadata.CloneMetadata(), metadata.DreamDaemonSettings); eventConsumer.SetWatchdog(watchdog); diff --git a/src/Tgstation.Server.Host/Components/Session/DeadSessionController.cs b/src/Tgstation.Server.Host/Components/Session/DeadSessionController.cs index 2f5ff7f5c4..a425b5aac3 100644 --- a/src/Tgstation.Server.Host/Components/Session/DeadSessionController.cs +++ b/src/Tgstation.Server.Host/Components/Session/DeadSessionController.cs @@ -123,5 +123,8 @@ namespace Tgstation.Server.Host.Components.Session /// public Task InstanceRenamed(string newInstanceName, CancellationToken cancellationToken) => Task.CompletedTask; + + /// + public Task CreateDump(string outputFile, CancellationToken cancellationToken) => throw new NotSupportedException(); } } diff --git a/src/Tgstation.Server.Host/Components/Session/SessionController.cs b/src/Tgstation.Server.Host/Components/Session/SessionController.cs index dca20a3a14..47cbac6bd8 100644 --- a/src/Tgstation.Server.Host/Components/Session/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Session/SessionController.cs @@ -696,5 +696,8 @@ namespace Tgstation.Server.Host.Components.Session new TopicParameters( new ChatUpdate(newChannels)), cancellationToken); + + /// + public Task CreateDump(string outputFile, CancellationToken cancellationToken) => process.CreateDump(outputFile, cancellationToken); } } diff --git a/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs index 96fbbbdf30..73d7a74d22 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs @@ -10,6 +10,7 @@ using Tgstation.Server.Host.Components.Deployment; using Tgstation.Server.Host.Components.Session; using Tgstation.Server.Host.Core; using Tgstation.Server.Host.Database; +using Tgstation.Server.Host.IO; using Tgstation.Server.Host.Jobs; namespace Tgstation.Server.Host.Components.Watchdog @@ -49,6 +50,7 @@ namespace Tgstation.Server.Host.Components.Watchdog /// The for the . /// The for the . /// The for the . + /// The for the . /// The for the . /// The for the . /// The for the . @@ -62,6 +64,7 @@ namespace Tgstation.Server.Host.Components.Watchdog IJobManager jobManager, IServerControl serverControl, IAsyncDelayer asyncDelayer, + IIOManager diagnosticsIOManager, ILogger logger, DreamDaemonLaunchParameters initialLaunchParameters, Api.Models.Instance instance, @@ -75,6 +78,7 @@ namespace Tgstation.Server.Host.Components.Watchdog jobManager, serverControl, asyncDelayer, + diagnosticsIOManager, logger, initialLaunchParameters, instance, diff --git a/src/Tgstation.Server.Host/Components/Watchdog/ExperimentalWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/ExperimentalWatchdog.cs index a399db4e8d..0addffef36 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/ExperimentalWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/ExperimentalWatchdog.cs @@ -11,6 +11,7 @@ using Tgstation.Server.Host.Components.Deployment; using Tgstation.Server.Host.Components.Session; using Tgstation.Server.Host.Core; using Tgstation.Server.Host.Database; +using Tgstation.Server.Host.IO; using Tgstation.Server.Host.Jobs; namespace Tgstation.Server.Host.Components.Watchdog @@ -60,6 +61,7 @@ namespace Tgstation.Server.Host.Components.Watchdog /// The for the . /// The for the . /// The for the . + /// The for the . /// The for the . /// The for the . /// The for the . @@ -73,6 +75,7 @@ namespace Tgstation.Server.Host.Components.Watchdog IJobManager jobManager, IServerControl serverControl, IAsyncDelayer asyncDelayer, + IIOManager diagnosticsIOManager, ILogger logger, DreamDaemonLaunchParameters initialLaunchParameters, Api.Models.Instance instance, bool autoStart) @@ -85,6 +88,7 @@ namespace Tgstation.Server.Host.Components.Watchdog jobManager, serverControl, asyncDelayer, + diagnosticsIOManager, logger, initialLaunchParameters, instance, diff --git a/src/Tgstation.Server.Host/Components/Watchdog/IWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/IWatchdog.cs index 6c40b1bd97..bc2fdbd8a7 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/IWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/IWatchdog.cs @@ -80,5 +80,12 @@ namespace Tgstation.Server.Host.Components.Watchdog /// The for the operation /// A representing the running operation Task ResetRebootState(CancellationToken cancellationToken); + + /// + /// Attempt to create a process dump for DreamDaemon. + /// + /// The for the operation. + /// A representing the running operation. + Task CreateDump(CancellationToken cancellationToken); } } diff --git a/src/Tgstation.Server.Host/Components/Watchdog/IWatchdogFactory.cs b/src/Tgstation.Server.Host/Components/Watchdog/IWatchdogFactory.cs index 4734c17f85..63f30ec362 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/IWatchdogFactory.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/IWatchdogFactory.cs @@ -18,7 +18,8 @@ namespace Tgstation.Server.Host.Components.Watchdog /// The for the with /// The for the /// The for the - /// The for the . + /// The pointing to the Game directory for the . + /// The pointing to the Diagnostics directory for the . /// The for the /// The initial for the /// A new @@ -27,7 +28,8 @@ namespace Tgstation.Server.Host.Components.Watchdog IDmbFactory dmbFactory, IReattachInfoHandler reattachInfoHandler, ISessionControllerFactory sessionControllerFactory, - IIOManager ioManager, + IIOManager gameIOManager, + IIOManager diagnosticsIOManager, Api.Models.Instance instance, DreamDaemonSettings settings); } diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs index 6f2b0341d1..d8e6139afa 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs @@ -18,6 +18,7 @@ using Tgstation.Server.Host.Components.Session; using Tgstation.Server.Host.Core; using Tgstation.Server.Host.Database; using Tgstation.Server.Host.Extensions; +using Tgstation.Server.Host.IO; using Tgstation.Server.Host.Jobs; namespace Tgstation.Server.Host.Components.Watchdog @@ -114,6 +115,11 @@ namespace Tgstation.Server.Host.Components.Watchdog /// readonly IRestartRegistration restartRegistration; + /// + /// The pointing to the Diagnostics directory. + /// + readonly IIOManager diagnosticsIOManager; + /// /// used for . /// @@ -170,6 +176,7 @@ namespace Tgstation.Server.Host.Components.Watchdog /// The value of /// The to populate with /// The value of . + /// The value of . /// The value of /// The initial value of . May be modified /// The value of @@ -183,6 +190,7 @@ namespace Tgstation.Server.Host.Components.Watchdog IJobManager jobManager, IServerControl serverControl, IAsyncDelayer asyncDelayer, + IIOManager diagnosticsIOManager, ILogger logger, DreamDaemonLaunchParameters initialLaunchParameters, Api.Models.Instance instance, @@ -195,6 +203,7 @@ namespace Tgstation.Server.Host.Components.Watchdog this.databaseContextFactory = databaseContextFactory ?? throw new ArgumentNullException(nameof(databaseContextFactory)); this.jobManager = jobManager ?? throw new ArgumentNullException(nameof(jobManager)); AsyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer)); + this.diagnosticsIOManager = diagnosticsIOManager ?? throw new ArgumentNullException(nameof(diagnosticsIOManager)); Logger = logger ?? throw new ArgumentNullException(nameof(logger)); ActiveLaunchParameters = initialLaunchParameters ?? throw new ArgumentNullException(nameof(initialLaunchParameters)); this.instance = instance ?? throw new ArgumentNullException(nameof(instance)); @@ -919,5 +928,22 @@ namespace Tgstation.Server.Host.Components.Watchdog /// public abstract Task InstanceRenamed(string newInstanceName, CancellationToken cancellationToken); + + /// + public async Task CreateDump(CancellationToken cancellationToken) + { + var session = GetActiveController(); + + const string DumpDirectory = "ProcessDumps"; + await diagnosticsIOManager.CreateDirectory(DumpDirectory, cancellationToken).ConfigureAwait(false); + + var dumpFileName = diagnosticsIOManager.ResolvePath( + diagnosticsIOManager.ConcatPath( + DumpDirectory, + $"DreamDaemon-{DateTimeOffset.Now.ToFileStamp()}.dmp")); + + Logger.LogInformation("Dumping session to {0}...", dumpFileName); + await session.CreateDump(dumpFileName, cancellationToken).ConfigureAwait(false); + } } } diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogFactory.cs b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogFactory.cs index 7f5f08e88e..b7fcaaa3bd 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogFactory.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogFactory.cs @@ -77,7 +77,8 @@ namespace Tgstation.Server.Host.Components.Watchdog IDmbFactory dmbFactory, IReattachInfoHandler reattachInfoHandler, ISessionControllerFactory sessionControllerFactory, - IIOManager ioManager, + IIOManager gameIOManager, + IIOManager diagnosticsIOManager, Api.Models.Instance instance, DreamDaemonSettings settings) { @@ -91,12 +92,21 @@ namespace Tgstation.Server.Host.Components.Watchdog JobManager, ServerControl, AsyncDelayer, + diagnosticsIOManager, LoggerFactory.CreateLogger(), settings, instance, settings.AutoStart.Value); - return CreateNonExperimentalWatchdog(chat, dmbFactory, reattachInfoHandler, sessionControllerFactory, ioManager, instance, settings); + return CreateNonExperimentalWatchdog( + chat, + dmbFactory, + reattachInfoHandler, + sessionControllerFactory, + gameIOManager, + diagnosticsIOManager, + instance, + settings); } /// @@ -106,7 +116,8 @@ namespace Tgstation.Server.Host.Components.Watchdog /// The for the with /// The for the /// The for the - /// The for the . + /// The pointing to the Game directory for the . + /// The pointing to the Diagnostics directory for the . /// The for the /// The initial for the /// A new @@ -115,7 +126,8 @@ namespace Tgstation.Server.Host.Components.Watchdog IDmbFactory dmbFactory, IReattachInfoHandler reattachInfoHandler, ISessionControllerFactory sessionControllerFactory, - IIOManager ioManager, + IIOManager gameIOManager, + IIOManager diagnosticsIOManager, Api.Models.Instance instance, DreamDaemonSettings settings) => new BasicWatchdog( @@ -127,6 +139,7 @@ namespace Tgstation.Server.Host.Components.Watchdog JobManager, ServerControl, AsyncDelayer, + diagnosticsIOManager, LoggerFactory.CreateLogger(), settings, instance, diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs index 473b3f36cc..206995886c 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs @@ -19,9 +19,9 @@ namespace Tgstation.Server.Host.Components.Watchdog sealed class WindowsWatchdog : BasicWatchdog { /// - /// The for the . + /// The for the pointing to the Game directory. /// - readonly IIOManager ioManager; + readonly IIOManager gameIOManager; /// /// The for the . @@ -54,7 +54,8 @@ namespace Tgstation.Server.Host.Components.Watchdog /// The for the . /// The for the . /// The for the . - /// The value of . + /// The for the . + /// The value of . /// The value of . /// The for the . /// The for the . @@ -69,7 +70,8 @@ namespace Tgstation.Server.Host.Components.Watchdog IJobManager jobManager, IServerControl serverControl, IAsyncDelayer asyncDelayer, - IIOManager ioManager, + IIOManager diagnosticsIOManager, + IIOManager gameIOManager, ISymlinkFactory symlinkFactory, ILogger logger, DreamDaemonLaunchParameters initialLaunchParameters, @@ -83,6 +85,7 @@ namespace Tgstation.Server.Host.Components.Watchdog jobManager, serverControl, asyncDelayer, + diagnosticsIOManager, logger, initialLaunchParameters, instance, @@ -90,7 +93,7 @@ namespace Tgstation.Server.Host.Components.Watchdog { try { - this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); + this.gameIOManager = gameIOManager ?? throw new ArgumentNullException(nameof(gameIOManager)); this.symlinkFactory = symlinkFactory ?? throw new ArgumentNullException(nameof(symlinkFactory)); } catch @@ -151,7 +154,7 @@ namespace Tgstation.Server.Host.Components.Watchdog bool suspended = false; try { - windowsProvider = new WindowsSwappableDmbProvider(compileJobProvider, ioManager, symlinkFactory); + windowsProvider = new WindowsSwappableDmbProvider(compileJobProvider, gameIOManager, symlinkFactory); Logger.LogDebug("Swapping to compile job {0}...", windowsProvider.CompileJob.Id); try @@ -195,7 +198,7 @@ namespace Tgstation.Server.Host.Components.Watchdog // Add another lock to the startup DMB because it'll be used throughout the lifetime of the watchdog startupDmbProvider = await DmbFactory.FromCompileJob(dmbToUse.CompileJob, cancellationToken).ConfigureAwait(false); - activeSwappable = pendingSwappable ?? new WindowsSwappableDmbProvider(dmbToUse, ioManager, symlinkFactory); + activeSwappable = pendingSwappable ?? new WindowsSwappableDmbProvider(dmbToUse, gameIOManager, symlinkFactory); pendingSwappable = null; try diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs index 2d8273386d..d8c4314280 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs @@ -58,7 +58,8 @@ namespace Tgstation.Server.Host.Components.Watchdog IDmbFactory dmbFactory, IReattachInfoHandler reattachInfoHandler, ISessionControllerFactory sessionControllerFactory, - IIOManager ioManager, + IIOManager gameIOManager, + IIOManager diagnosticsIOManager, Api.Models.Instance instance, DreamDaemonSettings settings) => new WindowsWatchdog( @@ -70,7 +71,8 @@ namespace Tgstation.Server.Host.Components.Watchdog JobManager, ServerControl, AsyncDelayer, - ioManager, + diagnosticsIOManager, + gameIOManager, symlinkFactory, LoggerFactory.CreateLogger(), settings, diff --git a/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs b/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs index 707f1d9309..1e03b0163d 100644 --- a/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs +++ b/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs @@ -290,5 +290,37 @@ namespace Tgstation.Server.Host.Controllers await jobManager.RegisterOperation(job, (paramJob, databaseContextFactory, progressReporter, ct) => watchdog.Restart(false, ct), cancellationToken).ConfigureAwait(false); return Accepted(job.ToApi()); } + + /// + /// Creates a to generate a DreamDaemon process dump. + /// + /// The for the operation + /// A resulting in the of the request + /// Dump started successfully. + [HttpPatch(Routes.Diagnostics)] + [TgsAuthorize(DreamDaemonRights.CreateDump)] + [ProducesResponseType(typeof(Api.Models.Job), 202)] + public async Task CreateDump(CancellationToken cancellationToken) + { + var job = new Models.Job + { + Instance = Instance, + CancelRightsType = RightsType.DreamDaemon, + CancelRight = (ulong)DreamDaemonRights.CreateDump, + StartedBy = AuthenticationContext.User, + Description = "Create DreamDaemon Process Dump" + }; + + var watchdog = instanceManager.GetInstance(Instance).Watchdog; + + if (!watchdog.Running) + return Conflict(new ErrorMessage(ErrorCode.WatchdogNotRunning)); + + await jobManager.RegisterOperation( + job, + (paramJob, databaseContextFactory, progressReporter, ct) => watchdog.CreateDump(ct), cancellationToken) + .ConfigureAwait(false); + return Accepted(job.ToApi()); + } } } diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index 8bc5133bc8..5e96354cfb 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -273,7 +273,11 @@ namespace Tgstation.Server.Host.Core services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); + + // PosixProcessFeatures also needs a IProcessExecutor for gcore + services.AddSingleton(x => new Lazy(() => x.GetRequiredService(), true)); services.AddSingleton(); } diff --git a/src/Tgstation.Server.Host/Database/SqliteDatabaseContext.cs b/src/Tgstation.Server.Host/Database/SqliteDatabaseContext.cs index 4d9fecf844..229a30a8dc 100644 --- a/src/Tgstation.Server.Host/Database/SqliteDatabaseContext.cs +++ b/src/Tgstation.Server.Host/Database/SqliteDatabaseContext.cs @@ -9,7 +9,7 @@ using Tgstation.Server.Host.Configuration; namespace Tgstation.Server.Host.Database { /// - /// for MySQL + /// for Sqlite. /// sealed class SqliteDatabaseContext : DatabaseContext { diff --git a/src/Tgstation.Server.Host/Extensions/DateTimeOffsetExtensions.cs b/src/Tgstation.Server.Host/Extensions/DateTimeOffsetExtensions.cs new file mode 100644 index 0000000000..f3f36d3b23 --- /dev/null +++ b/src/Tgstation.Server.Host/Extensions/DateTimeOffsetExtensions.cs @@ -0,0 +1,19 @@ +using System; +using System.Globalization; + +namespace Tgstation.Server.Host.Extensions +{ + /// + /// Extension methods for the . + /// + static class DateTimeOffsetExtensions + { + /// + /// Convert a given into a that can be used to stamp file creation times. + /// + /// The to convert. + /// as a file stamp . + public static string ToFileStamp(this DateTimeOffset dateTimeOffset) + => dateTimeOffset.ToString("yyyyMMddhhmmss", CultureInfo.InvariantCulture); + } +} diff --git a/src/Tgstation.Server.Host/NativeMethods.cs b/src/Tgstation.Server.Host/NativeMethods.cs index afc7211071..90fe18ec03 100644 --- a/src/Tgstation.Server.Host/NativeMethods.cs +++ b/src/Tgstation.Server.Host/NativeMethods.cs @@ -5,7 +5,7 @@ using System.Text; namespace Tgstation.Server.Host { /// - /// Native Windows methods used by the code + /// Native Windows methods used by the code. /// #pragma warning disable SA1600 #pragma warning disable SA1602 @@ -32,6 +32,19 @@ namespace Tgstation.Server.Host SuspendResume = 0x0002, } + /// + /// See https://docs.microsoft.com/en-us/windows/win32/api/minidumpapiset/ne-minidumpapiset-minidump_type + /// + [Flags] + public enum MiniDumpType : uint + { + WithDataSegs = 0x00000001, + WithFullMemory = 0x00000002, + WithHandleData = 0x00000004, + WithUnloadedModules = 0x00000020, + WithThreadInfo = 0x00001000, + } + /// /// See https://docs.microsoft.com/en-us/windows/desktop/api/winuser/nf-winuser-getwindowthreadprocessid /// @@ -102,5 +115,18 @@ namespace Tgstation.Server.Host /// [DllImport("kernel32.dll", SetLastError = true, CharSet = CharSet.Unicode)] public static extern uint ResumeThread(IntPtr hThread); + + /// + /// See https://docs.microsoft.com/en-us/windows/win32/api/minidumpapiset/nf-minidumpapiset-minidumpwritedump + /// + [DllImport("dbghelp.dll", SetLastError = true, CharSet = CharSet.Unicode)] + public static extern bool MiniDumpWriteDump( + IntPtr hProcess, + uint processId, + SafeHandle hFile, + MiniDumpType dumpType, + IntPtr expParam, + IntPtr userStreamParam, + IntPtr callbackParam); } } diff --git a/src/Tgstation.Server.Host/System/IProcessBase.cs b/src/Tgstation.Server.Host/System/IProcessBase.cs index a9013d567a..c861f8ae5f 100644 --- a/src/Tgstation.Server.Host/System/IProcessBase.cs +++ b/src/Tgstation.Server.Host/System/IProcessBase.cs @@ -1,4 +1,5 @@ using System; +using System.Threading; using System.Threading.Tasks; namespace Tgstation.Server.Host.System @@ -27,5 +28,13 @@ namespace Tgstation.Server.Host.System /// Resumes the process. /// void Resume(); + + /// + /// Create a dump file of the process. + /// + /// The full path to the output file. + /// The for the operation. + /// A representing the running operation. + Task CreateDump(string outputFile, CancellationToken cancellationToken); } } diff --git a/src/Tgstation.Server.Host/System/IProcessExecutor.cs b/src/Tgstation.Server.Host/System/IProcessExecutor.cs index 3dea91680b..e7f0d98388 100644 --- a/src/Tgstation.Server.Host/System/IProcessExecutor.cs +++ b/src/Tgstation.Server.Host/System/IProcessExecutor.cs @@ -13,7 +13,7 @@ /// The arguments for the /// If standard output should be read /// If standard error should be read - /// If shell execute should not be used. Ignored if or are set + /// If shell execute should not be used. Must be set if or are set. /// A new IProcess LaunchProcess(string fileName, string workingDirectory, string arguments = null, bool readOutput = false, bool readError = false, bool noShellExecute = false); diff --git a/src/Tgstation.Server.Host/System/IProcessFeatures.cs b/src/Tgstation.Server.Host/System/IProcessFeatures.cs index c56d03f5d0..b45d0831e0 100644 --- a/src/Tgstation.Server.Host/System/IProcessFeatures.cs +++ b/src/Tgstation.Server.Host/System/IProcessFeatures.cs @@ -17,7 +17,7 @@ namespace Tgstation.Server.Host.System Task GetExecutingUsername(global::System.Diagnostics.Process process, CancellationToken cancellationToken); /// - /// Suspend a given . + /// Suspend a given . /// /// The to suspend. void SuspendProcess(global::System.Diagnostics.Process process); @@ -27,5 +27,14 @@ namespace Tgstation.Server.Host.System /// /// The to susperesumend. void ResumeProcess(global::System.Diagnostics.Process process); + + /// + /// Create a dump file for a given . + /// + /// The to dump. + /// The full path to the output file. + /// The for the operation. + /// A representing the running operation. + Task CreateDump(global::System.Diagnostics.Process process, string outputFile, CancellationToken cancellationToken); } } diff --git a/src/Tgstation.Server.Host/System/PosixProcessFeatures.cs b/src/Tgstation.Server.Host/System/PosixProcessFeatures.cs index 751bd47eff..f808e588de 100644 --- a/src/Tgstation.Server.Host/System/PosixProcessFeatures.cs +++ b/src/Tgstation.Server.Host/System/PosixProcessFeatures.cs @@ -7,13 +7,20 @@ using System.Linq; using System.Text; using System.Threading; using System.Threading.Tasks; +using Tgstation.Server.Api.Models; using Tgstation.Server.Host.IO; +using Tgstation.Server.Host.Jobs; namespace Tgstation.Server.Host.System { /// sealed class PosixProcessFeatures : IProcessFeatures { + /// + /// loaded . + /// + readonly Lazy lazyLoadedProcessExecutor; + /// /// The for the . /// @@ -27,10 +34,12 @@ namespace Tgstation.Server.Host.System /// /// Initializes a new instance of the . /// + /// The value of . /// The value of . /// The value of . - public PosixProcessFeatures(IIOManager ioManager, ILogger logger) + public PosixProcessFeatures(Lazy lazyLoadedProcessExecutor, IIOManager ioManager, ILogger logger) { + this.lazyLoadedProcessExecutor = lazyLoadedProcessExecutor ?? throw new ArgumentNullException(nameof(lazyLoadedProcessExecutor)); this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); } @@ -93,5 +102,46 @@ namespace Tgstation.Server.Host.System .FirstOrDefault(x => !String.IsNullOrWhiteSpace(x)) ?? "UNPARSABLE"; } + + /// + public async Task CreateDump(global::System.Diagnostics.Process process, string outputFile, CancellationToken cancellationToken) + { + if (process == null) + throw new ArgumentNullException(nameof(process)); + if (outputFile == null) + throw new ArgumentNullException(nameof(outputFile)); + + const string GCorePath = "/usr/bin/gcore"; + if (!await ioManager.FileExists(GCorePath, cancellationToken).ConfigureAwait(false)) + throw new JobException(ErrorCode.MissingGCore); + + var pid = process.Id; + string output; + int exitCode; + using (var gcoreProc = lazyLoadedProcessExecutor.Value.LaunchProcess( + GCorePath, + Environment.CurrentDirectory, + $"-o {outputFile} {process.Id}", + true, + true, + true)) + { + using (cancellationToken.Register(() => gcoreProc.Terminate())) + exitCode = await gcoreProc.Lifetime.ConfigureAwait(false); + + output = gcoreProc.GetCombinedOutput(); + logger.LogDebug("gcore output:{0}{1}", Environment.NewLine, output); + } + + if (exitCode != 0) + throw new JobException( + ErrorCode.GCoreFailure, + new JobException( + $"Exit Code: {exitCode}{Environment.NewLine}Output:{Environment.NewLine}{output}")); + + // gcore outputs name.pid so remove the pid part + var generatedGCoreFile = $"{outputFile}.{pid}"; + await ioManager.MoveFile(generatedGCoreFile, outputFile, cancellationToken).ConfigureAwait(false); + } } } diff --git a/src/Tgstation.Server.Host/System/Process.cs b/src/Tgstation.Server.Host/System/Process.cs index 044f5fb34c..5f2c3cf5a6 100644 --- a/src/Tgstation.Server.Host/System/Process.cs +++ b/src/Tgstation.Server.Host/System/Process.cs @@ -165,5 +165,15 @@ namespace Tgstation.Server.Host.System logger.LogTrace("PID {0} Username: {1}", Id, result); return result; } + + /// + public Task CreateDump(string outputFile, CancellationToken cancellationToken) + { + if (outputFile == null) + throw new ArgumentNullException(nameof(outputFile)); + + logger.LogTrace("Dumping PID {0} to {1}...", Id, outputFile); + return processFeatures.CreateDump(handle, outputFile, cancellationToken); + } } } diff --git a/src/Tgstation.Server.Host/System/WindowsProcessFeatures.cs b/src/Tgstation.Server.Host/System/WindowsProcessFeatures.cs index 4ea3a49a74..339e847e75 100644 --- a/src/Tgstation.Server.Host/System/WindowsProcessFeatures.cs +++ b/src/Tgstation.Server.Host/System/WindowsProcessFeatures.cs @@ -2,6 +2,7 @@ using Microsoft.Extensions.Logging; using System; using System.Diagnostics; +using System.IO; using System.Linq; using System.Management; using System.Threading; @@ -110,5 +111,29 @@ namespace Tgstation.Server.Host.System return Task.FromResult("NO OWNER"); } + + /// + public Task CreateDump(global::System.Diagnostics.Process process, string outputFile, CancellationToken cancellationToken) + => Task.Factory.StartNew( + () => + { + using var fileStream = new FileStream(outputFile, FileMode.CreateNew); + if (!NativeMethods.MiniDumpWriteDump( + process.Handle, + (uint)process.Id, + fileStream.SafeFileHandle, + NativeMethods.MiniDumpType.WithDataSegs + | NativeMethods.MiniDumpType.WithFullMemory + | NativeMethods.MiniDumpType.WithHandleData + | NativeMethods.MiniDumpType.WithThreadInfo + | NativeMethods.MiniDumpType.WithUnloadedModules, + IntPtr.Zero, + IntPtr.Zero, + IntPtr.Zero)) + throw new Win32Exception(); + }, + cancellationToken, + TaskCreationOptions.LongRunning, + TaskScheduler.Current); } } diff --git a/tests/Tgstation.Server.Host.Tests/System/TestProcessFeatures.cs b/tests/Tgstation.Server.Host.Tests/System/TestProcessFeatures.cs index 065813a50c..cfa6173cce 100644 --- a/tests/Tgstation.Server.Host.Tests/System/TestProcessFeatures.cs +++ b/tests/Tgstation.Server.Host.Tests/System/TestProcessFeatures.cs @@ -21,7 +21,7 @@ namespace Tgstation.Server.Host.System.Tests { features = new PlatformIdentifier().IsWindows ? (IProcessFeatures)new WindowsProcessFeatures(Mock.Of>()) - : new PosixProcessFeatures(new DefaultIOManager(), Mock.Of>()); + : new PosixProcessFeatures(new Lazy(() => null), new DefaultIOManager(), Mock.Of>()); } [TestMethod] diff --git a/tests/Tgstation.Server.Tests/Instance/WatchdogTest.cs b/tests/Tgstation.Server.Tests/Instance/WatchdogTest.cs index 9955bc1af4..4823cebd9d 100644 --- a/tests/Tgstation.Server.Tests/Instance/WatchdogTest.cs +++ b/tests/Tgstation.Server.Tests/Instance/WatchdogTest.cs @@ -43,6 +43,9 @@ namespace Tgstation.Server.Tests.Instance SoftRestart = true }, cancellationToken), ErrorCode.DreamDaemonDoubleSoft); + await ApiAssert.ThrowsException(() => instanceClient.DreamDaemon.CreateDump(cancellationToken), ErrorCode.WatchdogNotRunning); + await ApiAssert.ThrowsException(() => instanceClient.DreamDaemon.Restart(cancellationToken), ErrorCode.WatchdogNotRunning); + await RunBasicTest(cancellationToken); // await RunLongRunningTestThenUpdate(cancellationToken); @@ -53,6 +56,15 @@ namespace Tgstation.Server.Tests.Instance await RunHeartbeatTest(cancellationToken); await StartAndLeaveRunning(cancellationToken); + + var dumpJob = await instanceClient.DreamDaemon.CreateDump(cancellationToken); + await WaitForJob(dumpJob, 3000, false, cancellationToken); + + var dumpFiles = Directory.GetFiles(Path.Combine( + instanceClient.Metadata.Path, "Diagnostics", "ProcessDumps"), "*.dmp"); + Assert.AreEqual(1, dumpFiles.Length); + File.Delete(dumpFiles.Single()); + global::System.Console.WriteLine("TEST: END WATCHDOG TESTS"); } @@ -103,12 +115,14 @@ namespace Tgstation.Server.Tests.Instance Assert.Inconclusive($"Incorrect number of DD processes: {ddProcs.Count}"); using var ddProc = ddProcs.Single(); - using var ourProcessHandler = new ProcessExecutor( + IProcessExecutor executor = null; + executor = new ProcessExecutor( new PlatformIdentifier().IsWindows ? (IProcessFeatures)new WindowsProcessFeatures(Mock.Of>()) - : new PosixProcessFeatures(Mock.Of(), Mock.Of>()), + : new PosixProcessFeatures(new Lazy(() => executor), Mock.Of(), Mock.Of>()), Mock.Of>(), - LoggerFactory.Create(x => { })) + LoggerFactory.Create(x => { })); + using var ourProcessHandler = executor .GetProcess(ddProc.Id); // Ensure it's responding to heartbeats