Merge pull request #1051 from tgstation/612-Dumps

Implement dumps

.travis.yml

@@ -32,6 +32,7 @@ jobs:
           packages:
             - libc6-i386
             - libstdc++6:i386
+            - gdb
     - env:
       - DoxGeneration=false
       - DockerBuild=false
@@ -51,6 +52,7 @@ jobs:
           packages:
             - libc6-i386
             - libstdc++6:i386
+            - gdb
     - env:
       - DoxGeneration=false
       - DockerBuild=false
@@ -74,6 +76,7 @@ jobs:
             - postgresql-12
             - libc6-i386
             - libstdc++6:i386
+            - gdb
     - env:
       - DoxGeneration=false
       - DockerBuild=false

README.md

@@ -39,6 +39,7 @@ The following dependencies are required to run tgstation-server on Linux alongsi
 - libc6-i386
 - libstdc++6:i386
 - libssl1.0.0
+- gdb (for using gcore to create core dumps)
 - gcc-multilib (Only on 64-bit systems)
 
 Note that tgstation-server has only ever been tested on Linux via it's [docker environment](build/Dockerfile#L22). If you are having trouble with something in a native installation, or figure out a required workaround, please contact project maintainers so this documentation may be better updated.

build/Dockerfile

@@ -58,7 +58,8 @@ FROM mcr.microsoft.com/dotnet/core/aspnet:3.1-bionic
 #needed for byond
 RUN apt-get update \
 	&& apt-get install -y \
-	gcc-multilib \
+		gcc-multilib \
+		gdb \
 	&& rm -rf /var/lib/apt/lists/*
 	
 EXPOSE 5000

build/integration_test.sh

@@ -1,6 +1,9 @@
 #!/bin/bash
 set -e
 
+# Needed so gcore can work
+echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope
+
 export TGS4_TEST_DISCORD_CHANNEL=493119635319947269
 export TGS4_TEST_IRC_CHANNEL=\#botbus
 export TGS4_TEST_TEMP_DIRECTORY=~/tgs4_test

src/Tgstation.Server.Api/Models/DreamDaemon.cs

@@ -48,5 +48,10 @@ namespace Tgstation.Server.Api.Models
 		/// If the server is undergoing a soft shutdown
 		/// </summary>
 		public bool? SoftShutdown { get; set; }
+
+		/// <summary>
+		/// If a dump of the active DreamDaemon executable should be created.
+		/// </summary>
+		public bool? CreateDump { get; set; }
 	}
 }

src/Tgstation.Server.Api/Models/ErrorCode.cs

@@ -519,5 +519,17 @@ namespace Tgstation.Server.Api.Models
 		/// </summary>
 		[Description("Cannot cancel the job as it is no longer running.")]
 		JobStopped,
+
+		/// <summary>
+		/// Missing GCore executable.
+		/// </summary>
+		[Description("Attempted to create a process dump but /usr/bin/gcore could not be located!")]
+		MissingGCore,
+
+		/// <summary>
+		/// Non-zero gcore exit code.
+		/// </summary>
+		[Description("Could not create dump as gcore exited with a non-zero exit code!")]
+		GCoreFailure,
 	}
 }

src/Tgstation.Server.Api/Rights/DreamDaemonRights.cs

@@ -77,5 +77,10 @@ namespace Tgstation.Server.Api.Rights
 		/// User can change <see cref="Models.Internal.DreamDaemonLaunchParameters.HeartbeatSeconds"/>
 		/// </summary>
 		SetHeartbeatInterval = 4096,
+
+		/// <summary>
+		/// User can create DreamDaemon process dumps.
+		/// </summary>
+		CreateDump = 8192,
 	}
 }

src/Tgstation.Server.Api/Routes.cs

@@ -43,6 +43,11 @@ namespace Tgstation.Server.Api
 		/// </summary>
 		public const string DreamDaemon = Root + nameof(Models.DreamDaemon);
 
+		/// <summary>
+		/// For accessing DD diagnostics
+		/// </summary>
+		public const string Diagnostics = DreamDaemon + "/Diagnostics";
+
 		/// <summary>
 		/// The <see cref="Models.ConfigurationFile"/> controller
 		/// </summary>

src/Tgstation.Server.Client/Components/DreamDaemonClient.cs

@@ -44,5 +44,8 @@ namespace Tgstation.Server.Client.Components
 
 		/// <inheritdoc />
 		public Task<DreamDaemon> Update(DreamDaemon dreamDaemon, CancellationToken cancellationToken) => apiClient.Update<DreamDaemon, DreamDaemon>(Routes.DreamDaemon, dreamDaemon ?? throw new ArgumentNullException(nameof(dreamDaemon)), instance.Id, cancellationToken);
+
+		/// <inheritdoc />
+		public Task<Job> CreateDump(CancellationToken cancellationToken) => apiClient.Patch<Job>(Routes.Diagnostics, instance.Id, cancellationToken);
 	}
 }

src/Tgstation.Server.Client/Components/IDreamDaemonClient.cs

@@ -44,5 +44,12 @@ namespace Tgstation.Server.Client.Components
 		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
 		/// <returns>A <see cref="Task{TResult}"/> resulting in the <see cref="DreamDaemon"/> information</returns>
 		Task<DreamDaemon> Update(DreamDaemon dreamDaemon, CancellationToken cancellationToken);
+
+		/// <summary>
+		/// Start a job to create a process dump of the active DreamDaemon executable.
+		/// </summary>
+		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
+		/// <returns>A <see cref="Task{TResult}"/> resulting in the <see cref="Job"/> of the running operation.</returns>
+		Task<Job> CreateDump(CancellationToken cancellationToken);
 	}
 }

src/Tgstation.Server.Host/Components/InstanceFactory.cs

@@ -202,6 +202,7 @@ namespace Tgstation.Server.Host.Components
 			var repoIoManager = new ResolvingIOManager(instanceIoManager, "Repository");
 			var byondIOManager = new ResolvingIOManager(instanceIoManager, "Byond");
 			var gameIoManager = new ResolvingIOManager(instanceIoManager, "Game");
+			var diagnosticsIOManager = new ResolvingIOManager(instanceIoManager, "Diagnostics");
 			var configurationIoManager = new ResolvingIOManager(instanceIoManager, "Configuration");
 
 			var configuration = new StaticFiles.Configuration(configurationIoManager, synchronousIOManager, symlinkFactory, processExecutor, postWriteHandler, platformIdentifier, loggerFactory.CreateLogger<StaticFiles.Configuration>());
@@ -248,6 +249,7 @@ namespace Tgstation.Server.Host.Components
 							reattachInfoHandler,
 							sessionControllerFactory,
 							gameIoManager,
+							diagnosticsIOManager,
 							metadata.CloneMetadata(),
 							metadata.DreamDaemonSettings);
 						eventConsumer.SetWatchdog(watchdog);

src/Tgstation.Server.Host/Components/Session/DeadSessionController.cs

@@ -123,5 +123,8 @@ namespace Tgstation.Server.Host.Components.Session
 
 		/// <inheritdoc />
 		public Task InstanceRenamed(string newInstanceName, CancellationToken cancellationToken) => Task.CompletedTask;
+
+		/// <inheritdoc />
+		public Task CreateDump(string outputFile, CancellationToken cancellationToken) => throw new NotSupportedException();
 	}
 }

src/Tgstation.Server.Host/Components/Session/SessionController.cs

@@ -696,5 +696,8 @@ namespace Tgstation.Server.Host.Components.Session
 				new TopicParameters(
 					new ChatUpdate(newChannels)),
 				cancellationToken);
+
+		/// <inheritdoc />
+		public Task CreateDump(string outputFile, CancellationToken cancellationToken) => process.CreateDump(outputFile, cancellationToken);
 	}
 }

src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs

@@ -10,6 +10,7 @@ using Tgstation.Server.Host.Components.Deployment;
 using Tgstation.Server.Host.Components.Session;
 using Tgstation.Server.Host.Core;
 using Tgstation.Server.Host.Database;
+using Tgstation.Server.Host.IO;
 using Tgstation.Server.Host.Jobs;
 
 namespace Tgstation.Server.Host.Components.Watchdog
@@ -49,6 +50,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// <param name="jobManager">The <see cref="IJobManager"/> for the <see cref="WatchdogBase"/>.</param>
 		/// <param name="serverControl">The <see cref="IServerControl"/> for the <see cref="WatchdogBase"/>.</param>
 		/// <param name="asyncDelayer">The <see cref="IAsyncDelayer"/> for the <see cref="WatchdogBase"/>.</param>
+		/// <param name="diagnosticsIOManager">The <see cref="IIOManager"/> for the <see cref="WatchdogBase"/>.</param>
 		/// <param name="logger">The <see cref="ILogger"/> for the <see cref="WatchdogBase"/>.</param>
 		/// <param name="initialLaunchParameters">The <see cref="DreamDaemonLaunchParameters"/> for the <see cref="WatchdogBase"/>.</param>
 		/// <param name="instance">The <see cref="Api.Models.Instance"/> for the <see cref="WatchdogBase"/>.</param>
@@ -62,6 +64,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			IJobManager jobManager,
 			IServerControl serverControl,
 			IAsyncDelayer asyncDelayer,
+			IIOManager diagnosticsIOManager,
 			ILogger<BasicWatchdog> logger,
 			DreamDaemonLaunchParameters initialLaunchParameters,
 			Api.Models.Instance instance,
@@ -75,6 +78,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 				 jobManager,
 				 serverControl,
 				 asyncDelayer,
+				 diagnosticsIOManager,
 				 logger,
 				 initialLaunchParameters,
 				 instance,

src/Tgstation.Server.Host/Components/Watchdog/ExperimentalWatchdog.cs

@@ -11,6 +11,7 @@ using Tgstation.Server.Host.Components.Deployment;
 using Tgstation.Server.Host.Components.Session;
 using Tgstation.Server.Host.Core;
 using Tgstation.Server.Host.Database;
+using Tgstation.Server.Host.IO;
 using Tgstation.Server.Host.Jobs;
 
 namespace Tgstation.Server.Host.Components.Watchdog
@@ -60,6 +61,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// <param name="jobManager">The <see cref="IJobManager"/> for the <see cref="WatchdogBase"/>.</param>
 		/// <param name="serverControl">The <see cref="IServerControl"/> for the <see cref="WatchdogBase"/>.</param>
 		/// <param name="asyncDelayer">The <see cref="IAsyncDelayer"/> for the <see cref="WatchdogBase"/>.</param>
+		/// <param name="diagnosticsIOManager">The <see cref="IIOManager"/> for the <see cref="WatchdogBase"/>.</param>
 		/// <param name="logger">The <see cref="ILogger"/> for the <see cref="WatchdogBase"/>.</param>
 		/// <param name="initialLaunchParameters">The <see cref="DreamDaemonLaunchParameters"/> for the <see cref="WatchdogBase"/>.</param>
 		/// <param name="instance">The <see cref="Api.Models.Instance"/> for the <see cref="WatchdogBase"/>.</param>
@@ -73,6 +75,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			IJobManager jobManager,
 			IServerControl serverControl,
 			IAsyncDelayer asyncDelayer,
+			IIOManager diagnosticsIOManager,
 			ILogger<ExperimentalWatchdog> logger,
 			DreamDaemonLaunchParameters initialLaunchParameters,
 			Api.Models.Instance instance, bool autoStart)
@@ -85,6 +88,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 				 jobManager,
 				 serverControl,
 				 asyncDelayer,
+				 diagnosticsIOManager,
 				 logger,
 				 initialLaunchParameters,
 				 instance,

src/Tgstation.Server.Host/Components/Watchdog/IWatchdog.cs

@@ -80,5 +80,12 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
 		/// <returns>A <see cref="Task"/> representing the running operation</returns>
 		Task ResetRebootState(CancellationToken cancellationToken);
+
+		/// <summary>
+		/// Attempt to create a process dump for DreamDaemon.
+		/// </summary>
+		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
+		/// <returns>A <see cref="Task"/> representing the running operation.</returns>
+		Task CreateDump(CancellationToken cancellationToken);
 	}
 }

src/Tgstation.Server.Host/Components/Watchdog/IWatchdogFactory.cs

@@ -18,7 +18,8 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// <param name="dmbFactory">The <see cref="IDmbFactory"/> for the <see cref="IWatchdog"/> with</param>
 		/// <param name="reattachInfoHandler">The <see cref="IReattachInfoHandler"/> for the <see cref="IWatchdog"/></param>
 		/// <param name="sessionControllerFactory">The <see cref="ISessionControllerFactory"/> for the <see cref="IWatchdog"/></param>
-		/// <param name="ioManager">The <see cref="IIOManager"/> for the <see cref="IWatchdog"/>.</param>
+		/// <param name="gameIOManager">The <see cref="IIOManager"/> pointing to the Game directory for the <see cref="IWatchdog"/>.</param>
+		/// <param name="diagnosticsIOManager">The <see cref="IIOManager"/> pointing to the Diagnostics directory for the <see cref="IWatchdog"/>.</param>
 		/// <param name="instance">The <see cref="Instance"/> for the <see cref="IWatchdog"/></param>
 		/// <param name="settings">The initial <see cref="DreamDaemonSettings"/> for the <see cref="IWatchdog"/></param>
 		/// <returns>A new <see cref="IWatchdog"/></returns>
@@ -27,7 +28,8 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			IDmbFactory dmbFactory,
 			IReattachInfoHandler reattachInfoHandler,
 			ISessionControllerFactory sessionControllerFactory,
-			IIOManager ioManager,
+			IIOManager gameIOManager,
+			IIOManager diagnosticsIOManager,
 			Api.Models.Instance instance,
 			DreamDaemonSettings settings);
 	}

src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs

@@ -18,6 +18,7 @@ using Tgstation.Server.Host.Components.Session;
 using Tgstation.Server.Host.Core;
 using Tgstation.Server.Host.Database;
 using Tgstation.Server.Host.Extensions;
+using Tgstation.Server.Host.IO;
 using Tgstation.Server.Host.Jobs;
 
 namespace Tgstation.Server.Host.Components.Watchdog
@@ -114,6 +115,11 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// </summary>
 		readonly IRestartRegistration restartRegistration;
 
+		/// <summary>
+		/// The <see cref="IIOManager"/> pointing to the Diagnostics directory.
+		/// </summary>
+		readonly IIOManager diagnosticsIOManager;
+
 		/// <summary>
 		/// <see langword="lock"/> <see cref="object"/> used for <see cref="DisposeAndNullControllers"/>.
 		/// </summary>
@@ -170,6 +176,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// <param name="jobManager">The value of <see cref="jobManager"/></param>
 		/// <param name="serverControl">The <see cref="IServerControl"/> to populate <see cref="restartRegistration"/> with</param>
 		/// <param name="asyncDelayer">The value of <see cref="AsyncDelayer"/>.</param>
+		/// <param name="diagnosticsIOManager">The value of <see cref="diagnosticsIOManager"/>.</param>
 		/// <param name="logger">The value of <see cref="Logger"/></param>
 		/// <param name="initialLaunchParameters">The initial value of <see cref="ActiveLaunchParameters"/>. May be modified</param>
 		/// <param name="instance">The value of <see cref="instance"/></param>
@@ -183,6 +190,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			IJobManager jobManager,
 			IServerControl serverControl,
 			IAsyncDelayer asyncDelayer,
+			IIOManager diagnosticsIOManager,
 			ILogger logger,
 			DreamDaemonLaunchParameters initialLaunchParameters,
 			Api.Models.Instance instance,
@@ -195,6 +203,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			this.databaseContextFactory = databaseContextFactory ?? throw new ArgumentNullException(nameof(databaseContextFactory));
 			this.jobManager = jobManager ?? throw new ArgumentNullException(nameof(jobManager));
 			AsyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer));
+			this.diagnosticsIOManager = diagnosticsIOManager ?? throw new ArgumentNullException(nameof(diagnosticsIOManager));
 			Logger = logger ?? throw new ArgumentNullException(nameof(logger));
 			ActiveLaunchParameters = initialLaunchParameters ?? throw new ArgumentNullException(nameof(initialLaunchParameters));
 			this.instance = instance ?? throw new ArgumentNullException(nameof(instance));
@@ -919,5 +928,22 @@ namespace Tgstation.Server.Host.Components.Watchdog
 
 		/// <inheritdoc />
 		public abstract Task InstanceRenamed(string newInstanceName, CancellationToken cancellationToken);
+
+		/// <inheritdoc />
+		public async Task CreateDump(CancellationToken cancellationToken)
+		{
+			var session = GetActiveController();
+
+			const string DumpDirectory = "ProcessDumps";
+			await diagnosticsIOManager.CreateDirectory(DumpDirectory, cancellationToken).ConfigureAwait(false);
+
+			var dumpFileName = diagnosticsIOManager.ResolvePath(
+				diagnosticsIOManager.ConcatPath(
+					DumpDirectory,
+					$"DreamDaemon-{DateTimeOffset.Now.ToFileStamp()}.dmp"));
+
+			Logger.LogInformation("Dumping session to {0}...", dumpFileName);
+			await session.CreateDump(dumpFileName, cancellationToken).ConfigureAwait(false);
+		}
 	}
 }

src/Tgstation.Server.Host/Components/Watchdog/WatchdogFactory.cs

@@ -77,7 +77,8 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			IDmbFactory dmbFactory,
 			IReattachInfoHandler reattachInfoHandler,
 			ISessionControllerFactory sessionControllerFactory,
-			IIOManager ioManager,
+			IIOManager gameIOManager,
+			IIOManager diagnosticsIOManager,
 			Api.Models.Instance instance,
 			DreamDaemonSettings settings)
 		{
@@ -91,12 +92,21 @@ namespace Tgstation.Server.Host.Components.Watchdog
 					JobManager,
 					ServerControl,
 					AsyncDelayer,
+					diagnosticsIOManager,
 					LoggerFactory.CreateLogger<ExperimentalWatchdog>(),
 					settings,
 					instance,
 					settings.AutoStart.Value);
 
-			return CreateNonExperimentalWatchdog(chat, dmbFactory, reattachInfoHandler, sessionControllerFactory, ioManager, instance, settings);
+			return CreateNonExperimentalWatchdog(
+				chat,
+				dmbFactory,
+				reattachInfoHandler,
+				sessionControllerFactory,
+				gameIOManager,
+				diagnosticsIOManager,
+				instance,
+				settings);
 		}
 
 		/// <summary>
@@ -106,7 +116,8 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// <param name="dmbFactory">The <see cref="IDmbFactory"/> for the <see cref="IWatchdog"/> with</param>
 		/// <param name="reattachInfoHandler">The <see cref="IReattachInfoHandler"/> for the <see cref="IWatchdog"/></param>
 		/// <param name="sessionControllerFactory">The <see cref="ISessionControllerFactory"/> for the <see cref="IWatchdog"/></param>
-		/// <param name="ioManager">The <see cref="IIOManager"/> for the <see cref="IWatchdog"/>.</param>
+		/// <param name="gameIOManager">The <see cref="IIOManager"/> pointing to the Game directory for the <see cref="IWatchdog"/>.</param>
+		/// <param name="diagnosticsIOManager">The <see cref="IIOManager"/> pointing to the Diagnostics directory for the <see cref="IWatchdog"/>.</param>
 		/// <param name="instance">The <see cref="Instance"/> for the <see cref="IWatchdog"/></param>
 		/// <param name="settings">The initial <see cref="DreamDaemonSettings"/> for the <see cref="IWatchdog"/></param>
 		/// <returns>A new <see cref="IWatchdog"/></returns>
@@ -115,7 +126,8 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			IDmbFactory dmbFactory,
 			IReattachInfoHandler reattachInfoHandler,
 			ISessionControllerFactory sessionControllerFactory,
-			IIOManager ioManager,
+			IIOManager gameIOManager,
+			IIOManager diagnosticsIOManager,
 			Api.Models.Instance instance,
 			DreamDaemonSettings settings)
 			=> new BasicWatchdog(
@@ -127,6 +139,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 				JobManager,
 				ServerControl,
 				AsyncDelayer,
+				diagnosticsIOManager,
 				LoggerFactory.CreateLogger<BasicWatchdog>(),
 				settings,
 				instance,

src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs

@@ -19,9 +19,9 @@ namespace Tgstation.Server.Host.Components.Watchdog
 	sealed class WindowsWatchdog : BasicWatchdog
 	{
 		/// <summary>
-		/// The <see cref="IIOManager"/> for the <see cref="WindowsWatchdog"/>.
+		/// The <see cref="IIOManager"/> for the <see cref="WindowsWatchdog"/> pointing to the Game directory.
 		/// </summary>
-		readonly IIOManager ioManager;
+		readonly IIOManager gameIOManager;
 
 		/// <summary>
 		/// The <see cref="ISymlinkFactory"/> for the <see cref="WindowsWatchdog"/>.
@@ -54,7 +54,8 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// <param name="jobManager">The <see cref="IJobManager"/> for the <see cref="WatchdogBase"/>.</param>
 		/// <param name="serverControl">The <see cref="IServerControl"/> for the <see cref="WatchdogBase"/>.</param>
 		/// <param name="asyncDelayer">The <see cref="IAsyncDelayer"/> for the <see cref="WatchdogBase"/>.</param>
-		/// <param name="ioManager">The value of <see cref="ioManager"/>.</param>
+		/// <param name="diagnosticsIOManager">The <see cref="IIOManager"/> for the <see cref="WatchdogBase"/>.</param>
+		/// <param name="gameIOManager">The value of <see cref="gameIOManager"/>.</param>
 		/// <param name="symlinkFactory">The value of <see cref="symlinkFactory"/>.</param>
 		/// <param name="logger">The <see cref="ILogger"/> for the <see cref="WatchdogBase"/>.</param>
 		/// <param name="initialLaunchParameters">The <see cref="DreamDaemonLaunchParameters"/> for the <see cref="WatchdogBase"/>.</param>
@@ -69,7 +70,8 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			IJobManager jobManager,
 			IServerControl serverControl,
 			IAsyncDelayer asyncDelayer,
-			IIOManager ioManager,
+			IIOManager diagnosticsIOManager,
+			IIOManager gameIOManager,
 			ISymlinkFactory symlinkFactory,
 			ILogger<WindowsWatchdog> logger,
 			DreamDaemonLaunchParameters initialLaunchParameters,
@@ -83,6 +85,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 				jobManager,
 				serverControl,
 				asyncDelayer,
+				diagnosticsIOManager,
 				logger,
 				initialLaunchParameters,
 				instance,
@@ -90,7 +93,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		{
 			try
 			{
-				this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager));
+				this.gameIOManager = gameIOManager ?? throw new ArgumentNullException(nameof(gameIOManager));
 				this.symlinkFactory = symlinkFactory ?? throw new ArgumentNullException(nameof(symlinkFactory));
 			}
 			catch
@@ -151,7 +154,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			bool suspended = false;
 			try
 			{
-				windowsProvider = new WindowsSwappableDmbProvider(compileJobProvider, ioManager, symlinkFactory);
+				windowsProvider = new WindowsSwappableDmbProvider(compileJobProvider, gameIOManager, symlinkFactory);
 
 				Logger.LogDebug("Swapping to compile job {0}...", windowsProvider.CompileJob.Id);
 				try
@@ -195,7 +198,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			// Add another lock to the startup DMB because it'll be used throughout the lifetime of the watchdog
 			startupDmbProvider = await DmbFactory.FromCompileJob(dmbToUse.CompileJob, cancellationToken).ConfigureAwait(false);
 
-			activeSwappable = pendingSwappable ?? new WindowsSwappableDmbProvider(dmbToUse, ioManager, symlinkFactory);
+			activeSwappable = pendingSwappable ?? new WindowsSwappableDmbProvider(dmbToUse, gameIOManager, symlinkFactory);
 			pendingSwappable = null;
 
 			try

src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdogFactory.cs

@@ -58,7 +58,8 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			IDmbFactory dmbFactory,
 			IReattachInfoHandler reattachInfoHandler,
 			ISessionControllerFactory sessionControllerFactory,
-			IIOManager ioManager,
+			IIOManager gameIOManager,
+			IIOManager diagnosticsIOManager,
 			Api.Models.Instance instance,
 			DreamDaemonSettings settings)
 			=> new WindowsWatchdog(
@@ -70,7 +71,8 @@ namespace Tgstation.Server.Host.Components.Watchdog
 				JobManager,
 				ServerControl,
 				AsyncDelayer,
-				ioManager,
+				diagnosticsIOManager,
+				gameIOManager,
 				symlinkFactory,
 				LoggerFactory.CreateLogger<WindowsWatchdog>(),
 				settings,

src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs

@@ -290,5 +290,37 @@ namespace Tgstation.Server.Host.Controllers
 			await jobManager.RegisterOperation(job, (paramJob, databaseContextFactory, progressReporter, ct) => watchdog.Restart(false, ct), cancellationToken).ConfigureAwait(false);
 			return Accepted(job.ToApi());
 		}
+
+		/// <summary>
+		/// Creates a <see cref="Api.Models.Job"/> to generate a DreamDaemon process dump.
+		/// </summary>
+		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
+		/// <returns>A <see cref="Task{TResult}"/> resulting in the <see cref="IActionResult"/> of the request</returns>
+		/// <response code="202">Dump <see cref="Api.Models.Job"/> started successfully.</response>
+		[HttpPatch(Routes.Diagnostics)]
+		[TgsAuthorize(DreamDaemonRights.CreateDump)]
+		[ProducesResponseType(typeof(Api.Models.Job), 202)]
+		public async Task<IActionResult> CreateDump(CancellationToken cancellationToken)
+		{
+			var job = new Models.Job
+			{
+				Instance = Instance,
+				CancelRightsType = RightsType.DreamDaemon,
+				CancelRight = (ulong)DreamDaemonRights.CreateDump,
+				StartedBy = AuthenticationContext.User,
+				Description = "Create DreamDaemon Process Dump"
+			};
+
+			var watchdog = instanceManager.GetInstance(Instance).Watchdog;
+
+			if (!watchdog.Running)
+				return Conflict(new ErrorMessage(ErrorCode.WatchdogNotRunning));
+
+			await jobManager.RegisterOperation(
+				job,
+				(paramJob, databaseContextFactory, progressReporter, ct) => watchdog.CreateDump(ct), cancellationToken)
+				.ConfigureAwait(false);
+			return Accepted(job.ToApi());
+		}
 	}
 }

src/Tgstation.Server.Host/Core/Application.cs

@@ -273,7 +273,11 @@ namespace Tgstation.Server.Host.Core
 				services.AddSingleton<ISymlinkFactory, PosixSymlinkFactory>();
 				services.AddSingleton<IByondInstaller, PosixByondInstaller>();
 				services.AddSingleton<IPostWriteHandler, PosixPostWriteHandler>();
+
 				services.AddSingleton<IProcessFeatures, PosixProcessFeatures>();
+
+				// PosixProcessFeatures also needs a IProcessExecutor for gcore
+				services.AddSingleton(x => new Lazy<IProcessExecutor>(() => x.GetRequiredService<IProcessExecutor>(), true));
 				services.AddSingleton<INetworkPromptReaper, PosixNetworkPromptReaper>();
 			}
 

src/Tgstation.Server.Host/Database/SqliteDatabaseContext.cs

@@ -9,7 +9,7 @@ using Tgstation.Server.Host.Configuration;
 namespace Tgstation.Server.Host.Database
 {
 	/// <summary>
-	/// <see cref="DatabaseContext"/> for MySQL
+	/// <see cref="DatabaseContext"/> for Sqlite.
 	/// </summary>
 	sealed class SqliteDatabaseContext : DatabaseContext
 	{

src/Tgstation.Server.Host/Extensions/DateTimeOffsetExtensions.cs

@@ -0,0 +1,19 @@
+using System;
+using System.Globalization;
+
+namespace Tgstation.Server.Host.Extensions
+{
+	/// <summary>
+	/// Extension methods for the <see cref="DateTimeOffset"/> <see langword="class"/>.
+	/// </summary>
+	static class DateTimeOffsetExtensions
+	{
+		/// <summary>
+		/// Convert a given <paramref name="dateTimeOffset"/> into a <see cref="string"/> that can be used to stamp file creation times.
+		/// </summary>
+		/// <param name="dateTimeOffset">The <see cref="DateTimeOffset"/> to convert.</param>
+		/// <returns><paramref name="dateTimeOffset"/> as a file stamp <see cref="string"/>.</returns>
+		public static string ToFileStamp(this DateTimeOffset dateTimeOffset)
+			=> dateTimeOffset.ToString("yyyyMMddhhmmss", CultureInfo.InvariantCulture);
+	}
+}

src/Tgstation.Server.Host/NativeMethods.cs

@@ -5,7 +5,7 @@ using System.Text;
 namespace Tgstation.Server.Host
 {
 	/// <summary>
-	/// Native Windows methods used by the code
+	/// Native Windows methods used by the code.
 	/// </summary>
 	#pragma warning disable SA1600
 	#pragma warning disable SA1602
@@ -32,6 +32,19 @@ namespace Tgstation.Server.Host
 			SuspendResume = 0x0002,
 		}
 
+		/// <summary>
+		/// See https://docs.microsoft.com/en-us/windows/win32/api/minidumpapiset/ne-minidumpapiset-minidump_type
+		/// </summary>
+		[Flags]
+		public enum MiniDumpType : uint
+		{
+			WithDataSegs = 0x00000001,
+			WithFullMemory = 0x00000002,
+			WithHandleData = 0x00000004,
+			WithUnloadedModules = 0x00000020,
+			WithThreadInfo = 0x00001000,
+		}
+
 		/// <summary>
 		/// See https://docs.microsoft.com/en-us/windows/desktop/api/winuser/nf-winuser-getwindowthreadprocessid
 		/// </summary>
@@ -102,5 +115,18 @@ namespace Tgstation.Server.Host
 		/// </summary>
 		[DllImport("kernel32.dll", SetLastError = true, CharSet = CharSet.Unicode)]
 		public static extern uint ResumeThread(IntPtr hThread);
+
+		/// <summary>
+		/// See https://docs.microsoft.com/en-us/windows/win32/api/minidumpapiset/nf-minidumpapiset-minidumpwritedump
+		/// </summary>
+		[DllImport("dbghelp.dll", SetLastError = true, CharSet = CharSet.Unicode)]
+		public static extern bool MiniDumpWriteDump(
+			IntPtr hProcess,
+			uint processId,
+			SafeHandle hFile,
+			MiniDumpType dumpType,
+			IntPtr expParam,
+			IntPtr userStreamParam,
+			IntPtr callbackParam);
 	}
 }

src/Tgstation.Server.Host/System/IProcessBase.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Threading;
 using System.Threading.Tasks;
 
 namespace Tgstation.Server.Host.System
@@ -27,5 +28,13 @@ namespace Tgstation.Server.Host.System
 		/// Resumes the process.
 		/// </summary>
 		void Resume();
+
+		/// <summary>
+		/// Create a dump file of the process.
+		/// </summary>
+		/// <param name="outputFile">The full path to the output file.</param>
+		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
+		/// <returns>A <see cref="Task"/> representing the running operation.</returns>
+		Task CreateDump(string outputFile, CancellationToken cancellationToken);
 	}
 }

src/Tgstation.Server.Host/System/IProcessExecutor.cs

@@ -13,7 +13,7 @@
 		/// <param name="arguments">The arguments for the <see cref="IProcess"/></param>
 		/// <param name="readOutput">If standard output should be read</param>
 		/// <param name="readError">If standard error should be read</param>
-		/// <param name="noShellExecute">If shell execute should not be used. Ignored if <paramref name="readError"/> or <paramref name="readOutput"/> are set</param>
+		/// <param name="noShellExecute">If shell execute should not be used. Must be set if <paramref name="readError"/> or <paramref name="readOutput"/> are set.</param>
 		/// <returns>A new <see cref="IProcess"/></returns>
 		IProcess LaunchProcess(string fileName, string workingDirectory, string arguments = null, bool readOutput = false, bool readError = false, bool noShellExecute = false);
 

src/Tgstation.Server.Host/System/IProcessFeatures.cs

@@ -17,7 +17,7 @@ namespace Tgstation.Server.Host.System
 		Task<string> GetExecutingUsername(global::System.Diagnostics.Process process, CancellationToken cancellationToken);
 
 		/// <summary>
-		/// Suspend a given <see cref="Process"/>.
+		/// Suspend a given <paramref name="process"/>.
 		/// </summary>
 		/// <param name="process">The <see cref="Process"/> to suspend.</param>
 		void SuspendProcess(global::System.Diagnostics.Process process);
@@ -27,5 +27,14 @@ namespace Tgstation.Server.Host.System
 		/// </summary>
 		/// <param name="process">The <see cref="Process"/> to susperesumend.</param>
 		void ResumeProcess(global::System.Diagnostics.Process process);
+
+		/// <summary>
+		/// Create a dump file for a given <paramref name="process"/>.
+		/// </summary>
+		/// <param name="process">The <see cref="Process"/> to dump.</param>
+		/// <param name="outputFile">The full path to the output file.</param>
+		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
+		/// <returns>A <see cref="Task"/> representing the running operation.</returns>
+		Task CreateDump(global::System.Diagnostics.Process process, string outputFile, CancellationToken cancellationToken);
 	}
 }

src/Tgstation.Server.Host/System/PosixProcessFeatures.cs

@@ -7,13 +7,20 @@ using System.Linq;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
+using Tgstation.Server.Api.Models;
 using Tgstation.Server.Host.IO;
+using Tgstation.Server.Host.Jobs;
 
 namespace Tgstation.Server.Host.System
 {
 	/// <inheritdoc />
 	sealed class PosixProcessFeatures : IProcessFeatures
 	{
+		/// <summary>
+		/// <see cref="Lazy{T}"/> loaded <see cref="IProcessExecutor"/>.
+		/// </summary>
+		readonly Lazy<IProcessExecutor> lazyLoadedProcessExecutor;
+
 		/// <summary>
 		/// The <see cref="IIOManager"/> for the <see cref="PosixProcessFeatures"/>.
 		/// </summary>
@@ -27,10 +34,12 @@ namespace Tgstation.Server.Host.System
 		/// <summary>
 		/// Initializes a new instance of the <see cref="PosixProcessFeatures"/> <see langword="class"/>.
 		/// </summary>
+		/// <param name="lazyLoadedProcessExecutor">The value of <see cref="lazyLoadedProcessExecutor"/>.</param>
 		/// <param name="ioManager">The value of <see cref="ioManager"/>.</param>
 		/// <param name="logger">The value of <see cref="logger"/>.</param>
-		public PosixProcessFeatures(IIOManager ioManager, ILogger<PosixProcessFeatures> logger)
+		public PosixProcessFeatures(Lazy<IProcessExecutor> lazyLoadedProcessExecutor, IIOManager ioManager, ILogger<PosixProcessFeatures> logger)
 		{
+			this.lazyLoadedProcessExecutor = lazyLoadedProcessExecutor ?? throw new ArgumentNullException(nameof(lazyLoadedProcessExecutor));
 			this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager));
 			this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
 		}
@@ -93,5 +102,46 @@ namespace Tgstation.Server.Host.System
 				.FirstOrDefault(x => !String.IsNullOrWhiteSpace(x))
 				?? "UNPARSABLE";
 		}
+
+		/// <inheritdoc />
+		public async Task CreateDump(global::System.Diagnostics.Process process, string outputFile, CancellationToken cancellationToken)
+		{
+			if (process == null)
+				throw new ArgumentNullException(nameof(process));
+			if (outputFile == null)
+				throw new ArgumentNullException(nameof(outputFile));
+
+			const string GCorePath = "/usr/bin/gcore";
+			if (!await ioManager.FileExists(GCorePath, cancellationToken).ConfigureAwait(false))
+				throw new JobException(ErrorCode.MissingGCore);
+
+			var pid = process.Id;
+			string output;
+			int exitCode;
+			using (var gcoreProc = lazyLoadedProcessExecutor.Value.LaunchProcess(
+				GCorePath,
+				Environment.CurrentDirectory,
+				$"-o {outputFile} {process.Id}",
+				true,
+				true,
+				true))
+			{
+				using (cancellationToken.Register(() => gcoreProc.Terminate()))
+					exitCode = await gcoreProc.Lifetime.ConfigureAwait(false);
+
+				output = gcoreProc.GetCombinedOutput();
+				logger.LogDebug("gcore output:{0}{1}", Environment.NewLine, output);
+			}
+
+			if (exitCode != 0)
+				throw new JobException(
+					ErrorCode.GCoreFailure,
+					new JobException(
+						$"Exit Code: {exitCode}{Environment.NewLine}Output:{Environment.NewLine}{output}"));
+
+			// gcore outputs name.pid so remove the pid part
+			var generatedGCoreFile = $"{outputFile}.{pid}";
+			await ioManager.MoveFile(generatedGCoreFile, outputFile, cancellationToken).ConfigureAwait(false);
+		}
 	}
 }

src/Tgstation.Server.Host/System/Process.cs

@@ -165,5 +165,15 @@ namespace Tgstation.Server.Host.System
 			logger.LogTrace("PID {0} Username: {1}", Id, result);
 			return result;
 		}
+
+		/// <inheritdoc />
+		public Task CreateDump(string outputFile, CancellationToken cancellationToken)
+		{
+			if (outputFile == null)
+				throw new ArgumentNullException(nameof(outputFile));
+
+			logger.LogTrace("Dumping PID {0} to {1}...", Id, outputFile);
+			return processFeatures.CreateDump(handle, outputFile, cancellationToken);
+		}
 	}
 }

src/Tgstation.Server.Host/System/WindowsProcessFeatures.cs

@@ -2,6 +2,7 @@
 using Microsoft.Extensions.Logging;
 using System;
 using System.Diagnostics;
+using System.IO;
 using System.Linq;
 using System.Management;
 using System.Threading;
@@ -110,5 +111,29 @@ namespace Tgstation.Server.Host.System
 
 			return Task.FromResult("NO OWNER");
 		}
+
+		/// <inheritdoc />
+		public Task CreateDump(global::System.Diagnostics.Process process, string outputFile, CancellationToken cancellationToken)
+			=> Task.Factory.StartNew(
+				() =>
+				{
+					using var fileStream = new FileStream(outputFile, FileMode.CreateNew);
+					if (!NativeMethods.MiniDumpWriteDump(
+						process.Handle,
+						(uint)process.Id,
+						fileStream.SafeFileHandle,
+						NativeMethods.MiniDumpType.WithDataSegs
+						| NativeMethods.MiniDumpType.WithFullMemory
+						| NativeMethods.MiniDumpType.WithHandleData
+						| NativeMethods.MiniDumpType.WithThreadInfo
+						| NativeMethods.MiniDumpType.WithUnloadedModules,
+						IntPtr.Zero,
+						IntPtr.Zero,
+						IntPtr.Zero))
+						throw new Win32Exception();
+				},
+				cancellationToken,
+				TaskCreationOptions.LongRunning,
+				TaskScheduler.Current);
 	}
 }

tests/Tgstation.Server.Host.Tests/System/TestProcessFeatures.cs

@@ -21,7 +21,7 @@ namespace Tgstation.Server.Host.System.Tests
 		{
 			features = new PlatformIdentifier().IsWindows
 				? (IProcessFeatures)new WindowsProcessFeatures(Mock.Of<ILogger<WindowsProcessFeatures>>())
-				: new PosixProcessFeatures(new DefaultIOManager(), Mock.Of<ILogger<PosixProcessFeatures>>());
+				: new PosixProcessFeatures(new Lazy<IProcessExecutor>(() => null), new DefaultIOManager(), Mock.Of<ILogger<PosixProcessFeatures>>());
 		}
 
 		[TestMethod]

tests/Tgstation.Server.Tests/Instance/WatchdogTest.cs

@@ -43,6 +43,9 @@ namespace Tgstation.Server.Tests.Instance
 				SoftRestart = true
 			}, cancellationToken), ErrorCode.DreamDaemonDoubleSoft);
 
+			await ApiAssert.ThrowsException<ConflictException>(() => instanceClient.DreamDaemon.CreateDump(cancellationToken), ErrorCode.WatchdogNotRunning);
+			await ApiAssert.ThrowsException<ConflictException>(() => instanceClient.DreamDaemon.Restart(cancellationToken), ErrorCode.WatchdogNotRunning);
+
 			await RunBasicTest(cancellationToken);
 
 			// await RunLongRunningTestThenUpdate(cancellationToken);
@@ -53,6 +56,15 @@ namespace Tgstation.Server.Tests.Instance
 			await RunHeartbeatTest(cancellationToken);
 
 			await StartAndLeaveRunning(cancellationToken);
+
+			var dumpJob = await instanceClient.DreamDaemon.CreateDump(cancellationToken);
+			await WaitForJob(dumpJob, 3000, false, cancellationToken);
+
+			var dumpFiles = Directory.GetFiles(Path.Combine(
+				instanceClient.Metadata.Path, "Diagnostics", "ProcessDumps"), "*.dmp");
+			Assert.AreEqual(1, dumpFiles.Length);
+			File.Delete(dumpFiles.Single());
+
 			global::System.Console.WriteLine("TEST: END WATCHDOG TESTS");
 		}
 
@@ -103,12 +115,14 @@ namespace Tgstation.Server.Tests.Instance
 				Assert.Inconclusive($"Incorrect number of DD processes: {ddProcs.Count}");
 
 			using var ddProc = ddProcs.Single();
-			using var ourProcessHandler = new ProcessExecutor(
+			IProcessExecutor executor = null;
+			executor = new ProcessExecutor(
 				new PlatformIdentifier().IsWindows
 					? (IProcessFeatures)new WindowsProcessFeatures(Mock.Of<ILogger<WindowsProcessFeatures>>())
-					: new PosixProcessFeatures(Mock.Of<IIOManager>(), Mock.Of<ILogger<PosixProcessFeatures>>()),
+					: new PosixProcessFeatures(new Lazy<IProcessExecutor>(() => executor), Mock.Of<IIOManager>(), Mock.Of<ILogger<PosixProcessFeatures>>()),
 				Mock.Of<ILogger<ProcessExecutor>>(),
-				LoggerFactory.Create(x => { }))
+				LoggerFactory.Create(x => { }));
+			using var ourProcessHandler = executor
 				.GetProcess(ddProc.Id);
 
 			// Ensure it's responding to heartbeats