New hard link system for PosixWatchdog

- IDmbProvider is now `AsyncDisposable` as opposed to `IDisposable`.
- Add hard link support to `ISymlinkFactory`.
- `PosixWatchdog` now has to mirror deployment structure as hard links before swapping. Because of this deployments may not immediately be applied if the reboots immediately after they were completed.

src/Tgstation.Server.Host/Components/Deployment/DmbFactory.cs

@@ -148,15 +148,18 @@ namespace Tgstation.Server.Host.Components.Deployment
 					cancellationToken);
 			}
 
+			ValueTask dmbDisposeTask;
 			lock (jobLockCounts)
 			{
-				nextDmbProvider?.Dispose();
+				dmbDisposeTask = nextDmbProvider?.DisposeAsync() ?? ValueTask.CompletedTask;
 				nextDmbProvider = newProvider;
 
 				// Oh god dammit
 				var temp = Interlocked.Exchange(ref newerDmbTcs, new TaskCompletionSource());
 				temp.SetResult();
 			}
+
+			await dmbDisposeTask;
 		}
 
 		/// <inheritdoc />
@@ -320,7 +323,7 @@ namespace Tgstation.Server.Host.Components.Deployment
 			finally
 			{
 				if (!providerSubmitted)
-					newProvider.Dispose();
+					await newProvider.DisposeAsync();
 			}
 		}
 #pragma warning restore CA1506

src/Tgstation.Server.Host/Components/Deployment/DmbProvider.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Threading.Tasks;
 
 using Tgstation.Server.Host.IO;
 using Tgstation.Server.Host.Models;
@@ -30,7 +31,7 @@ namespace Tgstation.Server.Host.Components.Deployment
 		readonly string directoryAppend;
 
 		/// <summary>
-		/// The <see cref="Action"/> to run when <see cref="Dispose"/> is called.
+		/// The <see cref="Action"/> to run when <see cref="DisposeAsync"/> is called.
 		/// </summary>
 		Action onDispose;
 
@@ -50,7 +51,11 @@ namespace Tgstation.Server.Host.Components.Deployment
 		}
 
 		/// <inheritdoc />
-		public void Dispose() => onDispose?.Invoke();
+		public ValueTask DisposeAsync()
+		{
+			onDispose?.Invoke();
+			return ValueTask.CompletedTask;
+		}
 
 		/// <inheritdoc />
 		public void KeepAlive() => onDispose = null;

src/Tgstation.Server.Host/Components/Deployment/DreamMaker.cs

@@ -802,7 +802,7 @@ namespace Tgstation.Server.Host.Components.Deployment
 			job.MinimumSecurityLevel = securityLevel; // needed for the TempDmbProvider
 
 			ApiValidationStatus validationStatus;
-			using (var provider = new TemporaryDmbProvider(ioManager.ResolvePath(job.DirectoryName.ToString()), String.Concat(job.DmeName, DmbExtension), job))
+			await using (var provider = new TemporaryDmbProvider(ioManager.ResolvePath(job.DirectoryName.ToString()), String.Concat(job.DmeName, DmbExtension), job))
 			await using (var controller = await sessionControllerFactory.LaunchNew(provider, byondLock, launchParameters, true, cancellationToken))
 			{
 				var launchResult = await controller.LaunchResult.WaitAsync(cancellationToken);

src/Tgstation.Server.Host/Components/Deployment/HardLinkDmbProvider.cs

@@ -0,0 +1,222 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Globalization;
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+
+using Microsoft.Extensions.Logging;
+
+using Tgstation.Server.Host.Configuration;
+using Tgstation.Server.Host.Extensions;
+using Tgstation.Server.Host.IO;
+using Tgstation.Server.Host.Utils;
+
+namespace Tgstation.Server.Host.Components.Deployment
+{
+	/// <summary>
+	/// A <see cref="IDmbProvider"/> that uses hard links.
+	/// </summary>
+	sealed class HardLinkDmbProvider : SwappableDmbProvider
+	{
+		/// <summary>
+		/// The <see cref="CancellationTokenSource"/> for <see cref="mirroringTask"/>.
+		/// </summary>
+		readonly CancellationTokenSource cancellationTokenSource;
+
+		/// <summary>
+		/// The <see cref="Task"/> representing the base provider mirroring operation.
+		/// </summary>
+		readonly Task<string> mirroringTask;
+
+		/// <summary>
+		/// The <see cref="ILogger"/> for the <see cref="HardLinkDmbProvider"/>.
+		/// </summary>
+		readonly ILogger logger;
+
+		/// <summary>
+		/// Initializes a new instance of the <see cref="HardLinkDmbProvider"/> class.
+		/// </summary>
+		/// <param name="baseProvider">The <see cref="IDmbProvider"/> for the <see cref="SwappableDmbProvider"/>.</param>
+		/// <param name="ioManager">The <see cref="IIOManager"/> for the <see cref="SwappableDmbProvider"/>.</param>
+		/// <param name="symlinkFactory">The <see cref="ISymlinkFactory"/> for the <see cref="SwappableDmbProvider"/>.</param>
+		/// <param name="logger">The value of <see cref="logger"/>.</param>
+		/// <param name="generalConfiguration">The <see cref="GeneralConfiguration"/> for the <see cref="HardLinkDmbProvider"/>.</param>
+		public HardLinkDmbProvider(
+			IDmbProvider baseProvider,
+			IIOManager ioManager,
+			ISymlinkFactory symlinkFactory,
+			ILogger logger,
+			GeneralConfiguration generalConfiguration)
+			: base(
+				 baseProvider,
+				 ioManager,
+				 symlinkFactory)
+		{
+			this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
+			cancellationTokenSource = new CancellationTokenSource();
+			try
+			{
+				mirroringTask = MirrorSourceDirectory(generalConfiguration.GetCopyDirectoryTaskThrottle(), cancellationTokenSource.Token);
+			}
+			catch
+			{
+				cancellationTokenSource.Dispose();
+				throw;
+			}
+		}
+
+		/// <inheritdoc />
+		public override async ValueTask DisposeAsync()
+		{
+			cancellationTokenSource.Cancel();
+			cancellationTokenSource.Dispose();
+			try
+			{
+				await mirroringTask;
+			}
+			catch (OperationCanceledException ex)
+			{
+				logger.LogDebug(ex, "Mirroring task cancelled!");
+			}
+
+			await base.DisposeAsync();
+		}
+
+		/// <inheritdoc />
+		public override Task FinishActivationPreparation(CancellationToken cancellationToken)
+		{
+			if (!mirroringTask.IsCompleted)
+				logger.LogTrace("Waiting for mirroring to complete...");
+
+			return mirroringTask.WaitAsync(cancellationToken);
+		}
+
+		/// <inheritdoc />
+		protected override async Task DoSwap(CancellationToken cancellationToken)
+		{
+			var mirroredDir = await mirroringTask.WaitAsync(cancellationToken);
+			var goAheadTcs = new TaskCompletionSource();
+
+			// I feel dirty...
+			async void DisposeOfOldDirectory()
+			{
+				var directoryMoved = false;
+				var disposePath = Guid.NewGuid().ToString();
+				try
+				{
+					await IOManager.MoveDirectory(LiveGameDirectory, disposePath, cancellationToken);
+					directoryMoved = true;
+					goAheadTcs.SetResult();
+					await IOManager.DeleteDirectory(disposePath, CancellationToken.None); // DCT: We're detached at this point
+				}
+				catch (Exception ex)
+				{
+					if (directoryMoved)
+						logger.LogWarning(ex, "Failed to delete hard linked directory: {disposePath}", disposePath);
+					else
+					{
+						logger.LogDebug(ex, "Live directory appears to not exist");
+						goAheadTcs.SetResult();
+					}
+				}
+			}
+
+			DisposeOfOldDirectory();
+			await goAheadTcs.Task;
+			await IOManager.MoveDirectory(mirroredDir, LiveGameDirectory, cancellationToken);
+		}
+
+		/// <summary>
+		/// Mirror the <see cref="Models.CompileJob"/>.
+		/// </summary>
+		/// <param name="taskThrottle">The optional maximum number of simultaneous tasks allowed to execute.</param>
+		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
+		/// <returns>A <see cref="Task{TResult}"/> resulting in the full path to the mirrored directory.</returns>
+		async Task<string> MirrorSourceDirectory(int? taskThrottle, CancellationToken cancellationToken)
+		{
+			var stopwatch = Stopwatch.StartNew();
+			var mirrorGuid = Guid.NewGuid();
+			logger.LogDebug("Starting to mirror {sourceDir} as hard links to {mirrorGuid}...", CompileJob.DirectoryName, mirrorGuid);
+			if (taskThrottle.HasValue && taskThrottle < 1)
+				throw new ArgumentOutOfRangeException(nameof(taskThrottle), taskThrottle, "taskThrottle must be at least 1!");
+
+			var src = IOManager.ResolvePath(CompileJob.DirectoryName.ToString());
+			var dest = IOManager.ResolvePath(mirrorGuid.ToString());
+
+			using var semaphore = taskThrottle.HasValue ? new SemaphoreSlim(taskThrottle.Value) : null;
+			await Task.WhenAll(MirrorDirectoryImpl(src, dest, semaphore, cancellationToken));
+			stopwatch.Stop();
+
+			logger.LogDebug(
+				"Finished mirror of {sourceDir} to {mirrorGuid} in {seconds}s...",
+				CompileJob.DirectoryName,
+				mirrorGuid,
+				stopwatch.Elapsed.TotalSeconds.ToString("0.##", CultureInfo.InvariantCulture));
+
+			return dest;
+		}
+
+		/// <summary>
+		/// Recursively create tasks to create a hard link directory mirror of <paramref name="src"/> to <paramref name="dest"/>.
+		/// </summary>
+		/// <param name="src">The source directory path.</param>
+		/// <param name="dest">The destination directory path.</param>
+		/// <param name="semaphore">Optional <see cref="SemaphoreSlim"/> used to limit degree of parallelism.</param>
+		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
+		/// <returns>A <see cref="IEnumerable{T}"/> of <see cref="Task"/>s representing the running operations. The first <see cref="Task"/> returned is always the necessary call to <see cref="IIOManager.CreateDirectory(string, CancellationToken)"/>.</returns>
+		/// <remarks>I genuinely don't know how this will work with symlinked files. Waiting for the issue report I guess.</remarks>
+		IEnumerable<Task> MirrorDirectoryImpl(string src, string dest, SemaphoreSlim semaphore, CancellationToken cancellationToken)
+		{
+			var dir = new DirectoryInfo(src);
+			Task subdirCreationTask = null;
+			foreach (var subDirectory in dir.EnumerateDirectories())
+			{
+				// check if we are a symbolic link
+				if (!subDirectory.Attributes.HasFlag(FileAttributes.Directory) || subDirectory.Attributes.HasFlag(FileAttributes.ReparsePoint))
+				{
+					logger.LogTrace("Skipping symlink to {subdir}", subDirectory.Name);
+					continue;
+				}
+
+				var checkingSubdirCreationTask = true;
+				foreach (var copyTask in MirrorDirectoryImpl(subDirectory.FullName, Path.Combine(dest, subDirectory.Name), semaphore, cancellationToken))
+				{
+					if (subdirCreationTask == null)
+					{
+						subdirCreationTask = copyTask;
+						yield return subdirCreationTask;
+					}
+					else if (!checkingSubdirCreationTask)
+						yield return copyTask;
+
+					checkingSubdirCreationTask = false;
+				}
+			}
+
+			foreach (var fileInfo in dir.EnumerateFiles())
+			{
+				if (subdirCreationTask == null)
+				{
+					subdirCreationTask = IOManager.CreateDirectory(dest, cancellationToken);
+					yield return subdirCreationTask;
+				}
+
+				var sourceFile = fileInfo.FullName;
+				var destFile = IOManager.ConcatPath(dest, fileInfo.Name);
+
+				async Task LinkThisFile()
+				{
+					await subdirCreationTask.WaitAsync(cancellationToken);
+					using var lockContext = semaphore != null
+						? await SemaphoreSlimContext.Lock(semaphore, cancellationToken)
+						: null;
+					await SymlinkFactory.CreateHardLink(sourceFile, destFile, cancellationToken);
+				}
+
+				yield return LinkThisFile();
+			}
+		}
+	}
+}

src/Tgstation.Server.Host/Components/Deployment/IDmbProvider.cs

@@ -7,7 +7,7 @@ namespace Tgstation.Server.Host.Components.Deployment
 	/// <summary>
 	/// Provides absolute paths to the latest compiled .dmbs.
 	/// </summary>
-	public interface IDmbProvider : IDisposable
+	public interface IDmbProvider : IAsyncDisposable
 	{
 		/// <summary>
 		/// The file name of the .dmb.

src/Tgstation.Server.Host/Components/Deployment/SwappableDmbProvider.cs

@@ -10,7 +10,7 @@ namespace Tgstation.Server.Host.Components.Deployment
 	/// <summary>
 	/// A <see cref="IDmbProvider"/> that uses symlinks.
 	/// </summary>
-	sealed class SwappableDmbProvider : IDmbProvider
+	class SwappableDmbProvider : IDmbProvider
 	{
 		/// <summary>
 		/// The directory where the <see cref="baseProvider"/> is symlinked to.
@@ -21,7 +21,7 @@ namespace Tgstation.Server.Host.Components.Deployment
 		public string DmbName => baseProvider.DmbName;
 
 		/// <inheritdoc />
-		public string Directory => ioManager.ResolvePath(LiveGameDirectory);
+		public string Directory => IOManager.ResolvePath(LiveGameDirectory);
 
 		/// <inheritdoc />
 		public CompileJob CompileJob => baseProvider.CompileJob;
@@ -31,20 +31,20 @@ namespace Tgstation.Server.Host.Components.Deployment
 		/// </summary>
 		public bool Swapped => swapped != 0;
 
-		/// <summary>
-		/// The <see cref="IDmbProvider"/> we are swapping for.
-		/// </summary>
-		readonly IDmbProvider baseProvider;
-
 		/// <summary>
 		/// The <see cref="IIOManager"/> to use.
 		/// </summary>
-		readonly IIOManager ioManager;
+		protected IIOManager IOManager { get; }
 
 		/// <summary>
 		/// The <see cref="ISymlinkFactory"/> to use.
 		/// </summary>
-		readonly ISymlinkFactory symlinkFactory;
+		protected ISymlinkFactory SymlinkFactory { get; }
+
+		/// <summary>
+		/// The <see cref="IDmbProvider"/> we are swapping for.
+		/// </summary>
+		readonly IDmbProvider baseProvider;
 
 		/// <summary>
 		/// Backing field for <see cref="Swapped"/>.
@@ -55,17 +55,17 @@ namespace Tgstation.Server.Host.Components.Deployment
 		/// Initializes a new instance of the <see cref="SwappableDmbProvider"/> class.
 		/// </summary>
 		/// <param name="baseProvider">The value of <see cref="baseProvider"/>.</param>
-		/// <param name="ioManager">The value of <see cref="ioManager"/>.</param>
-		/// <param name="symlinkFactory">The value of <see cref="symlinkFactory"/>.</param>
+		/// <param name="ioManager">The value of <see cref="IOManager"/>.</param>
+		/// <param name="symlinkFactory">The value of <see cref="SymlinkFactory"/>.</param>
 		public SwappableDmbProvider(IDmbProvider baseProvider, IIOManager ioManager, ISymlinkFactory symlinkFactory)
 		{
 			this.baseProvider = baseProvider ?? throw new ArgumentNullException(nameof(baseProvider));
-			this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager));
-			this.symlinkFactory = symlinkFactory ?? throw new ArgumentNullException(nameof(symlinkFactory));
+			IOManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager));
+			SymlinkFactory = symlinkFactory ?? throw new ArgumentNullException(nameof(symlinkFactory));
 		}
 
 		/// <inheritdoc />
-		public void Dispose() => baseProvider.Dispose();
+		public virtual ValueTask DisposeAsync() => baseProvider.DisposeAsync();
 
 		/// <inheritdoc />
 		public void KeepAlive() => baseProvider.KeepAlive();
@@ -75,19 +75,37 @@ namespace Tgstation.Server.Host.Components.Deployment
 		/// </summary>
 		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
 		/// <returns>A <see cref="Task"/> representing the running operation.</returns>
-		public async Task MakeActive(CancellationToken cancellationToken)
+		public Task MakeActive(CancellationToken cancellationToken)
 		{
 			if (Interlocked.Exchange(ref swapped, 1) != 0)
 				throw new InvalidOperationException("Already swapped!");
 
-			if (symlinkFactory.SymlinkedDirectoriesAreDeletedAsFiles)
-				await ioManager.DeleteFile(LiveGameDirectory, cancellationToken);
-			else
-				await ioManager.DeleteDirectory(LiveGameDirectory, cancellationToken);
+			return DoSwap(cancellationToken);
+		}
 
-			await symlinkFactory.CreateSymbolicLink(
-				ioManager.ResolvePath(baseProvider.Directory),
-				ioManager.ResolvePath(LiveGameDirectory),
+		/// <summary>
+		/// Should be <see langword="await"/>. before calling <see cref="MakeActive(CancellationToken)"/> to ensure the <see cref="SwappableDmbProvider"/> is ready to instantly swap. Can be called multiple times.
+		/// </summary>
+		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
+		/// <returns>A <see cref="Task"/> representing the preparation process.</returns>
+		public virtual Task FinishActivationPreparation(CancellationToken cancellationToken)
+			=> Task.CompletedTask;
+
+		/// <summary>
+		/// Perform the swapping action.
+		/// </summary>
+		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
+		/// <returns>A <see cref="Task"/> representing the running operation.</returns>
+		protected virtual async Task DoSwap(CancellationToken cancellationToken)
+		{
+			if (SymlinkFactory.SymlinkedDirectoriesAreDeletedAsFiles)
+				await IOManager.DeleteFile(LiveGameDirectory, cancellationToken);
+			else
+				await IOManager.DeleteDirectory(LiveGameDirectory, cancellationToken);
+
+			await SymlinkFactory.CreateSymbolicLink(
+				IOManager.ResolvePath(baseProvider.Directory),
+				IOManager.ResolvePath(LiveGameDirectory),
 				cancellationToken);
 		}
 	}

src/Tgstation.Server.Host/Components/Deployment/TemporaryDmbProvider.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Threading.Tasks;
 
 using Tgstation.Server.Host.Models;
 
@@ -32,9 +33,7 @@ namespace Tgstation.Server.Host.Components.Deployment
 		}
 
 		/// <inheritdoc />
-		public void Dispose()
-		{
-		}
+		public ValueTask DisposeAsync() => ValueTask.CompletedTask;
 
 		/// <inheritdoc />
 		public void KeepAlive() => throw new NotSupportedException();

src/Tgstation.Server.Host/Components/Session/ISessionController.cs

@@ -128,7 +128,7 @@ namespace Tgstation.Server.Host.Components.Session
 		/// Replace the <see cref="IDmbProvider"/> in use with a given <paramref name="newProvider"/>, disposing the old one.
 		/// </summary>
 		/// <param name="newProvider">The new <see cref="IDmbProvider"/>.</param>
-		/// <returns>An <see cref="IDisposable"/> to be disposed once certain that the original <see cref="IDmbProvider"/> is no longer in use.</returns>
-		IDisposable ReplaceDmbProvider(IDmbProvider newProvider);
+		/// <returns>An <see cref="IAsyncDisposable"/> to be disposed once certain that the original <see cref="IDmbProvider"/> is no longer in use.</returns>
+		IAsyncDisposable ReplaceDmbProvider(IDmbProvider newProvider);
 	}
 }

src/Tgstation.Server.Host/Components/Session/SessionController.cs

@@ -350,8 +350,13 @@ namespace Tgstation.Server.Host.Components.Session
 			await process.DisposeAsync();
 			byondLock.Dispose();
 			bridgeRegistration?.Dispose();
-			ReattachInformation.Dmb.Dispose();
-			ReattachInformation.InitialDmb?.Dispose();
+			var regularDmbDisposeTask = ReattachInformation.Dmb.DisposeAsync();
+			var initialDmb = ReattachInformation.InitialDmb;
+			if (initialDmb != null)
+				await initialDmb.DisposeAsync();
+
+			await regularDmbDisposeTask;
+
 			chatTrackingContext.Dispose();
 			reattachTopicCts.Dispose();
 
@@ -552,7 +557,7 @@ namespace Tgstation.Server.Host.Components.Session
 		public void Resume() => process.Resume();
 
 		/// <inheritdoc />
-		public IDisposable ReplaceDmbProvider(IDmbProvider dmbProvider)
+		public IAsyncDisposable ReplaceDmbProvider(IDmbProvider dmbProvider)
 		{
 			var oldDmb = ReattachInformation.Dmb;
 			ReattachInformation.Dmb = dmbProvider ?? throw new ArgumentNullException(nameof(dmbProvider));

src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs

@@ -268,7 +268,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 
 				// server didn't get control of this dmb
 				if (dmbToUse != null && !serverWasActive)
-					dmbToUse.Dispose();
+					await dmbToUse.DisposeAsync();
 
 				throw;
 			}

src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdog.cs

@@ -1,4 +1,5 @@
-using System.Threading;
+using System;
+using System.Threading;
 using System.Threading.Tasks;
 
 using Microsoft.Extensions.Logging;
@@ -9,6 +10,7 @@ using Tgstation.Server.Host.Components.Deployment;
 using Tgstation.Server.Host.Components.Deployment.Remote;
 using Tgstation.Server.Host.Components.Events;
 using Tgstation.Server.Host.Components.Session;
+using Tgstation.Server.Host.Configuration;
 using Tgstation.Server.Host.Core;
 using Tgstation.Server.Host.IO;
 using Tgstation.Server.Host.Jobs;
@@ -21,6 +23,11 @@ namespace Tgstation.Server.Host.Components.Watchdog
 	/// </summary>
 	sealed class PosixWatchdog : WindowsWatchdog
 	{
+		/// <summary>
+		/// The <see cref="GeneralConfiguration"/> for the <see cref="PosixWatchdog"/>.
+		/// </summary>
+		readonly GeneralConfiguration generalConfiguration;
+
 		/// <summary>
 		/// Initializes a new instance of the <see cref="PosixWatchdog"/> class.
 		/// </summary>
@@ -39,6 +46,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// <param name="logger">The <see cref="ILogger"/> for the <see cref="WatchdogBase"/>.</param>
 		/// <param name="initialLaunchParameters">The <see cref="DreamDaemonLaunchParameters"/> for the <see cref="WatchdogBase"/>.</param>
 		/// <param name="instance">The <see cref="Api.Models.Instance"/> for the <see cref="WatchdogBase"/>.</param>
+		/// <param name="generalConfiguration">The value of <see cref="GeneralConfiguration"/>.</param>
 		/// <param name="autoStart">The autostart value for the <see cref="WatchdogBase"/>.</param>
 		public PosixWatchdog(
 			IChatManager chat,
@@ -56,6 +64,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			ILogger<PosixWatchdog> logger,
 			DreamDaemonLaunchParameters initialLaunchParameters,
 			Api.Models.Instance instance,
+			GeneralConfiguration generalConfiguration,
 			bool autoStart)
 			: base(
 				  chat,
@@ -75,13 +84,15 @@ namespace Tgstation.Server.Host.Components.Watchdog
 				  instance,
 				  autoStart)
 		{
+			this.generalConfiguration = generalConfiguration ?? throw new ArgumentNullException(nameof(generalConfiguration));
 		}
 
 		/// <inheritdoc />
 		protected override Task ApplyInitialDmb(CancellationToken cancellationToken)
-		{
-			// not necessary to hold initial .dmb on Linux because of based inode deletes
-			return Task.CompletedTask;
-		}
+			=> Task.CompletedTask; // not necessary to hold initial .dmb on Linux because of based inode deletes
+
+		/// <inheritdoc />
+		protected override SwappableDmbProvider CreateSwappableDmbProvider(IDmbProvider dmbProvider)
+			=> new HardLinkDmbProvider(dmbProvider, GameIOManager, SymlinkFactory, Logger, generalConfiguration);
 	}
 }

src/Tgstation.Server.Host/Components/Watchdog/PosixWatchdogFactory.cs

@@ -76,6 +76,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 				LoggerFactory.CreateLogger<PosixWatchdog>(),
 				settings,
 				instance,
+				GeneralConfiguration,
 				settings.AutoStart ?? throw new ArgumentNullException(nameof(settings)));
 	}
 }

src/Tgstation.Server.Host/Components/Watchdog/WindowsWatchdog.cs

@@ -37,7 +37,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// <summary>
 		/// The <see cref="ISymlinkFactory"/> for the <see cref="WindowsWatchdog"/>.
 		/// </summary>
-		readonly ISymlinkFactory symlinkFactory;
+		protected ISymlinkFactory SymlinkFactory { get; }
 
 		/// <summary>
 		/// <see cref="List{T}"/> of <see cref="Task"/>s that are waiting to clean up old deployments.
@@ -68,7 +68,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// <param name="eventConsumer">The <see cref="IEventConsumer"/> for the <see cref="WatchdogBase"/>.</param>
 		/// <param name="remoteDeploymentManagerFactory">The <see cref="IRemoteDeploymentManagerFactory"/> for the <see cref="WatchdogBase"/>.</param>
 		/// <param name="gameIOManager">The value of <see cref="GameIOManager"/>.</param>
-		/// <param name="symlinkFactory">The value of <see cref="symlinkFactory"/>.</param>
+		/// <param name="symlinkFactory">The value of <see cref="SymlinkFactory"/>.</param>
 		/// <param name="logger">The <see cref="ILogger"/> for the <see cref="WatchdogBase"/>.</param>
 		/// <param name="initialLaunchParameters">The <see cref="DreamDaemonLaunchParameters"/> for the <see cref="WatchdogBase"/>.</param>
 		/// <param name="instance">The <see cref="Api.Models.Instance"/> for the <see cref="WatchdogBase"/>.</param>
@@ -109,7 +109,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			try
 			{
 				GameIOManager = gameIOManager ?? throw new ArgumentNullException(nameof(gameIOManager));
-				this.symlinkFactory = symlinkFactory ?? throw new ArgumentNullException(nameof(symlinkFactory));
+				SymlinkFactory = symlinkFactory ?? throw new ArgumentNullException(nameof(symlinkFactory));
 
 				deploymentCleanupTasks = new List<Task>();
 			}
@@ -131,7 +131,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 
 			// If we reach this point, we can guarantee PrepServerForLaunch will be called before starting again.
 			ActiveSwappable = null;
-			pendingSwappable?.Dispose();
+			await (pendingSwappable?.DisposeAsync() ?? ValueTask.CompletedTask);
 			pendingSwappable = null;
 
 			await DrainDeploymentCleanupTasks(true);
@@ -142,9 +142,10 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		{
 			if (pendingSwappable != null)
 			{
-				var updateTask = BeforeApplyDmb(pendingSwappable.CompileJob, cancellationToken);
+				Task RunPrequel() => BeforeApplyDmb(pendingSwappable.CompileJob, cancellationToken);
 
-				if (!pendingSwappable.Swapped)
+				var needToSwap = !pendingSwappable.Swapped;
+				if (needToSwap)
 				{
 					// IMPORTANT: THE SESSIONCONTROLLER SHOULD STILL BE PROCESSING THE BRIDGE REQUEST SO WE KNOW DD IS SLEEPING
 					// OTHERWISE, IT COULD RETURN TO /world/Reboot() TOO EARLY AND LOAD THE WRONG .DMB
@@ -153,18 +154,29 @@ namespace Tgstation.Server.Host.Components.Watchdog
 						// integration test logging will catch this
 						Logger.LogError(
 							"The reboot bridge request completed before the watchdog could suspend the server! This can lead to buggy DreamDaemon behaviour and should be reported! To ensure stability, we will need to hard reboot the server");
-						await updateTask;
+						await RunPrequel();
 						return MonitorAction.Restart;
 					}
 
-					await PerformDmbSwap(pendingSwappable, cancellationToken);
+					// DCT: Not necessary
+					if (!pendingSwappable.FinishActivationPreparation(CancellationToken.None).IsCompleted)
+					{
+						// rare pokemon
+						Logger.LogInformation("Deployed .dme is not ready to swap, delaying until next reboot!");
+						Chat.QueueWatchdogMessage("The pending deployment was not ready to be activated this reboot. It will be applied at the next one.");
+						return MonitorAction.Continue;
+					}
 				}
 
+				var updateTask = RunPrequel();
+				if (needToSwap)
+					await PerformDmbSwap(pendingSwappable, cancellationToken);
+
 				var currentCompileJobId = Server.ReattachInformation.Dmb.CompileJob.Id;
 
 				await DrainDeploymentCleanupTasks(false);
 
-				IDisposable lingeringDeployment;
+				IAsyncDisposable lingeringDeployment;
 				var localDeploymentCleanupGate = new TaskCompletionSource();
 				async Task CleanupLingeringDeployment()
 				{
@@ -191,7 +203,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 							? " due to timeout!"
 							: "...");
 
-					lingeringDeployment.Dispose();
+					await lingeringDeployment.DisposeAsync();
 				}
 
 				var oldDeploymentCleanupGate = Interlocked.Exchange(ref deploymentCleanupGate, localDeploymentCleanupGate);
@@ -247,31 +259,31 @@ namespace Tgstation.Server.Host.Components.Watchdog
 
 			if (!canSeamlesslySwap)
 			{
-				compileJobProvider.Dispose();
+				await compileJobProvider.DisposeAsync();
 				await base.HandleNewDmbAvailable(cancellationToken);
 				return;
 			}
 
-			SwappableDmbProvider windowsProvider = null;
+			SwappableDmbProvider swappableProvider = null;
 			try
 			{
-				windowsProvider = new SwappableDmbProvider(compileJobProvider, GameIOManager, symlinkFactory);
+				swappableProvider = CreateSwappableDmbProvider(compileJobProvider);
 				if (ActiveCompileJob.DMApiVersion == null)
 				{
 					Logger.LogWarning("Active compile job has no DMAPI! Commencing immediate .dmb swap. Note this behavior is known to be buggy in some DM code contexts. See https://github.com/tgstation/tgstation-server/issues/1550");
-					await PerformDmbSwap(windowsProvider, cancellationToken);
+					await PerformDmbSwap(swappableProvider, cancellationToken);
 				}
 			}
 			catch (Exception ex)
 			{
 				Logger.LogError(ex, "Exception while swapping");
-				IDmbProvider providerToDispose = windowsProvider ?? compileJobProvider;
-				providerToDispose.Dispose();
+				IDmbProvider providerToDispose = swappableProvider ?? compileJobProvider;
+				await providerToDispose.DisposeAsync();
 				throw;
 			}
 
-			pendingSwappable?.Dispose();
-			pendingSwappable = windowsProvider;
+			await (pendingSwappable?.DisposeAsync() ?? ValueTask.CompletedTask);
+			pendingSwappable = swappableProvider;
 		}
 
 		/// <inheritdoc />
@@ -284,7 +296,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
 
 			Logger.LogTrace("Prep for server launch");
 
-			ActiveSwappable = new SwappableDmbProvider(dmbToUse, GameIOManager, symlinkFactory);
+			ActiveSwappable = CreateSwappableDmbProvider(dmbToUse);
 			try
 			{
 				await InitialLink(cancellationToken);
@@ -310,6 +322,14 @@ namespace Tgstation.Server.Host.Components.Watchdog
 			Server.ReattachInformation.InitialDmb = await DmbFactory.FromCompileJob(Server.CompileJob, cancellationToken);
 		}
 
+		/// <summary>
+		/// Create a <see cref="SwappableDmbProvider"/> for a given <paramref name="dmbProvider"/>.
+		/// </summary>
+		/// <param name="dmbProvider">The <see cref="IDmbProvider"/> to create a <see cref="SwappableDmbProvider"/> for.</param>
+		/// <returns>A new <see cref="SwappableDmbProvider"/>.</returns>
+		protected virtual SwappableDmbProvider CreateSwappableDmbProvider(IDmbProvider dmbProvider)
+			=> new SwappableDmbProvider(dmbProvider, GameIOManager, SymlinkFactory);
+
 		/// <inheritdoc />
 		protected override async Task SessionStartupPersist(CancellationToken cancellationToken)
 		{
@@ -332,10 +352,11 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		/// </summary>
 		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
 		/// <returns>A <see cref="Task"/> representing the running operation.</returns>
-		Task InitialLink(CancellationToken cancellationToken)
+		async ValueTask InitialLink(CancellationToken cancellationToken)
 		{
-			Logger.LogTrace("Symlinking compile job...");
-			return ActiveSwappable.MakeActive(cancellationToken);
+			await ActiveSwappable.FinishActivationPreparation(cancellationToken);
+			Logger.LogTrace("Linking compile job...");
+			await ActiveSwappable.MakeActive(cancellationToken);
 		}
 
 		/// <summary>
@@ -348,6 +369,8 @@ namespace Tgstation.Server.Host.Components.Watchdog
 		{
 			Logger.LogDebug("Swapping to compile job {id}...", newProvider.CompileJob.Id);
 
+			await newProvider.FinishActivationPreparation(cancellationToken);
+
 			var suspended = false;
 			var server = Server;
 			try

src/Tgstation.Server.Host/IO/ISymlinkFactory.cs

@@ -22,5 +22,14 @@ namespace Tgstation.Server.Host.IO
 		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
 		/// <returns>A <see cref="Task"/> representing the running operation.</returns>
 		Task CreateSymbolicLink(string targetPath, string linkPath, CancellationToken cancellationToken);
+
+		/// <summary>
+		/// Creates a hard link.
+		/// </summary>
+		/// <param name="targetPath">The path to the hard target.</param>
+		/// <param name="linkPath">The path to the link.</param>
+		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
+		/// <returns>A <see cref="Task"/> representing the running operation.</returns>
+		Task CreateHardLink(string targetPath, string linkPath, CancellationToken cancellationToken);
 	}
 }

src/Tgstation.Server.Host/IO/PosixSymlinkFactory.cs

@@ -15,6 +15,22 @@ namespace Tgstation.Server.Host.IO
 		/// <inheritdoc />
 		public bool SymlinkedDirectoriesAreDeletedAsFiles => true;
 
+		/// <inheritdoc />
+		public Task CreateHardLink(string targetPath, string linkPath, CancellationToken cancellationToken) => Task.Factory.StartNew(
+			() =>
+			{
+				ArgumentNullException.ThrowIfNull(targetPath);
+				ArgumentNullException.ThrowIfNull(linkPath);
+
+				cancellationToken.ThrowIfCancellationRequested();
+				var fsInfo = new UnixFileInfo(targetPath);
+				cancellationToken.ThrowIfCancellationRequested();
+				fsInfo.CreateLink(linkPath);
+			},
+			cancellationToken,
+			DefaultIOManager.BlockingTaskCreationOptions,
+			TaskScheduler.Current);
+
 		/// <inheritdoc />
 		public Task CreateSymbolicLink(string targetPath, string linkPath, CancellationToken cancellationToken) => Task.Factory.StartNew(
 			() =>

src/Tgstation.Server.Host/IO/WindowsSymlinkFactory.cs

@@ -16,6 +16,10 @@ namespace Tgstation.Server.Host.IO
 		/// <inheritdoc />
 		public bool SymlinkedDirectoriesAreDeletedAsFiles => false;
 
+		/// <inheritdoc />
+		public Task CreateHardLink(string targetPath, string linkPath, CancellationToken cancellationToken)
+			=> throw new NotSupportedException();
+
 		/// <inheritdoc />
 		public Task CreateSymbolicLink(string targetPath, string linkPath, CancellationToken cancellationToken) => Task.Factory.StartNew(
 			() =>