Even safe instance accessing

- Break up IInstance for better separation of church and state
- Jobs now provide the IInstanceCore to the operation callback
- Added instance references to prevent race conditions with controllers and onlining/offlining

src/Tgstation.Server.Host/Components/IInstance.cs

@@ -1,55 +1,12 @@
 using Microsoft.Extensions.Hosting;
 using System;
-using System.Threading.Tasks;
-using Tgstation.Server.Host.Components.Byond;
-using Tgstation.Server.Host.Components.Chat;
-using Tgstation.Server.Host.Components.Deployment;
-using Tgstation.Server.Host.Components.Repository;
-using Tgstation.Server.Host.Components.StaticFiles;
-using Tgstation.Server.Host.Components.Watchdog;
 
 namespace Tgstation.Server.Host.Components
 {
 	/// <summary>
-	/// For interacting with the instance services
+	/// Component version of <see cref="IInstanceCore"/>.
 	/// </summary>
-	public interface IInstance : ILatestCompileJobProvider, IHostedService, IRenameNotifyee, IAsyncDisposable
+	interface IInstance : IInstanceCore, IHostedService, IAsyncDisposable
 	{
-		/// <summary>
-		/// The <see cref="IRepositoryManager"/> for the <see cref="IInstance"/>
-		/// </summary>
-		IRepositoryManager RepositoryManager { get; }
-
-		/// <summary>
-		/// The <see cref="IByondManager"/> for the <see cref="IInstance"/>
-		/// </summary>
-		IByondManager ByondManager { get; }
-
-		/// <summary>
-		/// The <see cref="IDreamMaker"/> for the <see cref="IInstance"/>.
-		/// </summary>
-		IDreamMaker DreamMaker { get; }
-
-		/// <summary>
-		/// The <see cref="IWatchdog"/> for the <see cref="IInstance"/>
-		/// </summary>
-		IWatchdog Watchdog { get; }
-
-		/// <summary>
-		/// The <see cref="IChatManager"/> for the <see cref="IInstance"/>
-		/// </summary>
-		IChatManager Chat { get; }
-
-		/// <summary>
-		/// The <see cref="IConfiguration"/> for the <see cref="IInstance"/>
-		/// </summary>
-		IConfiguration Configuration { get; }
-
-		/// <summary>
-		/// Change the <see cref="Api.Models.Instance.AutoUpdateInterval"/> for the <see cref="IInstance"/>
-		/// </summary>
-		/// <param name="newInterval">The new auto update inteval</param>
-		/// <returns>A <see cref="Task"/> representing the running operation</returns>
-		Task SetAutoUpdateInterval(uint newInterval);
 	}
-}
+}

src/Tgstation.Server.Host/Components/IInstanceCore.cs

@@ -0,0 +1,53 @@
+using System.Threading.Tasks;
+using Tgstation.Server.Host.Components.Byond;
+using Tgstation.Server.Host.Components.Chat;
+using Tgstation.Server.Host.Components.Deployment;
+using Tgstation.Server.Host.Components.Repository;
+using Tgstation.Server.Host.Components.StaticFiles;
+using Tgstation.Server.Host.Components.Watchdog;
+
+namespace Tgstation.Server.Host.Components
+{
+	/// <summary>
+	/// For interacting with the instance services
+	/// </summary>
+	public interface IInstanceCore : ILatestCompileJobProvider, IRenameNotifyee
+	{
+		/// <summary>
+		/// The <see cref="IRepositoryManager"/> for the <see cref="IInstanceCore"/>
+		/// </summary>
+		IRepositoryManager RepositoryManager { get; }
+
+		/// <summary>
+		/// The <see cref="IByondManager"/> for the <see cref="IInstanceCore"/>
+		/// </summary>
+		IByondManager ByondManager { get; }
+
+		/// <summary>
+		/// The <see cref="IDreamMaker"/> for the <see cref="IInstanceCore"/>.
+		/// </summary>
+		IDreamMaker DreamMaker { get; }
+
+		/// <summary>
+		/// The <see cref="IWatchdog"/> for the <see cref="IInstanceCore"/>
+		/// </summary>
+		IWatchdog Watchdog { get; }
+
+		/// <summary>
+		/// The <see cref="IChatManager"/> for the <see cref="IInstanceCore"/>
+		/// </summary>
+		IChatManager Chat { get; }
+
+		/// <summary>
+		/// The <see cref="IConfiguration"/> for the <see cref="IInstanceCore"/>
+		/// </summary>
+		IConfiguration Configuration { get; }
+
+		/// <summary>
+		/// Change the <see cref="Api.Models.Instance.AutoUpdateInterval"/> for the <see cref="IInstanceCore"/>
+		/// </summary>
+		/// <param name="newInterval">The new auto update inteval</param>
+		/// <returns>A <see cref="Task"/> representing the running operation</returns>
+		Task SetAutoUpdateInterval(uint newInterval);
+	}
+}

src/Tgstation.Server.Host/Components/IInstanceCoreProvider.cs

@@ -0,0 +1,15 @@
+namespace Tgstation.Server.Host.Components
+{
+	/// <summary>
+	/// Provider for <see cref="IInstanceCore"/>s
+	/// </summary>
+	interface IInstanceCoreProvider
+	{
+		/// <summary>
+		/// Get the <see cref="IInstanceCore"/> for a given <paramref name="instance"/> if it's online.
+		/// </summary>
+		/// <param name="instance">The <see cref="Instance"/> to get the <see cref="IInstanceCore"/> for.</param>
+		/// <returns>The <see cref="IInstanceCore"/> if it is online, <see langword="null"/> otherwise.</returns>
+		IInstanceCore GetInstance(Models.Instance instance);
+	}
+}

src/Tgstation.Server.Host/Components/IInstanceManager.cs

@@ -16,11 +16,11 @@ namespace Tgstation.Server.Host.Components
 		Task Ready { get; }
 
 		/// <summary>
-		/// Get the <see cref="IInstance"/> associated with given <paramref name="metadata"/>
+		/// Get the <see cref="IInstanceReference"/> associated with given <paramref name="metadata"/>
 		/// </summary>
 		/// <param name="metadata">The <see cref="Models.Instance"/> of the desired <see cref="IInstance"/></param>
 		/// <returns>The <see cref="IInstance"/> associated with the given <paramref name="metadata"/> if it is online, <see langword="null"/> otherwise.</returns>
-		IInstance GetInstance(Models.Instance metadata);
+		IInstanceReference GetInstanceReference(Models.Instance metadata);
 
 		/// <summary>
 		/// Online an <see cref="IInstance"/>

src/Tgstation.Server.Host/Components/IInstanceReference.cs

@@ -0,0 +1,15 @@
+using System;
+
+namespace Tgstation.Server.Host.Components
+{
+	/// <summary>
+	/// Controller version of <see cref="IInstanceCore"/>.
+	/// </summary>
+	public interface IInstanceReference : IInstanceCore, IDisposable
+	{
+		/// <summary>
+		/// A unique ID for the <see cref="IInstanceReference"/>.
+		/// </summary>
+		public Guid Uid { get; }
+	}
+}

src/Tgstation.Server.Host/Components/Instance.cs

@@ -23,6 +23,11 @@ namespace Tgstation.Server.Host.Components
 	#pragma warning disable CA1506 // TODO: Decomplexify
 	sealed class Instance : IInstance
 	{
+		/// <summary>
+		/// Message for the <see cref="InvalidOperationException"/> if ever a job starts on a different <see cref="IInstanceCore"/> than the one that queued it.
+		/// </summary>
+		public const string DifferentCoreExceptionMessage = "Job started on different instance core!";
+
 		/// <inheritdoc />
 		public IRepositoryManager RepositoryManager { get; }
 
@@ -186,8 +191,11 @@ namespace Tgstation.Server.Host.Components
 						};
 
 						string deploySha = null;
-						await jobManager.RegisterOperation(repositoryUpdateJob, async (paramJob, databaseContextFactory, progressReporter, jobCancellationToken) =>
+						await jobManager.RegisterOperation(repositoryUpdateJob, async (core, databaseContextFactory, paramJob, progressReporter, jobCancellationToken) =>
 						{
+							if (core != this)
+								throw new InvalidOperationException(DifferentCoreExceptionMessage);
+
 							// assume 5 steps with synchronize
 							const int ProgressSections = 7;
 							const int ProgressStep = 100 / ProgressSections;
@@ -373,7 +381,16 @@ namespace Tgstation.Server.Host.Components
 
 						await jobManager.RegisterOperation(
 							compileProcessJob,
-							DreamMaker.DeploymentProcess,
+							(core, databaseContextFactory, job, progressReporter, jobCancellationToken) =>
+							{
+								if (core != this)
+									throw new InvalidOperationException(DifferentCoreExceptionMessage);
+								return DreamMaker.DeploymentProcess(
+									job,
+									databaseContextFactory,
+									progressReporter,
+									jobCancellationToken);
+							},
 							cancellationToken).ConfigureAwait(false);
 
 						await jobManager.WaitForJobCompletion(compileProcessJob, systemUser, default, cancellationToken).ConfigureAwait(false);

src/Tgstation.Server.Host/Components/InstanceContainer.cs

@@ -0,0 +1,86 @@
+using System;
+using System.Threading.Tasks;
+
+namespace Tgstation.Server.Host.Components
+{
+	/// <summary>
+	/// Wrapper for managing <see cref="IInstance"/>s
+	/// </summary>
+	sealed class InstanceContainer
+	{
+		/// <summary>
+		/// The <see cref="IInstance"/>.
+		/// </summary>
+		public IInstance Instance { get; }
+
+		/// <summary>
+		/// A <see cref="Task"/> that completes when there are no <see cref="IInstanceReference"/>s active for the <see cref="Instance"/>.
+		/// </summary>
+		public Task OnZeroReferences
+		{
+			get
+			{
+				lock (referenceCountLock)
+				{
+					if (referenceCount == 0)
+						return Task.CompletedTask;
+					return onZeroReferencesTcs.Task;
+				}
+			}
+		}
+
+		/// <summary>
+		/// <see langword="lock"/> <see cref="object"/> for <see cref="referenceCount"/>.
+		/// </summary>
+		readonly object referenceCountLock;
+
+		/// <summary>
+		/// Backing <see cref="TaskCompletionSource{TResult}"/> for <see cref="OnZeroReferences"/>.
+		/// </summary>
+		TaskCompletionSource<object> onZeroReferencesTcs;
+
+		/// <summary>
+		/// Count of active <see cref="IInstanceReference"/>s.
+		/// </summary>
+		ulong referenceCount;
+
+		/// <summary>
+		/// Initializes a new instance of the <see cref="InstanceContainer"/> <see langword="class"/>.
+		/// </summary>
+		/// <param name="instance">The value of <see cref="Instance"/>.</param>
+		public InstanceContainer(IInstance instance)
+		{
+			Instance = instance ?? throw new ArgumentNullException(nameof(instance));
+
+			referenceCountLock = new object();
+		}
+
+		/// <summary>
+		/// Create a new <see cref="IInstanceReference"/>.
+		/// </summary>
+		/// <returns>A new <see cref="IInstanceReference"/>.</returns>
+		public IInstanceReference AddReference()
+		{
+			lock (referenceCountLock)
+			{
+				if (referenceCount++ == 0)
+					onZeroReferencesTcs = new TaskCompletionSource<object>();
+
+				try
+				{
+					return new InstanceWrapper(Instance, () =>
+					{
+						lock (referenceCountLock)
+							if (--referenceCount == 0)
+								onZeroReferencesTcs.SetResult(null);
+					});
+				}
+				catch
+				{
+					--referenceCount;
+					throw;
+				}
+			}
+		}
+	}
+}

src/Tgstation.Server.Host/Components/InstanceManager.cs

@@ -24,7 +24,13 @@ using Tgstation.Server.Host.System;
 namespace Tgstation.Server.Host.Components
 {
 	/// <inheritdoc />
-	sealed class InstanceManager : IInstanceManager, IRestartHandler, IHostedService, IBridgeRegistrar, IAsyncDisposable
+	sealed class InstanceManager :
+		IInstanceManager,
+		IInstanceCoreProvider,
+		IRestartHandler,
+		IHostedService,
+		IBridgeRegistrar,
+		IAsyncDisposable
 	{
 		/// <inheritdoc />
 		public Task Ready => readyTcs.Task;
@@ -90,15 +96,20 @@ namespace Tgstation.Server.Host.Components
 		readonly ILogger<InstanceManager> logger;
 
 		/// <summary>
-		/// Map of instance <see cref="EntityId.Id"/>s to respective <see cref="IInstance"/>s. Also used as a <see langword="lock"/> <see cref="object"/>.
+		/// Map of instance <see cref="EntityId.Id"/>s to respective <see cref="InstanceContainer"/>s. Also used as a <see langword="lock"/> <see cref="object"/>.
 		/// </summary>
-		readonly IDictionary<long, IInstance> instances;
+		readonly IDictionary<long, InstanceContainer> instances;
 
 		/// <summary>
 		/// Map of <see cref="DMApiParameters.AccessIdentifier"/>s to their respective <see cref="IBridgeHandler"/>s.
 		/// </summary>
 		readonly IDictionary<string, IBridgeHandler> bridgeHandlers;
 
+		/// <summary>
+		/// <see cref="SemaphoreSlim"/> used to guard calls to <see cref="OnlineInstance(Models.Instance, CancellationToken)"/> and <see cref="OfflineInstance(Models.Instance, Models.User, CancellationToken)"/>.
+		/// </summary>
+		readonly SemaphoreSlim instanceStateChangeSemaphore;
+
 		/// <summary>
 		/// The <see cref="GeneralConfiguration"/> for the <see cref="InstanceManager"/>.
 		/// </summary>
@@ -163,9 +174,10 @@ namespace Tgstation.Server.Host.Components
 
 			lazyRestartRegistration = new Lazy<IRestartRegistration>(() => serverControl.RegisterForRestart(this));
 
-			instances = new Dictionary<long, IInstance>();
+			instances = new Dictionary<long, InstanceContainer>();
 			bridgeHandlers = new Dictionary<string, IBridgeHandler>();
 			readyTcs = new TaskCompletionSource<object>();
+			instanceStateChangeSemaphore = new SemaphoreSlim(1);
 		}
 
 		/// <inheritdoc />
@@ -179,22 +191,29 @@ namespace Tgstation.Server.Host.Components
 			}
 
 			foreach (var I in instances)
-				await I.Value.DisposeAsync().ConfigureAwait(false);
+				await I.Value.Instance.DisposeAsync().ConfigureAwait(false);
 
 			lazyRestartRegistration.Value.Dispose();
+			instanceStateChangeSemaphore.Dispose();
 
 			logger.LogInformation("Server shutdown");
 		}
 
 		/// <inheritdoc />
-		public IInstance GetInstance(Models.Instance metadata)
+		public IInstanceReference GetInstanceReference(Models.Instance metadata)
 		{
 			if (metadata == null)
 				throw new ArgumentNullException(nameof(metadata));
+
 			lock (instances)
 			{
-				instances.TryGetValue(metadata.Id, out IInstance instance);
-				return instance; // null if above is false
+				if (!instances.TryGetValue(metadata.Id, out var instance))
+				{
+					logger.LogTrace("Cannot reference instance {0} as it is not online!", metadata.Id);
+					return null;
+				}
+
+				return instance.AddReference();
 			}
 		}
 
@@ -203,7 +222,7 @@ namespace Tgstation.Server.Host.Components
 		{
 			if (oldPath == null)
 				throw new ArgumentNullException(nameof(oldPath));
-			if (GetInstance(instance) != null)
+			if (GetInstanceReference(instance) != null)
 				throw new InvalidOperationException("Cannot move an online instance!");
 			var newPath = instance.Path;
 			try
@@ -268,17 +287,26 @@ namespace Tgstation.Server.Host.Components
 		{
 			if (metadata == null)
 				throw new ArgumentNullException(nameof(metadata));
+
+			using var _ = await SemaphoreSlimContext.Lock(instanceStateChangeSemaphore, cancellationToken).ConfigureAwait(false);
+
 			logger.LogInformation("Offlining instance ID {0}", metadata.Id);
-			IInstance instance;
+			InstanceContainer container;
 			lock (instances)
 			{
-				if (!instances.TryGetValue(metadata.Id, out instance))
-					throw new InvalidOperationException("Instance not online!");
+				if (!instances.TryGetValue(metadata.Id, out container))
+				{
+					logger.LogDebug("Not offlining removed instance {0}", metadata.Id);
+					return;
+				}
+
 				instances.Remove(metadata.Id);
 			}
 
 			try
 			{
+				await container.OnZeroReferences.ConfigureAwait(false);
+
 				// we are the one responsible for cancelling his jobs
 				var tasks = new List<Task>();
 				await databaseContextFactory.UseContext(async db =>
@@ -300,11 +328,11 @@ namespace Tgstation.Server.Host.Components
 
 				await Task.WhenAll(tasks).ConfigureAwait(false);
 
-				await instance.StopAsync(cancellationToken).ConfigureAwait(false);
+				await container.Instance.StopAsync(cancellationToken).ConfigureAwait(false);
 			}
 			finally
 			{
-				await instance.DisposeAsync().ConfigureAwait(false);
+				await container.Instance.DisposeAsync().ConfigureAwait(false);
 			}
 		}
 
@@ -313,15 +341,37 @@ namespace Tgstation.Server.Host.Components
 		{
 			if (metadata == null)
 				throw new ArgumentNullException(nameof(metadata));
+
+			using var _ = await SemaphoreSlimContext.Lock(instanceStateChangeSemaphore, cancellationToken).ConfigureAwait(false);
+			lock (instances)
+				if (instances.ContainsKey(metadata.Id))
+				{
+					logger.LogDebug("Aborting instance creation due to it seemingly already being online");
+					return;
+				}
+
 			logger.LogInformation("Onlining instance ID {0} ({1}) at {2}", metadata.Id, metadata.Name, metadata.Path);
 			var instance = await instanceFactory.CreateInstance(this, metadata).ConfigureAwait(false);
 			try
 			{
-				lock (instances)
+				await instance.StartAsync(cancellationToken).ConfigureAwait(false);
+
+				try
 				{
-					if (instances.ContainsKey(metadata.Id))
-						throw new InvalidOperationException("Instance already online!");
-					instances.Add(metadata.Id, instance);
+					lock (instances)
+						instances.Add(metadata.Id, new InstanceContainer(instance));
+				}
+				catch (Exception ex)
+				{
+					logger.LogError("Unable to commit onlined instance {0} into service, offlining!", metadata.Id);
+					try
+					{
+						await instance.StopAsync(default).ConfigureAwait(false);
+					}
+					catch (Exception innerEx)
+					{
+						throw new AggregateException(innerEx, ex);
+					}
 				}
 			}
 			catch
@@ -329,8 +379,6 @@ namespace Tgstation.Server.Host.Components
 				await instance.DisposeAsync().ConfigureAwait(false);
 				throw;
 			}
-
-			await instance.StartAsync(cancellationToken).ConfigureAwait(false);
 		}
 
 		/// <inheritdoc />
@@ -405,7 +453,7 @@ namespace Tgstation.Server.Host.Components
 		public async Task StopAsync(CancellationToken cancellationToken)
 		{
 			await jobManager.StopAsync(cancellationToken).ConfigureAwait(false);
-			await Task.WhenAll(instances.Select(x => x.Value.StopAsync(cancellationToken))).ConfigureAwait(false);
+			await Task.WhenAll(instances.Select(x => x.Value.Instance.StopAsync(cancellationToken))).ConfigureAwait(false);
 			await instanceFactory.StopAsync(cancellationToken).ConfigureAwait(false);
 
 			// downgrade the db if necessary
@@ -489,5 +537,15 @@ namespace Tgstation.Server.Host.Components
 				}
 			});
 		}
+
+		/// <inheritdoc />
+		public IInstanceCore GetInstance(Models.Instance metadata)
+		{
+			lock (instances)
+			{
+				instances.TryGetValue(metadata.Id, out var container);
+				return container?.Instance;
+			}
+		}
 	}
 }

src/Tgstation.Server.Host/Components/InstanceWrapper.cs

@@ -0,0 +1,99 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using Tgstation.Server.Host.Components.Byond;
+using Tgstation.Server.Host.Components.Chat;
+using Tgstation.Server.Host.Components.Deployment;
+using Tgstation.Server.Host.Components.Repository;
+using Tgstation.Server.Host.Components.StaticFiles;
+using Tgstation.Server.Host.Components.Watchdog;
+using Tgstation.Server.Host.Models;
+
+namespace Tgstation.Server.Host.Components
+{
+	/// <summary>
+	/// Warpper around a given <see cref="IInstance"/> with a <see cref="Dispose"/> <see cref="Action"/>.
+	/// </summary>
+	sealed class InstanceWrapper : IInstanceReference
+	{
+		/// <inheritdoc />
+		public Guid Uid { get; }
+
+		/// <summary>
+		/// The <see langword="lock"/> object for <see cref="Dispose"/>.
+		/// </summary>
+		readonly object disposeLock;
+
+		/// <summary>
+		/// The <see cref="Action"/> to take when <see cref="Dispose"/> is called.
+		/// </summary>
+		Action onDisposed;
+
+		/// <summary>
+		/// The <see cref="IInstance"/> calls are forwarded to.
+		/// </summary>
+		IInstanceCore actualInstance;
+
+		/// <summary>
+		/// Initializes a new instance of the <see cref="InstanceWrapper"/> <see langword="class"/>.
+		/// </summary>
+		/// <param name="actualInstance">The value of <see cref="actualInstance"/>.</param>
+		/// <param name="onDisposed">The value of <see cref="onDisposed"/>.</param>
+		public InstanceWrapper(IInstanceCore actualInstance, Action onDisposed)
+		{
+			this.actualInstance = actualInstance ?? throw new ArgumentNullException(nameof(actualInstance));
+			this.onDisposed = onDisposed ?? throw new ArgumentNullException(nameof(onDisposed));
+			Uid = Guid.NewGuid();
+			disposeLock = new object();
+		}
+
+		/// <inheritdoc />
+		public void Dispose()
+		{
+			lock (disposeLock)
+			{
+				onDisposed?.Invoke();
+				onDisposed = null;
+				actualInstance = null;
+			}
+		}
+
+		/// <inheritdoc />
+		public IRepositoryManager RepositoryManager => actualInstance?.RepositoryManager ?? throw new ObjectDisposedException(nameof(InstanceWrapper));
+
+		/// <inheritdoc />
+		public IByondManager ByondManager => actualInstance?.ByondManager ?? throw new ObjectDisposedException(nameof(InstanceWrapper));
+
+		/// <inheritdoc />
+		public IDreamMaker DreamMaker => actualInstance?.DreamMaker ?? throw new ObjectDisposedException(nameof(InstanceWrapper));
+
+		/// <inheritdoc />
+		public IWatchdog Watchdog => actualInstance?.Watchdog ?? throw new ObjectDisposedException(nameof(InstanceWrapper));
+
+		/// <inheritdoc />
+		public IChatManager Chat => actualInstance?.Chat ?? throw new ObjectDisposedException(nameof(InstanceWrapper));
+
+		/// <inheritdoc />
+		public IConfiguration Configuration => actualInstance?.Configuration ?? throw new ObjectDisposedException(nameof(InstanceWrapper));
+
+		/// <inheritdoc />
+		public Task InstanceRenamed(string newInstanceName, CancellationToken cancellationToken)
+			=> actualInstance?.InstanceRenamed(newInstanceName, cancellationToken) ?? throw new ObjectDisposedException(nameof(InstanceWrapper));
+
+		/// <inheritdoc />
+		public CompileJob LatestCompileJob()
+		{
+			if (actualInstance == null)
+				throw new ObjectDisposedException(nameof(InstanceWrapper));
+			return actualInstance.LatestCompileJob();
+		}
+
+		/// <inheritdoc />
+		public Task SetAutoUpdateInterval(uint newInterval)
+		{
+			if (actualInstance == null)
+				throw new ObjectDisposedException(nameof(InstanceWrapper));
+			return actualInstance.SetAutoUpdateInterval(newInterval);
+		}
+	}
+}

src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs

@@ -912,11 +912,16 @@ namespace Tgstation.Server.Host.Components.Watchdog
 				CancelRight = (ulong)DreamDaemonRights.Shutdown,
 				CancelRightsType = RightsType.DreamDaemon
 			};
-			await jobManager.RegisterOperation(job, async (j, databaseContextFactory, progressFunction, ct) =>
-			{
-				using (await SemaphoreSlimContext.Lock(synchronizationSemaphore, ct).ConfigureAwait(false))
-					await LaunchNoLock(true, true, true, reattachInfo, ct).ConfigureAwait(false);
-			}, cancellationToken).ConfigureAwait(false);
+			await jobManager.RegisterOperation(
+				job,
+				async (core, databaseContextFactory, paramJob, progressFunction, ct) =>
+				{
+					// core will certainly be null here since jobs started before the instance is onlined can't provide one
+					using (await SemaphoreSlimContext.Lock(synchronizationSemaphore, ct).ConfigureAwait(false))
+						await LaunchNoLock(true, true, true, reattachInfo, ct).ConfigureAwait(false);
+				},
+				cancellationToken)
+				.ConfigureAwait(false);
 		}
 
 		/// <inheritdoc />

src/Tgstation.Server.Host/Controllers/ByondController.cs

@@ -154,7 +154,7 @@ namespace Tgstation.Server.Host.Controllers
 						};
 						await jobManager.RegisterOperation(
 							job,
-							(paramJob, databaseContextFactory, progressHandler, jobCancellationToken) => byondManager.ChangeVersion(
+							(core, databaseContextFactory, paramJob, progressHandler, jobCancellationToken) => core.ByondManager.ChangeVersion(
 								model.Version,
 								model.Content,
 								jobCancellationToken),

src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs

@@ -78,7 +78,7 @@ namespace Tgstation.Server.Host.Controllers
 				};
 				await jobManager.RegisterOperation(
 					job,
-					(paramJob, databaseContextFactory, progressHandler, innerCt) => instance.Watchdog.Launch(innerCt),
+					(core, databaseContextFactory, paramJob, progressHandler, innerCt) => core.Watchdog.Launch(innerCt),
 					cancellationToken)
 					.ConfigureAwait(false);
 				return Accepted(job.ToApi());
@@ -302,7 +302,7 @@ namespace Tgstation.Server.Host.Controllers
 
 				await jobManager.RegisterOperation(
 					job,
-					(paramJob, databaseContextFactory, progressReporter, ct) => watchdog.Restart(false, ct),
+					(core, paramJob, databaseContextFactory, progressReporter, ct) => core.Watchdog.Restart(false, ct),
 					cancellationToken)
 					.ConfigureAwait(false);
 				return Accepted(job.ToApi());
@@ -336,7 +336,7 @@ namespace Tgstation.Server.Host.Controllers
 
 				await jobManager.RegisterOperation(
 					job,
-					(paramJob, databaseContextFactory, progressReporter, ct) => watchdog.CreateDump(ct), cancellationToken)
+					(core, databaseContextFactory, paramJob, progressReporter, ct) => core.Watchdog.CreateDump(ct), cancellationToken)
 					.ConfigureAwait(false);
 				return Accepted(job.ToApi());
 			});

src/Tgstation.Server.Host/Controllers/DreamMakerController.cs

@@ -133,26 +133,25 @@ namespace Tgstation.Server.Host.Controllers
 		[HttpPut]
 		[TgsAuthorize(DreamMakerRights.Compile)]
 		[ProducesResponseType(typeof(Api.Models.Job), 202)]
-		public Task<IActionResult> Create(CancellationToken cancellationToken)
-			=> WithComponentInstance(
-				async instance =>
-				{
-					var job = new Models.Job
-					{
-						Description = "Compile active repository code",
-						StartedBy = AuthenticationContext.User,
-						CancelRightsType = RightsType.DreamMaker,
-						CancelRight = (ulong)DreamMakerRights.CancelCompile,
-						Instance = Instance
-					};
+		public async Task<IActionResult> Create(CancellationToken cancellationToken)
+		{
+			var job = new Models.Job
+			{
+				Description = "Compile active repository code",
+				StartedBy = AuthenticationContext.User,
+				CancelRightsType = RightsType.DreamMaker,
+				CancelRight = (ulong)DreamMakerRights.CancelCompile,
+				Instance = Instance
+			};
 
-					await jobManager.RegisterOperation(
-						job,
-						instance.DreamMaker.DeploymentProcess,
-						cancellationToken)
-						.ConfigureAwait(false);
-					return Accepted(job.ToApi());
-				});
+			await jobManager.RegisterOperation(
+				job,
+				(core, databaseContextFactory, paramJob, progressReporter, jobCancellationToken)
+					=> core.DreamMaker.DeploymentProcess(paramJob, databaseContextFactory, progressReporter, jobCancellationToken),
+				cancellationToken)
+				.ConfigureAwait(false);
+			return Accepted(job.ToApi());
+		}
 
 		/// <summary>
 		/// Update deployment settings.

src/Tgstation.Server.Host/Controllers/InstanceController.cs

@@ -481,7 +481,7 @@ namespace Tgstation.Server.Host.Controllers
 
 			if (renamed)
 			{
-				var componentInstance = instanceManager.GetInstance(originalModel);
+				var componentInstance = instanceManager.GetInstanceReference(originalModel);
 				if (componentInstance != null)
 					await componentInstance.InstanceRenamed(originalModel.Name, cancellationToken).ConfigureAwait(false);
 			}
@@ -532,7 +532,8 @@ namespace Tgstation.Server.Host.Controllers
 
 				await jobManager.RegisterOperation(
 					job,
-					(paramJob, databaseContextFactory, progressHandler, ct) => instanceManager.MoveInstance(originalModel, originalModelPath, ct),
+					(core, databaseContextFactory, paramJob, progressHandler, ct) // core will be null here since the instance is offline
+						=> instanceManager.MoveInstance(originalModel, originalModelPath, ct),
 					cancellationToken)
 					.ConfigureAwait(false);
 				api.MoveJob = job.ToApi();
@@ -540,7 +541,7 @@ namespace Tgstation.Server.Host.Controllers
 
 			if (model.AutoUpdateInterval.HasValue && oldAutoUpdateInterval != model.AutoUpdateInterval)
 			{
-				var componentInstance = instanceManager.GetInstance(originalModel);
+				var componentInstance = instanceManager.GetInstanceReference(originalModel);
 				if (componentInstance != null)
 					await componentInstance.SetAutoUpdateInterval(model.AutoUpdateInterval.Value).ConfigureAwait(false);
 			}

src/Tgstation.Server.Host/Controllers/InstanceRequiredController.cs

@@ -1,5 +1,6 @@
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Logging;
+using Serilog.Context;
 using System;
 using System.Threading;
 using System.Threading.Tasks;
@@ -48,7 +49,7 @@ namespace Tgstation.Server.Host.Controllers
 			if (ValidateInstanceOnlineStatus(instanceManager, Logger, Instance))
 				await DatabaseContext.Save(cancellationToken).ConfigureAwait(false);
 
-			if (instanceManager.GetInstance(Instance) == null)
+			if (instanceManager.GetInstanceReference(Instance) == null)
 				return Conflict(new ErrorMessage(ErrorCode.InstanceOffline));
 			return null;
 		}
@@ -59,15 +60,18 @@ namespace Tgstation.Server.Host.Controllers
 		/// <param name="action">A <see cref="Func{T, TResult}"/> accepting the <see cref="IInstance"/> and returning a <see cref="Task{TResult}"/> with the <see cref="IActionResult"/>.</param>
 		/// <returns>A <see cref="Task{TResult}"/> resulting in the <see cref="IActionResult"/> that should be returned.</returns>
 		/// <remarks>The context of <paramref name="action"/> should be as small as possinle so as to avoid race conditions.</remarks>
-		protected async Task<IActionResult> WithComponentInstance(Func<IInstance, Task<IActionResult>> action)
+		protected async Task<IActionResult> WithComponentInstance(Func<IInstanceCore, Task<IActionResult>> action)
 		{
 			if (action == null)
 				throw new ArgumentNullException(nameof(action));
 
-			var componentInstance = instanceManager.GetInstance(Instance);
-			if (componentInstance == null)
-				return Conflict(new ErrorMessage(ErrorCode.InstanceOffline));
-			return await action(componentInstance).ConfigureAwait(false);
+			using var instanceReference = instanceManager.GetInstanceReference(Instance);
+			using (LogContext.PushProperty("InstanceReference", instanceReference.Uid))
+			{
+				if (instanceReference == null)
+					return Conflict(new ErrorMessage(ErrorCode.InstanceOffline));
+				return await action(instanceReference).ConfigureAwait(false);
+			}
 		}
 
 		/// <summary>
@@ -84,7 +88,7 @@ namespace Tgstation.Server.Host.Controllers
 			if (metadata == null)
 				throw new ArgumentNullException(nameof(metadata));
 
-			var online = instanceManager.GetInstance(metadata) != null;
+			var online = instanceManager.GetInstanceReference(metadata) != null;
 
 			if (metadata.Online.Value == online)
 				return false;

src/Tgstation.Server.Host/Controllers/RepositoryController.cs

@@ -211,8 +211,9 @@ namespace Tgstation.Server.Host.Controllers
 						Instance = Instance
 					};
 					var api = currentModel.ToApi();
-					await jobManager.RegisterOperation(job, async (paramJob, databaseContextFactory, progressReporter, ct) =>
+					await jobManager.RegisterOperation(job, async (core, databaseContextFactory, paramJob, progressReporter, ct) =>
 					{
+						var repoManager = core.RepositoryManager;
 						using var repos = await repoManager.CloneRepository(
 							new Uri(origin),
 							cloneBranch,
@@ -284,18 +285,13 @@ namespace Tgstation.Server.Host.Controllers
 				Instance = Instance
 			};
 			var api = currentModel.ToApi();
-			return await WithComponentInstance(
-				async instance =>
-				{
-					await jobManager.RegisterOperation(
-						job,
-						(paramJob, databaseContextFactory, progressReporter, ct) => instance.RepositoryManager.DeleteRepository(ct),
-						cancellationToken)
-					.ConfigureAwait(false);
-					api.ActiveJob = job.ToApi();
-					return Accepted(api);
-				})
-				.ConfigureAwait(false);
+			await jobManager.RegisterOperation(
+				job,
+				(core, databaseContextFactory, paramJob, progressReporter, ct) => core.RepositoryManager.DeleteRepository(ct),
+				cancellationToken)
+			.ConfigureAwait(false);
+			api.ActiveJob = job.ToApi();
+			return Accepted(api);
 		}
 
 		/// <summary>
@@ -363,8 +359,8 @@ namespace Tgstation.Server.Host.Controllers
 		[ProducesResponseType(typeof(Repository), 200)]
 		[ProducesResponseType(typeof(Repository), 202)]
 		[ProducesResponseType(typeof(ErrorMessage), 410)]
-		#pragma warning disable CA1502, CA1505 // TODO: Decomplexify
-		public async Task<IActionResult> Update([FromBody]Repository model, CancellationToken cancellationToken)
+#pragma warning disable CA1502, CA1505 // TODO: Decomplexify
+		public async Task<IActionResult> Update([FromBody] Repository model, CancellationToken cancellationToken)
 		{
 			if (model == null)
 				throw new ArgumentNullException(nameof(model));
@@ -494,7 +490,7 @@ namespace Tgstation.Server.Host.Controllers
 				return Json(api); // no git changes
 
 			async Task<IActionResult> UpdateCallbackThatDesperatelyNeedsRefactoring(
-				IInstance instance,
+				IInstanceCore instance,
 				IDatabaseContextFactory databaseContextFactory,
 				Action<int> progressReporter,
 				CancellationToken ct)
@@ -897,13 +893,12 @@ namespace Tgstation.Server.Host.Controllers
 			// Time to access git, do it in a job
 			await jobManager.RegisterOperation(
 				job,
-				(paramJob, databaseContextFactory, progressReporter, ct)
-					=> WithComponentInstance( // Will never fail in the context of a job
-						instance => UpdateCallbackThatDesperatelyNeedsRefactoring(
-							instance,
-							databaseContextFactory,
-							progressReporter,
-							ct)),
+				(core, databaseContextFactory, paramJob, progressReporter, ct) =>
+					UpdateCallbackThatDesperatelyNeedsRefactoring(
+						core,
+						databaseContextFactory,
+						progressReporter,
+						ct),
 				cancellationToken)
 				.ConfigureAwait(false);
 

src/Tgstation.Server.Host/Core/Application.cs

@@ -132,7 +132,7 @@ namespace Tgstation.Server.Host.Core
 					var formatter = new MessageTemplateTextFormatter(
 						"{Timestamp:o} "
 						+ ServiceCollectionExtensions.SerilogContextTemplate
-						+ ": [{Level:u3}] {SourceContext:l}: {Message} ({EventId:x8}){NewLine}{Exception}",
+						+ "|IR:{InstanceReference}){): [{Level:u3}] {SourceContext:l}: {Message} ({EventId:x8}){NewLine}{Exception}",
 						null);
 
 					logPath = IOManager.ConcatPath(logPath, "tgs-.log");
@@ -301,6 +301,7 @@ namespace Tgstation.Server.Host.Core
 
 			services.AddSingleton<InstanceManager>();
 			services.AddSingleton<IBridgeDispatcher>(x => x.GetRequiredService<InstanceManager>());
+			services.AddSingleton(x => new Lazy<IInstanceCoreProvider>(() => x.GetRequiredService<InstanceManager>()));
 			services.AddSingleton<IInstanceManager>(x => x.GetRequiredService<InstanceManager>());
 		}
 

src/Tgstation.Server.Host/Extensions/ServiceCollectionExtensions.cs

@@ -18,7 +18,7 @@ namespace Tgstation.Server.Host.Extensions
 		/// <summary>
 		/// Common template used for adding our custom log context to serilog.
 		/// </summary>
-		public const string SerilogContextTemplate = "(Instance:{Instance}|Job:{Job}|Request:{Request}|User:{User}|Monitor:{Monitor}|Bridge:{Bridge}|Chat:{ChatMessage})";
+		public const string SerilogContextTemplate = "(Instance:{Instance}|Job:{Job}|Request:{Request}|User:{User}|Monitor:{Monitor}|Bridge:{Bridge}|Chat:{ChatMessage}";
 
 		/// <summary>
 		/// Add a standard <typeparamref name="TConfig"/> binding
@@ -79,7 +79,7 @@ namespace Tgstation.Server.Host.Extensions
 						sinkConfiguration.Console(
 							outputTemplate: "[{Timestamp:HH:mm:ss}] {Level:w3}: {SourceContext:l} "
 								+ SerilogContextTemplate
-								+ "{NewLine}    {Message:lj}{NewLine}{Exception}");
+								+ "|IR:{InstanceReference}){NewLine}    {Message:lj}{NewLine}{Exception}");
 						sinkConfigurationAction?.Invoke(sinkConfiguration);
 					});
 

src/Tgstation.Server.Host/Jobs/IJobManager.cs

@@ -1,8 +1,6 @@
 using Microsoft.Extensions.Hosting;
-using System;
 using System.Threading;
 using System.Threading.Tasks;
-using Tgstation.Server.Host.Database;
 using Tgstation.Server.Host.Models;
 
 namespace Tgstation.Server.Host.Jobs
@@ -23,10 +21,10 @@ namespace Tgstation.Server.Host.Jobs
 		/// Registers a given <see cref="Job"/> and begins running it
 		/// </summary>
 		/// <param name="job">The <see cref="Job"/></param>
-		/// <param name="operation">The operation to run taking the started <see cref="Job"/>, a <see cref="IDatabaseContextFactory"/>, progress reporter <see cref="Action{T1}"/> and a <see cref="CancellationToken"/></param>
+		/// <param name="operation">The <see cref="JobEntrypoint"/> for the <paramref name="job"/>.</param>
 		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
 		/// <returns>A <see cref="Task"/> representing a running operation</returns>
-		Task RegisterOperation(Job job, Func<Job, IDatabaseContextFactory, Action<int>, CancellationToken, Task> operation, CancellationToken cancellationToken);
+		Task RegisterOperation(Job job, JobEntrypoint operation, CancellationToken cancellationToken);
 
 		/// <summary>
 		/// Wait for a given <paramref name="job"/> to complete

src/Tgstation.Server.Host/Jobs/JobEntrypoint.cs

@@ -0,0 +1,25 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using Tgstation.Server.Host.Components;
+using Tgstation.Server.Host.Database;
+using Tgstation.Server.Host.Models;
+
+namespace Tgstation.Server.Host.Jobs
+{
+	/// <summary>
+	/// Entrypoint for running a given job.
+	/// </summary>
+	/// <param name="instance">The <see cref="IInstanceCore"/> the job is running on. <see langword="null"/> only when performing an instance move operation.</param>
+	/// <param name="databaseContextFactory">The <see cref="IDatabaseContextFactory"/> for the operation.</param>
+	/// <param name="job">The running <see cref="Job"/>.</param>
+	/// <param name="progressReporter">A <see cref="Action{T1}"/> that will update the progress of the job.</param>
+	/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
+	/// <returns>A <see cref="Task"/> representing the running operation.</returns>
+	public delegate Task JobEntrypoint(
+		IInstanceCore instance,
+		IDatabaseContextFactory databaseContextFactory,
+		Job job,
+		Action<int> progressReporter,
+		CancellationToken cancellationToken);
+}

src/Tgstation.Server.Host/Jobs/JobHandler.cs

@@ -14,21 +14,24 @@ namespace Tgstation.Server.Host.Jobs
 		/// </summary>
 		readonly CancellationTokenSource cancellationTokenSource;
 
+		/// <summary>
+		/// A <see cref="Func{T, TResult}"/> taking a <see cref="CancellationToken"/> and returning a <see cref="Task"/> that the <see cref="JobHandler"/> will wrap
+		/// </summary>
+		readonly Func<CancellationToken, Task> jobActivator;
+
 		/// <summary>
 		/// The <see cref="Task"/> being run
 		/// </summary>
-		readonly Task task;
+		Task task;
 
 		/// <summary>
 		/// Construct a <see cref="JobHandler"/>
 		/// </summary>
-		/// <param name="job">A <see cref="Func{T, TResult}"/> taking a <see cref="CancellationToken"/> and returning a <see cref="Task"/> that the <see cref="JobHandler"/> will wrap</param>
-		public JobHandler(Func<CancellationToken, Task> job)
+		/// <param name="jobActivator">The value of <see cref="jobActivator"/>.</param>
+		public JobHandler(Func<CancellationToken, Task> jobActivator)
 		{
-			if (job == null)
-				throw new ArgumentNullException(nameof(job));
+			this.jobActivator = jobActivator ?? throw new ArgumentNullException(nameof(jobActivator));
 			cancellationTokenSource = new CancellationTokenSource();
-			task = job(cancellationTokenSource.Token);
 		}
 
 		/// <inheritdoc />
@@ -46,6 +49,9 @@ namespace Tgstation.Server.Host.Jobs
 		/// <returns>A <see cref="Task"/> representing the running operation</returns>
 		public async Task Wait(CancellationToken cancellationToken)
 		{
+			if (task == null)
+				throw new InvalidOperationException("Job not started!");
+
 			TaskCompletionSource<object> tcs = new TaskCompletionSource<object>();
 			using (cancellationToken.Register(() => tcs.SetCanceled()))
 				await Task.WhenAny(tcs.Task, task).ConfigureAwait(false);
@@ -56,5 +62,18 @@ namespace Tgstation.Server.Host.Jobs
 		/// Cancels <see cref="task"/>
 		/// </summary>
 		public void Cancel() => cancellationTokenSource.Cancel();
+
+		/// <summary>
+		/// Starts the job.
+		/// </summary>
+		public void Start()
+		{
+			lock (cancellationTokenSource)
+			{
+				if (task != null)
+					throw new InvalidOperationException("Job already started");
+				task = jobActivator(cancellationTokenSource.Token);
+			}
+		}
 	}
 }

src/Tgstation.Server.Host/Jobs/JobManager.cs

@@ -6,6 +6,7 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
+using Tgstation.Server.Host.Components;
 using Tgstation.Server.Host.Database;
 using Tgstation.Server.Host.Models;
 
@@ -24,6 +25,11 @@ namespace Tgstation.Server.Host.Jobs
 		/// </summary>
 		readonly ILogger<JobManager> logger;
 
+		/// <summary>
+		/// The <see cref="IInstanceCoreProvider"/> for the <see cref="JobManager"/>.
+		/// </summary>
+		readonly Lazy<IInstanceCoreProvider> instanceCoreProvider;
+
 		/// <summary>
 		/// <see cref="Dictionary{TKey, TValue}"/> of <see cref="Job"/> <see cref="Api.Models.EntityId.Id"/>s to running <see cref="JobHandler"/>s
 		/// </summary>
@@ -38,10 +44,12 @@ namespace Tgstation.Server.Host.Jobs
 		/// Construct a <see cref="JobManager"/>
 		/// </summary>
 		/// <param name="databaseContextFactory">The value of <see cref="databaseContextFactory"/></param>
+		/// <param name="instanceCoreProvider">The value of <see cref="instanceCoreProvider"/>.</param>
 		/// <param name="logger">The value of <see cref="logger"/></param>
-		public JobManager(IDatabaseContextFactory databaseContextFactory, ILogger<JobManager> logger)
+		public JobManager(IDatabaseContextFactory databaseContextFactory, Lazy<IInstanceCoreProvider> instanceCoreProvider, ILogger<JobManager> logger)
 		{
 			this.databaseContextFactory = databaseContextFactory ?? throw new ArgumentNullException(nameof(databaseContextFactory));
+			this.instanceCoreProvider = instanceCoreProvider ?? throw new ArgumentNullException(nameof(instanceCoreProvider));
 			this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
 			jobs = new Dictionary<long, JobHandler>();
 			synchronizationLock = new object();
@@ -73,10 +81,10 @@ namespace Tgstation.Server.Host.Jobs
 		/// Runner for <see cref="JobHandler"/>s
 		/// </summary>
 		/// <param name="job">The <see cref="Job"/> being run</param>
-		/// <param name="operation">The operation for the <paramref name="job"/></param>
+		/// <param name="operation">The <see cref="JobEntrypoint"/> for the <paramref name="job"/></param>
 		/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
 		/// <returns>A <see cref="Task"/> representing the running operation</returns>
-		async Task RunJob(Job job, Func<Job, IDatabaseContextFactory, CancellationToken, Task> operation, CancellationToken cancellationToken)
+		async Task RunJob(Job job, JobEntrypoint operation, CancellationToken cancellationToken)
 		{
 			using (LogContext.PushProperty("Job", job.Id))
 				try
@@ -87,7 +95,20 @@ namespace Tgstation.Server.Host.Jobs
 						var oldJob = job;
 						job = new Job { Id = oldJob.Id };
 
-						await operation(job, databaseContextFactory, cancellationToken).ConfigureAwait(false);
+						void UpdateProgress(int progress)
+						{
+							lock (synchronizationLock)
+								if (jobs.TryGetValue(oldJob.Id, out var handler))
+									handler.Progress = progress;
+						}
+
+						await operation(
+							instanceCoreProvider.Value.GetInstance(oldJob.Instance),
+							databaseContextFactory,
+							job,
+							UpdateProgress,
+							cancellationToken)
+							.ConfigureAwait(false);
 
 						logger.LogDebug("Job {0} completed!", job.Id);
 					}
@@ -144,43 +165,50 @@ namespace Tgstation.Server.Host.Jobs
 		}
 
 		/// <inheritdoc />
-		public Task RegisterOperation(Job job, Func<Job, IDatabaseContextFactory, Action<int>, CancellationToken, Task> operation, CancellationToken cancellationToken) => databaseContextFactory.UseContext(async databaseContext =>
-		{
-			if (job == null)
-				throw new ArgumentNullException(nameof(job));
-			if (operation == null)
-				throw new ArgumentNullException(nameof(operation));
+		public Task RegisterOperation(Job job, JobEntrypoint operation, CancellationToken cancellationToken)
+			=> databaseContextFactory.UseContext(
+				async databaseContext =>
+				{
+					if (job == null)
+						throw new ArgumentNullException(nameof(job));
+					if (operation == null)
+						throw new ArgumentNullException(nameof(operation));
 
-			job.StartedAt = DateTimeOffset.Now;
-			job.Cancelled = false;
+					job.StartedAt = DateTimeOffset.Now;
+					job.Cancelled = false;
 
-			job.Instance = new Instance
-			{
-				Id = job.Instance.Id
-			};
-			databaseContext.Instances.Attach(job.Instance);
+					job.Instance = new Models.Instance
+					{
+						Id = job.Instance.Id
+					};
+					databaseContext.Instances.Attach(job.Instance);
 
-			job.StartedBy = new User
-			{
-				Id = job.StartedBy.Id
-			};
-			databaseContext.Users.Attach(job.StartedBy);
+					job.StartedBy = new User
+					{
+						Id = job.StartedBy.Id
+					};
+					databaseContext.Users.Attach(job.StartedBy);
 
-			databaseContext.Jobs.Add(job);
+					databaseContext.Jobs.Add(job);
 
-			await databaseContext.Save(cancellationToken).ConfigureAwait(false);
-			logger.LogDebug("Starting job {0}: {1}...", job.Id, job.Description);
-			var jobHandler = new JobHandler(x => RunJob(job, (jobParam, serviceProvider, ct) =>
-			operation(jobParam, serviceProvider, y =>
-			{
-				lock (synchronizationLock)
-					if (jobs.TryGetValue(job.Id, out var handler))
-						handler.Progress = y;
-			}, ct),
-			x));
-			lock (synchronizationLock)
-				jobs.Add(job.Id, jobHandler);
-		});
+					await databaseContext.Save(cancellationToken).ConfigureAwait(false);
+
+					logger.LogDebug("Starting job {0}: {1}...", job.Id, job.Description);
+					var jobHandler = new JobHandler(jobCancellationToken => RunJob(job, operation, jobCancellationToken));
+					try
+					{
+						lock (synchronizationLock)
+						{
+							jobs.Add(job.Id, jobHandler);
+							jobHandler.Start();
+						}
+					}
+					catch
+					{
+						jobHandler.Dispose();
+						throw;
+					}
+				});
 
 		/// <inheritdoc />
 		public async Task StartAsync(CancellationToken cancellationToken)