diff --git a/src/Tgstation.Server.Host/Components/IInstance.cs b/src/Tgstation.Server.Host/Components/IInstance.cs index 05a0c2fa91..b7e64db9b2 100644 --- a/src/Tgstation.Server.Host/Components/IInstance.cs +++ b/src/Tgstation.Server.Host/Components/IInstance.cs @@ -1,55 +1,12 @@ using Microsoft.Extensions.Hosting; using System; -using System.Threading.Tasks; -using Tgstation.Server.Host.Components.Byond; -using Tgstation.Server.Host.Components.Chat; -using Tgstation.Server.Host.Components.Deployment; -using Tgstation.Server.Host.Components.Repository; -using Tgstation.Server.Host.Components.StaticFiles; -using Tgstation.Server.Host.Components.Watchdog; namespace Tgstation.Server.Host.Components { /// - /// For interacting with the instance services + /// Component version of . /// - public interface IInstance : ILatestCompileJobProvider, IHostedService, IRenameNotifyee, IAsyncDisposable + interface IInstance : IInstanceCore, IHostedService, IAsyncDisposable { - /// - /// The for the - /// - IRepositoryManager RepositoryManager { get; } - - /// - /// The for the - /// - IByondManager ByondManager { get; } - - /// - /// The for the . - /// - IDreamMaker DreamMaker { get; } - - /// - /// The for the - /// - IWatchdog Watchdog { get; } - - /// - /// The for the - /// - IChatManager Chat { get; } - - /// - /// The for the - /// - IConfiguration Configuration { get; } - - /// - /// Change the for the - /// - /// The new auto update inteval - /// A representing the running operation - Task SetAutoUpdateInterval(uint newInterval); } -} \ No newline at end of file +} diff --git a/src/Tgstation.Server.Host/Components/IInstanceCore.cs b/src/Tgstation.Server.Host/Components/IInstanceCore.cs new file mode 100644 index 0000000000..27552e5bc6 --- /dev/null +++ b/src/Tgstation.Server.Host/Components/IInstanceCore.cs @@ -0,0 +1,53 @@ +using System.Threading.Tasks; +using Tgstation.Server.Host.Components.Byond; +using Tgstation.Server.Host.Components.Chat; +using Tgstation.Server.Host.Components.Deployment; +using Tgstation.Server.Host.Components.Repository; +using Tgstation.Server.Host.Components.StaticFiles; +using Tgstation.Server.Host.Components.Watchdog; + +namespace Tgstation.Server.Host.Components +{ + /// + /// For interacting with the instance services + /// + public interface IInstanceCore : ILatestCompileJobProvider, IRenameNotifyee + { + /// + /// The for the + /// + IRepositoryManager RepositoryManager { get; } + + /// + /// The for the + /// + IByondManager ByondManager { get; } + + /// + /// The for the . + /// + IDreamMaker DreamMaker { get; } + + /// + /// The for the + /// + IWatchdog Watchdog { get; } + + /// + /// The for the + /// + IChatManager Chat { get; } + + /// + /// The for the + /// + IConfiguration Configuration { get; } + + /// + /// Change the for the + /// + /// The new auto update inteval + /// A representing the running operation + Task SetAutoUpdateInterval(uint newInterval); + } +} \ No newline at end of file diff --git a/src/Tgstation.Server.Host/Components/IInstanceCoreProvider.cs b/src/Tgstation.Server.Host/Components/IInstanceCoreProvider.cs new file mode 100644 index 0000000000..494aaf130f --- /dev/null +++ b/src/Tgstation.Server.Host/Components/IInstanceCoreProvider.cs @@ -0,0 +1,15 @@ +namespace Tgstation.Server.Host.Components +{ + /// + /// Provider for s + /// + interface IInstanceCoreProvider + { + /// + /// Get the for a given if it's online. + /// + /// The to get the for. + /// The if it is online, otherwise. + IInstanceCore GetInstance(Models.Instance instance); + } +} diff --git a/src/Tgstation.Server.Host/Components/IInstanceManager.cs b/src/Tgstation.Server.Host/Components/IInstanceManager.cs index a4ff54c906..1731be40cf 100644 --- a/src/Tgstation.Server.Host/Components/IInstanceManager.cs +++ b/src/Tgstation.Server.Host/Components/IInstanceManager.cs @@ -16,11 +16,11 @@ namespace Tgstation.Server.Host.Components Task Ready { get; } /// - /// Get the associated with given + /// Get the associated with given /// /// The of the desired /// The associated with the given if it is online, otherwise. - IInstance GetInstance(Models.Instance metadata); + IInstanceReference GetInstanceReference(Models.Instance metadata); /// /// Online an diff --git a/src/Tgstation.Server.Host/Components/IInstanceReference.cs b/src/Tgstation.Server.Host/Components/IInstanceReference.cs new file mode 100644 index 0000000000..a6453747f8 --- /dev/null +++ b/src/Tgstation.Server.Host/Components/IInstanceReference.cs @@ -0,0 +1,15 @@ +using System; + +namespace Tgstation.Server.Host.Components +{ + /// + /// Controller version of . + /// + public interface IInstanceReference : IInstanceCore, IDisposable + { + /// + /// A unique ID for the . + /// + public Guid Uid { get; } + } +} diff --git a/src/Tgstation.Server.Host/Components/Instance.cs b/src/Tgstation.Server.Host/Components/Instance.cs index 1617658bf5..8eba501262 100644 --- a/src/Tgstation.Server.Host/Components/Instance.cs +++ b/src/Tgstation.Server.Host/Components/Instance.cs @@ -23,6 +23,11 @@ namespace Tgstation.Server.Host.Components #pragma warning disable CA1506 // TODO: Decomplexify sealed class Instance : IInstance { + /// + /// Message for the if ever a job starts on a different than the one that queued it. + /// + public const string DifferentCoreExceptionMessage = "Job started on different instance core!"; + /// public IRepositoryManager RepositoryManager { get; } @@ -186,8 +191,11 @@ namespace Tgstation.Server.Host.Components }; string deploySha = null; - await jobManager.RegisterOperation(repositoryUpdateJob, async (paramJob, databaseContextFactory, progressReporter, jobCancellationToken) => + await jobManager.RegisterOperation(repositoryUpdateJob, async (core, databaseContextFactory, paramJob, progressReporter, jobCancellationToken) => { + if (core != this) + throw new InvalidOperationException(DifferentCoreExceptionMessage); + // assume 5 steps with synchronize const int ProgressSections = 7; const int ProgressStep = 100 / ProgressSections; @@ -373,7 +381,16 @@ namespace Tgstation.Server.Host.Components await jobManager.RegisterOperation( compileProcessJob, - DreamMaker.DeploymentProcess, + (core, databaseContextFactory, job, progressReporter, jobCancellationToken) => + { + if (core != this) + throw new InvalidOperationException(DifferentCoreExceptionMessage); + return DreamMaker.DeploymentProcess( + job, + databaseContextFactory, + progressReporter, + jobCancellationToken); + }, cancellationToken).ConfigureAwait(false); await jobManager.WaitForJobCompletion(compileProcessJob, systemUser, default, cancellationToken).ConfigureAwait(false); diff --git a/src/Tgstation.Server.Host/Components/InstanceContainer.cs b/src/Tgstation.Server.Host/Components/InstanceContainer.cs new file mode 100644 index 0000000000..3a1e64f92a --- /dev/null +++ b/src/Tgstation.Server.Host/Components/InstanceContainer.cs @@ -0,0 +1,86 @@ +using System; +using System.Threading.Tasks; + +namespace Tgstation.Server.Host.Components +{ + /// + /// Wrapper for managing s + /// + sealed class InstanceContainer + { + /// + /// The . + /// + public IInstance Instance { get; } + + /// + /// A that completes when there are no s active for the . + /// + public Task OnZeroReferences + { + get + { + lock (referenceCountLock) + { + if (referenceCount == 0) + return Task.CompletedTask; + return onZeroReferencesTcs.Task; + } + } + } + + /// + /// for . + /// + readonly object referenceCountLock; + + /// + /// Backing for . + /// + TaskCompletionSource onZeroReferencesTcs; + + /// + /// Count of active s. + /// + ulong referenceCount; + + /// + /// Initializes a new instance of the . + /// + /// The value of . + public InstanceContainer(IInstance instance) + { + Instance = instance ?? throw new ArgumentNullException(nameof(instance)); + + referenceCountLock = new object(); + } + + /// + /// Create a new . + /// + /// A new . + public IInstanceReference AddReference() + { + lock (referenceCountLock) + { + if (referenceCount++ == 0) + onZeroReferencesTcs = new TaskCompletionSource(); + + try + { + return new InstanceWrapper(Instance, () => + { + lock (referenceCountLock) + if (--referenceCount == 0) + onZeroReferencesTcs.SetResult(null); + }); + } + catch + { + --referenceCount; + throw; + } + } + } + } +} diff --git a/src/Tgstation.Server.Host/Components/InstanceManager.cs b/src/Tgstation.Server.Host/Components/InstanceManager.cs index 81bd54d97c..ce3d3f775f 100644 --- a/src/Tgstation.Server.Host/Components/InstanceManager.cs +++ b/src/Tgstation.Server.Host/Components/InstanceManager.cs @@ -24,7 +24,13 @@ using Tgstation.Server.Host.System; namespace Tgstation.Server.Host.Components { /// - sealed class InstanceManager : IInstanceManager, IRestartHandler, IHostedService, IBridgeRegistrar, IAsyncDisposable + sealed class InstanceManager : + IInstanceManager, + IInstanceCoreProvider, + IRestartHandler, + IHostedService, + IBridgeRegistrar, + IAsyncDisposable { /// public Task Ready => readyTcs.Task; @@ -90,15 +96,20 @@ namespace Tgstation.Server.Host.Components readonly ILogger logger; /// - /// Map of instance s to respective s. Also used as a . + /// Map of instance s to respective s. Also used as a . /// - readonly IDictionary instances; + readonly IDictionary instances; /// /// Map of s to their respective s. /// readonly IDictionary bridgeHandlers; + /// + /// used to guard calls to and . + /// + readonly SemaphoreSlim instanceStateChangeSemaphore; + /// /// The for the . /// @@ -163,9 +174,10 @@ namespace Tgstation.Server.Host.Components lazyRestartRegistration = new Lazy(() => serverControl.RegisterForRestart(this)); - instances = new Dictionary(); + instances = new Dictionary(); bridgeHandlers = new Dictionary(); readyTcs = new TaskCompletionSource(); + instanceStateChangeSemaphore = new SemaphoreSlim(1); } /// @@ -179,22 +191,29 @@ namespace Tgstation.Server.Host.Components } foreach (var I in instances) - await I.Value.DisposeAsync().ConfigureAwait(false); + await I.Value.Instance.DisposeAsync().ConfigureAwait(false); lazyRestartRegistration.Value.Dispose(); + instanceStateChangeSemaphore.Dispose(); logger.LogInformation("Server shutdown"); } /// - public IInstance GetInstance(Models.Instance metadata) + public IInstanceReference GetInstanceReference(Models.Instance metadata) { if (metadata == null) throw new ArgumentNullException(nameof(metadata)); + lock (instances) { - instances.TryGetValue(metadata.Id, out IInstance instance); - return instance; // null if above is false + if (!instances.TryGetValue(metadata.Id, out var instance)) + { + logger.LogTrace("Cannot reference instance {0} as it is not online!", metadata.Id); + return null; + } + + return instance.AddReference(); } } @@ -203,7 +222,7 @@ namespace Tgstation.Server.Host.Components { if (oldPath == null) throw new ArgumentNullException(nameof(oldPath)); - if (GetInstance(instance) != null) + if (GetInstanceReference(instance) != null) throw new InvalidOperationException("Cannot move an online instance!"); var newPath = instance.Path; try @@ -268,17 +287,26 @@ namespace Tgstation.Server.Host.Components { if (metadata == null) throw new ArgumentNullException(nameof(metadata)); + + using var _ = await SemaphoreSlimContext.Lock(instanceStateChangeSemaphore, cancellationToken).ConfigureAwait(false); + logger.LogInformation("Offlining instance ID {0}", metadata.Id); - IInstance instance; + InstanceContainer container; lock (instances) { - if (!instances.TryGetValue(metadata.Id, out instance)) - throw new InvalidOperationException("Instance not online!"); + if (!instances.TryGetValue(metadata.Id, out container)) + { + logger.LogDebug("Not offlining removed instance {0}", metadata.Id); + return; + } + instances.Remove(metadata.Id); } try { + await container.OnZeroReferences.ConfigureAwait(false); + // we are the one responsible for cancelling his jobs var tasks = new List(); await databaseContextFactory.UseContext(async db => @@ -300,11 +328,11 @@ namespace Tgstation.Server.Host.Components await Task.WhenAll(tasks).ConfigureAwait(false); - await instance.StopAsync(cancellationToken).ConfigureAwait(false); + await container.Instance.StopAsync(cancellationToken).ConfigureAwait(false); } finally { - await instance.DisposeAsync().ConfigureAwait(false); + await container.Instance.DisposeAsync().ConfigureAwait(false); } } @@ -313,15 +341,37 @@ namespace Tgstation.Server.Host.Components { if (metadata == null) throw new ArgumentNullException(nameof(metadata)); + + using var _ = await SemaphoreSlimContext.Lock(instanceStateChangeSemaphore, cancellationToken).ConfigureAwait(false); + lock (instances) + if (instances.ContainsKey(metadata.Id)) + { + logger.LogDebug("Aborting instance creation due to it seemingly already being online"); + return; + } + logger.LogInformation("Onlining instance ID {0} ({1}) at {2}", metadata.Id, metadata.Name, metadata.Path); var instance = await instanceFactory.CreateInstance(this, metadata).ConfigureAwait(false); try { - lock (instances) + await instance.StartAsync(cancellationToken).ConfigureAwait(false); + + try { - if (instances.ContainsKey(metadata.Id)) - throw new InvalidOperationException("Instance already online!"); - instances.Add(metadata.Id, instance); + lock (instances) + instances.Add(metadata.Id, new InstanceContainer(instance)); + } + catch (Exception ex) + { + logger.LogError("Unable to commit onlined instance {0} into service, offlining!", metadata.Id); + try + { + await instance.StopAsync(default).ConfigureAwait(false); + } + catch (Exception innerEx) + { + throw new AggregateException(innerEx, ex); + } } } catch @@ -329,8 +379,6 @@ namespace Tgstation.Server.Host.Components await instance.DisposeAsync().ConfigureAwait(false); throw; } - - await instance.StartAsync(cancellationToken).ConfigureAwait(false); } /// @@ -405,7 +453,7 @@ namespace Tgstation.Server.Host.Components public async Task StopAsync(CancellationToken cancellationToken) { await jobManager.StopAsync(cancellationToken).ConfigureAwait(false); - await Task.WhenAll(instances.Select(x => x.Value.StopAsync(cancellationToken))).ConfigureAwait(false); + await Task.WhenAll(instances.Select(x => x.Value.Instance.StopAsync(cancellationToken))).ConfigureAwait(false); await instanceFactory.StopAsync(cancellationToken).ConfigureAwait(false); // downgrade the db if necessary @@ -489,5 +537,15 @@ namespace Tgstation.Server.Host.Components } }); } + + /// + public IInstanceCore GetInstance(Models.Instance metadata) + { + lock (instances) + { + instances.TryGetValue(metadata.Id, out var container); + return container?.Instance; + } + } } } diff --git a/src/Tgstation.Server.Host/Components/InstanceWrapper.cs b/src/Tgstation.Server.Host/Components/InstanceWrapper.cs new file mode 100644 index 0000000000..0830098235 --- /dev/null +++ b/src/Tgstation.Server.Host/Components/InstanceWrapper.cs @@ -0,0 +1,99 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Tgstation.Server.Host.Components.Byond; +using Tgstation.Server.Host.Components.Chat; +using Tgstation.Server.Host.Components.Deployment; +using Tgstation.Server.Host.Components.Repository; +using Tgstation.Server.Host.Components.StaticFiles; +using Tgstation.Server.Host.Components.Watchdog; +using Tgstation.Server.Host.Models; + +namespace Tgstation.Server.Host.Components +{ + /// + /// Warpper around a given with a . + /// + sealed class InstanceWrapper : IInstanceReference + { + /// + public Guid Uid { get; } + + /// + /// The object for . + /// + readonly object disposeLock; + + /// + /// The to take when is called. + /// + Action onDisposed; + + /// + /// The calls are forwarded to. + /// + IInstanceCore actualInstance; + + /// + /// Initializes a new instance of the . + /// + /// The value of . + /// The value of . + public InstanceWrapper(IInstanceCore actualInstance, Action onDisposed) + { + this.actualInstance = actualInstance ?? throw new ArgumentNullException(nameof(actualInstance)); + this.onDisposed = onDisposed ?? throw new ArgumentNullException(nameof(onDisposed)); + Uid = Guid.NewGuid(); + disposeLock = new object(); + } + + /// + public void Dispose() + { + lock (disposeLock) + { + onDisposed?.Invoke(); + onDisposed = null; + actualInstance = null; + } + } + + /// + public IRepositoryManager RepositoryManager => actualInstance?.RepositoryManager ?? throw new ObjectDisposedException(nameof(InstanceWrapper)); + + /// + public IByondManager ByondManager => actualInstance?.ByondManager ?? throw new ObjectDisposedException(nameof(InstanceWrapper)); + + /// + public IDreamMaker DreamMaker => actualInstance?.DreamMaker ?? throw new ObjectDisposedException(nameof(InstanceWrapper)); + + /// + public IWatchdog Watchdog => actualInstance?.Watchdog ?? throw new ObjectDisposedException(nameof(InstanceWrapper)); + + /// + public IChatManager Chat => actualInstance?.Chat ?? throw new ObjectDisposedException(nameof(InstanceWrapper)); + + /// + public IConfiguration Configuration => actualInstance?.Configuration ?? throw new ObjectDisposedException(nameof(InstanceWrapper)); + + /// + public Task InstanceRenamed(string newInstanceName, CancellationToken cancellationToken) + => actualInstance?.InstanceRenamed(newInstanceName, cancellationToken) ?? throw new ObjectDisposedException(nameof(InstanceWrapper)); + + /// + public CompileJob LatestCompileJob() + { + if (actualInstance == null) + throw new ObjectDisposedException(nameof(InstanceWrapper)); + return actualInstance.LatestCompileJob(); + } + + /// + public Task SetAutoUpdateInterval(uint newInterval) + { + if (actualInstance == null) + throw new ObjectDisposedException(nameof(InstanceWrapper)); + return actualInstance.SetAutoUpdateInterval(newInterval); + } + } +} diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs index e2b4bbb176..5b061c5108 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs @@ -912,11 +912,16 @@ namespace Tgstation.Server.Host.Components.Watchdog CancelRight = (ulong)DreamDaemonRights.Shutdown, CancelRightsType = RightsType.DreamDaemon }; - await jobManager.RegisterOperation(job, async (j, databaseContextFactory, progressFunction, ct) => - { - using (await SemaphoreSlimContext.Lock(synchronizationSemaphore, ct).ConfigureAwait(false)) - await LaunchNoLock(true, true, true, reattachInfo, ct).ConfigureAwait(false); - }, cancellationToken).ConfigureAwait(false); + await jobManager.RegisterOperation( + job, + async (core, databaseContextFactory, paramJob, progressFunction, ct) => + { + // core will certainly be null here since jobs started before the instance is onlined can't provide one + using (await SemaphoreSlimContext.Lock(synchronizationSemaphore, ct).ConfigureAwait(false)) + await LaunchNoLock(true, true, true, reattachInfo, ct).ConfigureAwait(false); + }, + cancellationToken) + .ConfigureAwait(false); } /// diff --git a/src/Tgstation.Server.Host/Controllers/ByondController.cs b/src/Tgstation.Server.Host/Controllers/ByondController.cs index 0db373c6e0..cab2dd1fb2 100644 --- a/src/Tgstation.Server.Host/Controllers/ByondController.cs +++ b/src/Tgstation.Server.Host/Controllers/ByondController.cs @@ -154,7 +154,7 @@ namespace Tgstation.Server.Host.Controllers }; await jobManager.RegisterOperation( job, - (paramJob, databaseContextFactory, progressHandler, jobCancellationToken) => byondManager.ChangeVersion( + (core, databaseContextFactory, paramJob, progressHandler, jobCancellationToken) => core.ByondManager.ChangeVersion( model.Version, model.Content, jobCancellationToken), diff --git a/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs b/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs index 1cd85be3c2..82e6866312 100644 --- a/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs +++ b/src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs @@ -78,7 +78,7 @@ namespace Tgstation.Server.Host.Controllers }; await jobManager.RegisterOperation( job, - (paramJob, databaseContextFactory, progressHandler, innerCt) => instance.Watchdog.Launch(innerCt), + (core, databaseContextFactory, paramJob, progressHandler, innerCt) => core.Watchdog.Launch(innerCt), cancellationToken) .ConfigureAwait(false); return Accepted(job.ToApi()); @@ -302,7 +302,7 @@ namespace Tgstation.Server.Host.Controllers await jobManager.RegisterOperation( job, - (paramJob, databaseContextFactory, progressReporter, ct) => watchdog.Restart(false, ct), + (core, paramJob, databaseContextFactory, progressReporter, ct) => core.Watchdog.Restart(false, ct), cancellationToken) .ConfigureAwait(false); return Accepted(job.ToApi()); @@ -336,7 +336,7 @@ namespace Tgstation.Server.Host.Controllers await jobManager.RegisterOperation( job, - (paramJob, databaseContextFactory, progressReporter, ct) => watchdog.CreateDump(ct), cancellationToken) + (core, databaseContextFactory, paramJob, progressReporter, ct) => core.Watchdog.CreateDump(ct), cancellationToken) .ConfigureAwait(false); return Accepted(job.ToApi()); }); diff --git a/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs b/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs index 6a27f95aba..7a4ec3bd0c 100644 --- a/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs +++ b/src/Tgstation.Server.Host/Controllers/DreamMakerController.cs @@ -133,26 +133,25 @@ namespace Tgstation.Server.Host.Controllers [HttpPut] [TgsAuthorize(DreamMakerRights.Compile)] [ProducesResponseType(typeof(Api.Models.Job), 202)] - public Task Create(CancellationToken cancellationToken) - => WithComponentInstance( - async instance => - { - var job = new Models.Job - { - Description = "Compile active repository code", - StartedBy = AuthenticationContext.User, - CancelRightsType = RightsType.DreamMaker, - CancelRight = (ulong)DreamMakerRights.CancelCompile, - Instance = Instance - }; + public async Task Create(CancellationToken cancellationToken) + { + var job = new Models.Job + { + Description = "Compile active repository code", + StartedBy = AuthenticationContext.User, + CancelRightsType = RightsType.DreamMaker, + CancelRight = (ulong)DreamMakerRights.CancelCompile, + Instance = Instance + }; - await jobManager.RegisterOperation( - job, - instance.DreamMaker.DeploymentProcess, - cancellationToken) - .ConfigureAwait(false); - return Accepted(job.ToApi()); - }); + await jobManager.RegisterOperation( + job, + (core, databaseContextFactory, paramJob, progressReporter, jobCancellationToken) + => core.DreamMaker.DeploymentProcess(paramJob, databaseContextFactory, progressReporter, jobCancellationToken), + cancellationToken) + .ConfigureAwait(false); + return Accepted(job.ToApi()); + } /// /// Update deployment settings. diff --git a/src/Tgstation.Server.Host/Controllers/InstanceController.cs b/src/Tgstation.Server.Host/Controllers/InstanceController.cs index afed0c82a1..19c98d6f2c 100644 --- a/src/Tgstation.Server.Host/Controllers/InstanceController.cs +++ b/src/Tgstation.Server.Host/Controllers/InstanceController.cs @@ -481,7 +481,7 @@ namespace Tgstation.Server.Host.Controllers if (renamed) { - var componentInstance = instanceManager.GetInstance(originalModel); + var componentInstance = instanceManager.GetInstanceReference(originalModel); if (componentInstance != null) await componentInstance.InstanceRenamed(originalModel.Name, cancellationToken).ConfigureAwait(false); } @@ -532,7 +532,8 @@ namespace Tgstation.Server.Host.Controllers await jobManager.RegisterOperation( job, - (paramJob, databaseContextFactory, progressHandler, ct) => instanceManager.MoveInstance(originalModel, originalModelPath, ct), + (core, databaseContextFactory, paramJob, progressHandler, ct) // core will be null here since the instance is offline + => instanceManager.MoveInstance(originalModel, originalModelPath, ct), cancellationToken) .ConfigureAwait(false); api.MoveJob = job.ToApi(); @@ -540,7 +541,7 @@ namespace Tgstation.Server.Host.Controllers if (model.AutoUpdateInterval.HasValue && oldAutoUpdateInterval != model.AutoUpdateInterval) { - var componentInstance = instanceManager.GetInstance(originalModel); + var componentInstance = instanceManager.GetInstanceReference(originalModel); if (componentInstance != null) await componentInstance.SetAutoUpdateInterval(model.AutoUpdateInterval.Value).ConfigureAwait(false); } diff --git a/src/Tgstation.Server.Host/Controllers/InstanceRequiredController.cs b/src/Tgstation.Server.Host/Controllers/InstanceRequiredController.cs index d247daf857..c3d0ab996d 100644 --- a/src/Tgstation.Server.Host/Controllers/InstanceRequiredController.cs +++ b/src/Tgstation.Server.Host/Controllers/InstanceRequiredController.cs @@ -1,5 +1,6 @@ using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; +using Serilog.Context; using System; using System.Threading; using System.Threading.Tasks; @@ -48,7 +49,7 @@ namespace Tgstation.Server.Host.Controllers if (ValidateInstanceOnlineStatus(instanceManager, Logger, Instance)) await DatabaseContext.Save(cancellationToken).ConfigureAwait(false); - if (instanceManager.GetInstance(Instance) == null) + if (instanceManager.GetInstanceReference(Instance) == null) return Conflict(new ErrorMessage(ErrorCode.InstanceOffline)); return null; } @@ -59,15 +60,18 @@ namespace Tgstation.Server.Host.Controllers /// A accepting the and returning a with the . /// A resulting in the that should be returned. /// The context of should be as small as possinle so as to avoid race conditions. - protected async Task WithComponentInstance(Func> action) + protected async Task WithComponentInstance(Func> action) { if (action == null) throw new ArgumentNullException(nameof(action)); - var componentInstance = instanceManager.GetInstance(Instance); - if (componentInstance == null) - return Conflict(new ErrorMessage(ErrorCode.InstanceOffline)); - return await action(componentInstance).ConfigureAwait(false); + using var instanceReference = instanceManager.GetInstanceReference(Instance); + using (LogContext.PushProperty("InstanceReference", instanceReference.Uid)) + { + if (instanceReference == null) + return Conflict(new ErrorMessage(ErrorCode.InstanceOffline)); + return await action(instanceReference).ConfigureAwait(false); + } } /// @@ -84,7 +88,7 @@ namespace Tgstation.Server.Host.Controllers if (metadata == null) throw new ArgumentNullException(nameof(metadata)); - var online = instanceManager.GetInstance(metadata) != null; + var online = instanceManager.GetInstanceReference(metadata) != null; if (metadata.Online.Value == online) return false; diff --git a/src/Tgstation.Server.Host/Controllers/RepositoryController.cs b/src/Tgstation.Server.Host/Controllers/RepositoryController.cs index 10aea1aac6..423818dc6e 100644 --- a/src/Tgstation.Server.Host/Controllers/RepositoryController.cs +++ b/src/Tgstation.Server.Host/Controllers/RepositoryController.cs @@ -211,8 +211,9 @@ namespace Tgstation.Server.Host.Controllers Instance = Instance }; var api = currentModel.ToApi(); - await jobManager.RegisterOperation(job, async (paramJob, databaseContextFactory, progressReporter, ct) => + await jobManager.RegisterOperation(job, async (core, databaseContextFactory, paramJob, progressReporter, ct) => { + var repoManager = core.RepositoryManager; using var repos = await repoManager.CloneRepository( new Uri(origin), cloneBranch, @@ -284,18 +285,13 @@ namespace Tgstation.Server.Host.Controllers Instance = Instance }; var api = currentModel.ToApi(); - return await WithComponentInstance( - async instance => - { - await jobManager.RegisterOperation( - job, - (paramJob, databaseContextFactory, progressReporter, ct) => instance.RepositoryManager.DeleteRepository(ct), - cancellationToken) - .ConfigureAwait(false); - api.ActiveJob = job.ToApi(); - return Accepted(api); - }) - .ConfigureAwait(false); + await jobManager.RegisterOperation( + job, + (core, databaseContextFactory, paramJob, progressReporter, ct) => core.RepositoryManager.DeleteRepository(ct), + cancellationToken) + .ConfigureAwait(false); + api.ActiveJob = job.ToApi(); + return Accepted(api); } /// @@ -363,8 +359,8 @@ namespace Tgstation.Server.Host.Controllers [ProducesResponseType(typeof(Repository), 200)] [ProducesResponseType(typeof(Repository), 202)] [ProducesResponseType(typeof(ErrorMessage), 410)] - #pragma warning disable CA1502, CA1505 // TODO: Decomplexify - public async Task Update([FromBody]Repository model, CancellationToken cancellationToken) +#pragma warning disable CA1502, CA1505 // TODO: Decomplexify + public async Task Update([FromBody] Repository model, CancellationToken cancellationToken) { if (model == null) throw new ArgumentNullException(nameof(model)); @@ -494,7 +490,7 @@ namespace Tgstation.Server.Host.Controllers return Json(api); // no git changes async Task UpdateCallbackThatDesperatelyNeedsRefactoring( - IInstance instance, + IInstanceCore instance, IDatabaseContextFactory databaseContextFactory, Action progressReporter, CancellationToken ct) @@ -897,13 +893,12 @@ namespace Tgstation.Server.Host.Controllers // Time to access git, do it in a job await jobManager.RegisterOperation( job, - (paramJob, databaseContextFactory, progressReporter, ct) - => WithComponentInstance( // Will never fail in the context of a job - instance => UpdateCallbackThatDesperatelyNeedsRefactoring( - instance, - databaseContextFactory, - progressReporter, - ct)), + (core, databaseContextFactory, paramJob, progressReporter, ct) => + UpdateCallbackThatDesperatelyNeedsRefactoring( + core, + databaseContextFactory, + progressReporter, + ct), cancellationToken) .ConfigureAwait(false); diff --git a/src/Tgstation.Server.Host/Core/Application.cs b/src/Tgstation.Server.Host/Core/Application.cs index cad2b6970f..fa29699de4 100644 --- a/src/Tgstation.Server.Host/Core/Application.cs +++ b/src/Tgstation.Server.Host/Core/Application.cs @@ -132,7 +132,7 @@ namespace Tgstation.Server.Host.Core var formatter = new MessageTemplateTextFormatter( "{Timestamp:o} " + ServiceCollectionExtensions.SerilogContextTemplate - + ": [{Level:u3}] {SourceContext:l}: {Message} ({EventId:x8}){NewLine}{Exception}", + + "|IR:{InstanceReference}){): [{Level:u3}] {SourceContext:l}: {Message} ({EventId:x8}){NewLine}{Exception}", null); logPath = IOManager.ConcatPath(logPath, "tgs-.log"); @@ -301,6 +301,7 @@ namespace Tgstation.Server.Host.Core services.AddSingleton(); services.AddSingleton(x => x.GetRequiredService()); + services.AddSingleton(x => new Lazy(() => x.GetRequiredService())); services.AddSingleton(x => x.GetRequiredService()); } diff --git a/src/Tgstation.Server.Host/Extensions/ServiceCollectionExtensions.cs b/src/Tgstation.Server.Host/Extensions/ServiceCollectionExtensions.cs index 8ff326c398..b6c6e9e5d1 100644 --- a/src/Tgstation.Server.Host/Extensions/ServiceCollectionExtensions.cs +++ b/src/Tgstation.Server.Host/Extensions/ServiceCollectionExtensions.cs @@ -18,7 +18,7 @@ namespace Tgstation.Server.Host.Extensions /// /// Common template used for adding our custom log context to serilog. /// - public const string SerilogContextTemplate = "(Instance:{Instance}|Job:{Job}|Request:{Request}|User:{User}|Monitor:{Monitor}|Bridge:{Bridge}|Chat:{ChatMessage})"; + public const string SerilogContextTemplate = "(Instance:{Instance}|Job:{Job}|Request:{Request}|User:{User}|Monitor:{Monitor}|Bridge:{Bridge}|Chat:{ChatMessage}"; /// /// Add a standard binding @@ -79,7 +79,7 @@ namespace Tgstation.Server.Host.Extensions sinkConfiguration.Console( outputTemplate: "[{Timestamp:HH:mm:ss}] {Level:w3}: {SourceContext:l} " + SerilogContextTemplate - + "{NewLine} {Message:lj}{NewLine}{Exception}"); + + "|IR:{InstanceReference}){NewLine} {Message:lj}{NewLine}{Exception}"); sinkConfigurationAction?.Invoke(sinkConfiguration); }); diff --git a/src/Tgstation.Server.Host/Jobs/IJobManager.cs b/src/Tgstation.Server.Host/Jobs/IJobManager.cs index a6d3610a9d..526ca4a437 100644 --- a/src/Tgstation.Server.Host/Jobs/IJobManager.cs +++ b/src/Tgstation.Server.Host/Jobs/IJobManager.cs @@ -1,8 +1,6 @@ using Microsoft.Extensions.Hosting; -using System; using System.Threading; using System.Threading.Tasks; -using Tgstation.Server.Host.Database; using Tgstation.Server.Host.Models; namespace Tgstation.Server.Host.Jobs @@ -23,10 +21,10 @@ namespace Tgstation.Server.Host.Jobs /// Registers a given and begins running it /// /// The - /// The operation to run taking the started , a , progress reporter and a + /// The for the . /// The for the operation /// A representing a running operation - Task RegisterOperation(Job job, Func, CancellationToken, Task> operation, CancellationToken cancellationToken); + Task RegisterOperation(Job job, JobEntrypoint operation, CancellationToken cancellationToken); /// /// Wait for a given to complete diff --git a/src/Tgstation.Server.Host/Jobs/JobEntrypoint.cs b/src/Tgstation.Server.Host/Jobs/JobEntrypoint.cs new file mode 100644 index 0000000000..0c1f9fc16f --- /dev/null +++ b/src/Tgstation.Server.Host/Jobs/JobEntrypoint.cs @@ -0,0 +1,25 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Tgstation.Server.Host.Components; +using Tgstation.Server.Host.Database; +using Tgstation.Server.Host.Models; + +namespace Tgstation.Server.Host.Jobs +{ + /// + /// Entrypoint for running a given job. + /// + /// The the job is running on. only when performing an instance move operation. + /// The for the operation. + /// The running . + /// A that will update the progress of the job. + /// The for the operation. + /// A representing the running operation. + public delegate Task JobEntrypoint( + IInstanceCore instance, + IDatabaseContextFactory databaseContextFactory, + Job job, + Action progressReporter, + CancellationToken cancellationToken); +} diff --git a/src/Tgstation.Server.Host/Jobs/JobHandler.cs b/src/Tgstation.Server.Host/Jobs/JobHandler.cs index 072b075fe2..86437139d5 100644 --- a/src/Tgstation.Server.Host/Jobs/JobHandler.cs +++ b/src/Tgstation.Server.Host/Jobs/JobHandler.cs @@ -14,21 +14,24 @@ namespace Tgstation.Server.Host.Jobs /// readonly CancellationTokenSource cancellationTokenSource; + /// + /// A taking a and returning a that the will wrap + /// + readonly Func jobActivator; + /// /// The being run /// - readonly Task task; + Task task; /// /// Construct a /// - /// A taking a and returning a that the will wrap - public JobHandler(Func job) + /// The value of . + public JobHandler(Func jobActivator) { - if (job == null) - throw new ArgumentNullException(nameof(job)); + this.jobActivator = jobActivator ?? throw new ArgumentNullException(nameof(jobActivator)); cancellationTokenSource = new CancellationTokenSource(); - task = job(cancellationTokenSource.Token); } /// @@ -46,6 +49,9 @@ namespace Tgstation.Server.Host.Jobs /// A representing the running operation public async Task Wait(CancellationToken cancellationToken) { + if (task == null) + throw new InvalidOperationException("Job not started!"); + TaskCompletionSource tcs = new TaskCompletionSource(); using (cancellationToken.Register(() => tcs.SetCanceled())) await Task.WhenAny(tcs.Task, task).ConfigureAwait(false); @@ -56,5 +62,18 @@ namespace Tgstation.Server.Host.Jobs /// Cancels /// public void Cancel() => cancellationTokenSource.Cancel(); + + /// + /// Starts the job. + /// + public void Start() + { + lock (cancellationTokenSource) + { + if (task != null) + throw new InvalidOperationException("Job already started"); + task = jobActivator(cancellationTokenSource.Token); + } + } } } \ No newline at end of file diff --git a/src/Tgstation.Server.Host/Jobs/JobManager.cs b/src/Tgstation.Server.Host/Jobs/JobManager.cs index 113ec63cf0..6ec0fdc7c3 100644 --- a/src/Tgstation.Server.Host/Jobs/JobManager.cs +++ b/src/Tgstation.Server.Host/Jobs/JobManager.cs @@ -6,6 +6,7 @@ using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; +using Tgstation.Server.Host.Components; using Tgstation.Server.Host.Database; using Tgstation.Server.Host.Models; @@ -24,6 +25,11 @@ namespace Tgstation.Server.Host.Jobs /// readonly ILogger logger; + /// + /// The for the . + /// + readonly Lazy instanceCoreProvider; + /// /// of s to running s /// @@ -38,10 +44,12 @@ namespace Tgstation.Server.Host.Jobs /// Construct a /// /// The value of + /// The value of . /// The value of - public JobManager(IDatabaseContextFactory databaseContextFactory, ILogger logger) + public JobManager(IDatabaseContextFactory databaseContextFactory, Lazy instanceCoreProvider, ILogger logger) { this.databaseContextFactory = databaseContextFactory ?? throw new ArgumentNullException(nameof(databaseContextFactory)); + this.instanceCoreProvider = instanceCoreProvider ?? throw new ArgumentNullException(nameof(instanceCoreProvider)); this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); jobs = new Dictionary(); synchronizationLock = new object(); @@ -73,10 +81,10 @@ namespace Tgstation.Server.Host.Jobs /// Runner for s /// /// The being run - /// The operation for the + /// The for the /// The for the operation /// A representing the running operation - async Task RunJob(Job job, Func operation, CancellationToken cancellationToken) + async Task RunJob(Job job, JobEntrypoint operation, CancellationToken cancellationToken) { using (LogContext.PushProperty("Job", job.Id)) try @@ -87,7 +95,20 @@ namespace Tgstation.Server.Host.Jobs var oldJob = job; job = new Job { Id = oldJob.Id }; - await operation(job, databaseContextFactory, cancellationToken).ConfigureAwait(false); + void UpdateProgress(int progress) + { + lock (synchronizationLock) + if (jobs.TryGetValue(oldJob.Id, out var handler)) + handler.Progress = progress; + } + + await operation( + instanceCoreProvider.Value.GetInstance(oldJob.Instance), + databaseContextFactory, + job, + UpdateProgress, + cancellationToken) + .ConfigureAwait(false); logger.LogDebug("Job {0} completed!", job.Id); } @@ -144,43 +165,50 @@ namespace Tgstation.Server.Host.Jobs } /// - public Task RegisterOperation(Job job, Func, CancellationToken, Task> operation, CancellationToken cancellationToken) => databaseContextFactory.UseContext(async databaseContext => - { - if (job == null) - throw new ArgumentNullException(nameof(job)); - if (operation == null) - throw new ArgumentNullException(nameof(operation)); + public Task RegisterOperation(Job job, JobEntrypoint operation, CancellationToken cancellationToken) + => databaseContextFactory.UseContext( + async databaseContext => + { + if (job == null) + throw new ArgumentNullException(nameof(job)); + if (operation == null) + throw new ArgumentNullException(nameof(operation)); - job.StartedAt = DateTimeOffset.Now; - job.Cancelled = false; + job.StartedAt = DateTimeOffset.Now; + job.Cancelled = false; - job.Instance = new Instance - { - Id = job.Instance.Id - }; - databaseContext.Instances.Attach(job.Instance); + job.Instance = new Models.Instance + { + Id = job.Instance.Id + }; + databaseContext.Instances.Attach(job.Instance); - job.StartedBy = new User - { - Id = job.StartedBy.Id - }; - databaseContext.Users.Attach(job.StartedBy); + job.StartedBy = new User + { + Id = job.StartedBy.Id + }; + databaseContext.Users.Attach(job.StartedBy); - databaseContext.Jobs.Add(job); + databaseContext.Jobs.Add(job); - await databaseContext.Save(cancellationToken).ConfigureAwait(false); - logger.LogDebug("Starting job {0}: {1}...", job.Id, job.Description); - var jobHandler = new JobHandler(x => RunJob(job, (jobParam, serviceProvider, ct) => - operation(jobParam, serviceProvider, y => - { - lock (synchronizationLock) - if (jobs.TryGetValue(job.Id, out var handler)) - handler.Progress = y; - }, ct), - x)); - lock (synchronizationLock) - jobs.Add(job.Id, jobHandler); - }); + await databaseContext.Save(cancellationToken).ConfigureAwait(false); + + logger.LogDebug("Starting job {0}: {1}...", job.Id, job.Description); + var jobHandler = new JobHandler(jobCancellationToken => RunJob(job, operation, jobCancellationToken)); + try + { + lock (synchronizationLock) + { + jobs.Add(job.Id, jobHandler); + jobHandler.Start(); + } + } + catch + { + jobHandler.Dispose(); + throw; + } + }); /// public async Task StartAsync(CancellationToken cancellationToken)