diff --git a/src/Tgstation.Server.Host/Components/DmbFactory.cs b/src/Tgstation.Server.Host/Components/DmbFactory.cs index 9d21f4747f..e16506e3ab 100644 --- a/src/Tgstation.Server.Host/Components/DmbFactory.cs +++ b/src/Tgstation.Server.Host/Components/DmbFactory.cs @@ -13,6 +13,16 @@ namespace Tgstation.Server.Host.Components /// sealed class DmbFactory : IDmbFactory, ICompileJobConsumer { + /// + public Task OnNewerDmb + { + get + { + lock (this) + return newerDmbTcs.Task; + } + } + /// /// The for the /// @@ -109,13 +119,6 @@ namespace Tgstation.Server.Host.Components return result; } - /// - public Task OnNewerDmb() - { - lock (this) - return newerDmbTcs.Task; - } - /// public Task StartAsync(CancellationToken cancellationToken) => databaseContextFactory.UseContext(async (db) => { diff --git a/src/Tgstation.Server.Host/Components/DreamMaker.cs b/src/Tgstation.Server.Host/Components/DreamMaker.cs index 173332c7d7..2dafc3b983 100644 --- a/src/Tgstation.Server.Host/Components/DreamMaker.cs +++ b/src/Tgstation.Server.Host/Components/DreamMaker.cs @@ -36,6 +36,10 @@ namespace Tgstation.Server.Host.Components /// public CompilerStatus Status { get; private set; } + /// + /// The for + /// + readonly IByond byond; /// /// The for /// @@ -49,10 +53,6 @@ namespace Tgstation.Server.Host.Components /// readonly ISessionControllerFactory sessionControllerFactory; /// - /// The for - /// - readonly IByond byond; - /// /// The for /// readonly ICompileJobConsumer compileJobConsumer; @@ -64,16 +64,17 @@ namespace Tgstation.Server.Host.Components /// /// Construct /// + /// The value of /// The value of /// The value of /// The value of - /// The value of /// The value of /// The value of /// The value of /// - public DreamMaker(IIOManager ioManager, IConfiguration configuration, ISessionControllerFactory sessionControllerFactory, ICompileJobConsumer compileJobConsumer, IApplication application) + public DreamMaker(IByond byond, IIOManager ioManager, IConfiguration configuration, ISessionControllerFactory sessionControllerFactory, ICompileJobConsumer compileJobConsumer, IApplication application) { + this.byond = byond; this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); this.configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); this.sessionControllerFactory = sessionControllerFactory ?? throw new ArgumentNullException(nameof(sessionControllerFactory)); diff --git a/src/Tgstation.Server.Host/Components/IDmbFactory.cs b/src/Tgstation.Server.Host/Components/IDmbFactory.cs index 2ed1a771ff..391c5e727a 100644 --- a/src/Tgstation.Server.Host/Components/IDmbFactory.cs +++ b/src/Tgstation.Server.Host/Components/IDmbFactory.cs @@ -7,18 +7,18 @@ namespace Tgstation.Server.Host.Components /// Factory for s /// interface IDmbFactory - { + { + /// + /// Get a that completes when the result of a call to will be different than the previous call if any + /// + /// A representing the running operation + Task OnNewerDmb { get; } + /// /// Gets the next /// /// The for the operation /// A representing the running operation Task LockNextDmb(CancellationToken cancellationToken); - - /// - /// Get a that completes when the result of a call to will be different than the previous call if any - /// - /// A representing the running operation - Task OnNewerDmb(); } } diff --git a/src/Tgstation.Server.Host/Components/InstanceManager.cs b/src/Tgstation.Server.Host/Components/InstanceManager.cs index ef150c8552..ff78b157b8 100644 --- a/src/Tgstation.Server.Host/Components/InstanceManager.cs +++ b/src/Tgstation.Server.Host/Components/InstanceManager.cs @@ -53,6 +53,7 @@ namespace Tgstation.Server.Host.Components this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); this.databaseContextFactory = databaseContextFactory ?? throw new ArgumentNullException(nameof(databaseContextFactory)); instances = new Dictionary(); + interopConsumers = new Dictionary(); } /// diff --git a/src/Tgstation.Server.Host/Components/Watchdog/IReattachInfoHandler.cs b/src/Tgstation.Server.Host/Components/Watchdog/IReattachInfoHandler.cs new file mode 100644 index 0000000000..a5994d1d5b --- /dev/null +++ b/src/Tgstation.Server.Host/Components/Watchdog/IReattachInfoHandler.cs @@ -0,0 +1,26 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace Tgstation.Server.Host.Components.Watchdog +{ + /// + /// Handles saving and loading + /// + interface IReattachInfoHandler + { + /// + /// Save some + /// + /// The to save + /// The for the operation + /// A representing the running operation + Task Save(WatchdogReattachInformation reattachInformation, CancellationToken cancellationToken); + + /// + /// Load a saved + /// + /// The for the operation + /// A resulting in the stored if any + Task Load(CancellationToken cancellationToken); + } +} \ No newline at end of file diff --git a/src/Tgstation.Server.Host/Components/Watchdog/LaunchResult.cs b/src/Tgstation.Server.Host/Components/Watchdog/LaunchResult.cs index 016d9f57d0..539181995f 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/LaunchResult.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/LaunchResult.cs @@ -1,4 +1,5 @@ using System; +using System.Globalization; namespace Tgstation.Server.Host.Components.Watchdog { @@ -21,5 +22,8 @@ namespace Tgstation.Server.Host.Components.Watchdog /// The peak virtual memory usage in bytes /// public long PeakMemory { get; set; } + + /// + public override string ToString() => String.Format(CultureInfo.InvariantCulture, "Exit Code: {0}, RAM: {1}, Time {2}ms", ExitCode, PeakMemory, StartupTime.TotalMilliseconds); } } \ No newline at end of file diff --git a/src/Tgstation.Server.Host/Components/Watchdog/MonitorAction.cs b/src/Tgstation.Server.Host/Components/Watchdog/MonitorAction.cs new file mode 100644 index 0000000000..3140b9e854 --- /dev/null +++ b/src/Tgstation.Server.Host/Components/Watchdog/MonitorAction.cs @@ -0,0 +1,13 @@ +namespace Tgstation.Server.Host.Components.Watchdog +{ + /// + /// The action for the monitor loop to take when control is returned to it + /// + enum MonitorAction + { + Continue, + Break, + Restart, + Exit + } +} \ No newline at end of file diff --git a/src/Tgstation.Server.Host/Components/Watchdog/MonitorState.cs b/src/Tgstation.Server.Host/Components/Watchdog/MonitorState.cs index dfb418f9f4..09f24d30c5 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/MonitorState.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/MonitorState.cs @@ -5,6 +5,8 @@ public bool RebootingInactiveServer { get; set; } public bool InactiveServerHasStagedDmb { get; set; } + public MonitorAction NextAction { get; set; } + public ISessionController ActiveServer { get; set; } public ISessionController InactiveServer { get; set; } } diff --git a/src/Tgstation.Server.Host/Components/Watchdog/SessionController.cs b/src/Tgstation.Server.Host/Components/Watchdog/SessionController.cs index 28bd36126b..88c4e1d662 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/SessionController.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/SessionController.cs @@ -53,7 +53,7 @@ namespace Tgstation.Server.Host.Components.Watchdog } /// - public bool ClosePortsOnReboot { get; set; } + public bool ClosePortOnReboot { get; set; } /// public bool ApiValidated @@ -104,6 +104,9 @@ namespace Tgstation.Server.Host.Components.Watchdog /// public Task Lifetime => session.Lifetime; + /// + public Task OnReboot => rebootTcs.Task; + /// /// The up to date /// @@ -142,6 +145,11 @@ namespace Tgstation.Server.Host.Components.Watchdog /// The port to assign DreamDaemon when it queries for it /// ushort nextPort; + + /// + /// The that completes when DD tells us about a reboot + /// + TaskCompletionSource rebootTcs; /// /// If we know DreamDaemon currently has it's port closed @@ -181,6 +189,8 @@ namespace Tgstation.Server.Host.Components.Watchdog portClosed = false; disposed = false; apiValidated = false; + + rebootTcs = new TaskCompletionSource(); } /// @@ -229,13 +239,16 @@ namespace Tgstation.Server.Host.Components.Watchdog apiValidated = true; break; case DMCommandWorldReboot: - if (ClosePortsOnReboot) + if (ClosePortOnReboot) { content = new Dictionary { { DMParameterNewPort, 0 } }; portClosed = true; } else - ClosePortsOnReboot = true; + ClosePortOnReboot = true; + var oldTcs = rebootTcs; + rebootTcs = new TaskCompletionSource(); + oldTcs.SetResult(null); break; default: status = HttpStatusCode.BadRequest; diff --git a/src/Tgstation.Server.Host/Components/Watchdog/SessionControllerFactory.cs b/src/Tgstation.Server.Host/Components/Watchdog/SessionControllerFactory.cs index aae84ea84a..b829806d86 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/SessionControllerFactory.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/SessionControllerFactory.cs @@ -60,6 +60,31 @@ namespace Tgstation.Server.Host.Components.Watchdog /// readonly IChat chat; + /// + /// Construct a + /// + /// The value of + /// The value of + /// The value of + /// The value of + /// The value of + /// The value of + /// The value of + /// The value of + /// The value of + public SessionControllerFactory(IExecutor executor, IByond byond, IByondTopicSender byondTopicSender, IInteropRegistrar interopRegistrar, ICryptographySuite cryptographySuite, IApplication application, IInstance instance, IIOManager ioManager, IChat chat) + { + this.executor = executor ?? throw new ArgumentNullException(nameof(executor)); + this.byond = byond ?? throw new ArgumentNullException(nameof(byond)); + this.byondTopicSender = byondTopicSender ?? throw new ArgumentNullException(nameof(byondTopicSender)); + this.interopRegistrar = interopRegistrar ?? throw new ArgumentNullException(nameof(interopRegistrar)); + this.cryptographySuite = cryptographySuite ?? throw new ArgumentNullException(nameof(cryptographySuite)); + this.application = application ?? throw new ArgumentNullException(nameof(application)); + this.instance = instance ?? throw new ArgumentNullException(nameof(instance)); + this.ioManager = ioManager ?? throw new ArgumentNullException(nameof(ioManager)); + this.chat = chat ?? throw new ArgumentNullException(nameof(chat)); + } + /// public async Task LaunchNew(DreamDaemonLaunchParameters launchParameters, IDmbProvider dmbProvider, bool primaryPort, bool primaryDirectory, bool apiValidate, CancellationToken cancellationToken) { @@ -139,15 +164,46 @@ namespace Tgstation.Server.Host.Components.Watchdog } catch { - chatJsonTrackingTask.Dispose(); + chatJsonTrackingContext.Dispose(); throw; } } /// - public Task Reattach(ReattachInformation reattachInformation, CancellationToken cancellationToken) + public async Task Reattach(ReattachInformation reattachInformation, CancellationToken cancellationToken) { - throw new NotImplementedException(); + if (reattachInformation == null) + throw new ArgumentNullException(nameof(reattachInformation)); + + var basePath = reattachInformation.IsPrimary ? reattachInformation.Dmb.PrimaryDirectory : reattachInformation.Dmb.SecondaryDirectory; + var chatJsonTrackingContext = await chat.TrackJsons(basePath, reattachInformation.ChatChannelsJson, reattachInformation.ChatCommandsJson, cancellationToken).ConfigureAwait(false); + try + { + var byondLock = byond.UseExecutables(reattachInformation.Dmb.CompileJob.ByondVersion); + try + { + var session = executor.AttachToDreamDaemon(reattachInformation.ProcessId, byondLock); + try + { + return new SessionController(reattachInformation, session, byondTopicSender, interopRegistrar, chatJsonTrackingContext, chat); + } + catch + { + session.Dispose(); + throw; + } + } + catch + { + byondLock.Dispose(); + throw; + } + } + catch + { + chatJsonTrackingContext.Dispose(); + throw; + } } } } diff --git a/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs index 5bf055c988..bc79f36450 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/Watchdog.cs @@ -1,16 +1,22 @@ -using System; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; +using System; using System.Globalization; using System.Threading; using System.Threading.Tasks; using Tgstation.Server.Api.Models.Internal; using Tgstation.Server.Host.Core; -using Tgstation.Server.Host.Models; namespace Tgstation.Server.Host.Components.Watchdog { /// sealed class Watchdog : IWatchdog { + /// + /// The time in milliseconds to wait from starting to start . Does not take responsiveness into account + /// + const int AlphaBravoStartupSeperationInterval = 3000; + /// public bool Running { get; private set; } @@ -33,7 +39,7 @@ namespace Tgstation.Server.Host.Components.Watchdog public DreamDaemonLaunchParameters LastLaunchParameters { get; private set; } /// - public RebootState? RebootState => Running ? (RebootState?)(AlphaIsActive ? alphaServer.RebootState : bravoServer.RebootState) : null; + public RebootState? RebootState => Running ? (RebootState?)(AlphaIsActive ? alphaServer?.RebootState : bravoServer?.RebootState) : null; /// /// The for the @@ -45,21 +51,21 @@ namespace Tgstation.Server.Host.Components.Watchdog /// readonly ISessionControllerFactory sessionControllerFactory; - /// - /// The for the - /// - readonly IEventConsumer eventConsumer; - - /// - /// The for the - /// - readonly IInteropRegistrar interopRegistrar; - /// /// The for the /// readonly IDmbFactory dmbFactory; + /// + /// The for the + /// + readonly ILogger logger; + + /// + /// The for the + /// + readonly IReattachInfoHandler reattachInfoHandler; + /// /// The for the /// @@ -82,19 +88,19 @@ namespace Tgstation.Server.Host.Components.Watchdog /// The value of /// The value of /// The value of - /// The value of - /// The value of /// The for the + /// The value of + /// The value of /// The initial value of - public Watchdog(IChat chat, ISessionControllerFactory sessionControllerFactory, IDmbFactory dmbFactory, IEventConsumer eventConsumer, IInteropRegistrar interopRegistrar, IServerUpdater serverUpdater, DreamDaemonLaunchParameters initialLaunchParameters) + public Watchdog(IChat chat, ISessionControllerFactory sessionControllerFactory, IDmbFactory dmbFactory, IServerUpdater serverUpdater, ILogger logger, IReattachInfoHandler reattachInfoHandler, DreamDaemonLaunchParameters initialLaunchParameters) { this.chat = chat ?? throw new ArgumentNullException(nameof(chat)); this.sessionControllerFactory = sessionControllerFactory ?? throw new ArgumentNullException(nameof(sessionControllerFactory)); this.dmbFactory = dmbFactory ?? throw new ArgumentNullException(nameof(dmbFactory)); - this.eventConsumer = eventConsumer ?? throw new ArgumentNullException(nameof(eventConsumer)); - this.interopRegistrar = interopRegistrar ?? throw new ArgumentNullException(nameof(interopRegistrar)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + this.reattachInfoHandler = reattachInfoHandler ?? throw new ArgumentNullException(nameof(reattachInfoHandler)); - if(serverUpdater == null) + if (serverUpdater == null) throw new ArgumentNullException(nameof(serverUpdater)); serverUpdater.RegisterForUpdate(() => releaseServers = true); @@ -112,9 +118,9 @@ namespace Tgstation.Server.Host.Components.Watchdog semaphore.Dispose(); } - void DisposeAndNullControllers() { + logger.LogTrace("DisposeAndNullControllers"); alphaServer?.Dispose(); alphaServer = null; bravoServer?.Dispose(); @@ -126,9 +132,17 @@ namespace Tgstation.Server.Host.Components.Watchdog var running = Running; if (!graceful || !running) { + Task chatTask; if (running) - await Terminate(false, cancellationToken).ConfigureAwait(false); - return await Launch(cancellationToken).ConfigureAwait(false); + { + chatTask = chat.SendWatchdogMessage("Manual restart triggered...", cancellationToken); + await TerminateNoLock(false, false, cancellationToken).ConfigureAwait(false); + } + else + chatTask = Task.CompletedTask; + var result = await LaunchNoLock(true, !running, false, cancellationToken).ConfigureAwait(false); + await chatTask.ConfigureAwait(false); + return result; } var toReboot = AlphaIsActive ? alphaServer : bravoServer; var other = AlphaIsActive ? bravoServer : alphaServer; @@ -138,53 +152,167 @@ namespace Tgstation.Server.Host.Components.Watchdog return null; } - async Task HandlerMonitorWakeup(MonitorActivationReason activationReason, MonitorState monitorState) + async Task TerminateNoLock(bool graceful, bool announce, CancellationToken cancellationToken) { - + if (!Running) + return; + if (!graceful) + { + var chatTask = announce ? chat.SendWatchdogMessage("Terminating...", cancellationToken) : Task.CompletedTask; + await StopMonitor().ConfigureAwait(false); + DisposeAndNullControllers(); + await chatTask.ConfigureAwait(false); + return; + } + var toKill = AlphaIsActive ? alphaServer : bravoServer; + var other = AlphaIsActive ? bravoServer : alphaServer; + if (toKill != null) + await toKill.SetRebootState(Components.Watchdog.RebootState.Shutdown, cancellationToken).ConfigureAwait(false); } + async Task HandlerMonitorWakeup(MonitorActivationReason activationReason, MonitorState monitorState) + { + logger.LogInformation("Monitor activation. Reason: {0}", activationReason); + await Task.Yield(); + } + + /// + /// The loop that watches the watchdog + /// + /// The for the operation + /// A representing the running operation async Task MonitorLifetimes(CancellationToken cancellationToken) { - var state = new MonitorState(); - while(true) + logger.LogDebug("Entered MonitorLifetimes"); + var iteration = 1; + for(var state = new MonitorState(); state.NextAction != MonitorAction.Exit; ++iteration) { + logger.LogDebug("New iteration of monitor loop"); try { - var alphaServerTask = alphaServer.Lifetime; - var bravoServerTask = bravoServer.Lifetime; - var alphaServerReboot = alphaServer.OnReboot; - var bravoServerReboot = bravoServer.OnReboot; + if(AlphaIsActive) + logger.LogDebug("Alpha is the active server"); + else + logger.LogDebug("Bravo is the active server"); - var activeServer = AlphaIsActive ? alphaServer : bravoServer; - var inactiveServer = AlphaIsActive ? bravoServer : alphaServer; + if(state.InactiveServerHasStagedDmb) + logger.LogDebug("Inactive server has staged .dmb"); + if (state.RebootingInactiveServer) + logger.LogDebug("Inactive server is rebooting"); + + state.ActiveServer = AlphaIsActive ? alphaServer : bravoServer; + state.InactiveServer = AlphaIsActive ? bravoServer : alphaServer; + + var activeServerLifetime = state.ActiveServer.Lifetime; + var inactiveServerLifetime = state.InactiveServer.Lifetime; + var activeServerReboot = state.ActiveServer.OnReboot; + var inactiveServerReboot = state.InactiveServer.OnReboot; + var inactiveServerStartup = state.InactiveServer.LaunchResult; + var newDmbAvailable = dmbFactory.OnNewerDmb; var cancelTcs = new TaskCompletionSource(); using (cancellationToken.Register(() => cancelTcs.SetCanceled())) { - var toWaitOn = Task.WhenAny(alphaServerTask, bravoServerTask, alphaServerReboot, bravoServerReboot, cancelTcs.Task); - if (!watchReboot) - toWaitOn = Task.WhenAny(toWaitOn, inactiveServer.LaunchResult); + var toWaitOn = Task.WhenAny(activeServerLifetime, inactiveServerLifetime, activeServerReboot, inactiveServerReboot, newDmbAvailable, cancelTcs.Task); + if (state.RebootingInactiveServer) + toWaitOn = Task.WhenAny(toWaitOn, inactiveServerStartup); await toWaitOn.ConfigureAwait(false); } - using (await SemaphoreContext.Lock(semaphore, default).ConfigureAwait(false)) + var chatTask = Task.CompletedTask; + using (await SemaphoreContext.Lock(semaphore, cancellationToken).ConfigureAwait(false)) { - state.ActiveServer = AlphaIsActive ? alphaServer : bravoServer; - state.InactiveServer = !AlphaIsActive ? alphaServer : bravoServer; - MonitorActivationReason activationReason; - if (activeServer.Lifetime.IsCompleted) - activationReason = MonitorActivationReason.ActiveServerCrashed; - else if(inactiveServer.Lifetime.IsCompleted) + MonitorActivationReason activationReason = default; + //multiple things may have happened, handle them one at a time + for (var moreActivationsToProcess = true; moreActivationsToProcess && state.NextAction == MonitorAction.Continue; await HandlerMonitorWakeup(activationReason, state).ConfigureAwait(false)) + { + if (activeServerLifetime?.IsCompleted == true) + { + activationReason = MonitorActivationReason.ActiveServerCrashed; + activeServerLifetime = null; + } + else if (inactiveServerLifetime?.IsCompleted == true) + { + activationReason = MonitorActivationReason.InactiveServerCrashed; + inactiveServerLifetime = null; + } + else if (activeServerReboot?.IsCompleted == true) + { + activationReason = MonitorActivationReason.ActiveServerRebooted; + activeServerReboot = null; + } + else if (inactiveServerReboot?.IsCompleted == true) + { + activationReason = MonitorActivationReason.InactiveServerRebooted; + inactiveServerReboot = null; + } + else if (inactiveServerStartup?.IsCompleted == true) + { + activationReason = MonitorActivationReason.InactiveServerStartupComplete; + inactiveServerStartup = null; + } + else if (newDmbAvailable?.IsCompleted == true) + { + activationReason = MonitorActivationReason.NewDmbAvailable; + newDmbAvailable = null; + } + else + moreActivationsToProcess = false; + } + //full reboot required + if (state.NextAction == MonitorAction.Restart) + { + logger.LogDebug("Next state action is to restart"); + DisposeAndNullControllers(); + Running = false; + chatTask = chat.SendWatchdogMessage("Restarting due to complications...", cancellationToken); + } + } + + for (var retryAttempts = 1; state.NextAction == MonitorAction.Restart; ++retryAttempts) + { + WatchdogLaunchResult result; + using (await SemaphoreContext.Lock(semaphore, cancellationToken).ConfigureAwait(false)) + result = await LaunchNoLock(false, false, false, cancellationToken).ConfigureAwait(false); + + await chatTask.ConfigureAwait(false); + if (Running) + state.NextAction = MonitorAction.Continue; + else + { + logger.LogWarning("Failed to automatically restart the watchdog! Alpha: {0}; Bravo: {1}", result.Alpha.ToString(), result.Bravo.ToString()); + var retryDelay = Math.Min(Math.Pow(2, retryAttempts), 3600); //max of one hour + chatTask = chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Failed to restart watchdog (Attempt: {0}), retrying in {1} seconds...", retryAttempts, retryDelay), cancellationToken); + await Task.WhenAll(Task.Delay((int)retryDelay, cancellationToken), chatTask).ConfigureAwait(false); + } } } - catch (OperationCanceledException) { } + catch (OperationCanceledException) + { + logger.LogDebug("Monitor cancelled"); + break; + } catch (Exception e) { - await chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Monitor crashed, this should NEVER happen! Restarting monitor... Error: {0}", e.Message), cancellationToken).ConfigureAwait(false); + logger.LogError("Monitor crashed! Iteration: {0}, State: {1}", iteration, JsonConvert.SerializeObject(state)); + await chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Monitor crashed, this should NEVER happen! Please report this, full details in logs! Restarting monitor... Error: {0}", e.Message), cancellationToken).ConfigureAwait(false); } } } + async Task StopMonitor() + { + logger.LogTrace("StopMonitor"); + if (monitorTask == null) + return false; + monitorCts.Cancel(); + await monitorTask.ConfigureAwait(false); + monitorCts.Dispose(); + monitorTask = null; + return true; + } + + /// public async Task ChangeSettings(DreamDaemonLaunchParameters launchParameters, CancellationToken cancellationToken) { @@ -196,99 +324,134 @@ namespace Tgstation.Server.Host.Components.Watchdog } } - async Task StopMonitor() - { - if (monitorTask == null) - return; - monitorCts.Cancel(); - await monitorTask.ConfigureAwait(false); - monitorCts.Dispose(); - monitorTask = null; - } - - async Task LaunchNoLock(bool startMonitor, CancellationToken cancellationToken) + async Task LaunchNoLock(bool startMonitor, bool announce, bool doReattach, CancellationToken cancellationToken) { using (var alphaStartCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken)) { + logger.LogTrace("Begin LaunchNoLock"); if (Running) + { + logger.LogTrace("Aborted due to already running!"); return null; + } + + + Task chatTask; + //this is necessary, the monitor could be in it's sleep loop trying to restart + if (startMonitor && await StopMonitor().ConfigureAwait(false)) + chatTask = chat.SendWatchdogMessage("Automatic retry sequence cancelled by manual launch. Restarting...", cancellationToken); + else if (announce) + chatTask = chat.SendWatchdogMessage("Starting...", cancellationToken); + else + chatTask = Task.CompletedTask; //start both servers LastLaunchParameters = ActiveLaunchParameters; - var dmbToUse = await dmbFactory.LockNextDmb(cancellationToken).ConfigureAwait(false); - Task alphaServerTask = null; - try { + //good ole sanity + if (alphaServer != null || bravoServer != null) + throw new InvalidOperationException("Entered LaunchNoLock with one or more of the servers not being null!"); + + WatchdogReattachInformation reattachInfo = doReattach ? await reattachInfoHandler.Load(cancellationToken).ConfigureAwait(false) : null; + var doesntNeedNewDmb = doReattach && reattachInfo.Alpha != null && reattachInfo.Bravo != null; + var dmbToUse = doesntNeedNewDmb ? null : await dmbFactory.LockNextDmb(cancellationToken).ConfigureAwait(false); + + Task alphaServerTask = null; try { - alphaServerTask = sessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbToUse, true, true, false, alphaStartCts.Token); - //do a few seconds of delay so that any backends the servers use know that alpha came first - await Task.Delay(5000, cancellationToken).ConfigureAwait(false); - var bravoServerTask = sessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbToUse, false, false, false, cancellationToken); - bravoServer = await bravoServerTask.ConfigureAwait(false); - alphaServer = await alphaServerTask.ConfigureAwait(false); + try + { + if (!doReattach || reattachInfo.Alpha == null) + alphaServerTask = sessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbToUse, true, true, false, alphaStartCts.Token); + else + alphaServerTask = sessionControllerFactory.Reattach(reattachInfo.Alpha, cancellationToken); + //do a few seconds of delay so that any backends the servers use know that alpha came first + await Task.Delay(AlphaBravoStartupSeperationInterval, cancellationToken).ConfigureAwait(false); + Task bravoServerTask; + if (!doReattach || reattachInfo.Bravo == null) + bravoServerTask = sessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbToUse, false, false, false, cancellationToken); + else + bravoServerTask = sessionControllerFactory.Reattach(reattachInfo.Bravo, cancellationToken); + + bravoServer = await bravoServerTask.ConfigureAwait(false); + alphaServer = await alphaServerTask.ConfigureAwait(false); + } + catch + { + if (alphaServerTask != null) + if (alphaServerTask.Status == TaskStatus.RanToCompletion) + alphaServer = await alphaServerTask.ConfigureAwait(false); + else + { + alphaStartCts.Cancel(); + try + { + alphaServer = await alphaServerTask.ConfigureAwait(false); + } + catch { } + } + throw; + } + + async Task CheckLaunch(ISessionController controller, string serverName) + { + var launch = await controller.LaunchResult.ConfigureAwait(false); + if (launch.ExitCode.HasValue) + //you killed us ray... + throw new Exception(String.Format(CultureInfo.InvariantCulture, "{1} server failed to start: {0}", launch.ToString(), serverName)); + return launch; + } + + var alphaLrt = CheckLaunch(alphaServer, "Alpha"); + var bravoLrt = CheckLaunch(bravoServer, "Bravo"); + //now we have two booting servers, get them up and running + var allTask = Task.WhenAll(alphaLrt, bravoLrt); + + //don't forget about the cancelationToken + var cancelTcs = new TaskCompletionSource(); + using (cancellationToken.Register(() => cancelTcs.SetCanceled())) + await Task.WhenAny(allTask, cancelTcs.Task).ConfigureAwait(false); + + //both servers are now running, alpha is the active server, huzzah + AlphaIsActive = doReattach ? reattachInfo.AlphaIsActive : true; + LiveCompileJob = AlphaIsActive ? alphaServer.Dmb.CompileJob : bravoServer.Dmb.CompileJob; + LastLaunchResult = alphaLrt.Result; + StagedCompileJob = null; + Running = true; + + if (startMonitor) + { + await StopMonitor().ConfigureAwait(false); + monitorCts = new CancellationTokenSource(); + monitorTask = MonitorLifetimes(monitorCts.Token); + } + return new WatchdogLaunchResult + { + Alpha = alphaLrt.Result, + Bravo = bravoLrt.Result + }; } catch { - if (alphaServerTask != null) - if (alphaServerTask.Status == TaskStatus.RanToCompletion) - alphaServer = await alphaServerTask.ConfigureAwait(false); - else - { - alphaStartCts.Cancel(); - try - { - alphaServer = await alphaServerTask.ConfigureAwait(false); - } - catch { } - } + if (alphaServer == null && bravoServer == null) + dmbToUse.Dispose(); //guaranteed to not be null here + DisposeAndNullControllers(); throw; } - - async Task CheckLaunch(ISessionController controller, string serverName) - { - var launch = await controller.LaunchResult.ConfigureAwait(false); - if (launch.ExitCode.HasValue) - //you killed us ray... - throw new Exception(String.Format(CultureInfo.InvariantCulture, "{2} server failed to start: Exit Code: {0}, RAM: {1}, Runtime {3}ms", launch.ExitCode, launch.StartupTime, serverName, launch.StartupTime.TotalMilliseconds)); - return launch; - } - - var alphaLrt = CheckLaunch(alphaServer, "Alpha"); - var bravoLrt = CheckLaunch(bravoServer, "Bravo"); - //now we have two booting servers, get them up and running - var allTask = Task.WhenAll(alphaLrt, bravoLrt); - - //don't forget about the cancelationToken - var cancelTcs = new TaskCompletionSource(); - using (cancellationToken.Register(() => cancelTcs.SetCanceled())) - await Task.WhenAny(allTask, cancelTcs.Task).ConfigureAwait(false); - - //both servers are now running, alpha is the active server, huzzah - LiveCompileJob = dmbToUse.CompileJob; - LastLaunchResult = alphaLrt.Result; - StagedCompileJob = null; - AlphaIsActive = true; - Running = true; - - if (startMonitor) - { - await StopMonitor().ConfigureAwait(false); - monitorCts = new CancellationTokenSource(); - monitorTask = MonitorLifetimes(monitorCts.Token); - } - - return new WatchdogLaunchResult - { - Alpha = alphaLrt.Result, - Bravo = bravoLrt.Result - }; } - catch + catch (Exception e) { - DisposeAndNullControllers(); + logger.LogWarning("Failed to start watchdog: {0}", e.ToString()); throw; } + finally + { + try + { + await chatTask.ConfigureAwait(false); + } + catch (OperationCanceledException) { } + } } } @@ -296,7 +459,7 @@ namespace Tgstation.Server.Host.Components.Watchdog public async Task Launch(CancellationToken cancellationToken) { using (await SemaphoreContext.Lock(semaphore, cancellationToken).ConfigureAwait(false)) - return await LaunchNoLock(true, cancellationToken).ConfigureAwait(false); + return await LaunchNoLock(true, true, false, cancellationToken).ConfigureAwait(false); } /// @@ -310,37 +473,23 @@ namespace Tgstation.Server.Host.Components.Watchdog public async Task Terminate(bool graceful, CancellationToken cancellationToken) { using (await SemaphoreContext.Lock(semaphore, cancellationToken).ConfigureAwait(false)) - { - if (!Running) - return; - await StopMonitor().ConfigureAwait(false); - if (!graceful) - { - DisposeAndNullControllers(); - return; - } - var toKill = AlphaIsActive ? alphaServer : bravoServer; - var other = AlphaIsActive ? bravoServer : alphaServer; - if (toKill != null) - await toKill.SetRebootState(Components.Watchdog.RebootState.Shutdown, cancellationToken).ConfigureAwait(false); - } + await TerminateNoLock(graceful, true, cancellationToken).ConfigureAwait(false); } /// - public Task StartAsync(CancellationToken cancellationToken) => Launch(cancellationToken); + public Task StartAsync(CancellationToken cancellationToken) => LaunchNoLock(true, true, true, cancellationToken); /// - public Task StopAsync(CancellationToken cancellationToken) + public async Task StopAsync(CancellationToken cancellationToken) { if (releaseServers) { - ReattachInformation reattachInformation; - if (AlphaIsActive) - reattachInformation = alphaServer?.Release(); - else - reattachInformation = bravoServer?.Release(); + var reattachInformation = new WatchdogReattachInformation { AlphaIsActive = AlphaIsActive }; + reattachInformation.Alpha = alphaServer?.Release(); + reattachInformation.Bravo = bravoServer?.Release(); + await reattachInfoHandler.Save(reattachInformation, cancellationToken).ConfigureAwait(false); } - return Terminate(false, cancellationToken); + await Terminate(false, cancellationToken).ConfigureAwait(false); } } } diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogFactory.cs b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogFactory.cs index dc93621c80..638f5f6940 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogFactory.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogFactory.cs @@ -1,4 +1,5 @@ -using System; +using Microsoft.Extensions.Logging; +using System; using Tgstation.Server.Api.Models.Internal; using Tgstation.Server.Host.Core; @@ -7,11 +8,6 @@ namespace Tgstation.Server.Host.Components.Watchdog /// sealed class WatchdogFactory : IWatchdogFactory { - /// - /// The for the - /// - readonly IByond byond; - /// /// The for the /// @@ -22,40 +18,40 @@ namespace Tgstation.Server.Host.Components.Watchdog /// readonly ISessionControllerFactory sessionManagerFactory; - /// - /// The for the - /// - readonly IEventConsumer eventConsumer; - - /// - /// The for the - /// - readonly IInteropRegistrar interopRegistrar; - /// /// The for the /// readonly IServerUpdater serverUpdater; + /// + /// The for the + /// + readonly ILoggerFactory loggerFactory; + + /// + /// The for the + /// + readonly IReattachInfoHandler reattachInfoHandler; + /// /// Construct a /// /// The value of /// The value of /// The value of - /// The value of - /// The value of /// The value of - public WatchdogFactory(IChat chat, ISessionControllerFactory sessionManagerFactory, IEventConsumer eventConsumer, IInteropRegistrar interopRegistrar, IServerUpdater serverUpdater) + /// The value of + /// The value of + public WatchdogFactory(IChat chat, ISessionControllerFactory sessionManagerFactory, IServerUpdater serverUpdater, ILoggerFactory loggerFactory, IReattachInfoHandler reattachInfoHandler) { this.chat = chat ?? throw new ArgumentNullException(nameof(chat)); this.sessionManagerFactory = sessionManagerFactory ?? throw new ArgumentNullException(nameof(sessionManagerFactory)); - this.eventConsumer = eventConsumer ?? throw new ArgumentNullException(nameof(eventConsumer)); - this.interopRegistrar = interopRegistrar ?? throw new ArgumentNullException(nameof(interopRegistrar)); this.serverUpdater = serverUpdater ?? throw new ArgumentNullException(nameof(serverUpdater)); + this.loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory)); + this.reattachInfoHandler = reattachInfoHandler ?? throw new ArgumentNullException(nameof(reattachInfoHandler)); } /// - public IWatchdog CreateWatchdog(IDmbFactory dmbFactory, DreamDaemonLaunchParameters launchParameters) => new Watchdog(chat, sessionManagerFactory, dmbFactory, eventConsumer, interopRegistrar, serverUpdater, launchParameters); + public IWatchdog CreateWatchdog(IDmbFactory dmbFactory, DreamDaemonLaunchParameters launchParameters) => new Watchdog(chat, sessionManagerFactory, dmbFactory, serverUpdater, loggerFactory.CreateLogger(), reattachInfoHandler, launchParameters); } } diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogReattachInformation.cs b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogReattachInformation.cs index 3109a1ef2b..6e63099862 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogReattachInformation.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogReattachInformation.cs @@ -8,16 +8,16 @@ /// /// If the Alpha session is the active session /// - bool AlphaIsActive { get; set; } + public bool AlphaIsActive { get; set; } /// /// for the Alpha session /// - ReattachInformation Alpha { get; set; } + public ReattachInformation Alpha { get; set; } /// /// for the Bravo session /// - ReattachInformation Bravo { get; set; } + public ReattachInformation Bravo { get; set; } } } diff --git a/tests/Tgstation.Server.Host.Tests/Core/TestApplication.cs b/tests/Tgstation.Server.Host.Tests/Core/TestApplication.cs index afd32b4923..519d590b0d 100644 --- a/tests/Tgstation.Server.Host.Tests/Core/TestApplication.cs +++ b/tests/Tgstation.Server.Host.Tests/Core/TestApplication.cs @@ -1,6 +1,7 @@ using Microsoft.AspNetCore; using Microsoft.AspNetCore.Hosting; using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; using System.IO; using System.Threading.Tasks; @@ -9,7 +10,9 @@ namespace Tgstation.Server.Host.Core.Tests [TestClass] public sealed class TestApplication : IServerUpdater { - public void ApplyUpdate(string updatePath) => throw new System.NotImplementedException(); + public void ApplyUpdate(string updatePath) => throw new NotImplementedException(); + + public void RegisterForUpdate(Action action) => throw new NotImplementedException(); [TestMethod] public async Task TestSuccessfulStartup()