WatchdogBase.cs

Go to the documentation of this file.

using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using Serilog.Context;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Tgstation.Server.Api.Models;
using Tgstation.Server.Api.Models.Internal;
using Tgstation.Server.Api.Rights;
using Tgstation.Server.Host.Components.Chat;
using Tgstation.Server.Host.Components.Deployment;
using Tgstation.Server.Host.Components.Events;
using Tgstation.Server.Host.Components.Interop.Topic;
using Tgstation.Server.Host.Components.Session;
using Tgstation.Server.Host.Core;
using Tgstation.Server.Host.Database;
using Tgstation.Server.Host.Extensions;
using Tgstation.Server.Host.IO;
using Tgstation.Server.Host.Jobs;

namespace Tgstation.Server.Host.Components.Watchdog
{
    #pragma warning disable CA1506 // TODO: Decomplexify
    abstract class WatchdogBase : IWatchdog, ICustomCommandHandler, IRestartHandler
    {
        public WatchdogStatus Status
        {
            get => status;
            set
            {
                status = value;
                Logger.LogTrace("Status set to {0}", status);
            }
        }

        public abstract bool AlphaIsActive { get; }

        public abstract Models.CompileJob ActiveCompileJob { get; }

        public DreamDaemonLaunchParameters ActiveLaunchParameters { get; protected set; }

        public DreamDaemonLaunchParameters LastLaunchParameters { get; protected set; }

        public abstract RebootState? RebootState { get; }

        protected TaskCompletionSource<object> ActiveParametersUpdated { get; set; }

        protected SemaphoreSlim Semaphore { get; }

        protected ILogger Logger { get; }

        protected IChatManager Chat { get; }

        protected ISessionControllerFactory SessionControllerFactory { get; }

        protected IDmbFactory DmbFactory { get; }

        protected IAsyncDelayer AsyncDelayer { get; }

        readonly Api.Models.Instance instance;

        readonly ISessionPersistor sessionPersistor;

        readonly IDatabaseContextFactory databaseContextFactory;

        readonly IJobManager jobManager;

        readonly IRestartRegistration restartRegistration;

        readonly IIOManager diagnosticsIOManager;

        readonly IEventConsumer eventConsumer;

        readonly object controllerDisposeLock;

        readonly bool autoStart;

        ReattachInformation releasedReattachInformation;

        CancellationTokenSource monitorCts;

        Task monitorTask;

        WatchdogStatus status;

        int heartbeatsMissed;

        bool releaseServers;

        bool disposed;

        protected WatchdogBase(
            IChatManager chat,
            ISessionControllerFactory sessionControllerFactory,
            IDmbFactory dmbFactory,
            ISessionPersistor sessionPersistor,
            IDatabaseContextFactory databaseContextFactory,
            IJobManager jobManager,
            IServerControl serverControl,
            IAsyncDelayer asyncDelayer,
            IIOManager diagnosticsIOManager,
            IEventConsumer eventConsumer,
            ILogger logger,
            DreamDaemonLaunchParameters initialLaunchParameters,
            Api.Models.Instance instance,
            bool autoStart)
        {
            Chat = chat ?? throw new ArgumentNullException(nameof(chat));
            SessionControllerFactory = sessionControllerFactory ?? throw new ArgumentNullException(nameof(sessionControllerFactory));
            DmbFactory = dmbFactory ?? throw new ArgumentNullException(nameof(dmbFactory));
            this.sessionPersistor = sessionPersistor ?? throw new ArgumentNullException(nameof(sessionPersistor));
            this.databaseContextFactory = databaseContextFactory ?? throw new ArgumentNullException(nameof(databaseContextFactory));
            this.jobManager = jobManager ?? throw new ArgumentNullException(nameof(jobManager));
            AsyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer));
            this.diagnosticsIOManager = diagnosticsIOManager ?? throw new ArgumentNullException(nameof(diagnosticsIOManager));
            this.eventConsumer = eventConsumer ?? throw new ArgumentNullException(nameof(eventConsumer));
            Logger = logger ?? throw new ArgumentNullException(nameof(logger));
            ActiveLaunchParameters = initialLaunchParameters ?? throw new ArgumentNullException(nameof(initialLaunchParameters));
            this.instance = instance ?? throw new ArgumentNullException(nameof(instance));
            this.autoStart = autoStart;

            if (serverControl == null)
                throw new ArgumentNullException(nameof(serverControl));

            chat.RegisterCommandHandler(this);

            ActiveLaunchParameters = initialLaunchParameters;
            releaseServers = false;
            ActiveParametersUpdated = new TaskCompletionSource<object>();
            controllerDisposeLock = new object();

            restartRegistration = serverControl.RegisterForRestart(this);
            try
            {
                Semaphore = new SemaphoreSlim(1);
            }
            catch
            {
                restartRegistration.Dispose();
                throw;
            }

            Logger.LogTrace("Created watchdog");
        }

        public void Dispose()
        {
            Logger.LogTrace("Disposing...");
            Semaphore.Dispose();
            restartRegistration.Dispose();
            DisposeAndNullControllers();
            monitorCts?.Dispose();
            disposed = true;
        }

        async Task TerminateNoLock(bool graceful, bool announce, CancellationToken cancellationToken)
        {
            if (Status == WatchdogStatus.Offline)
                return;
            if (!graceful)
            {
                var eventTask = eventConsumer.HandleEvent(releaseServers ? EventType.WatchdogDetach : EventType.WatchdogShutdown, null, cancellationToken);

                var chatTask = announce ? Chat.SendWatchdogMessage("Shutting down...", false, cancellationToken) : Task.CompletedTask;

                await eventTask.ConfigureAwait(false);

                await StopMonitor().ConfigureAwait(false);

                DisposeAndNullControllers();

                LastLaunchParameters = null;

                await chatTask.ConfigureAwait(false);
                return;
            }

            // merely set the reboot state
            var toKill = GetActiveController();
            if (toKill != null)
            {
                await toKill.SetRebootState(Session.RebootState.Shutdown, cancellationToken).ConfigureAwait(false);
                Logger.LogTrace("Graceful termination requested");
            }
            else
                Logger.LogTrace("Could not gracefully terminate as there is no active controller!");
        }

        async Task<MonitorAction> HandleHeartbeat(CancellationToken cancellationToken)
        {
            Logger.LogTrace("Sending heartbeat to active server...");
            var activeServer = GetActiveController();
            var response = await activeServer.SendCommand(new TopicParameters(), cancellationToken).ConfigureAwait(false);

            var shouldShutdown = activeServer.RebootState == Session.RebootState.Shutdown;
            if (response == null)
            {
                switch (++heartbeatsMissed)
                {
                    case 1:
                        Logger.LogDebug("DEFCON 4: DreamDaemon missed first heartbeat!");
                        break;
                    case 2:
                        var message2 = "DEFCON 3: DreamDaemon has missed 2 heartbeats!";
                        Logger.LogInformation(message2);
                        await Chat.SendWatchdogMessage(message2, true, cancellationToken).ConfigureAwait(false);
                        break;
                    case 3:
                        var actionToTake = shouldShutdown
                            ? "shutdown"
                            : "be restarted";
                        var message3 = $"DEFCON 2: DreamDaemon has missed 3 heartbeats! If it does not respond to the next one, the watchdog will {actionToTake}!";
                        Logger.LogWarning(message3);
                        await Chat.SendWatchdogMessage(message3, false, cancellationToken).ConfigureAwait(false);
                        break;
                    case 4:
                        var actionTaken = shouldShutdown
                            ? "Shutting down due to graceful termination request"
                            : "Restarting";
                        var message4 = $"DEFCON 1: Four heartbeats have been missed! {actionTaken}...";
                        Logger.LogWarning(message4);
                        await Chat.SendWatchdogMessage(message4, false, cancellationToken).ConfigureAwait(false);
                        DisposeAndNullControllers();
                        return shouldShutdown ? MonitorAction.Exit : MonitorAction.Restart;
                    default:
                        Logger.LogError("Invalid heartbeats missed count: {0}", heartbeatsMissed);
                        break;
                }
            }
            else
                heartbeatsMissed = 0;

            return MonitorAction.Continue;
        }

        protected async Task LaunchNoLock(
            bool startMonitor,
            bool announce,
            bool announceFailure,
            ReattachInformation reattachInfo,
            CancellationToken cancellationToken)
        {
            Logger.LogTrace("Begin LaunchImplNoLock");

            if (reattachInfo == null && !DmbFactory.DmbAvailable)
                throw new JobException(ErrorCode.WatchdogCompileJobCorrupted);

            // this is necessary, the monitor could be in it's sleep loop trying to restart, if so cancel THAT monitor and start our own with blackjack and hookers
            Task announceTask;
            if (announce)
            {
                announceTask = Chat.SendWatchdogMessage(reattachInfo == null ? "Launching..." : "Reattaching...", false, cancellationToken); // simple announce
                if (reattachInfo == null)
                    announceTask = Task.WhenAll(
                        eventConsumer.HandleEvent(EventType.WatchdogLaunch, Enumerable.Empty<string>(), cancellationToken),
                        announceTask);
            }
            else
                announceTask = Task.CompletedTask; // no announce

            // since neither server is running, this is safe to do
            LastLaunchParameters = ActiveLaunchParameters;
            heartbeatsMissed = 0;

            try
            {
                await InitControllers(announceTask, reattachInfo, cancellationToken).ConfigureAwait(false);
            }
            catch (OperationCanceledException)
            {
                Logger.LogTrace("Controller initialization canceled!");
                throw;
            }
            catch (Exception e)
            {
                // don't try to send chat tasks or warning logs if were suppressing exceptions or cancelled
                var originalChatTask = announceTask;
                async Task ChainChatTaskWithErrorMessage()
                {
                    await originalChatTask.ConfigureAwait(false);
                    if (announceFailure)
                        await Chat.SendWatchdogMessage("Startup failed!", false, cancellationToken).ConfigureAwait(false);
                }

                announceTask = ChainChatTaskWithErrorMessage();
                Logger.LogWarning("Failed to start watchdog: {0}", e.ToString());
                throw;
            }
            finally
            {
                // finish the chat task that's in flight
                try
                {
                    await announceTask.ConfigureAwait(false);
                }
                catch (OperationCanceledException)
                {
                    Logger.LogTrace("Announcement task canceled!");
                }
            }

            Logger.LogInformation("Controller(s) initialized successfully");

            if (startMonitor)
            {
                monitorCts = new CancellationTokenSource();
                monitorTask = MonitorLifetimes(monitorCts.Token);
            }
        }

        protected async Task<bool> StopMonitor()
        {
            Logger.LogTrace("StopMonitor");
            if (monitorTask == null)
                return false;
            var wasRunning = !monitorTask.IsCompleted;
            monitorCts.Cancel();
            await monitorTask.ConfigureAwait(false);
            Logger.LogTrace("Stopped Monitor");
            monitorCts.Dispose();
            monitorTask = null;
            monitorCts = null;
            return wasRunning;
        }

        protected async Task CheckLaunchResult(ISessionController controller, string serverName, CancellationToken cancellationToken)
        {
            var launchResult = await controller.LaunchResult.WithToken(cancellationToken).ConfigureAwait(false);

            // Dead sessions won't trigger this
            if (launchResult.ExitCode.HasValue) // you killed us ray...
                throw new JobException(
                    ErrorCode.WatchdogStartupFailed,
                    new JobException($"{serverName} failed to start: {launchResult}"));
            if (!launchResult.StartupTime.HasValue)
                throw new JobException(
                    ErrorCode.WatchdogStartupTimeout,
                    new JobException($"{serverName} timed out on startup: {ActiveLaunchParameters.StartupTimeout.Value}s"));
        }

        protected async Task ReattachFailure(Task chatTask, CancellationToken cancellationToken)
        {
            // we lost the server, just restart entirely
            DisposeAndNullControllers();
            const string FailReattachMessage = "Unable to properly reattach to server! Restarting watchdog...";
            Logger.LogWarning(FailReattachMessage);

            async Task ChainChatTask()
            {
                await chatTask.ConfigureAwait(false);
                await Chat.SendWatchdogMessage(FailReattachMessage, false, cancellationToken).ConfigureAwait(false);
            }

            await InitControllers(ChainChatTask(), null, cancellationToken).ConfigureAwait(false);
        }

        protected abstract void DisposeAndNullControllersImpl();

        protected void DisposeAndNullControllers()
        {
            Logger.LogTrace("DisposeAndNullControllers");
            lock (controllerDisposeLock)
                DisposeAndNullControllersImpl();
        }

        protected abstract ISessionController GetActiveController();

        protected abstract Task<MonitorAction> HandleMonitorWakeup(
            MonitorActivationReason activationReason,
            CancellationToken cancellationToken);

        private async Task MonitorRestart(CancellationToken cancellationToken)
        {
            Logger.LogTrace("Monitor restart!");
            DisposeAndNullControllers();

            var chatTask = Task.CompletedTask;
            for (var retryAttempts = 1; ; ++retryAttempts)
            {
                Status = WatchdogStatus.Restoring;
                Exception launchException;
                using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false))
                    try
                    {
                        // use LaunchImplNoLock without announcements or restarting the monitor
                        await LaunchNoLock(false, false, false, null, cancellationToken).ConfigureAwait(false);
                        Status = WatchdogStatus.Online;
                        Logger.LogDebug("Relaunch successful, resuming monitor...");
                        return;
                    }
                    catch (OperationCanceledException)
                    {
                        throw;
                    }
                    catch (Exception e)
                    {
                        launchException = e;
                    }
                    finally
                    {
                        await chatTask.ConfigureAwait(false);
                    }

                Logger.LogWarning("Failed to automatically restart the watchdog! Attempt: {0}, Exception: {1}", retryAttempts, launchException);
                Status = WatchdogStatus.DelayedRestart;

                var retryDelay = Math.Min(
                    Convert.ToInt32(
                        Math.Pow(2, retryAttempts)),
                    TimeSpan.FromHours(1).TotalSeconds); // max of one hour, increasing by a power of 2 each time

                chatTask = Chat.SendWatchdogMessage(
                    $"Failed to restart (Attempt: {retryAttempts}), retrying in {retryDelay}s...",
                    false,
                    cancellationToken);

                await Task.WhenAll(
                    AsyncDelayer.Delay(
                        TimeSpan.FromSeconds(retryDelay),
                        cancellationToken),
                    chatTask)
                    .ConfigureAwait(false);
            }
        }

        private async Task MonitorLifetimes(CancellationToken cancellationToken)
        {
            Logger.LogTrace("Entered MonitorLifetimes");
            Status = WatchdogStatus.Online;
            using var _ = cancellationToken.Register(() => Logger.LogTrace("Monitor cancellationToken triggered"));

            // this function is responsible for calling HandlerMonitorWakeup when necessary and manitaining the MonitorState
            try
            {
                MonitorAction nextAction = MonitorAction.Continue;
                for (ulong iteration = 1; nextAction != MonitorAction.Exit; ++iteration)
                    using (LogContext.PushProperty("Monitor", iteration))
                        try
                        {
                            Logger.LogTrace("Iteration {0} of monitor loop", iteration);
                            nextAction = MonitorAction.Continue;

                            var controller = GetActiveController();
                            Task activeServerLifetime = controller.Lifetime;
                            var activeServerReboot = controller.OnReboot;

                            Task activeLaunchParametersChanged = ActiveParametersUpdated.Task;
                            var newDmbAvailable = DmbFactory.OnNewerDmb;

                            var heartbeatSeconds = ActiveLaunchParameters.HeartbeatSeconds.Value;
                            var heartbeat = heartbeatSeconds == 0
                                || !controller.DMApiAvailable
                                ? Extensions.TaskExtensions.InfiniteTask()
                                : Task.Delay(TimeSpan.FromSeconds(heartbeatSeconds));

                            // cancel waiting if requested
                            var cancelTcs = new TaskCompletionSource<object>();
                            var toWaitOn = Task.WhenAny(
                                activeServerLifetime,
                                activeServerReboot,
                                heartbeat,
                                newDmbAvailable,
                                cancelTcs.Task,
                                activeLaunchParametersChanged);

                            // wait for something to happen
                            using (cancellationToken.Register(() => cancelTcs.SetCanceled()))
                                await toWaitOn.ConfigureAwait(false);

                            cancellationToken.ThrowIfCancellationRequested();
                            Logger.LogTrace("Monitor activated");

                            // always run HandleMonitorWakeup from the context of the semaphore lock
                            using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false))
                            {
                                // Set this sooner so chat sends don't hold us up
                                if (activeServerLifetime.IsCompleted)
                                    Status = WatchdogStatus.Restoring;

                                // multiple things may have happened, handle them one at a time
                                for (var moreActivationsToProcess = true; moreActivationsToProcess && (nextAction == MonitorAction.Continue || nextAction == MonitorAction.Skip);)
                                {
                                    MonitorActivationReason activationReason = default; // this will always be assigned before being used

                                    bool CheckActivationReason(ref Task task, MonitorActivationReason testActivationReason)
                                    {
                                        var taskCompleted = task?.IsCompleted == true;
                                        task = null;
                                        if (nextAction == MonitorAction.Skip)
                                            nextAction = MonitorAction.Continue;
                                        else if (taskCompleted)
                                        {
                                            activationReason = testActivationReason;
                                            return true;
                                        }

                                        return false;
                                    }

                                    // process the tasks in this order and call HandlerMonitorWakup for each depending on the new monitorState
                                    var anyActivation = CheckActivationReason(ref activeServerLifetime, MonitorActivationReason.ActiveServerCrashed)
                                        || CheckActivationReason(ref activeServerReboot, MonitorActivationReason.ActiveServerRebooted)
                                        || CheckActivationReason(ref newDmbAvailable, MonitorActivationReason.NewDmbAvailable)
                                        || CheckActivationReason(ref activeLaunchParametersChanged, MonitorActivationReason.ActiveLaunchParametersUpdated)
                                        || CheckActivationReason(ref heartbeat, MonitorActivationReason.Heartbeat);

                                    if (!anyActivation)
                                        moreActivationsToProcess = false;
                                    else
                                    {
                                        Logger.LogTrace("Reason: {0}", activationReason);
                                        if (activationReason == MonitorActivationReason.Heartbeat)
                                            nextAction = await HandleHeartbeat(
                                                cancellationToken)
                                                .ConfigureAwait(false);
                                        else
                                            nextAction = await HandleMonitorWakeup(
                                                activationReason,
                                                cancellationToken)
                                                .ConfigureAwait(false);
                                    }
                                }
                            }

                            Logger.LogTrace("Next monitor action is to {0}", nextAction);

                            // Restart if requested
                            if (nextAction == MonitorAction.Restart)
                            {
                                await MonitorRestart(cancellationToken).ConfigureAwait(false);
                                nextAction = MonitorAction.Continue;
                            }
                        }
                        catch (OperationCanceledException)
                        {
                            // let this bubble, other exceptions caught below
                            throw;
                        }
                        catch (Exception e)
                        {
                            // really, this should NEVER happen
                            Logger.LogError(
                                "Monitor crashed! Iteration: {0}, Exception: {1}",
                                iteration,
                                e);

                            var nextActionMessage = nextAction != MonitorAction.Exit
                                ? "Recovering"
                                : "Shutting down";
                            var chatTask = Chat.SendWatchdogMessage(
                                $"Monitor crashed, this should NEVER happen! Please report this, full details in logs! {nextActionMessage}. Error: {e.Message}",
                                false,
                                cancellationToken);

                            if (disposed)
                                nextAction = MonitorAction.Exit;
                            else if (nextAction != MonitorAction.Exit)
                            {
                                if (GetActiveController()?.Lifetime.IsCompleted != true)
                                    await MonitorRestart(cancellationToken).ConfigureAwait(false);
                                else
                                    Logger.LogDebug("Server seems to be okay, not restarting");
                                nextAction = MonitorAction.Continue;
                            }

                            await chatTask.ConfigureAwait(false);
                        }
            }
            catch (OperationCanceledException)
            {
                // stop signal
                Logger.LogDebug("Monitor cancelled");

                if (releaseServers)
                {
                    Logger.LogTrace("Detaching servers...");
                    releasedReattachInformation = GetActiveController().Release();
                }
            }

            DisposeAndNullControllers();
            Status = WatchdogStatus.Offline;

            Logger.LogTrace("Monitor exiting...");
        }

        protected abstract Task InitControllers(Task chatTask, ReattachInformation reattachInfo, CancellationToken cancellationToken);

        public async Task ChangeSettings(DreamDaemonLaunchParameters launchParameters, CancellationToken cancellationToken)
        {
            using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false))
            {
                bool match = launchParameters.CanApplyWithoutReboot(ActiveLaunchParameters);
                ActiveLaunchParameters = launchParameters;
                if (match || Status == WatchdogStatus.Offline)
                    return;

                ActiveParametersUpdated.TrySetResult(null); // queue an update
                ActiveParametersUpdated = new TaskCompletionSource<object>();
            }
        }

        async Task IEventConsumer.HandleEvent(EventType eventType, IEnumerable<string> parameters, CancellationToken cancellationToken)
        {
            // Method explicitly implemented to prevent accidental calls when this.eventConsumer should be used.
            var activeServer = GetActiveController();

            // Server may have ended
            if (activeServer == null)
                return;

            var notification = new EventNotification(eventType, parameters);
            var result = await activeServer.SendCommand(
                new TopicParameters(notification),
                cancellationToken)
                .ConfigureAwait(false);

            if (result?.InteropResponse?.ChatResponses != null)
                await Task.WhenAll(
                    result.InteropResponse.ChatResponses.Select(
                        x => Chat.SendMessage(
                            x.Text,
                            x.ChannelIds
                                .Select(channelIdString =>
                                {
                                    if (UInt64.TryParse(channelIdString, out var channelId))
                                        return (ulong?)channelId;

                                    return null;
                                })
                                .Where(nullableChannelId => nullableChannelId.HasValue)
                                .Select(nullableChannelId => nullableChannelId.Value),
                            cancellationToken)))
                    .ConfigureAwait(false);
        }

        public async Task<string> HandleChatCommand(string commandName, string arguments, ChatUser sender, CancellationToken cancellationToken)
        {
            using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false))
            {
                if (Status == WatchdogStatus.Offline)
                    return "TGS: Server offline!";

                var commandObject = new ChatCommand(sender, commandName, arguments);

                var command = new TopicParameters(commandObject);

                var activeServer = GetActiveController();
                var commandResult = await activeServer.SendCommand(command, cancellationToken).ConfigureAwait(false);

                if (commandResult == null)
                    return "TGS: Bad topic exchange!";

                if (commandResult.InteropResponse == null)
                    return "TGS: Bad topic response!";

                return commandResult.InteropResponse.CommandResponseMessage ??
                    "TGS: Command processed but no DMAPI response returned!";
            }
        }

        public async Task Launch(CancellationToken cancellationToken)
        {
            if (Status != WatchdogStatus.Offline)
                throw new JobException(ErrorCode.WatchdogRunning);
            using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false))
                await LaunchNoLock(true, true, true, null, cancellationToken).ConfigureAwait(false);
        }

        public virtual async Task ResetRebootState(CancellationToken cancellationToken)
        {
            using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false))
            {
                if (Status == WatchdogStatus.Offline)
                    return;
                var toClear = GetActiveController();
                if (toClear != null)
                    await toClear.SetRebootState(Session.RebootState.Normal, cancellationToken).ConfigureAwait(false);
            }
        }

        public async Task Restart(bool graceful, CancellationToken cancellationToken)
        {
            if (Status == WatchdogStatus.Offline)
                throw new JobException(ErrorCode.WatchdogNotRunning);

            Logger.LogTrace("Begin Restart. Graceful: {0}", graceful);
            using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false))
            {
                if (!graceful)
                {
                    var chatTask = Chat.SendWatchdogMessage("Manual restart triggered...", false, cancellationToken);
                    await TerminateNoLock(false, false, cancellationToken).ConfigureAwait(false);
                    await LaunchNoLock(true, false, true, null, cancellationToken).ConfigureAwait(false);
                    await chatTask.ConfigureAwait(false);
                    return;
                }

                var toReboot = GetActiveController();
                if (toReboot != null
                    && !await toReboot.SetRebootState(Session.RebootState.Restart, cancellationToken).ConfigureAwait(false))
                    Logger.LogWarning("Unable to send reboot state change event!");
            }
        }

        public async Task StartAsync(CancellationToken cancellationToken)
        {
            var reattachInfo = await sessionPersistor.Load(cancellationToken).ConfigureAwait(false);
            if (!autoStart && reattachInfo == null)
                return;

            long? adminUserId = null;

            await databaseContextFactory.UseContext(
                async db => adminUserId = await db
                    .Users
                    .AsQueryable()
                    .Where(x => x.CanonicalName == Models.User.CanonicalizeName(Api.Models.User.AdminName))
                    .Select(x => x.Id)
                    .FirstAsync(cancellationToken)
                    .ConfigureAwait(false))
                .ConfigureAwait(false);
            var job = new Models.Job
            {
                StartedBy = new Models.User
                {
                    Id = adminUserId.Value
                },
                Instance = new Models.Instance
                {
                    Id = instance.Id
                },
                Description = $"Instance startup watchdog {(reattachInfo != null ? "reattach" : "launch")}",
                CancelRight = (ulong)DreamDaemonRights.Shutdown,
                CancelRightsType = RightsType.DreamDaemon
            };
            await jobManager.RegisterOperation(job, async (j, databaseContextFactory, progressFunction, ct) =>
            {
                using (await SemaphoreSlimContext.Lock(Semaphore, ct).ConfigureAwait(false))
                    await LaunchNoLock(true, true, true, reattachInfo, ct).ConfigureAwait(false);
            }, cancellationToken).ConfigureAwait(false);
        }

        public async Task StopAsync(CancellationToken cancellationToken)
        {
            await TerminateNoLock(false, !releaseServers, cancellationToken).ConfigureAwait(false);
            if (releasedReattachInformation != null)
            {
                try
                {
                    await sessionPersistor.Save(releasedReattachInformation, cancellationToken).ConfigureAwait(false);
                }
                catch (Exception ex)
                {
                    Logger.LogCritical(
                        "Failed to persist session reattach information! To repair this, DreamDaemon will need to be manully stopped and then relaunched with TGS. Exception: {0}",
                        ex);
                }

                releasedReattachInformation = null;
                releaseServers = false;
            }
        }

        public async Task Terminate(bool graceful, CancellationToken cancellationToken)
        {
            using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false))
                await TerminateNoLock(graceful, !releaseServers, cancellationToken).ConfigureAwait(false);
        }

        public async Task HandleRestart(Version updateVersion, CancellationToken cancellationToken)
        {
            releaseServers = true;
            if (Status == WatchdogStatus.Online)
                await Chat.SendWatchdogMessage("Detaching...", false, cancellationToken).ConfigureAwait(false);
            else
                Logger.LogTrace("Not sending detach chat message as status is: {0}", Status);
        }

        public abstract Task InstanceRenamed(string newInstanceName, CancellationToken cancellationToken);

        public async Task CreateDump(CancellationToken cancellationToken)
        {
            const string DumpDirectory = "ProcessDumps";
            await diagnosticsIOManager.CreateDirectory(DumpDirectory, cancellationToken).ConfigureAwait(false);

            var dumpFileName = diagnosticsIOManager.ResolvePath(
                diagnosticsIOManager.ConcatPath(
                    DumpDirectory,
                    $"DreamDaemon-{DateTimeOffset.Now.ToFileStamp()}.dmp"));

            var session = GetActiveController();
            if (session?.Lifetime.IsCompleted != false)
                throw new JobException(ErrorCode.DreamDaemonOffline);

            Logger.LogInformation("Dumping session to {0}...", dumpFileName);
            await session.CreateDump(dumpFileName, cancellationToken).ConfigureAwait(false);
        }
    }
}