WatchdogBase.cs

Go to the documentation of this file.

using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using Serilog.Context;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Tgstation.Server.Api.Models;
using Tgstation.Server.Api.Models.Internal;
using Tgstation.Server.Api.Rights;
using Tgstation.Server.Host.Components.Chat;
using Tgstation.Server.Host.Components.Deployment;
using Tgstation.Server.Host.Components.Events;
using Tgstation.Server.Host.Components.Interop.Topic;
using Tgstation.Server.Host.Components.Session;
using Tgstation.Server.Host.Core;
using Tgstation.Server.Host.Database;
using Tgstation.Server.Host.Extensions;
using Tgstation.Server.Host.Jobs;

namespace Tgstation.Server.Host.Components.Watchdog
{
    #pragma warning disable CA1506 // TODO: Decomplexify
    abstract class WatchdogBase : IWatchdog, ICustomCommandHandler, IRestartHandler
    {
        public bool Running
        {
            get => running;
            set
            {
                running = value;
                Logger.LogTrace("Running set to {0}", running);
            }
        }

        public abstract bool AlphaIsActive { get; }

        public abstract Models.CompileJob ActiveCompileJob { get; }

        public DreamDaemonLaunchParameters ActiveLaunchParameters { get; protected set; }

        public DreamDaemonLaunchParameters LastLaunchParameters { get; protected set; }

        public abstract RebootState? RebootState { get; }

        protected TaskCompletionSource<object> ActiveParametersUpdated { get; set; }

        protected SemaphoreSlim Semaphore { get; }

        protected ILogger Logger { get; }

        protected IChatManager Chat { get; }

        protected ISessionControllerFactory SessionControllerFactory { get; }

        protected IDmbFactory DmbFactory { get; }

        protected IAsyncDelayer AsyncDelayer { get; }

        readonly Api.Models.Instance instance;

        readonly IReattachInfoHandler reattachInfoHandler;

        readonly IDatabaseContextFactory databaseContextFactory;

        readonly IJobManager jobManager;

        readonly IRestartRegistration restartRegistration;

        readonly object controllerDisposeLock;

        readonly bool autoStart;

        DualReattachInformation releasedReattachInformation;

        CancellationTokenSource monitorCts;

        Task monitorTask;

        int heartbeatsMissed;

        bool releaseServers;

        bool running;

        bool disposed;

        protected WatchdogBase(
            IChatManager chat,
            ISessionControllerFactory sessionControllerFactory,
            IDmbFactory dmbFactory,
            IReattachInfoHandler reattachInfoHandler,
            IDatabaseContextFactory databaseContextFactory,
            IJobManager jobManager,
            IServerControl serverControl,
            IAsyncDelayer asyncDelayer,
            ILogger logger,
            DreamDaemonLaunchParameters initialLaunchParameters,
            Api.Models.Instance instance,
            bool autoStart)
        {
            Chat = chat ?? throw new ArgumentNullException(nameof(chat));
            SessionControllerFactory = sessionControllerFactory ?? throw new ArgumentNullException(nameof(sessionControllerFactory));
            DmbFactory = dmbFactory ?? throw new ArgumentNullException(nameof(dmbFactory));
            this.reattachInfoHandler = reattachInfoHandler ?? throw new ArgumentNullException(nameof(reattachInfoHandler));
            this.databaseContextFactory = databaseContextFactory ?? throw new ArgumentNullException(nameof(databaseContextFactory));
            this.jobManager = jobManager ?? throw new ArgumentNullException(nameof(jobManager));
            AsyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer));
            Logger = logger ?? throw new ArgumentNullException(nameof(logger));
            ActiveLaunchParameters = initialLaunchParameters ?? throw new ArgumentNullException(nameof(initialLaunchParameters));
            this.instance = instance ?? throw new ArgumentNullException(nameof(instance));
            this.autoStart = autoStart;

            if (serverControl == null)
                throw new ArgumentNullException(nameof(serverControl));

            chat.RegisterCommandHandler(this);

            ActiveLaunchParameters = initialLaunchParameters;
            releaseServers = false;
            ActiveParametersUpdated = new TaskCompletionSource<object>();
            controllerDisposeLock = new object();

            restartRegistration = serverControl.RegisterForRestart(this);
            try
            {
                Semaphore = new SemaphoreSlim(1);
            }
            catch
            {
                restartRegistration.Dispose();
                throw;
            }
        }

        public void Dispose()
        {
            Logger.LogTrace("Disposing...");
            Semaphore.Dispose();
            restartRegistration.Dispose();
            DisposeAndNullControllers();
            monitorCts?.Dispose();
            disposed = true;
        }

        async Task TerminateNoLock(bool graceful, bool announce, CancellationToken cancellationToken)
        {
            if (!Running)
                return;
            if (!graceful)
            {
                var eventTask = HandleEvent(releaseServers ? EventType.WatchdogDetach : EventType.WatchdogShutdown, null, cancellationToken);

                var chatTask = announce ? Chat.SendWatchdogMessage("Shutting down...", false, cancellationToken) : Task.CompletedTask;

                await eventTask.ConfigureAwait(false);

                await StopMonitor().ConfigureAwait(false);

                DisposeAndNullControllers();

                LastLaunchParameters = null;

                await chatTask.ConfigureAwait(false);
                return;
            }

            // merely set the reboot state
            var toKill = GetActiveController();
            if (toKill != null)
            {
                await toKill.SetRebootState(Session.RebootState.Shutdown, cancellationToken).ConfigureAwait(false);
                Logger.LogTrace("Graceful termination requested");
            }
            else
                Logger.LogTrace("Could not gracefully terminate as there is no active controller!");
        }

        async Task<MonitorAction> HandleHeartbeat(ISessionController activeServer, CancellationToken cancellationToken)
        {
            Logger.LogTrace("Sending heartbeat to active server...");
            var response = await activeServer.SendCommand(new TopicParameters(), cancellationToken).ConfigureAwait(false);

            var shouldShutdown = activeServer.RebootState == Session.RebootState.Shutdown;
            if (response == null)
            {
                switch (++heartbeatsMissed)
                {
                    case 1:
                        Logger.LogDebug("DEFCON 4: DreamDaemon missed first heartbeat!");
                        break;
                    case 2:
                        var message2 = "DEFCON 3: DreamDaemon has missed 2 heartbeats!";
                        Logger.LogInformation(message2);
                        await Chat.SendWatchdogMessage(message2, true, cancellationToken).ConfigureAwait(false);
                        break;
                    case 3:
                        var actionToTake = shouldShutdown
                            ? "shutdown"
                            : "be restarted";
                        var message3 = $"DEFCON 2: DreamDaemon has missed 3 heartbeats! If it does not respond to the next one, the watchdog will {actionToTake}!";
                        Logger.LogWarning(message3);
                        await Chat.SendWatchdogMessage(message3, false, cancellationToken).ConfigureAwait(false);
                        break;
                    case 4:
                        var actionTaken = shouldShutdown
                            ? "Shutting down due to graceful termination request"
                            : "Restarting";
                        var message4 = $"DEFCON 1: Four heartbeats have been missed! {actionTaken}...";
                        Logger.LogWarning(message4);
                        DisposeAndNullControllers();
                        await Chat.SendWatchdogMessage(message4, false, cancellationToken).ConfigureAwait(false);
                        return shouldShutdown ? MonitorAction.Exit : MonitorAction.Restart;
                    default:
                        Logger.LogError("Invalid heartbeats missed count: {0}", heartbeatsMissed);
                        break;
                }
            }
            else
                heartbeatsMissed = 0;

            return MonitorAction.Continue;
        }

        protected async Task LaunchNoLock(bool startMonitor, bool announce, DualReattachInformation reattachInfo, CancellationToken cancellationToken)
        {
            Logger.LogTrace("Begin LaunchImplNoLock");

            if (Running)
                throw new JobException(ErrorCode.WatchdogRunning);

            if (reattachInfo == null && !DmbFactory.DmbAvailable)
                throw new JobException(ErrorCode.WatchdogCompileJobCorrupted);

            // this is necessary, the monitor could be in it's sleep loop trying to restart, if so cancel THAT monitor and start our own with blackjack and hookers
            Task chatTask;
            if (startMonitor && await StopMonitor().ConfigureAwait(false))
                chatTask = Chat.SendWatchdogMessage("Automatic retry sequence cancelled by manual launch. Restarting...", false, cancellationToken);
            else if (announce)
                chatTask = Chat.SendWatchdogMessage(reattachInfo == null ? "Launching..." : "Reattaching...", false, cancellationToken); // simple announce
            else
                chatTask = Task.CompletedTask; // no announce

            // since neither server is running, this is safe to do
            LastLaunchParameters = ActiveLaunchParameters;
            heartbeatsMissed = 0;

            // for when we call ourself and want to not catch thrown exceptions
            var recursiveCallToHappen = false;
            try
            {
                await InitControllers(() => recursiveCallToHappen = true, chatTask, reattachInfo, cancellationToken).ConfigureAwait(false);
                if (recursiveCallToHappen)
                    return;

                await chatTask.ConfigureAwait(false);

                Logger.LogInformation("Launched servers successfully");
                Running = true;

                if (startMonitor)
                {
                    monitorCts = new CancellationTokenSource();
                    monitorTask = MonitorLifetimes(monitorCts.Token);
                }
            }
            catch (Exception e)
            {
                // don't try to send chat tasks or warning logs if were suppressing exceptions or cancelled
                if (!recursiveCallToHappen && !cancellationToken.IsCancellationRequested)
                {
                    var originalChatTask = chatTask;
                    async Task ChainChatTaskWithErrorMessage()
                    {
                        await originalChatTask.ConfigureAwait(false);
                        await Chat.SendWatchdogMessage("Startup failed!", false, cancellationToken).ConfigureAwait(false);
                    }

                    chatTask = ChainChatTaskWithErrorMessage();
                    Logger.LogWarning("Failed to start watchdog: {0}", e.ToString());
                }

                throw;
            }
            finally
            {
                // finish the chat task that's in flight
                try
                {
                    await chatTask.ConfigureAwait(false);
                }
                catch (OperationCanceledException) { }
            }
        }

        protected async Task<bool> StopMonitor()
        {
            Logger.LogTrace("StopMonitor");
            if (monitorTask == null)
                return false;
            var wasRunning = !monitorTask.IsCompleted;
            monitorCts.Cancel();
            await monitorTask.ConfigureAwait(false);
            Logger.LogTrace("Stopped Monitor");
            monitorCts.Dispose();
            monitorTask = null;
            monitorCts = null;
            return wasRunning;
        }

        protected async Task CheckLaunchResult(ISessionController controller, string serverName, CancellationToken cancellationToken)
        {
            var launchResult = await controller.LaunchResult.WithToken(cancellationToken).ConfigureAwait(false);

            // Dead sessions won't trigger this
            if (launchResult.ExitCode.HasValue) // you killed us ray...
                throw new JobException(
                    ErrorCode.WatchdogStartupFailed,
                    new JobException($"{serverName} failed to start: {launchResult}"));
            if (!launchResult.StartupTime.HasValue)
                throw new JobException(
                    ErrorCode.WatchdogStartupTimeout,
                    new JobException($"{serverName} timed out on startup: {ActiveLaunchParameters.StartupTimeout.Value}s"));
        }

        protected abstract void DisposeAndNullControllersImpl();

        protected void DisposeAndNullControllers()
        {
            Logger.LogTrace("DisposeAndNullControllers");
            lock (controllerDisposeLock)
                DisposeAndNullControllersImpl();
        }

        protected abstract ISessionController GetActiveController();

        protected abstract DualReattachInformation CreateReattachInformation();

        protected abstract IReadOnlyDictionary<MonitorActivationReason, Task> GetMonitoredServerTasks(MonitorState monitorState);

        protected abstract Task HandleMonitorWakeup(
            MonitorActivationReason activationReason,
            MonitorState monitorState,
            CancellationToken cancellationToken);

        private async Task<MonitorState> MonitorRestart(CancellationToken cancellationToken)
        {
            Logger.LogTrace("Monitor restart!");
            DisposeAndNullControllers();

            var chatTask = Task.CompletedTask;
            for (var retryAttempts = 1; ; ++retryAttempts)
            {
                Exception launchException = null;
                using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false))
                    try
                    {
                        // use LaunchImplNoLock without announcements or restarting the monitor
                        await LaunchNoLock(false, false, null, cancellationToken).ConfigureAwait(false);
                        if (Running)
                        {
                            Logger.LogDebug("Relaunch successful, resetting monitor state...");
                            return new MonitorState();
                        }
                    }
                    catch (OperationCanceledException)
                    {
                        throw;
                    }
                    catch (Exception e)
                    {
                        launchException = e;
                    }

                await chatTask.ConfigureAwait(false);
                if (!Running)
                {
                    if (launchException == null)
                        Logger.LogWarning("Failed to automatically restart the watchdog! Attempt: {0}", retryAttempts);
                    else
                        Logger.LogWarning("Failed to automatically restart the watchdog! Attempt: {0}, Exception: {1}", retryAttempts, launchException);
                    var retryDelay = Math.Min(
                        Convert.ToInt32(
                            Math.Pow(2, retryAttempts)),
                        TimeSpan.FromHours(1).Seconds); // max of one hour, increasing by a power of 2 each time
                    chatTask = Chat.SendWatchdogMessage(
                        $"Failed to restart (Attempt: {retryAttempts}), retrying in {retryDelay}",
                        false,
                        cancellationToken);
                    await Task.WhenAll(
                        AsyncDelayer.Delay(
                            TimeSpan.FromSeconds(retryDelay),
                            cancellationToken),
                        chatTask)
                        .ConfigureAwait(false);
                }
            }
        }

        private async Task MonitorLifetimes(CancellationToken cancellationToken)
        {
            Logger.LogTrace("Entered MonitorLifetimes");
            using var _ = cancellationToken.Register(() => Logger.LogTrace("Monitor cancellationToken triggered"));

            // this function is responsible for calling HandlerMonitorWakeup when necessary and manitaining the MonitorState
            var iteration = 1;
            try
            {
                for (var monitorState = new MonitorState(); monitorState.NextAction != MonitorAction.Exit; ++iteration)
                    using (LogContext.PushProperty("Monitor", iteration))
                        try
                        {
                            Logger.LogTrace("Iteration {0} of monitor loop", iteration);

                            // load the activation tasks into local variables
                            var serverTasks = GetMonitoredServerTasks(monitorState);
                            if (serverTasks.Count != 5)
                                throw new InvalidOperationException("Expected 5 monitored server tasks!");

                            var activeServerLifetime = serverTasks[MonitorActivationReason.ActiveServerCrashed];
                            var activeServerReboot = serverTasks[MonitorActivationReason.ActiveServerRebooted];
                            var inactiveServerLifetime = serverTasks[MonitorActivationReason.InactiveServerCrashed];
                            var inactiveServerReboot = serverTasks[MonitorActivationReason.InactiveServerRebooted];
                            var inactiveStartupComplete = serverTasks[MonitorActivationReason.InactiveServerStartupComplete];

                            Task activeLaunchParametersChanged = ActiveParametersUpdated.Task;
                            var newDmbAvailable = DmbFactory.OnNewerDmb;

                            var heartbeatSeconds = ActiveLaunchParameters.HeartbeatSeconds.Value;
                            var heartbeat = heartbeatSeconds == 0
                                ? Extensions.TaskExtensions.InfiniteTask()
                                : Task.Delay(TimeSpan.FromSeconds(heartbeatSeconds));

                            // cancel waiting if requested
                            var cancelTcs = new TaskCompletionSource<object>();
                            var toWaitOn = Task.WhenAny(
                                activeServerLifetime,
                                activeServerReboot,
                                inactiveServerLifetime,
                                inactiveServerReboot,
                                inactiveStartupComplete,
                                heartbeat,
                                newDmbAvailable,
                                cancelTcs.Task,
                                activeLaunchParametersChanged);

                            // wait for something to happen
                            using (cancellationToken.Register(() => cancelTcs.SetCanceled()))
                                await toWaitOn.ConfigureAwait(false);

                            cancellationToken.ThrowIfCancellationRequested();
                            Logger.LogTrace("Monitor activated");

                            // always run HandleMonitorWakeup from the context of the semaphore lock
                            using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false))
                            {
                                // multiple things may have happened, handle them one at a time
                                for (var moreActivationsToProcess = true; moreActivationsToProcess && (monitorState.NextAction == MonitorAction.Continue || monitorState.NextAction == MonitorAction.Skip);)
                                {
                                    MonitorActivationReason activationReason = default; // this will always be assigned before being used

                                    bool CheckActivationReason(ref Task task, MonitorActivationReason testActivationReason)
                                    {
                                        var taskCompleted = task?.IsCompleted == true;
                                        task = null;
                                        if (monitorState.NextAction == MonitorAction.Skip)
                                            monitorState.NextAction = MonitorAction.Continue;
                                        else if (taskCompleted)
                                        {
                                            activationReason = testActivationReason;
                                            return true;
                                        }

                                        return false;
                                    }

                                    // process the tasks in this order and call HandlerMonitorWakup for each depending on the new monitorState
                                    var anyActivation = CheckActivationReason(ref activeServerLifetime, MonitorActivationReason.ActiveServerCrashed)
                                        || CheckActivationReason(ref activeServerReboot, MonitorActivationReason.ActiveServerRebooted)
                                        || CheckActivationReason(ref newDmbAvailable, MonitorActivationReason.NewDmbAvailable)
                                        || CheckActivationReason(ref inactiveServerLifetime, MonitorActivationReason.InactiveServerCrashed)
                                        || CheckActivationReason(ref inactiveServerReboot, MonitorActivationReason.InactiveServerRebooted)
                                        || CheckActivationReason(ref inactiveStartupComplete, MonitorActivationReason.InactiveServerStartupComplete)
                                        || CheckActivationReason(ref activeLaunchParametersChanged, MonitorActivationReason.ActiveLaunchParametersUpdated)
                                        || CheckActivationReason(ref heartbeat, MonitorActivationReason.Heartbeat);

                                    if (!anyActivation)
                                        moreActivationsToProcess = false;
                                    else
                                    {
                                        Logger.LogTrace("Reason: {0}", activationReason);
                                        if (activationReason == MonitorActivationReason.Heartbeat)
                                            monitorState.NextAction = await HandleHeartbeat(
                                                monitorState.ActiveServer,
                                                cancellationToken)
                                                .ConfigureAwait(false);
                                        else
                                            await HandleMonitorWakeup(
                                                activationReason,
                                                monitorState,
                                                cancellationToken)
                                                .ConfigureAwait(false);
                                    }
                                }
                            }

                            Logger.LogTrace("Next monitor action is to {0}", monitorState.NextAction);

                            // Restart if requested
                            if (monitorState.NextAction == MonitorAction.Restart)
                                monitorState = await MonitorRestart(cancellationToken).ConfigureAwait(false);
                        }
                        catch (OperationCanceledException)
                        {
                            // let this bubble, other exceptions caught below
                            throw;
                        }
                        catch (Exception e)
                        {
                            // really, this should NEVER happen
                            Logger.LogError(
                                "Monitor crashed! Iteration: {0}, Monitor State: {1}, Exception: {2}",
                                iteration,
                                JsonConvert.SerializeObject(monitorState),
                                e);

                            var nextActionMessage = monitorState.NextAction != MonitorAction.Exit
                                ? "Restarting"
                                : "Shutting down";
                            var chatTask = Chat.SendWatchdogMessage(
                                $"Monitor crashed, this should NEVER happen! Please report this, full details in logs! {nextActionMessage}. Error: {e.Message}",
                                false,
                                cancellationToken);

                            if (disposed)
                                monitorState.NextAction = MonitorAction.Exit;
                            else if (monitorState.NextAction != MonitorAction.Exit)
                                monitorState = await MonitorRestart(cancellationToken).ConfigureAwait(false);

                            await chatTask.ConfigureAwait(false);
                        }
            }
            catch (OperationCanceledException)
            {
                // stop signal
                Logger.LogDebug("Monitor cancelled");

                if (releaseServers)
                {
                    Logger.LogTrace("Detaching servers...");
                    releasedReattachInformation = CreateReattachInformation();
                }
            }

            DisposeAndNullControllers();

            Logger.LogTrace("Monitor exiting...");
        }

        protected abstract Task InitControllers(Action callBeforeRecurse, Task chatTask, DualReattachInformation reattachInfo, CancellationToken cancellationToken);

        public async Task ChangeSettings(DreamDaemonLaunchParameters launchParameters, CancellationToken cancellationToken)
        {
            using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false))
            {
                bool match = launchParameters.CanApplyWithoutReboot(ActiveLaunchParameters);
                ActiveLaunchParameters = launchParameters;
                if (match || !Running)
                    return;

                ActiveParametersUpdated.TrySetResult(null); // queue an update
                ActiveParametersUpdated = new TaskCompletionSource<object>();
            }
        }

        public async Task HandleEvent(EventType eventType, IEnumerable<string> parameters, CancellationToken cancellationToken)
        {
            var activeServer = GetActiveController();

            // Server may have ended
            if (activeServer == null)
                return;

            var notification = new EventNotification(eventType, parameters);
            var result = await activeServer.SendCommand(
                new TopicParameters(notification),
                cancellationToken)
                .ConfigureAwait(false);

            if (result?.InteropResponse?.ChatResponses != null)
                await Task.WhenAll(
                    result.InteropResponse.ChatResponses.Select(
                        x => Chat.SendMessage(
                            x.Text,
                            x.ChannelIds
                                .Select(channelIdString =>
                                {
                                    if (UInt64.TryParse(channelIdString, out var channelId))
                                        return (ulong?)channelId;

                                    return null;
                                })
                                .Where(nullableChannelId => nullableChannelId.HasValue)
                                .Select(nullableChannelId => nullableChannelId.Value),
                            cancellationToken)))
                    .ConfigureAwait(false);
        }

        public async Task<string> HandleChatCommand(string commandName, string arguments, ChatUser sender, CancellationToken cancellationToken)
        {
            using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false))
            {
                if (!Running)
                    return "ERROR: Server offline!";

                var commandObject = new ChatCommand(sender, commandName, arguments);

                var command = new TopicParameters(commandObject);

                var activeServer = GetActiveController();
                var commandResult = await activeServer.SendCommand(command, cancellationToken).ConfigureAwait(false);

                if (commandResult == null)
                    return "TGS: Bad topic exchange!";

                if (commandResult.InteropResponse == null)
                    return "TGS: Bad topic response!";

                return commandResult.InteropResponse.CommandResponseMessage ??
                    "TGS: Command processed but no DMAPI response returned!";
            }
        }

        public async Task Launch(CancellationToken cancellationToken)
        {
            using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false))
                await LaunchNoLock(true, true, null, cancellationToken).ConfigureAwait(false);
        }

        public virtual async Task ResetRebootState(CancellationToken cancellationToken)
        {
            using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false))
            {
                if (!Running)
                    return;
                var toClear = GetActiveController();
                if (toClear != null)
                    await toClear.SetRebootState(Session.RebootState.Normal, cancellationToken).ConfigureAwait(false);
            }
        }

        public async Task Restart(bool graceful, CancellationToken cancellationToken)
        {
            if (!Running)
                throw new JobException(ErrorCode.WatchdogRunning);

            Logger.LogTrace("Begin Restart. Graceful: {0}", graceful);
            using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false))
            {
                if (!graceful)
                {
                    Task chatTask;
                    if (Running)
                    {
                        chatTask = Chat.SendWatchdogMessage("Manual restart triggered...", false, cancellationToken);
                        await TerminateNoLock(false, false, cancellationToken).ConfigureAwait(false);
                    }
                    else
                        chatTask = Task.CompletedTask;
                    await LaunchNoLock(true, !Running, null, cancellationToken).ConfigureAwait(false);
                    await chatTask.ConfigureAwait(false);
                }

                var toReboot = GetActiveController();
                if (toReboot != null
                    && !await toReboot.SetRebootState(Session.RebootState.Restart, cancellationToken).ConfigureAwait(false))
                    Logger.LogWarning("Unable to send reboot state change event!");
            }
        }

        public async Task StartAsync(CancellationToken cancellationToken)
        {
            var reattachInfo = await reattachInfoHandler.Load(cancellationToken).ConfigureAwait(false);
            if (!autoStart && reattachInfo == null)
                return;

            long? adminUserId = null;

            await databaseContextFactory.UseContext(
                async db => adminUserId = await db
                    .Users
                    .AsQueryable()
                    .Where(x => x.CanonicalName == Models.User.CanonicalizeName(Api.Models.User.AdminName))
                    .Select(x => x.Id)
                    .FirstAsync(cancellationToken)
                    .ConfigureAwait(false))
                .ConfigureAwait(false);
            var job = new Models.Job
            {
                StartedBy = new Models.User
                {
                    Id = adminUserId.Value
                },
                Instance = new Models.Instance
                {
                    Id = instance.Id
                },
                Description = $"Instance startup watchdog {(reattachInfo != null ? "reattach" : "launch")}",
                CancelRight = (ulong)DreamDaemonRights.Shutdown,
                CancelRightsType = RightsType.DreamDaemon
            };
            await jobManager.RegisterOperation(job, async (j, databaseContextFactory, progressFunction, ct) =>
            {
                using (await SemaphoreSlimContext.Lock(Semaphore, ct).ConfigureAwait(false))
                    await LaunchNoLock(true, true, reattachInfo, ct).ConfigureAwait(false);
            }, cancellationToken).ConfigureAwait(false);
        }

        public async Task StopAsync(CancellationToken cancellationToken)
        {
            await TerminateNoLock(false, !releaseServers, cancellationToken).ConfigureAwait(false);
            if (releasedReattachInformation != null)
            {
                await reattachInfoHandler.Save(releasedReattachInformation, cancellationToken).ConfigureAwait(false);
                releasedReattachInformation = null;
                releaseServers = false;
            }
        }

        public async Task Terminate(bool graceful, CancellationToken cancellationToken)
        {
            using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false))
                await TerminateNoLock(graceful, !releaseServers, cancellationToken).ConfigureAwait(false);
        }

        public async Task HandleRestart(Version updateVersion, CancellationToken cancellationToken)
        {
            releaseServers = true;
            if (Running)
                await Chat.SendWatchdogMessage("Detaching...", false, cancellationToken).ConfigureAwait(false);
        }

        public abstract Task InstanceRenamed(string newInstanceName, CancellationToken cancellationToken);
    }
}