BasicWatchdog.cs

Go to the documentation of this file.

using Microsoft.Extensions.Logging;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Threading;
using System.Threading.Tasks;
using Tgstation.Server.Api.Models.Internal;
using Tgstation.Server.Host.Components.Chat;
using Tgstation.Server.Host.Components.Deployment;
using Tgstation.Server.Host.Components.Session;
using Tgstation.Server.Host.Core;
using Tgstation.Server.Host.Database;
using Tgstation.Server.Host.Jobs;

namespace Tgstation.Server.Host.Components.Watchdog
{
    class BasicWatchdog : WatchdogBase
    {
        public sealed override bool AlphaIsActive => true;

        public sealed override Models.CompileJob ActiveCompileJob => Server?.Dmb.CompileJob;

        public sealed override RebootState? RebootState => Server?.RebootState;

        protected ISessionController Server { get; private set; }

        bool gracefulRebootRequired;

        public BasicWatchdog(
            IChatManager chat,
            ISessionControllerFactory sessionControllerFactory,
            IDmbFactory dmbFactory,
            IReattachInfoHandler reattachInfoHandler,
            IDatabaseContextFactory databaseContextFactory,
            IJobManager jobManager,
            IServerControl serverControl,
            IAsyncDelayer asyncDelayer,
            ILogger<BasicWatchdog> logger,
            DreamDaemonLaunchParameters initialLaunchParameters,
            Api.Models.Instance instance,
            bool autoStart)
            : base(
                 chat,
                 sessionControllerFactory,
                 dmbFactory,
                 reattachInfoHandler,
                 databaseContextFactory,
                 jobManager,
                 serverControl,
                 asyncDelayer,
                 logger,
                 initialLaunchParameters,
                 instance,
                 autoStart)
        { }

        protected override IReadOnlyDictionary<MonitorActivationReason, Task> GetMonitoredServerTasks(MonitorState monitorState)
        {
            if (Server == null)
                return null;

            monitorState.ActiveServer = Server;

            return new Dictionary<MonitorActivationReason, Task>
            {
                { MonitorActivationReason.ActiveServerCrashed, Server.Lifetime },
                { MonitorActivationReason.ActiveServerRebooted, Server.OnReboot },
                { MonitorActivationReason.InactiveServerCrashed, Extensions.TaskExtensions.InfiniteTask() },
                { MonitorActivationReason.InactiveServerRebooted, Extensions.TaskExtensions.InfiniteTask() },
                { MonitorActivationReason.InactiveServerStartupComplete, Extensions.TaskExtensions.InfiniteTask() }
            };
        }

        protected override async Task HandleMonitorWakeup(MonitorActivationReason reason, MonitorState monitorState, CancellationToken cancellationToken)
        {
            switch (reason)
            {
                case MonitorActivationReason.ActiveServerCrashed:
                    string exitWord = Server.TerminationWasRequested ? "exited" : "crashed";
                    if (Server.RebootState == Session.RebootState.Shutdown)
                    {
                        // the time for graceful shutdown is now
                        await Chat.SendWatchdogMessage(
                            String.Format(
                                CultureInfo.InvariantCulture,
                                "Server {0}! Shutting down due to graceful termination request...",
                                exitWord),
                            false,
                            cancellationToken)
                            .ConfigureAwait(false);
                        monitorState.NextAction = MonitorAction.Exit;
                    }
                    else
                    {
                        await Chat.SendWatchdogMessage(
                            String.Format(
                                CultureInfo.InvariantCulture,
                                "Server {0}! Rebooting...",
                                exitWord),
                            false,
                            cancellationToken)
                            .ConfigureAwait(false);
                        monitorState.NextAction = MonitorAction.Restart;
                    }

                    break;
                case MonitorActivationReason.ActiveServerRebooted:
                    var rebootState = Server.RebootState;
                    if (gracefulRebootRequired && rebootState == Session.RebootState.Normal)
                    {
                        Logger.LogError("Watchdog reached normal reboot state with gracefulRebootRequired set!");
                        rebootState = Session.RebootState.Restart;
                    }

                    gracefulRebootRequired = false;
                    Server.ResetRebootState();

                    switch (rebootState)
                    {
                        case Session.RebootState.Normal:
                            monitorState.NextAction = HandleNormalReboot();
                            break;
                        case Session.RebootState.Restart:
                            monitorState.NextAction = MonitorAction.Restart;
                            break;
                        case Session.RebootState.Shutdown:
                            // graceful shutdown time
                            await Chat.SendWatchdogMessage(
                                "Active server rebooted! Shutting down due to graceful termination request...",
                                false,
                                cancellationToken)
                                .ConfigureAwait(false);
                            monitorState.NextAction = MonitorAction.Exit;
                            break;
                        default:
                            throw new InvalidOperationException($"Invalid reboot state: {rebootState}");
                    }

                    break;
                case MonitorActivationReason.ActiveLaunchParametersUpdated:
                    await Server.SetRebootState(Session.RebootState.Restart, cancellationToken).ConfigureAwait(false);
                    gracefulRebootRequired = true;
                    break;
                case MonitorActivationReason.NewDmbAvailable:
                    await HandleNewDmbAvailable(cancellationToken).ConfigureAwait(false);
                    break;
                case MonitorActivationReason.InactiveServerCrashed:
                case MonitorActivationReason.InactiveServerRebooted:
                case MonitorActivationReason.InactiveServerStartupComplete:
                    throw new NotSupportedException($"Unsupported activation reason: {reason}");
                case MonitorActivationReason.Heartbeat:
                default:
                    throw new InvalidOperationException($"Invalid activation reason: {reason}");
            }
        }

        protected sealed override DualReattachInformation CreateReattachInformation()
            => new DualReattachInformation
            {
                AlphaIsActive = true,
                Alpha = Server?.Release()
            };

        protected override void DisposeAndNullControllersImpl()
        {
            Server?.Dispose();
            Server = null;
            Running = false;
            gracefulRebootRequired = false;
        }

        protected sealed override ISessionController GetActiveController() => Server;

        protected sealed override async Task InitControllers(Action callBeforeRecurse, Task chatTask, DualReattachInformation reattachInfo, CancellationToken cancellationToken)
        {
            var serverToReattach = reattachInfo?.Alpha ?? reattachInfo?.Bravo;
            var serverToKill = reattachInfo?.Bravo ?? reattachInfo?.Alpha;

            // vice versa
            if (serverToKill == serverToReattach)
                serverToKill = null;

            if (reattachInfo?.AlphaIsActive == false)
            {
                var temp = serverToReattach;
                serverToReattach = serverToKill;
                serverToKill = temp;
            }

            // don't need a new dmb if reattaching
            var doesntNeedNewDmb = serverToReattach != null;
            var dmbToUse = doesntNeedNewDmb ? null : DmbFactory.LockNextDmb(1);

            // if this try catches something, both servers are killed
            bool inactiveServerWasKilled = false;
            try
            {
                // start the alpha server task, either by launch a new process or attaching to an existing one
                // The tasks returned are mainly for writing interop files to the directories among other things and should generally never fail
                // The tasks pertaining to server startup times are in the ISessionControllers
                Task<ISessionController> serverLaunchTask, inactiveReattachTask;
                if (!doesntNeedNewDmb)
                {
                    dmbToUse = await PrepServerForLaunch(dmbToUse, cancellationToken).ConfigureAwait(false);
                    serverLaunchTask = SessionControllerFactory.LaunchNew(
                        dmbToUse,
                        null,
                        ActiveLaunchParameters,
                        true,
                        true,
                        false,
                        cancellationToken);
                }
                else
                    serverLaunchTask = SessionControllerFactory.Reattach(serverToReattach, cancellationToken);

                bool thereIsAnInactiveServerToKill = serverToKill != null;
                if (thereIsAnInactiveServerToKill)
                    inactiveReattachTask = SessionControllerFactory.Reattach(serverToKill, cancellationToken);
                else
                    inactiveReattachTask = Task.FromResult<ISessionController>(null);

                // retrieve the session controller
                Server = await serverLaunchTask.ConfigureAwait(false);

                // failed reattaches will return null
                Server?.SetHighPriority();

                var inactiveServerController = await inactiveReattachTask.ConfigureAwait(false);
                inactiveServerController?.Dispose();
                inactiveServerWasKilled = inactiveServerController != null;

                // possiblity of null servers due to failed reattaches
                if (Server == null)
                {
                    callBeforeRecurse();
                    await NotifyOfFailedReattach(thereIsAnInactiveServerToKill && !inactiveServerWasKilled, cancellationToken).ConfigureAwait(false);
                    return;
                }

                await CheckLaunchResult(Server, "Server", cancellationToken).ConfigureAwait(false);

                Server.EnableCustomChatCommands();
            }
            catch
            {
                // kill the controllers
                bool serverWasActive = Server != null;
                DisposeAndNullControllers();

                // server didn't get control of this dmb
                if (dmbToUse != null && !serverWasActive)
                    dmbToUse.Dispose();

                if (serverToKill != null && !inactiveServerWasKilled)
                    serverToKill.Dmb.Dispose();
                throw;
            }
        }

        protected virtual MonitorAction HandleNormalReboot()
        {
            bool dmbUpdatePending = ActiveLaunchParameters != LastLaunchParameters;
            return dmbUpdatePending ? MonitorAction.Restart : MonitorAction.Continue;
        }

        protected virtual Task HandleNewDmbAvailable(CancellationToken cancellationToken)
        {
            gracefulRebootRequired = true;
            return Server.SetRebootState(Session.RebootState.Restart, cancellationToken);
        }

        protected virtual Task<IDmbProvider> PrepServerForLaunch(IDmbProvider dmbToUse, CancellationToken cancellationToken) => Task.FromResult(dmbToUse);

        public override Task ResetRebootState(CancellationToken cancellationToken)
        {
            if (!gracefulRebootRequired)
                return base.ResetRebootState(cancellationToken);

            return Restart(true, cancellationToken);
        }

        async Task NotifyOfFailedReattach(bool inactiveReattachSuccess, CancellationToken cancellationToken)
        {
            // we lost the server, just restart entirely
            DisposeAndNullControllers();
            const string FailReattachMessage = "Unable to properly reattach to server! Restarting...";
            Logger.LogWarning(FailReattachMessage);
            Logger.LogDebug(inactiveReattachSuccess ? "Also could not reattach to inactive server!" : "Inactive server was reattached successfully!");
            Task chatTask = Chat.SendWatchdogMessage(FailReattachMessage, false, cancellationToken);
            await LaunchNoLock(true, false, null, cancellationToken).ConfigureAwait(false);
            await chatTask.ConfigureAwait(false);
        }

        public sealed override Task InstanceRenamed(string newInstanceName, CancellationToken cancellationToken)
            => Server?.InstanceRenamed(newInstanceName, cancellationToken) ?? Task.CompletedTask;
    }
}