BasicWatchdog.cs

Go to the documentation of this file.

using Microsoft.Extensions.Logging;
using System;
using System.Globalization;
using System.Threading;
using System.Threading.Tasks;
using Tgstation.Server.Api.Models.Internal;
using Tgstation.Server.Host.Components.Chat;
using Tgstation.Server.Host.Components.Deployment;
using Tgstation.Server.Host.Components.Events;
using Tgstation.Server.Host.Components.Session;
using Tgstation.Server.Host.Core;
using Tgstation.Server.Host.Database;
using Tgstation.Server.Host.IO;
using Tgstation.Server.Host.Jobs;

namespace Tgstation.Server.Host.Components.Watchdog
{
    class BasicWatchdog : WatchdogBase
    {
        public sealed override bool AlphaIsActive => true;

        public sealed override Models.CompileJob ActiveCompileJob => Server?.Dmb.CompileJob;

        public sealed override RebootState? RebootState => Server?.RebootState;

        protected ISessionController Server { get; private set; }

        bool gracefulRebootRequired;

        public BasicWatchdog(
            IChatManager chat,
            ISessionControllerFactory sessionControllerFactory,
            IDmbFactory dmbFactory,
            ISessionPersistor sessionPersistor,
            IDatabaseContextFactory databaseContextFactory,
            IJobManager jobManager,
            IServerControl serverControl,
            IAsyncDelayer asyncDelayer,
            IIOManager diagnosticsIOManager,
            IEventConsumer eventConsumer,
            ILogger<BasicWatchdog> logger,
            DreamDaemonLaunchParameters initialLaunchParameters,
            Api.Models.Instance instance,
            bool autoStart)
            : base(
                 chat,
                 sessionControllerFactory,
                 dmbFactory,
                 sessionPersistor,
                 databaseContextFactory,
                 jobManager,
                 serverControl,
                 asyncDelayer,
                 diagnosticsIOManager,
                 eventConsumer,
                 logger,
                 initialLaunchParameters,
                 instance,
                 autoStart)
        { }

        protected override async Task<MonitorAction> HandleMonitorWakeup(MonitorActivationReason reason, CancellationToken cancellationToken)
        {
            switch (reason)
            {
                case MonitorActivationReason.ActiveServerCrashed:
                    string exitWord = Server.TerminationWasRequested ? "exited" : "crashed";
                    if (Server.RebootState == Session.RebootState.Shutdown)
                    {
                        // the time for graceful shutdown is now
                        await Chat.SendWatchdogMessage(
                            String.Format(
                                CultureInfo.InvariantCulture,
                                "Server {0}! Shutting down due to graceful termination request...",
                                exitWord),
                            false,
                            cancellationToken)
                            .ConfigureAwait(false);
                        return MonitorAction.Exit;
                    }

                    await Chat.SendWatchdogMessage(
                        String.Format(
                            CultureInfo.InvariantCulture,
                            "Server {0}! Rebooting...",
                            exitWord),
                        false,
                        cancellationToken)
                        .ConfigureAwait(false);
                    return MonitorAction.Restart;
                case MonitorActivationReason.ActiveServerRebooted:
                    var rebootState = Server.RebootState;
                    if (gracefulRebootRequired && rebootState == Session.RebootState.Normal)
                    {
                        Logger.LogError("Watchdog reached normal reboot state with gracefulRebootRequired set!");
                        rebootState = Session.RebootState.Restart;
                    }

                    gracefulRebootRequired = false;
                    Server.ResetRebootState();

                    switch (rebootState)
                    {
                        case Session.RebootState.Normal:
                            return HandleNormalReboot();
                        case Session.RebootState.Restart:
                            return MonitorAction.Restart;
                        case Session.RebootState.Shutdown:
                            // graceful shutdown time
                            await Chat.SendWatchdogMessage(
                                "Active server rebooted! Shutting down due to graceful termination request...",
                                false,
                                cancellationToken)
                                .ConfigureAwait(false);
                            return MonitorAction.Exit;
                        default:
                            throw new InvalidOperationException($"Invalid reboot state: {rebootState}");
                    }

                case MonitorActivationReason.ActiveLaunchParametersUpdated:
                    await Server.SetRebootState(Session.RebootState.Restart, cancellationToken).ConfigureAwait(false);
                    gracefulRebootRequired = true;
                    break;
                case MonitorActivationReason.NewDmbAvailable:
                    await HandleNewDmbAvailable(cancellationToken).ConfigureAwait(false);
                    break;
                case MonitorActivationReason.Heartbeat:
                default:
                    throw new InvalidOperationException($"Invalid activation reason: {reason}");
            }

            return MonitorAction.Continue;
        }

        protected override void DisposeAndNullControllersImpl()
        {
            Server?.Dispose();
            Server = null;
            gracefulRebootRequired = false;
        }

        protected sealed override ISessionController GetActiveController() => Server;

        protected override async Task InitControllers(
            Task chatTask,
            ReattachInformation reattachInfo,
            CancellationToken cancellationToken)
        {
            // don't need a new dmb if reattaching
            var reattachInProgress = reattachInfo != null;
            var dmbToUse = reattachInProgress ? null : DmbFactory.LockNextDmb(1);

            // if this try catches something, both servers are killed
            try
            {
                // start the alpha server task, either by launch a new process or attaching to an existing one
                // The tasks returned are mainly for writing interop files to the directories among other things and should generally never fail
                // The tasks pertaining to server startup times are in the ISessionControllers
                Task<ISessionController> serverLaunchTask;
                if (!reattachInProgress)
                {
                    dmbToUse = await PrepServerForLaunch(dmbToUse, cancellationToken).ConfigureAwait(false);
                    serverLaunchTask = SessionControllerFactory.LaunchNew(
                        dmbToUse,
                        null,
                        ActiveLaunchParameters,
                        false,
                        cancellationToken);
                }
                else
                    serverLaunchTask = SessionControllerFactory.Reattach(reattachInfo, cancellationToken);

                // retrieve the session controller
                Server = await serverLaunchTask.ConfigureAwait(false);

                // failed reattaches will return null
                Server?.SetHighPriority();

                // possiblity of null servers due to failed reattaches
                if (Server == null)
                {
                    await ReattachFailure(
                        chatTask,
                        cancellationToken)
                        .ConfigureAwait(false);
                    return;
                }

                await CheckLaunchResult(Server, "Server", cancellationToken).ConfigureAwait(false);

                Server.EnableCustomChatCommands();
            }
            catch
            {
                // kill the controllers
                bool serverWasActive = Server != null;
                DisposeAndNullControllers();

                // server didn't get control of this dmb
                if (dmbToUse != null && !serverWasActive)
                    dmbToUse.Dispose();

                throw;
            }
        }

        protected virtual MonitorAction HandleNormalReboot()
        {
            bool dmbUpdatePending = ActiveLaunchParameters != LastLaunchParameters;
            return dmbUpdatePending ? MonitorAction.Restart : MonitorAction.Continue;
        }

        protected virtual Task HandleNewDmbAvailable(CancellationToken cancellationToken)
        {
            gracefulRebootRequired = true;
            if (Server.Dmb.CompileJob.DMApiVersion == null)
                return Chat.SendWatchdogMessage(
                    "A new deployment has been made but cannot be applied automatically as the currently running server has no DMAPI. Please manually reboot the server to apply the update.",
                    true,
                    cancellationToken);
            return Server.SetRebootState(Session.RebootState.Restart, cancellationToken);
        }

        protected virtual Task<IDmbProvider> PrepServerForLaunch(IDmbProvider dmbToUse, CancellationToken cancellationToken) => Task.FromResult(dmbToUse);

        public override Task ResetRebootState(CancellationToken cancellationToken)
        {
            if (!gracefulRebootRequired)
                return base.ResetRebootState(cancellationToken);

            return Restart(true, cancellationToken);
        }

        public sealed override Task InstanceRenamed(string newInstanceName, CancellationToken cancellationToken)
            => Server?.InstanceRenamed(newInstanceName, cancellationToken) ?? Task.CompletedTask;
    }
}