mirror of
https://github.com/tgstation/tgstation-server.git
synced 2026-08-24 21:46:52 +01:00
Merge pull request #879 from tgstation/853-OkBuddyGuyFriendPal
Makes the existing watchdog experimental. Adds the basic watchdog
This commit is contained in:
Jordan Brown
@@ -372,7 +372,7 @@ namespace Tgstation.Server.Host.Components
|
||||
var startSha = repo.Head;
|
||||
if (!repo.Tracking)
|
||||
{
|
||||
logger.LogTrace("Aborting repo update, not tracking origin!");
|
||||
logger.LogTrace("Aborting repo update, active ref not tracking any remote branch!");
|
||||
deploySha = startSha;
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,359 @@
|
||||
using Byond.TopicSender;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using System;
|
||||
using System.Globalization;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Tgstation.Server.Api.Models.Internal;
|
||||
using Tgstation.Server.Host.Components.Chat;
|
||||
using Tgstation.Server.Host.Components.Compiler;
|
||||
using Tgstation.Server.Host.Core;
|
||||
|
||||
namespace Tgstation.Server.Host.Components.Watchdog
|
||||
{
|
||||
/// <summary>
|
||||
/// A <see cref="IWatchdog"/> that manages one server.
|
||||
/// </summary>
|
||||
sealed class BasicWatchdog : WatchdogBase
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public override bool AlphaIsActive => true;
|
||||
|
||||
/// <inheritdoc />
|
||||
public override Models.CompileJob ActiveCompileJob => server?.Dmb.CompileJob;
|
||||
|
||||
/// <inheritdoc />
|
||||
public override RebootState? RebootState => server?.RebootState;
|
||||
|
||||
/// <summary>
|
||||
/// The single <see cref="ISessionController"/>.
|
||||
/// </summary>
|
||||
ISessionController server;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="ExperimentalWatchdog"/> <see langword="class"/>.
|
||||
/// </summary>
|
||||
/// <param name="chat">The <see cref="IChat"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
/// <param name="sessionControllerFactory">The <see cref="ISessionControllerFactory"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
/// <param name="dmbFactory">The <see cref="IDmbFactory"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
/// <param name="reattachInfoHandler">The <see cref="IReattachInfoHandler"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
/// <param name="databaseContextFactory">The <see cref="IDatabaseContextFactory"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
/// <param name="byondTopicSender">The <see cref="IByondTopicSender"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
/// <param name="eventConsumer">The <see cref="IEventConsumer"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
/// <param name="jobManager">The <see cref="IJobManager"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
/// <param name="serverControl">The <see cref="IServerControl"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
/// <param name="asyncDelayer">The <see cref="IAsyncDelayer"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
/// <param name="logger">The <see cref="ILogger"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
/// <param name="initialLaunchParameters">The <see cref="DreamDaemonLaunchParameters"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
/// <param name="instance">The <see cref="Api.Models.Instance"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
/// <param name="autoStart">The autostart value for the <see cref="WatchdogBase"/>.</param>
|
||||
public BasicWatchdog(
|
||||
IChat chat,
|
||||
ISessionControllerFactory sessionControllerFactory,
|
||||
IDmbFactory dmbFactory,
|
||||
IReattachInfoHandler reattachInfoHandler,
|
||||
IDatabaseContextFactory databaseContextFactory,
|
||||
IByondTopicSender byondTopicSender,
|
||||
IEventConsumer eventConsumer,
|
||||
IJobManager jobManager,
|
||||
IServerControl serverControl,
|
||||
IAsyncDelayer asyncDelayer,
|
||||
ILogger<BasicWatchdog> logger,
|
||||
DreamDaemonLaunchParameters initialLaunchParameters,
|
||||
Api.Models.Instance instance,
|
||||
bool autoStart)
|
||||
: base(
|
||||
chat,
|
||||
sessionControllerFactory,
|
||||
dmbFactory,
|
||||
reattachInfoHandler,
|
||||
databaseContextFactory,
|
||||
byondTopicSender,
|
||||
eventConsumer,
|
||||
jobManager,
|
||||
serverControl,
|
||||
asyncDelayer,
|
||||
logger,
|
||||
initialLaunchParameters,
|
||||
instance,
|
||||
autoStart)
|
||||
{ }
|
||||
|
||||
async Task<MonitorAction> HandleMonitorWakeup(MonitorActivationReason reason, CancellationToken cancellationToken)
|
||||
{
|
||||
switch (reason)
|
||||
{
|
||||
case MonitorActivationReason.ActiveServerCrashed:
|
||||
string exitWord = server.TerminationWasRequested ? "exited" : "crashed";
|
||||
if (server.RebootState == Watchdog.RebootState.Shutdown)
|
||||
{
|
||||
// the time for graceful shutdown is now
|
||||
await Chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Server {0}! Exiting due to graceful termination request...", exitWord), cancellationToken).ConfigureAwait(false);
|
||||
DisposeAndNullControllers();
|
||||
return MonitorAction.Exit;
|
||||
}
|
||||
|
||||
await Chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Server {0}! Rebooting...", exitWord), cancellationToken).ConfigureAwait(false);
|
||||
return MonitorAction.Restart;
|
||||
case MonitorActivationReason.ActiveServerRebooted:
|
||||
var rebootState = server.RebootState;
|
||||
server.ResetRebootState();
|
||||
|
||||
switch (rebootState)
|
||||
{
|
||||
case Watchdog.RebootState.Normal:
|
||||
bool dmbUpdatePending = ActiveLaunchParameters != LastLaunchParameters;
|
||||
return dmbUpdatePending ? MonitorAction.Restart : MonitorAction.Continue;
|
||||
case Watchdog.RebootState.Restart:
|
||||
return MonitorAction.Restart;
|
||||
case Watchdog.RebootState.Shutdown:
|
||||
// graceful shutdown time
|
||||
await Chat.SendWatchdogMessage("Active server rebooted! Exiting due to graceful termination request...", cancellationToken).ConfigureAwait(false);
|
||||
DisposeAndNullControllers();
|
||||
return MonitorAction.Exit;
|
||||
default:
|
||||
throw new InvalidOperationException($"Invalid reboot state: {rebootState}");
|
||||
}
|
||||
|
||||
case MonitorActivationReason.ActiveLaunchParametersUpdated:
|
||||
case MonitorActivationReason.NewDmbAvailable:
|
||||
await server.SetRebootState(Watchdog.RebootState.Restart, cancellationToken).ConfigureAwait(false);
|
||||
return MonitorAction.Continue;
|
||||
case MonitorActivationReason.InactiveServerCrashed:
|
||||
case MonitorActivationReason.InactiveServerRebooted:
|
||||
case MonitorActivationReason.InactiveServerStartupComplete:
|
||||
throw new NotSupportedException($"Unsupported activation reason: {reason}");
|
||||
default:
|
||||
throw new InvalidOperationException($"Invalid activation reason: {reason}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override WatchdogReattachInformation CreateReattachInformation()
|
||||
=> new WatchdogReattachInformation
|
||||
{
|
||||
AlphaIsActive = true,
|
||||
Alpha = server?.Release()
|
||||
};
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void DisposeAndNullControllers()
|
||||
{
|
||||
server?.Dispose();
|
||||
server = null;
|
||||
Running = false;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override ISessionController GetActiveController() => server;
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override async Task InitControllers(Action callBeforeRecurse, Task chatTask, WatchdogReattachInformation reattachInfo, CancellationToken cancellationToken)
|
||||
{
|
||||
// good ole sanity, should never fucking trigger but i don't trust myself even though I should
|
||||
// TODO: Unit test this instead?
|
||||
if (server != null)
|
||||
throw new InvalidOperationException("Entered LaunchNoLock with server not being null!");
|
||||
|
||||
// don't need a new dmb if reattaching
|
||||
var doesntNeedNewDmb = reattachInfo?.Alpha != null && reattachInfo?.Bravo != null;
|
||||
var dmbToUse = doesntNeedNewDmb ? null : DmbFactory.LockNextDmb(1);
|
||||
|
||||
var serverToReattach = reattachInfo?.Alpha ?? reattachInfo?.Bravo;
|
||||
var serverToKill = reattachInfo?.Bravo ?? reattachInfo?.Alpha;
|
||||
|
||||
// vice versa
|
||||
if (reattachInfo?.AlphaIsActive == false)
|
||||
{
|
||||
var temp = serverToReattach;
|
||||
serverToReattach = serverToKill;
|
||||
serverToKill = temp;
|
||||
}
|
||||
|
||||
// if this try catches something, both servers are killed
|
||||
bool inactiveServerWasKilled = false;
|
||||
try
|
||||
{
|
||||
// start the alpha server task, either by launch a new process or attaching to an existing one
|
||||
// The tasks returned are mainly for writing interop files to the directories among other things and should generally never fail
|
||||
// The tasks pertaining to server startup times are in the ISessionControllers
|
||||
Task<ISessionController> serverLaunchTask, inactiveReattachTask;
|
||||
if (!doesntNeedNewDmb)
|
||||
serverLaunchTask = SessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbToUse, null, true, true, false, cancellationToken);
|
||||
else
|
||||
serverLaunchTask = SessionControllerFactory.Reattach(serverToReattach, cancellationToken);
|
||||
|
||||
bool thereIsAnInactiveServerToKill = serverToKill != null;
|
||||
if (thereIsAnInactiveServerToKill)
|
||||
inactiveReattachTask = SessionControllerFactory.Reattach(serverToKill, cancellationToken);
|
||||
else
|
||||
inactiveReattachTask = Task.FromResult<ISessionController>(null);
|
||||
|
||||
// retrieve the session controller
|
||||
server = await serverLaunchTask.ConfigureAwait(false);
|
||||
|
||||
// failed reattaches will return null
|
||||
server?.SetHighPriority();
|
||||
|
||||
var inactiveServerController = await inactiveReattachTask.ConfigureAwait(false);
|
||||
inactiveServerController?.Dispose();
|
||||
inactiveServerWasKilled = inactiveServerController != null;
|
||||
|
||||
// possiblity of null servers due to failed reattaches
|
||||
if (server == null)
|
||||
{
|
||||
callBeforeRecurse();
|
||||
await NotifyOfFailedReattach(thereIsAnInactiveServerToKill && !inactiveServerWasKilled, cancellationToken).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
await CheckLaunchResult(server, "Server", cancellationToken).ConfigureAwait(false);
|
||||
|
||||
server.EnableCustomChatCommands();
|
||||
}
|
||||
catch
|
||||
{
|
||||
// kill the controllers
|
||||
bool serverWasActive = server != null;
|
||||
DisposeAndNullControllers();
|
||||
|
||||
// server didn't get control of this dmb
|
||||
if (dmbToUse != null && !serverWasActive)
|
||||
dmbToUse.Dispose();
|
||||
|
||||
if (serverToKill != null && !inactiveServerWasKilled)
|
||||
serverToKill.Dmb.Dispose();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override async Task MonitorLifetimes(CancellationToken cancellationToken)
|
||||
{
|
||||
Logger.LogTrace("Entered MonitorLifetimes");
|
||||
|
||||
// this function is responsible for calling HandlerMonitorWakeup when necessary and manitaining the MonitorState
|
||||
var iteration = 1;
|
||||
for (MonitorAction nextAction = MonitorAction.Continue; nextAction != MonitorAction.Exit; ++iteration)
|
||||
{
|
||||
// always start out with continue
|
||||
nextAction = MonitorAction.Continue;
|
||||
|
||||
// dump some info to the logs
|
||||
Logger.LogDebug("Iteration {0} of monitor loop", iteration);
|
||||
try
|
||||
{
|
||||
Logger.LogDebug("Server Compile Job ID: {0}", server.Dmb.CompileJob.Id);
|
||||
|
||||
// load the activation tasks into local variables
|
||||
Task activeServerLifetime = server.Lifetime;
|
||||
var activeServerReboot = server.OnReboot;
|
||||
Task activeLaunchParametersChanged = ActiveParametersUpdated.Task;
|
||||
var newDmbAvailable = DmbFactory.OnNewerDmb;
|
||||
|
||||
// cancel waiting if requested
|
||||
var cancelTcs = new TaskCompletionSource<object>();
|
||||
using (cancellationToken.Register(() => cancelTcs.SetCanceled()))
|
||||
{
|
||||
var toWaitOn = Task.WhenAny(activeServerLifetime, activeServerReboot, newDmbAvailable, cancelTcs.Task, activeLaunchParametersChanged);
|
||||
|
||||
// wait for something to happen
|
||||
await toWaitOn.ConfigureAwait(false);
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
}
|
||||
|
||||
var chatTask = Task.CompletedTask;
|
||||
using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
// always run HandleMonitorWakeup from the context of the semaphore lock
|
||||
// multiple things may have happened, handle them one at a time
|
||||
for (var moreActivationsToProcess = true; moreActivationsToProcess && (nextAction == MonitorAction.Continue || nextAction == MonitorAction.Skip);)
|
||||
{
|
||||
MonitorActivationReason activationReason = default; // this will always be assigned before being used
|
||||
|
||||
// process the tasks in this order and call HandlerMonitorWakup for each
|
||||
bool CheckActivationReason(ref Task task, MonitorActivationReason testActivationReason)
|
||||
{
|
||||
var taskCompleted = task?.IsCompleted == true;
|
||||
task = null;
|
||||
if (nextAction == MonitorAction.Skip)
|
||||
nextAction = MonitorAction.Continue;
|
||||
else if (taskCompleted)
|
||||
{
|
||||
activationReason = testActivationReason;
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
if (CheckActivationReason(ref activeServerLifetime, MonitorActivationReason.ActiveServerCrashed)
|
||||
|| CheckActivationReason(ref activeServerReboot, MonitorActivationReason.ActiveServerRebooted)
|
||||
|| CheckActivationReason(ref newDmbAvailable, MonitorActivationReason.NewDmbAvailable)
|
||||
|| CheckActivationReason(ref activeLaunchParametersChanged, MonitorActivationReason.ActiveLaunchParametersUpdated))
|
||||
nextAction = await HandleMonitorWakeup(activationReason, cancellationToken).ConfigureAwait(false);
|
||||
else
|
||||
moreActivationsToProcess = false;
|
||||
}
|
||||
}
|
||||
|
||||
// full reboot required
|
||||
if (nextAction == MonitorAction.Restart)
|
||||
{
|
||||
Logger.LogDebug("Next state action is to restart");
|
||||
DisposeAndNullControllers();
|
||||
|
||||
for (var retryAttempts = 1; nextAction == MonitorAction.Restart; ++retryAttempts)
|
||||
{
|
||||
Exception launchException = null;
|
||||
using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false))
|
||||
try
|
||||
{
|
||||
// use LaunchImplNoLock without announcements or restarting the monitor
|
||||
await LaunchImplNoLock(false, false, null, cancellationToken).ConfigureAwait(false);
|
||||
if (Running)
|
||||
{
|
||||
Logger.LogDebug("Relaunch successful, resetting monitor state...");
|
||||
break; // continue on main loop
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
launchException = e;
|
||||
}
|
||||
|
||||
await chatTask.ConfigureAwait(false);
|
||||
if (!Running)
|
||||
{
|
||||
if (launchException == null)
|
||||
Logger.LogWarning("Failed to automatically restart the watchdog! Attempt: {0}", retryAttempts);
|
||||
else
|
||||
Logger.LogWarning("Failed to automatically restart the watchdog! Attempt: {0}, Exception: {1}", retryAttempts, launchException);
|
||||
var retryDelay = Math.Min(Math.Pow(2, retryAttempts), 3600); // max of one hour, increasing by a power of 2 each time
|
||||
chatTask = Chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Failed to restart watchdog (Attempt: {0}), retrying in {1} seconds...", retryAttempts, retryDelay), cancellationToken);
|
||||
await Task.WhenAll(AsyncDelayer.Delay(TimeSpan.FromSeconds(retryDelay), cancellationToken), chatTask).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
Logger.LogDebug("Monitor cancelled");
|
||||
break;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
// really, this should NEVER happen
|
||||
Logger.LogError("Monitor crashed! Iteration: {0}, NextAction: {1}, Exception: {2}", iteration, nextAction, e);
|
||||
await Chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Monitor crashed, this should NEVER happen! Please report this, full details in logs! Restarting monitor... Error: {0}", e.Message), cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
Logger.LogTrace("Monitor exiting...");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,649 @@
|
||||
using Byond.TopicSender;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Newtonsoft.Json;
|
||||
using System;
|
||||
using System.Diagnostics;
|
||||
using System.Globalization;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Tgstation.Server.Api.Models.Internal;
|
||||
using Tgstation.Server.Host.Components.Chat;
|
||||
using Tgstation.Server.Host.Components.Compiler;
|
||||
using Tgstation.Server.Host.Core;
|
||||
|
||||
namespace Tgstation.Server.Host.Components.Watchdog
|
||||
{
|
||||
/// <summary>
|
||||
/// A <see cref="IWatchdog"/> that tries to manage 2 servers at once for maximum uptime.
|
||||
/// </summary>
|
||||
sealed class ExperimentalWatchdog : WatchdogBase
|
||||
{
|
||||
/// <summary>
|
||||
/// The time in seconds to wait from starting <see cref="alphaServer"/> to start <see cref="bravoServer"/>. Does not take responsiveness into account
|
||||
/// </summary>
|
||||
const int AlphaBravoStartupSeperationInterval = 10; // TODO: Make this configurable
|
||||
|
||||
/// <inheritdoc />
|
||||
public override bool AlphaIsActive => alphaIsActive;
|
||||
|
||||
/// <inheritdoc />
|
||||
public override Models.CompileJob ActiveCompileJob => (AlphaIsActive ? alphaServer : bravoServer)?.Dmb.CompileJob;
|
||||
|
||||
/// <inheritdoc />
|
||||
public override RebootState? RebootState => Running ? (AlphaIsActive ? alphaServer?.RebootState : bravoServer?.RebootState) : null;
|
||||
|
||||
/// <summary>
|
||||
/// Server designation alpha
|
||||
/// </summary>
|
||||
ISessionController alphaServer;
|
||||
|
||||
/// <summary>
|
||||
/// Server designation bravo
|
||||
/// </summary>
|
||||
ISessionController bravoServer;
|
||||
|
||||
/// <summary>
|
||||
/// Backing field for <see cref="AlphaIsActive"/>.
|
||||
/// </summary>
|
||||
bool alphaIsActive;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="ExperimentalWatchdog"/> <see langword="class"/>.
|
||||
/// </summary>
|
||||
/// <param name="chat">The <see cref="IChat"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
/// <param name="sessionControllerFactory">The <see cref="ISessionControllerFactory"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
/// <param name="dmbFactory">The <see cref="IDmbFactory"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
/// <param name="reattachInfoHandler">The <see cref="IReattachInfoHandler"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
/// <param name="databaseContextFactory">The <see cref="IDatabaseContextFactory"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
/// <param name="byondTopicSender">The <see cref="IByondTopicSender"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
/// <param name="eventConsumer">The <see cref="IEventConsumer"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
/// <param name="jobManager">The <see cref="IJobManager"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
/// <param name="serverControl">The <see cref="IServerControl"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
/// <param name="asyncDelayer">The <see cref="IAsyncDelayer"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
/// <param name="logger">The <see cref="ILogger"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
/// <param name="initialLaunchParameters">The <see cref="DreamDaemonLaunchParameters"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
/// <param name="instance">The <see cref="Api.Models.Instance"/> for the <see cref="WatchdogBase"/>.</param>
|
||||
/// <param name="autoStart">The autostart value for the <see cref="WatchdogBase"/>.</param>
|
||||
public ExperimentalWatchdog(IChat chat, ISessionControllerFactory sessionControllerFactory, IDmbFactory dmbFactory, IReattachInfoHandler reattachInfoHandler, IDatabaseContextFactory databaseContextFactory, IByondTopicSender byondTopicSender, IEventConsumer eventConsumer, IJobManager jobManager, IServerControl serverControl, IAsyncDelayer asyncDelayer, ILogger<ExperimentalWatchdog> logger, DreamDaemonLaunchParameters initialLaunchParameters, Api.Models.Instance instance, bool autoStart)
|
||||
: base(
|
||||
chat,
|
||||
sessionControllerFactory,
|
||||
dmbFactory,
|
||||
reattachInfoHandler,
|
||||
databaseContextFactory,
|
||||
byondTopicSender,
|
||||
eventConsumer,
|
||||
jobManager,
|
||||
serverControl,
|
||||
asyncDelayer,
|
||||
logger,
|
||||
initialLaunchParameters,
|
||||
instance,
|
||||
autoStart)
|
||||
{
|
||||
alphaIsActive = true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Handles the actions to take when the monitor has to "wake up"
|
||||
/// </summary>
|
||||
/// <param name="activationReason">The <see cref="MonitorActivationReason"/> that caused the invocation</param>
|
||||
/// <param name="monitorState">The current <see cref="MonitorState"/>. Will be modified upon retrn</param>
|
||||
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
|
||||
/// <returns>A <see cref="Task"/> representing the running operation</returns>
|
||||
#pragma warning disable CA1502 // TODO: Decomplexify
|
||||
async Task HandlerMonitorWakeup(MonitorActivationReason activationReason, MonitorState monitorState, CancellationToken cancellationToken)
|
||||
{
|
||||
Logger.LogDebug("Monitor activation. Reason: {0}", activationReason);
|
||||
|
||||
// this is where the bulk of the watchdog handling code lives and is fraught with lambdas, sorry not sorry
|
||||
// I'll do my best to walk you through it
|
||||
|
||||
// returns true if the inactive server can't be used immediately
|
||||
// also sets monitor to restart if the above holds
|
||||
bool FullRestartDeadInactive()
|
||||
{
|
||||
if (monitorState.RebootingInactiveServer || monitorState.InactiveServerCritFail)
|
||||
{
|
||||
Logger.LogInformation("Inactive server is {0}! Restarting monitor...", monitorState.InactiveServerCritFail ? "critically failed" : "still rebooting");
|
||||
monitorState.NextAction = MonitorAction.Restart; // will dispose server
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
// trys to set inactive server's port to the public game port
|
||||
// doesn't handle closing active server's port
|
||||
// returns true on success and swaps inactiveserver and activeserver also sets LastLaunchParameters to ActiveLaunchParameters
|
||||
// on failure, sets monitor to restart
|
||||
async Task<bool> MakeInactiveActive()
|
||||
{
|
||||
Logger.LogDebug("Setting inactive server to port {0}...", ActiveLaunchParameters.PrimaryPort.Value);
|
||||
var result = await monitorState.InactiveServer.SetPort(ActiveLaunchParameters.PrimaryPort.Value, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (!result)
|
||||
{
|
||||
Logger.LogWarning("Failed to activate inactive server! Restarting monitor...");
|
||||
monitorState.NextAction = MonitorAction.Restart; // will dispose server
|
||||
return false;
|
||||
}
|
||||
|
||||
// inactive server should always be using active launch parameters
|
||||
LastLaunchParameters = ActiveLaunchParameters;
|
||||
|
||||
var tmp = monitorState.ActiveServer;
|
||||
monitorState.ActiveServer = monitorState.InactiveServer;
|
||||
monitorState.InactiveServer = tmp;
|
||||
alphaIsActive = !AlphaIsActive;
|
||||
monitorState.ActiveServer.EnableCustomChatCommands();
|
||||
return true;
|
||||
}
|
||||
|
||||
// Kills and tries to launch inactive server with the latest dmb
|
||||
// falls back to current dmb on failure
|
||||
// Sets critfail on inactive server failing that
|
||||
// returns false if the backup dmb was used successfully, true otherwise
|
||||
async Task UpdateAndRestartInactiveServer(bool breakAfter)
|
||||
{
|
||||
ActiveParametersUpdated = new TaskCompletionSource<object>();
|
||||
monitorState.InactiveServer.Dispose(); // kill or recycle it
|
||||
var desiredNextAction = breakAfter ? MonitorAction.Break : MonitorAction.Continue;
|
||||
monitorState.NextAction = desiredNextAction;
|
||||
|
||||
Logger.LogInformation("Rebooting inactive server...");
|
||||
var newDmb = DmbFactory.LockNextDmb(1);
|
||||
try
|
||||
{
|
||||
monitorState.InactiveServer = await SessionControllerFactory.LaunchNew(ActiveLaunchParameters, newDmb, null, false, !monitorState.ActiveServer.IsPrimary, false, cancellationToken).ConfigureAwait(false);
|
||||
monitorState.InactiveServer.SetHighPriority();
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Logger.LogError("Error occurred while recreating server! Attempting backup strategy of running DMB of running server! Exception: {0}", e.ToString());
|
||||
|
||||
// ahh jeez, what do we do here?
|
||||
// this is our fault, so it should never happen but
|
||||
// idk maybe a database error while handling the newest dmb?
|
||||
// either way try to start it using the active server's dmb as a backup
|
||||
try
|
||||
{
|
||||
var dmbBackup = await DmbFactory.FromCompileJob(monitorState.ActiveServer.Dmb.CompileJob, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (dmbBackup == null) // NANI!?
|
||||
throw new JobException("Creating backup DMB provider failed!"); // just give up, if THAT compile job is failing then the ActiveServer is gonna crash soon too or already has
|
||||
|
||||
monitorState.InactiveServer = await SessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbBackup, null, false, !monitorState.ActiveServer.IsPrimary, false, cancellationToken).ConfigureAwait(false);
|
||||
monitorState.InactiveServer.SetHighPriority();
|
||||
await Chat.SendWatchdogMessage("Staging newest DMB on inactive server failed: {0} Falling back to previous dmb...", cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception e2)
|
||||
{
|
||||
// fuuuuucckkk
|
||||
Logger.LogError("Backup strategy failed! Monitor will restart when active server reboots! Exception: {0}", e2.ToString());
|
||||
monitorState.InactiveServerCritFail = true;
|
||||
await Chat.SendWatchdogMessage("Attempted reboot of inactive server failed. Watchdog will reset when active server fails or exits", cancellationToken).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
Logger.LogInformation("Successfully relaunched inactive server!");
|
||||
monitorState.RebootingInactiveServer = true;
|
||||
}
|
||||
|
||||
string ExitWord(ISessionController controller) => controller.TerminationWasRequested ? "exited" : "crashed";
|
||||
|
||||
// reason handling
|
||||
switch (activationReason)
|
||||
{
|
||||
case MonitorActivationReason.ActiveServerCrashed:
|
||||
if (monitorState.ActiveServer.RebootState == Watchdog.RebootState.Shutdown)
|
||||
{
|
||||
// the time for graceful shutdown is now
|
||||
await Chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Active server {0}! Exiting due to graceful termination request...", ExitWord(monitorState.ActiveServer)), cancellationToken).ConfigureAwait(false);
|
||||
DisposeAndNullControllers();
|
||||
monitorState.NextAction = MonitorAction.Exit;
|
||||
break;
|
||||
}
|
||||
|
||||
if (FullRestartDeadInactive())
|
||||
{
|
||||
// tell chat about it and go ahead
|
||||
await Chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Active server {0}! Inactive server unable to online!", ExitWord(monitorState.ActiveServer)), cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// we've already been set to restart
|
||||
break;
|
||||
}
|
||||
|
||||
// tell chat about it
|
||||
await Chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Active server {0}! Onlining inactive server...", ExitWord(monitorState.ActiveServer)), cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// try to activate the inactive server
|
||||
if (!await MakeInactiveActive().ConfigureAwait(false))
|
||||
break; // failing that, we've already been set to restart
|
||||
|
||||
// bring up another inactive server
|
||||
await UpdateAndRestartInactiveServer(true).ConfigureAwait(false);
|
||||
break;
|
||||
case MonitorActivationReason.InactiveServerCrashed:
|
||||
// just announce and try to bring it back
|
||||
await Chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Inactive server {0}! Rebooting...", ExitWord(monitorState.InactiveServer)), cancellationToken).ConfigureAwait(false);
|
||||
await UpdateAndRestartInactiveServer(false).ConfigureAwait(false);
|
||||
break;
|
||||
case MonitorActivationReason.ActiveServerRebooted:
|
||||
// ideal goal: active server just closed its port
|
||||
// tell inactive server to open it's port and that's now the active server
|
||||
var rebootState = monitorState.ActiveServer.RebootState;
|
||||
monitorState.ActiveServer.ResetRebootState(); // the DMAPI has already done this internally
|
||||
|
||||
if (FullRestartDeadInactive() && rebootState != Watchdog.RebootState.Shutdown)
|
||||
break; // full restart if the inactive server is being fucky
|
||||
|
||||
// what matters here is the RebootState
|
||||
var restartOnceSwapped = false;
|
||||
|
||||
switch (rebootState)
|
||||
{
|
||||
case Watchdog.RebootState.Normal:
|
||||
// life as normal
|
||||
break;
|
||||
case Watchdog.RebootState.Restart:
|
||||
// reboot the current active server once the inactive one activates
|
||||
restartOnceSwapped = true;
|
||||
break;
|
||||
case Watchdog.RebootState.Shutdown:
|
||||
// graceful shutdown time
|
||||
await Chat.SendWatchdogMessage("Active server rebooted! Exiting due to graceful termination request...", cancellationToken).ConfigureAwait(false);
|
||||
DisposeAndNullControllers();
|
||||
monitorState.NextAction = MonitorAction.Exit;
|
||||
return;
|
||||
default:
|
||||
throw new InvalidOperationException($"Invalid reboot state: {rebootState}");
|
||||
}
|
||||
|
||||
// are both servers now running the same CompileJob?
|
||||
var sameCompileJob = monitorState.InactiveServer.Dmb.CompileJob.Id == monitorState.ActiveServer.Dmb.CompileJob.Id;
|
||||
|
||||
if (!sameCompileJob || ActiveLaunchParameters != LastLaunchParameters)
|
||||
restartOnceSwapped = true; // need a new launch to update either settings or compile job
|
||||
|
||||
if (restartOnceSwapped)
|
||||
/*
|
||||
* we need to manually restart active server
|
||||
* just kill it here, easier that way
|
||||
*/
|
||||
monitorState.ActiveServer.Dispose();
|
||||
|
||||
var activeServerStillHasPortOpen = !restartOnceSwapped && !monitorState.ActiveServer.ClosePortOnReboot;
|
||||
|
||||
if (activeServerStillHasPortOpen)
|
||||
/* we didn't want active server to swap for some reason and it still has it's port open
|
||||
* just continue as normal
|
||||
*/
|
||||
break;
|
||||
|
||||
if (!await MakeInactiveActive().ConfigureAwait(false))
|
||||
break; // monitor will restart
|
||||
|
||||
// servers now swapped
|
||||
// enable this now if inactive server is not still valid
|
||||
monitorState.ActiveServer.ClosePortOnReboot = restartOnceSwapped;
|
||||
|
||||
if (!restartOnceSwapped)
|
||||
/*
|
||||
* now try to reopen it on the private port
|
||||
* failing that, just reboot it
|
||||
*/
|
||||
restartOnceSwapped = !await monitorState.InactiveServer.SetPort(ActiveLaunchParameters.SecondaryPort.Value, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// break either way because any issues past this point would be solved by the reboot
|
||||
if (restartOnceSwapped) // for one reason or another
|
||||
await UpdateAndRestartInactiveServer(true).ConfigureAwait(false); // update and reboot
|
||||
else
|
||||
monitorState.NextAction = MonitorAction.Skip; // only skip checking inactive server rebooted, it's guaranteed InactiveServerStartup complete wouldn't fire this iteration
|
||||
break;
|
||||
case MonitorActivationReason.InactiveServerRebooted:
|
||||
// just don't let the active server close it's port if the inactive server isn't ready
|
||||
monitorState.RebootingInactiveServer = true;
|
||||
monitorState.InactiveServer.ResetRebootState();
|
||||
monitorState.ActiveServer.ClosePortOnReboot = false;
|
||||
monitorState.NextAction = MonitorAction.Continue;
|
||||
break;
|
||||
case MonitorActivationReason.InactiveServerStartupComplete:
|
||||
// opposite of above case
|
||||
monitorState.RebootingInactiveServer = false;
|
||||
monitorState.ActiveServer.ClosePortOnReboot = true;
|
||||
monitorState.NextAction = MonitorAction.Continue;
|
||||
break;
|
||||
case MonitorActivationReason.NewDmbAvailable:
|
||||
case MonitorActivationReason.ActiveLaunchParametersUpdated:
|
||||
// just reload the inactive server and wait for a swap to apply the changes
|
||||
await UpdateAndRestartInactiveServer(true).ConfigureAwait(false);
|
||||
break;
|
||||
default:
|
||||
Trace.Assert(false, String.Format(CultureInfo.InvariantCulture, "Invalid monitor activation reason: {0}!", activationReason));
|
||||
break;
|
||||
}
|
||||
}
|
||||
#pragma warning restore CA1502
|
||||
|
||||
/// <summary>
|
||||
/// Call <see cref="IDisposable.Dispose"/> on <see cref="alphaServer"/> and <see cref="bravoServer"/> and set them to <see langword="null"/>
|
||||
/// </summary>
|
||||
protected override void DisposeAndNullControllers()
|
||||
{
|
||||
alphaServer?.Dispose();
|
||||
alphaServer = null;
|
||||
bravoServer?.Dispose();
|
||||
bravoServer = null;
|
||||
Running = false;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
#pragma warning disable CA1502 // TODO: Decomplexify
|
||||
protected override async Task MonitorLifetimes(CancellationToken cancellationToken)
|
||||
{
|
||||
Logger.LogTrace("Entered MonitorLifetimes");
|
||||
|
||||
// this function is responsible for calling HandlerMonitorWakeup when necessary and manitaining the MonitorState
|
||||
var iteration = 1;
|
||||
for (var monitorState = new MonitorState(); monitorState.NextAction != MonitorAction.Exit; ++iteration)
|
||||
{
|
||||
// always start out with continue
|
||||
monitorState.NextAction = MonitorAction.Continue;
|
||||
|
||||
// dump some info to the logs
|
||||
Logger.LogDebug("Iteration {0} of monitor loop", iteration);
|
||||
try
|
||||
{
|
||||
if (AlphaIsActive)
|
||||
Logger.LogDebug("Alpha is the active server");
|
||||
else
|
||||
Logger.LogDebug("Bravo is the active server");
|
||||
|
||||
if (monitorState.RebootingInactiveServer)
|
||||
Logger.LogDebug("Inactive server is rebooting");
|
||||
|
||||
// update the monitor state with the inactive/active servers
|
||||
monitorState.ActiveServer = AlphaIsActive ? alphaServer : bravoServer;
|
||||
monitorState.InactiveServer = AlphaIsActive ? bravoServer : alphaServer;
|
||||
|
||||
if (monitorState.ActiveServer.ClosePortOnReboot)
|
||||
Logger.LogDebug("Active server will close port on reboot");
|
||||
if (monitorState.InactiveServer.ClosePortOnReboot)
|
||||
Logger.LogDebug("Inactive server will close port on reboot");
|
||||
|
||||
Logger.LogDebug("Active server Compile Job ID: {0}", monitorState.ActiveServer.Dmb.CompileJob.Id);
|
||||
Logger.LogDebug("Inactive server Compile Job ID: {0}", monitorState.InactiveServer.Dmb.CompileJob.Id);
|
||||
|
||||
// load the activation tasks into local variables
|
||||
Task activeServerLifetime = monitorState.ActiveServer.Lifetime;
|
||||
Task inactiveServerLifetime = monitorState.InactiveServer.Lifetime;
|
||||
var activeServerReboot = monitorState.ActiveServer.OnReboot;
|
||||
var inactiveServerReboot = monitorState.InactiveServer.OnReboot;
|
||||
Task inactiveServerStartup = monitorState.RebootingInactiveServer ? monitorState.InactiveServer.LaunchResult : null;
|
||||
Task activeLaunchParametersChanged = ActiveParametersUpdated.Task;
|
||||
var newDmbAvailable = DmbFactory.OnNewerDmb;
|
||||
|
||||
// cancel waiting if requested
|
||||
var cancelTcs = new TaskCompletionSource<object>();
|
||||
using (cancellationToken.Register(() => cancelTcs.SetCanceled()))
|
||||
{
|
||||
var toWaitOn = Task.WhenAny(activeServerLifetime, inactiveServerLifetime, activeServerReboot, inactiveServerReboot, newDmbAvailable, cancelTcs.Task, activeLaunchParametersChanged);
|
||||
if (monitorState.RebootingInactiveServer)
|
||||
toWaitOn = Task.WhenAny(toWaitOn, inactiveServerStartup);
|
||||
|
||||
// wait for something to happen
|
||||
await toWaitOn.ConfigureAwait(false);
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
}
|
||||
|
||||
var chatTask = Task.CompletedTask;
|
||||
using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
// always run HandleMonitorWakeup from the context of the semaphore lock
|
||||
// multiple things may have happened, handle them one at a time
|
||||
for (var moreActivationsToProcess = true; moreActivationsToProcess && (monitorState.NextAction == MonitorAction.Continue || monitorState.NextAction == MonitorAction.Skip);)
|
||||
{
|
||||
MonitorActivationReason activationReason = default; // this will always be assigned before being used
|
||||
|
||||
// process the tasks in this order and call HandlerMonitorWakup for each
|
||||
bool CheckActivationReason(ref Task task, MonitorActivationReason testActivationReason)
|
||||
{
|
||||
var taskCompleted = task?.IsCompleted == true;
|
||||
task = null;
|
||||
if (monitorState.NextAction == MonitorAction.Skip)
|
||||
monitorState.NextAction = MonitorAction.Continue;
|
||||
else if (taskCompleted)
|
||||
{
|
||||
activationReason = testActivationReason;
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
if (CheckActivationReason(ref activeServerLifetime, MonitorActivationReason.ActiveServerCrashed)
|
||||
|| CheckActivationReason(ref inactiveServerLifetime, MonitorActivationReason.InactiveServerCrashed)
|
||||
|| CheckActivationReason(ref activeServerReboot, MonitorActivationReason.ActiveServerRebooted)
|
||||
|| CheckActivationReason(ref inactiveServerReboot, MonitorActivationReason.InactiveServerRebooted)
|
||||
|| CheckActivationReason(ref inactiveServerStartup, MonitorActivationReason.InactiveServerStartupComplete)
|
||||
|| CheckActivationReason(ref newDmbAvailable, MonitorActivationReason.NewDmbAvailable)
|
||||
|| CheckActivationReason(ref activeLaunchParametersChanged, MonitorActivationReason.ActiveLaunchParametersUpdated))
|
||||
await HandlerMonitorWakeup(activationReason, monitorState, cancellationToken).ConfigureAwait(false);
|
||||
else
|
||||
moreActivationsToProcess = false;
|
||||
}
|
||||
|
||||
// writeback alphaServer and bravoServer from monitor state in case they changesd
|
||||
alphaServer = AlphaIsActive ? monitorState.ActiveServer : monitorState.InactiveServer;
|
||||
bravoServer = !AlphaIsActive ? monitorState.ActiveServer : monitorState.InactiveServer;
|
||||
}
|
||||
|
||||
// full reboot required
|
||||
if (monitorState.NextAction == MonitorAction.Restart)
|
||||
{
|
||||
Logger.LogDebug("Next state action is to restart");
|
||||
DisposeAndNullControllers();
|
||||
chatTask = Chat.SendWatchdogMessage("Restarting entirely due to complications...", cancellationToken);
|
||||
|
||||
for (var retryAttempts = 1; monitorState.NextAction == MonitorAction.Restart; ++retryAttempts)
|
||||
{
|
||||
Exception launchException = null;
|
||||
using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false))
|
||||
try
|
||||
{
|
||||
// use LaunchImplNoLock without announcements or restarting the monitor
|
||||
await LaunchImplNoLock(false, false, null, cancellationToken).ConfigureAwait(false);
|
||||
if (Running)
|
||||
{
|
||||
Logger.LogDebug("Relaunch successful, resetting monitor state...");
|
||||
monitorState = new MonitorState(); // clean the slate and continue
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
launchException = e;
|
||||
}
|
||||
|
||||
await chatTask.ConfigureAwait(false);
|
||||
if (!Running)
|
||||
{
|
||||
if (launchException == null)
|
||||
Logger.LogWarning("Failed to automatically restart the watchdog! Attempt: {0}", retryAttempts);
|
||||
else
|
||||
Logger.LogWarning("Failed to automatically restart the watchdog! Attempt: {0}, Exception: {1}", retryAttempts, launchException);
|
||||
var retryDelay = Math.Min(Math.Pow(2, retryAttempts), 3600); // max of one hour, increasing by a power of 2 each time
|
||||
chatTask = Chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Failed to restart watchdog (Attempt: {0}), retrying in {1} seconds...", retryAttempts, retryDelay), cancellationToken);
|
||||
await Task.WhenAll(AsyncDelayer.Delay(TimeSpan.FromSeconds(retryDelay), cancellationToken), chatTask).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
Logger.LogDebug("Monitor cancelled");
|
||||
break;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
// really, this should NEVER happen
|
||||
Logger.LogError("Monitor crashed! Iteration: {0}, State: {1}, Exception: {2}", iteration, JsonConvert.SerializeObject(monitorState), e);
|
||||
await Chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Monitor crashed, this should NEVER happen! Please report this, full details in logs! Restarting monitor... Error: {0}", e.Message), cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
Logger.LogTrace("Monitor exiting...");
|
||||
}
|
||||
#pragma warning restore CA1502
|
||||
|
||||
/// <inheritdoc />
|
||||
#pragma warning disable CA1502 // TODO: Decomplexify
|
||||
protected override async Task InitControllers(Action callBeforeRecurse, Task chatTask, WatchdogReattachInformation reattachInfo, CancellationToken cancellationToken)
|
||||
{
|
||||
// good ole sanity, should never fucking trigger but i don't trust myself even though I should
|
||||
// TODO: Unit test this instead?
|
||||
if (alphaServer != null || bravoServer != null)
|
||||
throw new InvalidOperationException("Entered LaunchNoLock with one or more of the servers not being null!");
|
||||
|
||||
// don't need a new dmb if reattaching
|
||||
var doesntNeedNewDmb = reattachInfo?.Alpha != null && reattachInfo?.Bravo != null;
|
||||
var dmbToUse = doesntNeedNewDmb ? null : DmbFactory.LockNextDmb(2);
|
||||
|
||||
// if this try catches something, both servers are killed
|
||||
try
|
||||
{
|
||||
// start the alpha server task, either by launch a new process or attaching to an existing one
|
||||
// The tasks returned are mainly for writing interop files to the directories among other things and should generally never fail
|
||||
// The tasks pertaining to server startup times are in the ISessionControllers
|
||||
Task<ISessionController> alphaServerTask;
|
||||
if (!doesntNeedNewDmb)
|
||||
alphaServerTask = SessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbToUse, null, true, true, false, cancellationToken);
|
||||
else
|
||||
alphaServerTask = SessionControllerFactory.Reattach(reattachInfo.Alpha, cancellationToken);
|
||||
|
||||
// retrieve the session controller
|
||||
var startTime = DateTimeOffset.Now;
|
||||
alphaServer = await alphaServerTask.ConfigureAwait(false);
|
||||
|
||||
// failed reattaches will return null
|
||||
alphaServer?.SetHighPriority();
|
||||
|
||||
// extra delay for total ordering
|
||||
var now = DateTimeOffset.Now;
|
||||
var delay = now - startTime;
|
||||
|
||||
// definitely not if reattaching though
|
||||
if (reattachInfo == null && delay.TotalSeconds < AlphaBravoStartupSeperationInterval)
|
||||
await AsyncDelayer.Delay(startTime.AddSeconds(AlphaBravoStartupSeperationInterval) - now, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// now bring bravo up
|
||||
if (!doesntNeedNewDmb)
|
||||
bravoServer = await SessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbToUse, null, false, false, false, cancellationToken).ConfigureAwait(false);
|
||||
else
|
||||
bravoServer = await SessionControllerFactory.Reattach(reattachInfo.Bravo, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// failed reattaches will return null
|
||||
bravoServer?.SetHighPriority();
|
||||
|
||||
// possiblity of null servers due to failed reattaches
|
||||
if (alphaServer == null || bravoServer == null)
|
||||
{
|
||||
await chatTask.ConfigureAwait(false);
|
||||
var bothServersDead = alphaServer == null && bravoServer == null;
|
||||
if (bothServersDead
|
||||
|| (alphaServer == null && reattachInfo.AlphaIsActive)
|
||||
|| (bravoServer == null && !reattachInfo.AlphaIsActive))
|
||||
{
|
||||
// we lost the active server, just restart entirely
|
||||
DisposeAndNullControllers();
|
||||
const string FailReattachMessage = "Unable to properly reattach to active server! Restarting...";
|
||||
Logger.LogWarning(FailReattachMessage);
|
||||
Logger.LogDebug(bothServersDead ? "Also could not reattach to inactive server!" : "Inactive server was reattached successfully!");
|
||||
chatTask = Chat.SendWatchdogMessage(FailReattachMessage, cancellationToken);
|
||||
callBeforeRecurse();
|
||||
await LaunchImplNoLock(true, false, null, cancellationToken).ConfigureAwait(false);
|
||||
await chatTask.ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
// we still have the active server but the other one is dead to us, hand it off to the monitor to restart
|
||||
const string InactiveReattachFailureMessage = "Unable to reattach to inactive server. Leaving for monitor to reboot...";
|
||||
chatTask = Chat.SendWatchdogMessage(InactiveReattachFailureMessage, cancellationToken);
|
||||
Logger.LogWarning(InactiveReattachFailureMessage);
|
||||
|
||||
if (reattachInfo.AlphaIsActive)
|
||||
bravoServer = SessionControllerFactory.CreateDeadSession(reattachInfo.Bravo.Dmb);
|
||||
else
|
||||
alphaServer = SessionControllerFactory.CreateDeadSession(reattachInfo.Alpha.Dmb);
|
||||
}
|
||||
|
||||
var alphaLrt = CheckLaunchResult(alphaServer, "Alpha", cancellationToken);
|
||||
var bravoLrt = CheckLaunchResult(bravoServer, "Bravo", cancellationToken);
|
||||
|
||||
// this task completes when both serers have finished booting
|
||||
var allTask = Task.WhenAll(alphaLrt, bravoLrt);
|
||||
|
||||
await allTask.ConfigureAwait(false);
|
||||
|
||||
// both servers are now running, alpha is the active server(unless reattach), huzzah
|
||||
alphaIsActive = reattachInfo?.AlphaIsActive ?? true;
|
||||
|
||||
var activeServer = AlphaIsActive ? alphaServer : bravoServer;
|
||||
activeServer.EnableCustomChatCommands();
|
||||
activeServer.ClosePortOnReboot = true;
|
||||
}
|
||||
catch
|
||||
{
|
||||
if (dmbToUse != null)
|
||||
{
|
||||
// we locked 2 dmbs
|
||||
if (bravoServer == null)
|
||||
{
|
||||
// bravo didn't get control of his
|
||||
dmbToUse.Dispose();
|
||||
if (alphaServer == null)
|
||||
dmbToUse.Dispose(); // alpha didn't get control of his
|
||||
}
|
||||
}
|
||||
else if (doesntNeedNewDmb) // we have reattachInfo
|
||||
if (bravoServer == null)
|
||||
{
|
||||
// bravo didn't get control of his
|
||||
reattachInfo.Bravo?.Dmb.Dispose();
|
||||
if (alphaServer == null)
|
||||
reattachInfo.Alpha?.Dmb.Dispose(); // alpha didn't get control of his
|
||||
}
|
||||
|
||||
// kill the controllers
|
||||
DisposeAndNullControllers();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
#pragma warning restore CA1502
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override ISessionController GetActiveController() => AlphaIsActive ? alphaServer : bravoServer;
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override WatchdogReattachInformation CreateReattachInformation()
|
||||
=> new WatchdogReattachInformation
|
||||
{
|
||||
AlphaIsActive = AlphaIsActive,
|
||||
Alpha = alphaServer?.Release(),
|
||||
Bravo = bravoServer?.Release()
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,7 @@
|
||||
namespace Tgstation.Server.Host.Components.Watchdog
|
||||
{
|
||||
/// <summary>
|
||||
/// The (absolute) state of the <see cref="Watchdog"/>
|
||||
/// The (absolute) state of the <see cref="ExperimentalWatchdog"/>
|
||||
/// </summary>
|
||||
sealed class MonitorState
|
||||
{
|
||||
@@ -18,7 +18,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
public bool InactiveServerCritFail { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The next <see cref="MonitorAction"/> to take in <see cref="Watchdog.MonitorLifetimes(System.Threading.CancellationToken)"/>
|
||||
/// The next <see cref="MonitorAction"/> to take in <see cref="ExperimentalWatchdog.MonitorLifetimes(System.Threading.CancellationToken)"/>
|
||||
/// </summary>
|
||||
public MonitorAction NextAction { get; set; }
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,614 @@
|
||||
using Byond.TopicSender;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Newtonsoft.Json;
|
||||
using Newtonsoft.Json.Serialization;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Tgstation.Server.Api.Models.Internal;
|
||||
using Tgstation.Server.Api.Rights;
|
||||
using Tgstation.Server.Host.Components.Chat;
|
||||
using Tgstation.Server.Host.Components.Compiler;
|
||||
using Tgstation.Server.Host.Components.Interop;
|
||||
using Tgstation.Server.Host.Core;
|
||||
|
||||
namespace Tgstation.Server.Host.Components.Watchdog
|
||||
{
|
||||
/// <summary>
|
||||
/// Base class for <see cref="IWatchdog"/>s.
|
||||
/// </summary>
|
||||
#pragma warning disable CA1506 // TODO: Decomplexify
|
||||
abstract class WatchdogBase : IWatchdog, ICustomCommandHandler, IRestartHandler
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public bool Running { get; protected set; }
|
||||
|
||||
/// <inheritdoc />
|
||||
public abstract bool AlphaIsActive { get; }
|
||||
|
||||
/// <inheritdoc />
|
||||
public abstract Models.CompileJob ActiveCompileJob { get; }
|
||||
|
||||
/// <inheritdoc />
|
||||
public DreamDaemonLaunchParameters ActiveLaunchParameters { get; protected set; }
|
||||
|
||||
/// <inheritdoc />
|
||||
public DreamDaemonLaunchParameters LastLaunchParameters { get; protected set; }
|
||||
|
||||
/// <inheritdoc />
|
||||
public abstract RebootState? RebootState { get; }
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="TaskCompletionSource{TResult}"/> that completes when <see cref="ActiveLaunchParameters"/> are changed and we are <see cref="Running"/>.
|
||||
/// </summary>
|
||||
protected TaskCompletionSource<object> ActiveParametersUpdated { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The <see cref="SemaphoreSlim"/> for the <see cref="WatchdogBase"/>.
|
||||
/// </summary>
|
||||
protected SemaphoreSlim Semaphore { get; }
|
||||
|
||||
/// <summary>
|
||||
/// The <see cref="ILogger"/> for the <see cref="WatchdogBase"/>.
|
||||
/// </summary>
|
||||
protected ILogger Logger { get; }
|
||||
|
||||
/// <summary>
|
||||
/// The <see cref="IChat"/> for the <see cref="WatchdogBase"/>
|
||||
/// </summary>
|
||||
protected IChat Chat { get; }
|
||||
|
||||
/// <summary>
|
||||
/// The <see cref="ISessionControllerFactory"/> for the <see cref="WatchdogBase"/>
|
||||
/// </summary>
|
||||
protected ISessionControllerFactory SessionControllerFactory { get; }
|
||||
|
||||
/// <summary>
|
||||
/// The <see cref="IDmbFactory"/> for the <see cref="WatchdogBase"/>
|
||||
/// </summary>
|
||||
protected IDmbFactory DmbFactory { get; }
|
||||
|
||||
/// <summary>
|
||||
/// The <see cref="IAsyncDelayer"/> for the <see cref="WatchdogBase"/>.
|
||||
/// </summary>
|
||||
protected IAsyncDelayer AsyncDelayer { get; }
|
||||
|
||||
/// <summary>
|
||||
/// The <see cref="Api.Models.Instance"/> for the <see cref="WatchdogBase"/>.
|
||||
/// </summary>
|
||||
readonly Api.Models.Instance instance;
|
||||
|
||||
/// <summary>
|
||||
/// The <see cref="IReattachInfoHandler"/> for the <see cref="WatchdogBase"/>
|
||||
/// </summary>
|
||||
readonly IReattachInfoHandler reattachInfoHandler;
|
||||
|
||||
/// <summary>
|
||||
/// The <see cref="IDatabaseContextFactory"/> for the <see cref="ExperimentalWatchdog"/>
|
||||
/// </summary>
|
||||
readonly IDatabaseContextFactory databaseContextFactory;
|
||||
|
||||
/// <summary>
|
||||
/// The <see cref="IByondTopicSender"/> for the <see cref="ExperimentalWatchdog"/>
|
||||
/// </summary>
|
||||
readonly IByondTopicSender byondTopicSender;
|
||||
|
||||
/// <summary>
|
||||
/// The <see cref="IEventConsumer"/> for the <see cref="ExperimentalWatchdog"/>
|
||||
/// </summary>
|
||||
readonly IEventConsumer eventConsumer;
|
||||
|
||||
/// <summary>
|
||||
/// The <see cref="IJobManager"/> for the <see cref="ExperimentalWatchdog"/>
|
||||
/// </summary>
|
||||
readonly IJobManager jobManager;
|
||||
|
||||
/// <summary>
|
||||
/// The <see cref="IRestartRegistration"/> for the <see cref="ExperimentalWatchdog"/>
|
||||
/// </summary>
|
||||
readonly IRestartRegistration restartRegistration;
|
||||
|
||||
/// <summary>
|
||||
/// If the <see cref="WatchdogBase"/> should <see cref="LaunchImplNoLock(bool, bool, WatchdogReattachInformation, CancellationToken)"/> in <see cref="StartAsync(CancellationToken)"/>
|
||||
/// </summary>
|
||||
readonly bool autoStart;
|
||||
|
||||
/// <summary>
|
||||
/// The <see cref="CancellationTokenSource"/> for the monitor loop
|
||||
/// </summary>
|
||||
CancellationTokenSource monitorCts;
|
||||
|
||||
/// <summary>
|
||||
/// The <see cref="Task"/> running the monitor loop
|
||||
/// </summary>
|
||||
Task monitorTask;
|
||||
|
||||
/// <summary>
|
||||
/// If the servers should be released instead of shutdown
|
||||
/// </summary>
|
||||
bool releaseServers;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="WatchdogBase"/> <see langword="class"/>.
|
||||
/// </summary>
|
||||
/// <param name="chat">The value of <see cref="Chat"/></param>
|
||||
/// <param name="sessionControllerFactory">The value of <see cref="SessionControllerFactory"/></param>
|
||||
/// <param name="dmbFactory">The value of <see cref="DmbFactory"/></param>
|
||||
/// <param name="reattachInfoHandler">The value of <see cref="reattachInfoHandler"/></param>
|
||||
/// <param name="databaseContextFactory">The value of <see cref="databaseContextFactory"/></param>
|
||||
/// <param name="byondTopicSender">The value of <see cref="byondTopicSender"/></param>
|
||||
/// <param name="eventConsumer">The value of <see cref="eventConsumer"/></param>
|
||||
/// <param name="jobManager">The value of <see cref="jobManager"/></param>
|
||||
/// <param name="serverControl">The <see cref="IServerControl"/> to populate <see cref="restartRegistration"/> with</param>
|
||||
/// <param name="asyncDelayer">The value of <see cref="AsyncDelayer"/>.</param>
|
||||
/// <param name="logger">The value of <see cref="Logger"/></param>
|
||||
/// <param name="initialLaunchParameters">The initial value of <see cref="ActiveLaunchParameters"/>. May be modified</param>
|
||||
/// <param name="instance">The value of <see cref="instance"/></param>
|
||||
/// <param name="autoStart">The value of <see cref="autoStart"/></param>
|
||||
protected WatchdogBase(
|
||||
IChat chat,
|
||||
ISessionControllerFactory sessionControllerFactory,
|
||||
IDmbFactory dmbFactory,
|
||||
IReattachInfoHandler reattachInfoHandler,
|
||||
IDatabaseContextFactory databaseContextFactory,
|
||||
IByondTopicSender byondTopicSender,
|
||||
IEventConsumer eventConsumer,
|
||||
IJobManager jobManager,
|
||||
IServerControl serverControl,
|
||||
IAsyncDelayer asyncDelayer,
|
||||
ILogger logger,
|
||||
DreamDaemonLaunchParameters initialLaunchParameters,
|
||||
Api.Models.Instance instance,
|
||||
bool autoStart)
|
||||
{
|
||||
Chat = chat ?? throw new ArgumentNullException(nameof(chat));
|
||||
SessionControllerFactory = sessionControllerFactory ?? throw new ArgumentNullException(nameof(sessionControllerFactory));
|
||||
DmbFactory = dmbFactory ?? throw new ArgumentNullException(nameof(dmbFactory));
|
||||
AsyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer));
|
||||
this.reattachInfoHandler = reattachInfoHandler ?? throw new ArgumentNullException(nameof(reattachInfoHandler));
|
||||
this.databaseContextFactory = databaseContextFactory ?? throw new ArgumentNullException(nameof(databaseContextFactory));
|
||||
this.byondTopicSender = byondTopicSender ?? throw new ArgumentNullException(nameof(byondTopicSender));
|
||||
this.eventConsumer = eventConsumer ?? throw new ArgumentNullException(nameof(eventConsumer));
|
||||
this.jobManager = jobManager ?? throw new ArgumentNullException(nameof(jobManager));
|
||||
Logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
ActiveLaunchParameters = initialLaunchParameters ?? throw new ArgumentNullException(nameof(initialLaunchParameters));
|
||||
this.instance = instance ?? throw new ArgumentNullException(nameof(instance));
|
||||
this.autoStart = autoStart;
|
||||
|
||||
if (serverControl == null)
|
||||
throw new ArgumentNullException(nameof(serverControl));
|
||||
|
||||
chat.RegisterCommandHandler(this);
|
||||
|
||||
ActiveLaunchParameters = initialLaunchParameters;
|
||||
releaseServers = false;
|
||||
ActiveParametersUpdated = new TaskCompletionSource<object>();
|
||||
|
||||
restartRegistration = serverControl.RegisterForRestart(this);
|
||||
try
|
||||
{
|
||||
Semaphore = new SemaphoreSlim(1);
|
||||
}
|
||||
catch
|
||||
{
|
||||
restartRegistration.Dispose();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Dispose()
|
||||
{
|
||||
Semaphore.Dispose();
|
||||
restartRegistration.Dispose();
|
||||
DisposeAndNullControllers();
|
||||
|
||||
Debug.Assert(monitorCts == null, "Expected monitorCts to be null!");
|
||||
monitorCts?.Dispose();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Implementation of <see cref="Terminate(bool, CancellationToken)"/>. Does not lock <see cref="Semaphore"/>
|
||||
/// </summary>
|
||||
/// <param name="graceful">If <see langword="true"/> the termination will be delayed until a reboot is detected in the active server's DMAPI and this function will return immediately</param>
|
||||
/// <param name="announce">If <see langword="true"/> the termination will be announced using <see cref="Chat"/></param>
|
||||
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
|
||||
/// <returns>A <see cref="Task"/> representing the running operation.</returns>
|
||||
async Task TerminateNoLock(bool graceful, bool announce, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!Running)
|
||||
return;
|
||||
if (!graceful)
|
||||
{
|
||||
var chatTask = announce ? Chat.SendWatchdogMessage("Terminating...", cancellationToken) : Task.CompletedTask;
|
||||
await StopMonitor().ConfigureAwait(false);
|
||||
DisposeAndNullControllers();
|
||||
LastLaunchParameters = null;
|
||||
await chatTask.ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
// merely set the reboot state
|
||||
var toKill = GetActiveController();
|
||||
if (toKill != null)
|
||||
await toKill.SetRebootState(Watchdog.RebootState.Shutdown, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Launches the watchdog.
|
||||
/// </summary>
|
||||
/// <param name="startMonitor">If <see cref="MonitorLifetimes(CancellationToken)"/> should be started by this function</param>
|
||||
/// <param name="announce">If the launch should be announced to chat by this function</param>
|
||||
/// <param name="reattachInfo"><see cref="WatchdogReattachInformation"/> to use, if any</param>
|
||||
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
|
||||
/// <returns>A <see cref="Task"/> representing the running operation</returns>
|
||||
protected async Task LaunchImplNoLock(bool startMonitor, bool announce, WatchdogReattachInformation reattachInfo, CancellationToken cancellationToken)
|
||||
{
|
||||
Logger.LogTrace("Begin LaunchNoLock");
|
||||
|
||||
if (Running)
|
||||
throw new JobException("Watchdog already running!");
|
||||
|
||||
// this is necessary, the monitor could be in it's sleep loop trying to restart, if so cancel THAT monitor and start our own with blackjack and hookers
|
||||
Task chatTask;
|
||||
if (startMonitor && await StopMonitor().ConfigureAwait(false))
|
||||
chatTask = Chat.SendWatchdogMessage("Automatic retry sequence cancelled by manual launch. Restarting...", cancellationToken);
|
||||
else if (announce)
|
||||
chatTask = Chat.SendWatchdogMessage(reattachInfo == null ? "Starting..." : "Reattaching...", cancellationToken); // simple announce
|
||||
else
|
||||
chatTask = Task.CompletedTask; // no announce
|
||||
|
||||
// since neither server is running, this is safe to do
|
||||
LastLaunchParameters = ActiveLaunchParameters;
|
||||
|
||||
// for when we call ourself and want to not catch thrown exceptions
|
||||
var ignoreNestedException = false;
|
||||
try
|
||||
{
|
||||
await InitControllers(() => ignoreNestedException = true, chatTask, reattachInfo, cancellationToken).ConfigureAwait(false);
|
||||
await chatTask.ConfigureAwait(false);
|
||||
|
||||
Logger.LogInformation("Launched servers successfully");
|
||||
Running = true;
|
||||
|
||||
if (startMonitor)
|
||||
{
|
||||
StartMonitor();
|
||||
}
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
// don't try to send chat tasks or warning logs if were suppressing exceptions or cancelled
|
||||
if (!ignoreNestedException && !cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
var originalChatTask = chatTask;
|
||||
async Task ChainChatTaskWithErrorMessage()
|
||||
{
|
||||
await originalChatTask.ConfigureAwait(false);
|
||||
await Chat.SendWatchdogMessage("Startup failed!", cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
chatTask = ChainChatTaskWithErrorMessage();
|
||||
Logger.LogWarning("Failed to start watchdog: {0}", e.ToString());
|
||||
}
|
||||
|
||||
throw;
|
||||
}
|
||||
finally
|
||||
{
|
||||
// finish the chat task that's in flight
|
||||
try
|
||||
{
|
||||
await chatTask.ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException) { }
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Call <see cref="MonitorLifetimes(CancellationToken)"/> and setup <see cref="monitorCts"/> and <see cref="monitorTask"/>.
|
||||
/// </summary>
|
||||
protected void StartMonitor()
|
||||
{
|
||||
monitorCts = new CancellationTokenSource();
|
||||
monitorTask = MonitorLifetimes(monitorCts.Token);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Stops <see cref="MonitorLifetimes(CancellationToken)"/>. Doesn't kill the servers
|
||||
/// </summary>
|
||||
/// <returns><see langword="true"/> if the monitor was running, <see langword="false"/> otherwise</returns>
|
||||
protected async Task<bool> StopMonitor()
|
||||
{
|
||||
Logger.LogTrace("StopMonitor");
|
||||
if (monitorTask == null)
|
||||
return false;
|
||||
monitorCts.Cancel();
|
||||
await monitorTask.ConfigureAwait(false);
|
||||
monitorCts.Dispose();
|
||||
monitorTask = null;
|
||||
monitorCts = null;
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Send a chat message and log about a failed reattach operation and attempts another call to <see cref="LaunchImplNoLock(bool, bool, WatchdogReattachInformation, CancellationToken)"/>.
|
||||
/// </summary>
|
||||
/// <param name="inactiveReattachSuccess">If the inactive server was reattached successfully.</param>
|
||||
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation/</param>
|
||||
/// <returns>A <see cref="Task"/> representing the running operation.</returns>
|
||||
protected async Task NotifyOfFailedReattach(bool inactiveReattachSuccess, CancellationToken cancellationToken)
|
||||
{
|
||||
// we lost the server, just restart entirely
|
||||
DisposeAndNullControllers();
|
||||
const string FailReattachMessage = "Unable to properly reattach to server! Restarting...";
|
||||
Logger.LogWarning(FailReattachMessage);
|
||||
Logger.LogDebug(inactiveReattachSuccess ? "Also could not reattach to inactive server!" : "Inactive server was reattached successfully!");
|
||||
Task chatTask = Chat.SendWatchdogMessage(FailReattachMessage, cancellationToken);
|
||||
await LaunchImplNoLock(true, false, null, cancellationToken).ConfigureAwait(false);
|
||||
await chatTask.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Check the <see cref="LaunchResult"/> of a given <paramref name="controller"/> for errors and throw a <see cref="JobException"/> if any are detected.
|
||||
/// </summary>
|
||||
/// <param name="controller">The <see cref="ISessionController"/> to checkou.</param>
|
||||
/// <param name="serverName">The name of the server being checked.</param>
|
||||
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
|
||||
/// <returns>A <see cref="Task"/> representing the running operation.</returns>
|
||||
protected async Task CheckLaunchResult(ISessionController controller, string serverName, CancellationToken cancellationToken)
|
||||
{
|
||||
var launchResult = await controller.LaunchResult.WithToken(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// Dead sessions won't trigger this
|
||||
if (launchResult.ExitCode.HasValue) // you killed us ray...
|
||||
throw new JobException(String.Format(CultureInfo.InvariantCulture, "{0} failed to start: {1}", serverName, launchResult));
|
||||
if (!launchResult.StartupTime.HasValue)
|
||||
throw new JobException(String.Format(CultureInfo.InvariantCulture, "{0} timed out on startup: {1}s", serverName, ActiveLaunchParameters.StartupTimeout.Value));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Call <see cref="IDisposable.Dispose"/> and null the fields for all <see cref="ISessionController"/>s and set <see cref="Running"/> to <see langword="false"/>.
|
||||
/// </summary>
|
||||
protected abstract void DisposeAndNullControllers();
|
||||
|
||||
/// <summary>
|
||||
/// Get the active <see cref="ISessionController"/>.
|
||||
/// </summary>
|
||||
/// <returns>The active <see cref="ISessionController"/>.</returns>
|
||||
protected abstract ISessionController GetActiveController();
|
||||
|
||||
/// <summary>
|
||||
/// Create the <see cref="WatchdogReattachInformation"/> for the <see cref="ISessionController"/>s.
|
||||
/// </summary>
|
||||
/// <returns>A new <see cref="WatchdogReattachInformation"/>.</returns>
|
||||
protected abstract WatchdogReattachInformation CreateReattachInformation();
|
||||
|
||||
/// <summary>
|
||||
/// The loop that watches the watchdog
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
|
||||
/// <returns>A <see cref="Task"/> representing the running operation</returns>
|
||||
protected abstract Task MonitorLifetimes(CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Starts all <see cref="ISessionController"/>s.
|
||||
/// </summary>
|
||||
/// <param name="callBeforeRecurse">An <see cref="Action"/> that must be run before making a recursive call to <see cref="LaunchImplNoLock(bool, bool, WatchdogReattachInformation, CancellationToken)"/>.</param>
|
||||
/// <param name="chatTask">A, possibly active, <see cref="Task"/> for an outgoing chat message.</param>
|
||||
/// <param name="reattachInfo"><see cref="WatchdogReattachInformation"/> to use, if any</param>
|
||||
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
|
||||
/// <returns>A <see cref="Task"/> representing the running operation</returns>
|
||||
protected abstract Task InitControllers(Action callBeforeRecurse, Task chatTask, WatchdogReattachInformation reattachInfo, CancellationToken cancellationToken);
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task ChangeSettings(DreamDaemonLaunchParameters launchParameters, CancellationToken cancellationToken)
|
||||
{
|
||||
using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
if (launchParameters.Match(ActiveLaunchParameters))
|
||||
return;
|
||||
ActiveLaunchParameters = launchParameters;
|
||||
if (Running)
|
||||
ActiveParametersUpdated.TrySetResult(null); // queue an update
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<bool> HandleEvent(EventType eventType, IEnumerable<string> parameters, CancellationToken cancellationToken)
|
||||
{
|
||||
string results;
|
||||
using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
if (!Running)
|
||||
return true;
|
||||
|
||||
var builder = new StringBuilder(Constants.DMTopicEvent);
|
||||
builder.Append('&');
|
||||
var notification = new EventNotification
|
||||
{
|
||||
Type = eventType,
|
||||
Parameters = parameters
|
||||
};
|
||||
var json = JsonConvert.SerializeObject(notification);
|
||||
builder.Append(byondTopicSender.SanitizeString(Constants.DMParameterData));
|
||||
builder.Append('=');
|
||||
builder.Append(byondTopicSender.SanitizeString(json));
|
||||
|
||||
var activeServer = GetActiveController();
|
||||
results = await activeServer.SendCommand(builder.ToString(), cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
if (results == Constants.DMResponseSuccess)
|
||||
return true;
|
||||
|
||||
List<Response> responses;
|
||||
try
|
||||
{
|
||||
responses = JsonConvert.DeserializeObject<List<Response>>(results);
|
||||
}
|
||||
catch
|
||||
{
|
||||
Logger.LogInformation("Recieved invalid response from DD when parsing event {0}:{1}{2}", eventType, Environment.NewLine, results);
|
||||
return true;
|
||||
}
|
||||
|
||||
await Task.WhenAll(responses.Select(x => Chat.SendMessage(x.Message, x.ChannelIds, cancellationToken))).ConfigureAwait(false);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<string> HandleChatCommand(string commandName, string arguments, Chat.User sender, CancellationToken cancellationToken)
|
||||
{
|
||||
using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
if (!Running)
|
||||
return "ERROR: Server offline!";
|
||||
|
||||
var commandObject = new ChatCommand
|
||||
{
|
||||
Command = commandName,
|
||||
Params = arguments,
|
||||
User = sender
|
||||
};
|
||||
|
||||
var json = JsonConvert.SerializeObject(commandObject, new JsonSerializerSettings
|
||||
{
|
||||
ContractResolver = new CamelCasePropertyNamesContractResolver()
|
||||
});
|
||||
|
||||
var command = String.Format(CultureInfo.InvariantCulture, "{0}&{1}={2}", byondTopicSender.SanitizeString(Constants.DMTopicChatCommand), byondTopicSender.SanitizeString(Constants.DMParameterData), byondTopicSender.SanitizeString(json));
|
||||
|
||||
var activeServer = GetActiveController();
|
||||
return await activeServer.SendCommand(command, cancellationToken).ConfigureAwait(false) ?? "ERROR: Bad topic exchange!";
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task Launch(CancellationToken cancellationToken)
|
||||
{
|
||||
using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false))
|
||||
await LaunchImplNoLock(true, true, null, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task ResetRebootState(CancellationToken cancellationToken)
|
||||
{
|
||||
using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
if (!Running)
|
||||
return;
|
||||
var toClear = GetActiveController();
|
||||
if (toClear != null)
|
||||
toClear.ResetRebootState();
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task Restart(bool graceful, CancellationToken cancellationToken)
|
||||
{
|
||||
Logger.LogTrace("Begin Restart. Graceful: {0}", graceful);
|
||||
using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
if (!graceful || !Running)
|
||||
{
|
||||
Task chatTask;
|
||||
if (Running)
|
||||
{
|
||||
chatTask = Chat.SendWatchdogMessage("Manual restart triggered...", cancellationToken);
|
||||
await TerminateNoLock(false, false, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
chatTask = Task.CompletedTask;
|
||||
await LaunchImplNoLock(true, !Running, null, cancellationToken).ConfigureAwait(false);
|
||||
await chatTask.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var toReboot = GetActiveController();
|
||||
if (toReboot != null)
|
||||
{
|
||||
if (!await toReboot.SetRebootState(Watchdog.RebootState.Restart, cancellationToken).ConfigureAwait(false))
|
||||
Logger.LogWarning("Unable to send reboot state change event!");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task StartAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var reattachInfo = await reattachInfoHandler.Load(cancellationToken).ConfigureAwait(false);
|
||||
if (!autoStart && reattachInfo == null)
|
||||
return;
|
||||
|
||||
long? adminUserId = null;
|
||||
|
||||
await databaseContextFactory.UseContext(async db => adminUserId = await db.Users
|
||||
.Where(x => x.CanonicalName == Api.Models.User.AdminName.ToUpperInvariant())
|
||||
.Select(x => x.Id)
|
||||
.FirstAsync(cancellationToken).ConfigureAwait(false)).ConfigureAwait(false);
|
||||
var job = new Models.Job
|
||||
{
|
||||
StartedBy = new Models.User
|
||||
{
|
||||
Id = adminUserId.Value
|
||||
},
|
||||
Instance = new Models.Instance
|
||||
{
|
||||
Id = instance.Id
|
||||
},
|
||||
Description = "Instance startup watchdog launch",
|
||||
CancelRight = (ulong)DreamDaemonRights.Shutdown,
|
||||
CancelRightsType = RightsType.DreamDaemon
|
||||
};
|
||||
await jobManager.RegisterOperation(job, async (j, databaseContext, progressFunction, ct) =>
|
||||
{
|
||||
using (await SemaphoreSlimContext.Lock(Semaphore, ct).ConfigureAwait(false))
|
||||
await LaunchImplNoLock(true, true, reattachInfo, ct).ConfigureAwait(false);
|
||||
}, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task StopAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (releaseServers && Running)
|
||||
{
|
||||
await StopMonitor().ConfigureAwait(false);
|
||||
|
||||
var reattachInformation = CreateReattachInformation();
|
||||
await reattachInfoHandler.Save(reattachInformation, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
await Terminate(false, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch
|
||||
{
|
||||
releaseServers = false;
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task Terminate(bool graceful, CancellationToken cancellationToken)
|
||||
{
|
||||
using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false))
|
||||
await TerminateNoLock(graceful, !releaseServers, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task HandleRestart(Version updateVersion, CancellationToken cancellationToken)
|
||||
{
|
||||
releaseServers = true;
|
||||
if (Running)
|
||||
await Chat.SendWatchdogMessage("Detaching...", cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,9 +1,11 @@
|
||||
using Byond.TopicSender;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using System;
|
||||
using Tgstation.Server.Api.Models.Internal;
|
||||
using Tgstation.Server.Host.Components.Chat;
|
||||
using Tgstation.Server.Host.Components.Compiler;
|
||||
using Tgstation.Server.Host.Configuration;
|
||||
using Tgstation.Server.Host.Core;
|
||||
|
||||
namespace Tgstation.Server.Host.Components.Watchdog
|
||||
@@ -41,6 +43,11 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
/// </summary>
|
||||
readonly IAsyncDelayer asyncDelayer;
|
||||
|
||||
/// <summary>
|
||||
/// The <see cref="GeneralConfiguration"/> for the <see cref="WatchdogFactory"/>
|
||||
/// </summary>
|
||||
readonly GeneralConfiguration generalConfiguration;
|
||||
|
||||
/// <summary>
|
||||
/// Construct a <see cref="WatchdogFactory"/>
|
||||
/// </summary>
|
||||
@@ -50,7 +57,8 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
/// <param name="byondTopicSender">The value of <see cref="byondTopicSender"/></param>
|
||||
/// <param name="jobManager">The value of <see cref="jobManager"/></param>
|
||||
/// <param name="asyncDelayer">The value of <see cref="asyncDelayer"/></param>
|
||||
public WatchdogFactory(IServerControl serverControl, ILoggerFactory loggerFactory, IDatabaseContextFactory databaseContextFactory, IByondTopicSender byondTopicSender, IJobManager jobManager, IAsyncDelayer asyncDelayer)
|
||||
/// <param name="generalConfigurationOptions">The <see cref="IOptions{TOptions}"/> containing the value of <see cref="generalConfiguration"/></param>
|
||||
public WatchdogFactory(IServerControl serverControl, ILoggerFactory loggerFactory, IDatabaseContextFactory databaseContextFactory, IByondTopicSender byondTopicSender, IJobManager jobManager, IAsyncDelayer asyncDelayer, IOptions<GeneralConfiguration> generalConfigurationOptions)
|
||||
{
|
||||
this.serverControl = serverControl ?? throw new ArgumentNullException(nameof(serverControl));
|
||||
this.loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory));
|
||||
@@ -58,9 +66,16 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
this.byondTopicSender = byondTopicSender ?? throw new ArgumentNullException(nameof(byondTopicSender));
|
||||
this.jobManager = jobManager ?? throw new ArgumentNullException(nameof(jobManager));
|
||||
this.asyncDelayer = asyncDelayer ?? throw new ArgumentNullException(nameof(asyncDelayer));
|
||||
generalConfiguration = generalConfigurationOptions?.Value ?? throw new ArgumentNullException(nameof(generalConfigurationOptions));
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public IWatchdog CreateWatchdog(IChat chat, IDmbFactory dmbFactory, IReattachInfoHandler reattachInfoHandler, IEventConsumer eventConsumer, ISessionControllerFactory sessionControllerFactory, Api.Models.Instance instance, DreamDaemonSettings settings) => new Watchdog(chat, sessionControllerFactory, dmbFactory, reattachInfoHandler, databaseContextFactory, byondTopicSender, eventConsumer, jobManager, serverControl, asyncDelayer, loggerFactory.CreateLogger<Watchdog>(), settings, instance, settings.AutoStart.Value);
|
||||
public IWatchdog CreateWatchdog(IChat chat, IDmbFactory dmbFactory, IReattachInfoHandler reattachInfoHandler, IEventConsumer eventConsumer, ISessionControllerFactory sessionControllerFactory, Api.Models.Instance instance, DreamDaemonSettings settings)
|
||||
{
|
||||
if (generalConfiguration.UseExperimentalWatchdog)
|
||||
return new ExperimentalWatchdog(chat, sessionControllerFactory, dmbFactory, reattachInfoHandler, databaseContextFactory, byondTopicSender, eventConsumer, jobManager, serverControl, asyncDelayer, loggerFactory.CreateLogger<ExperimentalWatchdog>(), settings, instance, settings.AutoStart.Value);
|
||||
|
||||
return new BasicWatchdog(chat, sessionControllerFactory, dmbFactory, reattachInfoHandler, databaseContextFactory, byondTopicSender, eventConsumer, jobManager, serverControl, asyncDelayer, loggerFactory.CreateLogger<BasicWatchdog>(), settings, instance, settings.AutoStart.Value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -53,5 +53,10 @@ namespace Tgstation.Server.Host.Configuration
|
||||
/// The timeout milliseconds for restarting the server
|
||||
/// </summary>
|
||||
public int RestartTimeout { get; set; } = DefaultRestartTimeout;
|
||||
|
||||
/// <summary>
|
||||
/// If the <see cref="Components.Watchdog.ExperimentalWatchdog"/> should be used.
|
||||
/// </summary>
|
||||
public bool UseExperimentalWatchdog { get; set; }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,7 +4,8 @@
|
||||
"GitHubAccessToken": null,
|
||||
"SetupWizardMode": "AutoDetect",
|
||||
"ByondTopicTimeout": 5000,
|
||||
"RestartTimeout": 10000
|
||||
"RestartTimeout": 10000,
|
||||
"UseExperimentalWatchdog" : false
|
||||
},
|
||||
"FileLogging": {
|
||||
"Directory": null, //use the default path
|
||||
|
||||
+18
-18
@@ -14,55 +14,55 @@ using Tgstation.Server.Host.Core;
|
||||
namespace Tgstation.Server.Host.Components.Watchdog.Tests
|
||||
{
|
||||
[TestClass]
|
||||
public sealed class TestWatchdog
|
||||
public sealed class TestExperimentalWatchdog
|
||||
{
|
||||
[TestMethod]
|
||||
public void TestConstruction()
|
||||
{
|
||||
Assert.ThrowsException<ArgumentNullException>(() => new Watchdog(null, null, null, null, null, null, null, null, null, null, null, null, null, default));
|
||||
Assert.ThrowsException<ArgumentNullException>(() => new ExperimentalWatchdog(null, null, null, null, null, null, null, null, null, null, null, null, null, default));
|
||||
|
||||
var mockChat = new Mock<IChat>();
|
||||
mockChat.Setup(x => x.RegisterCommandHandler(It.IsNotNull<ICustomCommandHandler>())).Verifiable();
|
||||
Assert.ThrowsException<ArgumentNullException>(() => new Watchdog(mockChat.Object, null, null, null, null, null, null, null, null, null, null, null, null, default));
|
||||
Assert.ThrowsException<ArgumentNullException>(() => new ExperimentalWatchdog(mockChat.Object, null, null, null, null, null, null, null, null, null, null, null, null, default));
|
||||
|
||||
var mockSessionControllerFactory = new Mock<ISessionControllerFactory>();
|
||||
Assert.ThrowsException<ArgumentNullException>(() => new Watchdog(mockChat.Object, mockSessionControllerFactory.Object, null, null, null, null, null, null, null, null, null, null, null, default));
|
||||
Assert.ThrowsException<ArgumentNullException>(() => new ExperimentalWatchdog(mockChat.Object, mockSessionControllerFactory.Object, null, null, null, null, null, null, null, null, null, null, null, default));
|
||||
|
||||
var mockDmbFactory = new Mock<IDmbFactory>();
|
||||
Assert.ThrowsException<ArgumentNullException>(() => new Watchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, null, null, null, null, null, null, null, null, null, null, default));
|
||||
Assert.ThrowsException<ArgumentNullException>(() => new ExperimentalWatchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, null, null, null, null, null, null, null, null, null, null, default));
|
||||
|
||||
var mockReattachInfoHandler = new Mock<IReattachInfoHandler>();
|
||||
Assert.ThrowsException<ArgumentNullException>(() => new Watchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, mockReattachInfoHandler.Object, null, null, null, null, null, null, null, null, null, default));
|
||||
Assert.ThrowsException<ArgumentNullException>(() => new ExperimentalWatchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, mockReattachInfoHandler.Object, null, null, null, null, null, null, null, null, null, default));
|
||||
|
||||
var mockDatabaseContextFactory = new Mock<IDatabaseContextFactory>();
|
||||
Assert.ThrowsException<ArgumentNullException>(() => new Watchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, mockReattachInfoHandler.Object, mockDatabaseContextFactory.Object, null, null, null, null, null, null, null, null, default));
|
||||
Assert.ThrowsException<ArgumentNullException>(() => new ExperimentalWatchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, mockReattachInfoHandler.Object, mockDatabaseContextFactory.Object, null, null, null, null, null, null, null, null, default));
|
||||
|
||||
var mockByondTopicSender = new Mock<IByondTopicSender>();
|
||||
Assert.ThrowsException<ArgumentNullException>(() => new Watchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, mockReattachInfoHandler.Object, mockDatabaseContextFactory.Object, mockByondTopicSender.Object, null, null, null, null, null, null, null, default));
|
||||
Assert.ThrowsException<ArgumentNullException>(() => new ExperimentalWatchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, mockReattachInfoHandler.Object, mockDatabaseContextFactory.Object, mockByondTopicSender.Object, null, null, null, null, null, null, null, default));
|
||||
|
||||
var mockEventConsumer = new Mock<IEventConsumer>();
|
||||
Assert.ThrowsException<ArgumentNullException>(() => new Watchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, mockReattachInfoHandler.Object, mockDatabaseContextFactory.Object, mockByondTopicSender.Object, mockEventConsumer.Object, null, null, null, null, null, null, default));
|
||||
Assert.ThrowsException<ArgumentNullException>(() => new ExperimentalWatchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, mockReattachInfoHandler.Object, mockDatabaseContextFactory.Object, mockByondTopicSender.Object, mockEventConsumer.Object, null, null, null, null, null, null, default));
|
||||
|
||||
var mockJobManager = new Mock<IJobManager>();
|
||||
Assert.ThrowsException<ArgumentNullException>(() => new Watchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, mockReattachInfoHandler.Object, mockDatabaseContextFactory.Object, mockByondTopicSender.Object, mockEventConsumer.Object, mockJobManager.Object, null, null, null, null, null, default));
|
||||
Assert.ThrowsException<ArgumentNullException>(() => new ExperimentalWatchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, mockReattachInfoHandler.Object, mockDatabaseContextFactory.Object, mockByondTopicSender.Object, mockEventConsumer.Object, mockJobManager.Object, null, null, null, null, null, default));
|
||||
|
||||
var mockRestartRegistration = new Mock<IRestartRegistration>();
|
||||
mockRestartRegistration.Setup(x => x.Dispose()).Verifiable();
|
||||
var mockServerControl = new Mock<IServerControl>();
|
||||
mockServerControl.Setup(x => x.RegisterForRestart(It.IsNotNull<IRestartHandler>())).Returns(mockRestartRegistration.Object).Verifiable();
|
||||
Assert.ThrowsException<ArgumentNullException>(() => new Watchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, mockReattachInfoHandler.Object, mockDatabaseContextFactory.Object, mockByondTopicSender.Object, mockEventConsumer.Object, mockJobManager.Object, mockServerControl.Object, null, null, null, null, default));
|
||||
Assert.ThrowsException<ArgumentNullException>(() => new ExperimentalWatchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, mockReattachInfoHandler.Object, mockDatabaseContextFactory.Object, mockByondTopicSender.Object, mockEventConsumer.Object, mockJobManager.Object, mockServerControl.Object, null, null, null, null, default));
|
||||
|
||||
var mockAsyncDelayer = new Mock<IAsyncDelayer>();
|
||||
Assert.ThrowsException<ArgumentNullException>(() => new Watchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, mockReattachInfoHandler.Object, mockDatabaseContextFactory.Object, mockByondTopicSender.Object, mockEventConsumer.Object, mockJobManager.Object, mockServerControl.Object, mockAsyncDelayer.Object, null, null, null, default));
|
||||
Assert.ThrowsException<ArgumentNullException>(() => new ExperimentalWatchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, mockReattachInfoHandler.Object, mockDatabaseContextFactory.Object, mockByondTopicSender.Object, mockEventConsumer.Object, mockJobManager.Object, mockServerControl.Object, mockAsyncDelayer.Object, null, null, null, default));
|
||||
|
||||
var mockLogger = new Mock<ILogger<Watchdog>>();
|
||||
Assert.ThrowsException<ArgumentNullException>(() => new Watchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, mockReattachInfoHandler.Object, mockDatabaseContextFactory.Object, mockByondTopicSender.Object, mockEventConsumer.Object, mockJobManager.Object, mockServerControl.Object, mockAsyncDelayer.Object, mockLogger.Object, null, null, default));
|
||||
var mockLogger = new Mock<ILogger<ExperimentalWatchdog>>();
|
||||
Assert.ThrowsException<ArgumentNullException>(() => new ExperimentalWatchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, mockReattachInfoHandler.Object, mockDatabaseContextFactory.Object, mockByondTopicSender.Object, mockEventConsumer.Object, mockJobManager.Object, mockServerControl.Object, mockAsyncDelayer.Object, mockLogger.Object, null, null, default));
|
||||
|
||||
var mockLaunchParameters = new DreamDaemonLaunchParameters();
|
||||
Assert.ThrowsException<ArgumentNullException>(() => new Watchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, mockReattachInfoHandler.Object, mockDatabaseContextFactory.Object, mockByondTopicSender.Object, mockEventConsumer.Object, mockJobManager.Object, mockServerControl.Object, mockAsyncDelayer.Object, mockLogger.Object, mockLaunchParameters, null, default));
|
||||
Assert.ThrowsException<ArgumentNullException>(() => new ExperimentalWatchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, mockReattachInfoHandler.Object, mockDatabaseContextFactory.Object, mockByondTopicSender.Object, mockEventConsumer.Object, mockJobManager.Object, mockServerControl.Object, mockAsyncDelayer.Object, mockLogger.Object, mockLaunchParameters, null, default));
|
||||
|
||||
var mockInstance = new Models.Instance();
|
||||
new Watchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, mockReattachInfoHandler.Object, mockDatabaseContextFactory.Object, mockByondTopicSender.Object, mockEventConsumer.Object, mockJobManager.Object, mockServerControl.Object, mockAsyncDelayer.Object, mockLogger.Object, mockLaunchParameters, mockInstance, default).Dispose();
|
||||
new ExperimentalWatchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, mockReattachInfoHandler.Object, mockDatabaseContextFactory.Object, mockByondTopicSender.Object, mockEventConsumer.Object, mockJobManager.Object, mockServerControl.Object, mockAsyncDelayer.Object, mockLogger.Object, mockLaunchParameters, mockInstance, default).Dispose();
|
||||
|
||||
mockRestartRegistration.VerifyAll();
|
||||
mockServerControl.VerifyAll();
|
||||
@@ -76,7 +76,7 @@ namespace Tgstation.Server.Host.Components.Watchdog.Tests
|
||||
mockChat.Setup(x => x.RegisterCommandHandler(It.IsNotNull<ICustomCommandHandler>())).Verifiable();
|
||||
var mockSessionControllerFactory = new Mock<ISessionControllerFactory>();
|
||||
var mockDmbFactory = new Mock<IDmbFactory>();
|
||||
var mockLogger = new Mock<ILogger<Watchdog>>();
|
||||
var mockLogger = new Mock<ILogger<ExperimentalWatchdog>>();
|
||||
var mockReattachInfoHandler = new Mock<IReattachInfoHandler>();
|
||||
var mockDatabaseContextFactory = new Mock<IDatabaseContextFactory>();
|
||||
var mockByondTopicSender = new Mock<IByondTopicSender>();
|
||||
@@ -90,7 +90,7 @@ namespace Tgstation.Server.Host.Components.Watchdog.Tests
|
||||
var mockInstance = new Models.Instance();
|
||||
var mockAsyncDelayer = new Mock<IAsyncDelayer>();
|
||||
|
||||
using (var wd = new Watchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, mockReattachInfoHandler.Object, mockDatabaseContextFactory.Object, mockByondTopicSender.Object, mockEventConsumer.Object, mockJobManager.Object, mockServerControl.Object, mockAsyncDelayer.Object, mockLogger.Object, mockLaunchParameters, mockInstance, default))
|
||||
using (var wd = new ExperimentalWatchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, mockReattachInfoHandler.Object, mockDatabaseContextFactory.Object, mockByondTopicSender.Object, mockEventConsumer.Object, mockJobManager.Object, mockServerControl.Object, mockAsyncDelayer.Object, mockLogger.Object, mockLaunchParameters, mockInstance, default))
|
||||
using (var cts = new CancellationTokenSource())
|
||||
{
|
||||
var mockCompileJob = new Models.CompileJob();
|
||||
Reference in New Issue
Block a user