mirror of
https://github.com/tgstation/tgstation-server.git
synced 2026-08-29 08:00:19 +01:00
Refactor Watchdog out of DreamDaemon
This commit is contained in:
Cyberboss
@@ -3,12 +3,11 @@ using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Tgstation.Server.Api.Models;
|
||||
using Tgstation.Server.Api.Models.Internal;
|
||||
using Tgstation.Server.Host.Security;
|
||||
|
||||
namespace Tgstation.Server.Host.Components
|
||||
{
|
||||
/// <inheritdoc />
|
||||
sealed class DreamDaemon : IDreamDaemon, IDisposable
|
||||
sealed class DreamDaemon : IDreamDaemon, ILaunchParametersFactory, IDisposable
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public bool Running { get; private set; }
|
||||
@@ -33,29 +32,13 @@ namespace Tgstation.Server.Host.Components
|
||||
/// </summary>
|
||||
readonly IEventConsumer eventConsumer;
|
||||
/// <summary>
|
||||
/// The <see cref="IByond"/> for <see cref="DreamDaemon"/>
|
||||
/// </summary>
|
||||
readonly IByond byond;
|
||||
/// <summary>
|
||||
/// The <see cref="ICryptographySuite"/> for <see cref="DreamDaemon"/>
|
||||
/// </summary>
|
||||
readonly ICryptographySuite cryptographySuite;
|
||||
/// <summary>
|
||||
/// The <see cref="IInterop"/> for <see cref="DreamDaemon"/>
|
||||
/// </summary>
|
||||
readonly IInterop interop;
|
||||
/// <summary>
|
||||
/// The <see cref="IInstanceShutdownMethod"/> for <see cref="DreamDaemon"/>
|
||||
/// The <see cref="IWatchdog"/> for <see cref="DreamDaemon"/>
|
||||
/// </summary>
|
||||
readonly IInstanceShutdownMethod instanceShutdownMethod;
|
||||
/// <summary>
|
||||
/// The <see cref="IDreamDaemonExecutor"/> for <see cref="DreamDaemon"/>
|
||||
/// </summary>
|
||||
readonly IDreamDaemonExecutor dreamDaemonExecutor;
|
||||
/// <summary>
|
||||
/// The <see cref="IDmbFactory"/> for <see cref="DreamDaemon"/>
|
||||
/// </summary>
|
||||
readonly IDmbFactory dmbFactory;
|
||||
readonly IWatchdog watchdog;
|
||||
|
||||
/// <summary>
|
||||
/// Used for write control to class variables
|
||||
@@ -72,39 +55,18 @@ namespace Tgstation.Server.Host.Components
|
||||
/// </summary>
|
||||
DreamDaemonLaunchParameters currentLaunchParameters;
|
||||
|
||||
/// <summary>
|
||||
/// The <see cref="CancellationTokenSource"/> for <see cref="watchdogTask"/>
|
||||
/// </summary>
|
||||
CancellationTokenSource watchdogCancellationTokenSource;
|
||||
/// <summary>
|
||||
/// The monitor for the DD process
|
||||
/// </summary>
|
||||
Task watchdogTask;
|
||||
/// <summary>
|
||||
/// <see cref="TaskCompletionSource{TResult}"/> to complete when the primary server is primed
|
||||
/// </summary>
|
||||
TaskCompletionSource<object> onPrimaryServerPrimed;
|
||||
|
||||
/// <summary>
|
||||
/// Construct <see cref="DreamDaemon"/>
|
||||
/// </summary>
|
||||
/// <param name="eventConsumer">The value of <see cref="eventConsumer"/></param>
|
||||
/// <param name="byond">The value of <see cref="byond"/></param>
|
||||
/// <param name="cryptographySuite">The value of <see cref="cryptographySuite"/></param>
|
||||
/// <param name="interop">The value of <see cref="interop"/></param>
|
||||
/// <param name="instanceShutdownMethod">The value of <see cref="instanceShutdownMethod"/></param>
|
||||
/// <param name="dreamDaemonExecutor">The value of <see cref="dreamDaemonExecutor"/></param>
|
||||
/// <param name="dmbFactory">The value of <see cref="dmbFactory"/></param>
|
||||
/// <param name="watchdog">The value of <see cref="watchdog"/></param>
|
||||
/// <param name="initialSettings">The initial value of <see cref="currentLaunchParameters"/> and <see cref="autoStart"/></param>
|
||||
public DreamDaemon(IEventConsumer eventConsumer, IByond byond, ICryptographySuite cryptographySuite, IInterop interop, IInstanceShutdownMethod instanceShutdownMethod, IDreamDaemonExecutor dreamDaemonExecutor, IDmbFactory dmbFactory, DreamDaemonSettings initialSettings)
|
||||
public DreamDaemon(IEventConsumer eventConsumer, IInterop interop, IWatchdog watchdog, DreamDaemonSettings initialSettings)
|
||||
{
|
||||
this.eventConsumer = eventConsumer ?? throw new ArgumentNullException(nameof(eventConsumer));
|
||||
this.byond = byond ?? throw new ArgumentNullException(nameof(byond));
|
||||
this.cryptographySuite = cryptographySuite ?? throw new ArgumentNullException(nameof(cryptographySuite));
|
||||
this.interop = interop ?? throw new ArgumentNullException(nameof(interop));
|
||||
this.instanceShutdownMethod = instanceShutdownMethod ?? throw new ArgumentNullException(nameof(instanceShutdownMethod));
|
||||
this.dreamDaemonExecutor = dreamDaemonExecutor ?? throw new ArgumentNullException(nameof(dreamDaemonExecutor));
|
||||
this.dmbFactory = dmbFactory ?? throw new ArgumentNullException(nameof(dmbFactory));
|
||||
this.watchdog = watchdog ?? throw new ArgumentNullException(nameof(watchdog));
|
||||
currentLaunchParameters = initialSettings ?? throw new ArgumentNullException(nameof(initialSettings));
|
||||
|
||||
interop.SetServerControlHandler(OnServerControl);
|
||||
@@ -115,12 +77,7 @@ namespace Tgstation.Server.Host.Components
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Dispose()
|
||||
{
|
||||
if (watchdogCancellationTokenSource != null)
|
||||
watchdogCancellationTokenSource.Dispose();
|
||||
semaphore.Dispose();
|
||||
}
|
||||
public void Dispose() => semaphore.Dispose();
|
||||
|
||||
/// <summary>
|
||||
/// Handler for server control events
|
||||
@@ -137,208 +94,6 @@ namespace Tgstation.Server.Host.Components
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Main DD execution and monitoring <see cref="Task"/>
|
||||
/// </summary>
|
||||
/// <param name="onSuccessfulStartup">The <see cref="TaskCompletionSource{TResult}"/> to be completed when the server initially starts</param>
|
||||
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
|
||||
/// <returns>A <see cref="Task"/> representing the running operation</returns>
|
||||
async Task Watchdog(TaskCompletionSource<object> onSuccessfulStartup, CancellationToken cancellationToken)
|
||||
{
|
||||
if (await byond.GetVersion(cancellationToken).ConfigureAwait(false) == null)
|
||||
throw new InvalidOperationException("No byond version installed!");
|
||||
await byond.ClearCache(cancellationToken).ConfigureAwait(false);
|
||||
var accessToken = cryptographySuite.GetSecureString();
|
||||
|
||||
//lock the byond executable and run the server
|
||||
async Task<int> RunServer(DreamDaemonLaunchParameters launchParameters, string dreamDaemonPath, bool isPrimary, CancellationToken serverCancellationToken)
|
||||
{
|
||||
using (var dmb = await dmbFactory.LockNextDmb(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
return await dreamDaemonExecutor.RunDreamDaemon(launchParameters, onSuccessfulStartup, dreamDaemonPath, String.Concat(dmb.PrimaryDirectory, dmb.DmbName), accessToken, isPrimary, serverCancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
};
|
||||
|
||||
void StartServer(DreamDaemonLaunchParameters launchParameters, bool isPrimary, out Task<int> ddTask, out CancellationTokenSource cancellationTokenSource)
|
||||
{
|
||||
cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||
try
|
||||
{
|
||||
var ddToken = cancellationTokenSource.Token;
|
||||
ddTask = byond.UseExecutable(dreamDaemonPath => RunServer(launchParameters, dreamDaemonPath, isPrimary, ddToken), false, true);
|
||||
interop.SetRun(isPrimary ? launchParameters.PrimaryPort : launchParameters.SecondaryPort, accessToken, isPrimary);
|
||||
}
|
||||
catch
|
||||
{
|
||||
cancellationTokenSource.Dispose();
|
||||
throw;
|
||||
}
|
||||
};
|
||||
|
||||
var retries = 0;
|
||||
do
|
||||
{
|
||||
var retryDelay = (int)Math.Min(Math.Pow(2, retries), TimeSpan.FromHours(1).Milliseconds); //max of one hour
|
||||
await Task.Delay(retryDelay, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
//load the event tcs' and get the initial launch parameters
|
||||
var secondaryRebootedTcs = new TaskCompletionSource<object>();
|
||||
var primaryPrimedTcs = new TaskCompletionSource<object>();
|
||||
DreamDaemonLaunchParameters initialLaunchParameters;
|
||||
await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
onPrimaryServerPrimed = primaryPrimedTcs;
|
||||
initialLaunchParameters = currentLaunchParameters;
|
||||
}
|
||||
finally
|
||||
{
|
||||
semaphore.Release();
|
||||
}
|
||||
|
||||
//start the primary server
|
||||
StartServer(initialLaunchParameters, true, out Task<int> ddPrimaryTask, out CancellationTokenSource primaryCts);
|
||||
try
|
||||
{
|
||||
//wait to make sure we got this far
|
||||
await onSuccessfulStartup.Task.ConfigureAwait(false);
|
||||
onSuccessfulStartup = null;
|
||||
|
||||
//wait for either the server to exit or be primed
|
||||
await Task.WhenAny(ddPrimaryTask, primaryPrimedTcs.Task).ConfigureAwait(false);
|
||||
|
||||
//if the server has exited
|
||||
async Task<bool> HandleServerCrashed(Task<int> serverTask, bool isPrimary)
|
||||
{
|
||||
|
||||
int exitCode;
|
||||
try
|
||||
{
|
||||
//nothing to do except try and reboot it
|
||||
exitCode = await serverTask.ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
await eventConsumer.HandleEvent(exitCode == 0 ? (isPrimary ? EventType.DDExit : EventType.DDOtherExit) : (isPrimary ? EventType.DDCrash : EventType.DDOtherCrash), null, cancellationToken).ConfigureAwait(false);
|
||||
return false;
|
||||
};
|
||||
|
||||
if (ddPrimaryTask.IsCompleted)
|
||||
{
|
||||
if (await HandleServerCrashed(ddPrimaryTask, true).ConfigureAwait(false))
|
||||
return;
|
||||
++retries;
|
||||
continue;
|
||||
}
|
||||
|
||||
var launchParameters = initialLaunchParameters;
|
||||
Task<int> ddSecondaryTask = null;
|
||||
CancellationTokenSource secondaryCts = null;
|
||||
try
|
||||
{
|
||||
do
|
||||
{
|
||||
if (ddSecondaryTask == null)
|
||||
//start the secondary server
|
||||
StartServer(initialLaunchParameters, false, out ddSecondaryTask, out secondaryCts);
|
||||
|
||||
var newDmbTask = dmbFactory.OnNewerDmb();
|
||||
|
||||
//now we wait for something to happen
|
||||
await Task.WhenAny(ddSecondaryTask, ddPrimaryTask, newDmbTask).ConfigureAwait(false);
|
||||
|
||||
if (newDmbTask.IsCompleted)
|
||||
{
|
||||
//restart the other server but don't treat it as an error
|
||||
|
||||
//load new launch parameters
|
||||
await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
launchParameters = currentLaunchParameters;
|
||||
}
|
||||
finally
|
||||
{
|
||||
semaphore.Release();
|
||||
}
|
||||
|
||||
//restart other server
|
||||
if (interop.SecondaryIsOther)
|
||||
{
|
||||
secondaryCts.Cancel();
|
||||
await ddSecondaryTask.ConfigureAwait(false);
|
||||
ddSecondaryTask = null;
|
||||
secondaryCts.Dispose();
|
||||
continue;
|
||||
}
|
||||
else
|
||||
{
|
||||
primaryCts.Cancel();
|
||||
await ddPrimaryTask.ConfigureAwait(false);
|
||||
primaryCts.Dispose();
|
||||
StartServer(initialLaunchParameters, true, out ddPrimaryTask, out primaryCts);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
//crash of both servers
|
||||
if (ddSecondaryTask.IsCompleted && ddPrimaryTask.IsCompleted)
|
||||
{
|
||||
//catastrophic, start over
|
||||
var t1 = HandleServerCrashed(ddPrimaryTask, interop.SecondaryIsOther);
|
||||
var t2 = HandleServerCrashed(ddSecondaryTask, !interop.SecondaryIsOther);
|
||||
await Task.WhenAll(t1, t2).ConfigureAwait(false);
|
||||
if (t1.Result)
|
||||
return;
|
||||
++retries;
|
||||
continue;
|
||||
}
|
||||
|
||||
//activate the other server
|
||||
await interop.ActivateOtherServer(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
//load new launch parameters
|
||||
await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
launchParameters = currentLaunchParameters;
|
||||
}
|
||||
finally
|
||||
{
|
||||
semaphore.Release();
|
||||
}
|
||||
|
||||
//crash of secondary server, just reboot it
|
||||
if (ddSecondaryTask.IsCompleted)
|
||||
{
|
||||
await HandleServerCrashed(ddSecondaryTask, !interop.SecondaryIsOther).ConfigureAwait(false);
|
||||
secondaryCts.Dispose();
|
||||
//restart the server
|
||||
ddSecondaryTask = null;
|
||||
}
|
||||
//crash of primary server, bring it back
|
||||
else
|
||||
{
|
||||
await HandleServerCrashed(ddPrimaryTask, interop.SecondaryIsOther).ConfigureAwait(false);
|
||||
primaryCts.Dispose();
|
||||
StartServer(initialLaunchParameters, true, out ddPrimaryTask, out primaryCts);
|
||||
}
|
||||
} while (true);
|
||||
}
|
||||
finally
|
||||
{
|
||||
secondaryCts?.Dispose();
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
primaryCts.Dispose();
|
||||
}
|
||||
} while (true);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task CancelGracefulActions(CancellationToken cancellationToken)
|
||||
{
|
||||
@@ -381,25 +136,22 @@ namespace Tgstation.Server.Host.Components
|
||||
{
|
||||
if (launchParameters == null)
|
||||
throw new ArgumentNullException(nameof(launchParameters));
|
||||
TaskCompletionSource<object> startupTcs;
|
||||
await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
Task watchdogStartup;
|
||||
try
|
||||
{
|
||||
if (Running)
|
||||
throw new InvalidOperationException("DreamDaemon already running!");
|
||||
Running = true;
|
||||
await eventConsumer.HandleEvent(EventType.DDLaunched, null, cancellationToken).ConfigureAwait(false);
|
||||
watchdogCancellationTokenSource?.Dispose();
|
||||
watchdogCancellationTokenSource = new CancellationTokenSource();
|
||||
startupTcs = new TaskCompletionSource<object>();
|
||||
watchdogTask = Watchdog(startupTcs, watchdogCancellationTokenSource.Token);
|
||||
watchdogStartup = watchdog.Start(this, cancellationToken);
|
||||
}
|
||||
finally
|
||||
{
|
||||
semaphore.Release();
|
||||
}
|
||||
//important to leave the lock so the watchdog can enter it
|
||||
await startupTcs.Task.ConfigureAwait(false);
|
||||
await watchdogStartup.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
@@ -419,7 +171,7 @@ namespace Tgstation.Server.Host.Components
|
||||
}
|
||||
await eventConsumer.HandleEvent(EventType.DDRestart, null, cancellationToken).ConfigureAwait(false);
|
||||
if (Running)
|
||||
watchdogCancellationTokenSource.Cancel();
|
||||
await watchdog.Stop().ConfigureAwait(false);
|
||||
await Launch(currentLaunchParameters, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
finally
|
||||
@@ -449,8 +201,21 @@ namespace Tgstation.Server.Host.Components
|
||||
return;
|
||||
}
|
||||
await eventConsumer.HandleEvent(EventType.DDTerminated, null, cancellationToken).ConfigureAwait(false);
|
||||
watchdogCancellationTokenSource.Cancel();
|
||||
await watchdogTask.ConfigureAwait(false);
|
||||
await watchdog.Stop().ConfigureAwait(false);
|
||||
}
|
||||
finally
|
||||
{
|
||||
semaphore.Release();
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<DreamDaemonLaunchParameters> GetLaunchParameters(CancellationToken cancellationToken)
|
||||
{
|
||||
await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
return currentLaunchParameters;
|
||||
}
|
||||
finally
|
||||
{
|
||||
|
||||
@@ -21,5 +21,6 @@ namespace Tgstation.Server.Host.Components
|
||||
Task ActivateOtherServer(CancellationToken cancellationToken);
|
||||
|
||||
Task<string> ChatCommand(string command, string arguments, CancellationToken cancellationToken);
|
||||
}
|
||||
void OnServerPrimed(Action actionToTake);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Tgstation.Server.Api.Models.Internal;
|
||||
|
||||
namespace Tgstation.Server.Host.Components
|
||||
{
|
||||
/// <summary>
|
||||
/// For retrieving current <see cref="DreamDaemonLaunchParameters"/>
|
||||
/// </summary>
|
||||
interface ILaunchParametersFactory
|
||||
{
|
||||
/// <summary>
|
||||
/// Get the latest <see cref="DreamDaemonLaunchParameters"/>
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
|
||||
/// <returns>A <see cref="Task{TResult}"/> resulting in the latest <see cref="DreamDaemonLaunchParameters"/></returns>
|
||||
Task<DreamDaemonLaunchParameters> GetLaunchParameters(CancellationToken cancellationToken);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace Tgstation.Server.Host.Components
|
||||
{
|
||||
/// <summary>
|
||||
/// For monitoring DreamDaemon uptime
|
||||
/// </summary>
|
||||
interface IWatchdog
|
||||
{
|
||||
/// <summary>
|
||||
/// Start the <see cref="IWatchdog"/>
|
||||
/// </summary>
|
||||
/// <param name="launchParametersFactory">The <see cref="ILaunchParametersFactory"/> for the run</param>
|
||||
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
|
||||
/// <returns>A <see cref="Task"/> representing the running operation</returns>
|
||||
Task Start(ILaunchParametersFactory launchParametersFactory, CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Stop the <see cref="IWatchdog"/>
|
||||
/// </summary>
|
||||
/// <returns>A <see cref="Task"/> representing the running operation</returns>
|
||||
Task Stop();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,309 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Tgstation.Server.Api.Models.Internal;
|
||||
using Tgstation.Server.Host.Security;
|
||||
|
||||
namespace Tgstation.Server.Host.Components
|
||||
{
|
||||
/// <inheritdoc />
|
||||
sealed class Watchdog : IWatchdog, IDisposable
|
||||
{
|
||||
/// <summary>
|
||||
/// The <see cref="IByond"/> for the <see cref="Watchdog"/>
|
||||
/// </summary>
|
||||
readonly IByond byond;
|
||||
/// <summary>
|
||||
/// The <see cref="IDreamDaemonExecutor"/> for the <see cref="Watchdog"/>
|
||||
/// </summary>
|
||||
readonly IDreamDaemonExecutor dreamDaemonExecutor;
|
||||
/// <summary>
|
||||
/// The <see cref="IInterop"/> for the <see cref="Watchdog"/>
|
||||
/// </summary>
|
||||
readonly IInterop interop;
|
||||
/// <summary>
|
||||
/// The <see cref="IDmbFactory"/> for the <see cref="Watchdog"/>
|
||||
/// </summary>
|
||||
readonly IDmbFactory dmbFactory;
|
||||
/// <summary>
|
||||
/// The <see cref="ICryptographySuite"/> for the <see cref="Watchdog"/>
|
||||
/// </summary>
|
||||
readonly ICryptographySuite cryptographySuite;
|
||||
/// <summary>
|
||||
/// The <see cref="IEventConsumer"/> for the <see cref="Watchdog"/>
|
||||
/// </summary>
|
||||
readonly IEventConsumer eventConsumer;
|
||||
|
||||
/// <summary>
|
||||
/// Represents the currently running <see cref="Watchdog"/>
|
||||
/// </summary>
|
||||
Task watchdogTask;
|
||||
/// <summary>
|
||||
/// <see cref="CancellationTokenSource"/> for <see cref="watchdogTask"/>
|
||||
/// </summary>
|
||||
CancellationTokenSource watchdogCancellationTokenSource;
|
||||
|
||||
/// <summary>
|
||||
/// Construct a <see cref="IWatchdog"/>
|
||||
/// </summary>
|
||||
/// <param name="byond">The value of <see cref="byond"/></param>
|
||||
/// <param name="dreamDaemonExecutor">The value of <see cref="dreamDaemonExecutor"/></param>
|
||||
/// <param name="interop">The value of <see cref="interop"/></param>
|
||||
/// <param name="dmbFactory">The value of <see cref="dmbFactory"/></param>
|
||||
/// <param name="cryptographySuite">The value of <see cref="cryptographySuite"/></param>
|
||||
/// <param name="eventConsumer">The value of <see cref="eventConsumer"/></param>
|
||||
public Watchdog(IByond byond, IDreamDaemonExecutor dreamDaemonExecutor, IInterop interop, IDmbFactory dmbFactory, ICryptographySuite cryptographySuite, IEventConsumer eventConsumer)
|
||||
{
|
||||
this.byond = byond ?? throw new ArgumentNullException(nameof(byond));
|
||||
this.dreamDaemonExecutor = dreamDaemonExecutor ?? throw new ArgumentNullException(nameof(dreamDaemonExecutor));
|
||||
this.interop = interop ?? throw new ArgumentNullException(nameof(interop));
|
||||
this.dmbFactory = dmbFactory ?? throw new ArgumentNullException(nameof(dmbFactory));
|
||||
this.cryptographySuite = cryptographySuite ?? throw new ArgumentNullException(nameof(cryptographySuite));
|
||||
this.eventConsumer = eventConsumer ?? throw new ArgumentNullException(nameof(eventConsumer));
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Dispose() => watchdogCancellationTokenSource?.Dispose();
|
||||
|
||||
/// <summary>
|
||||
/// Loads a dmb and runs it through <see cref="dreamDaemonExecutor"/>
|
||||
/// </summary>
|
||||
/// <param name="launchParameters">The <see cref="DreamDaemonLaunchParameters"/> for the run</param>
|
||||
/// <param name="onSuccessfulStartup">The <see cref="TaskCompletionSource{TResult}"/> to be completed once the server starts if any</param>
|
||||
/// <param name="accessToken">The access token for the server</param>
|
||||
/// <param name="dreamDaemonPath">The path to the DreamDaemon executable</param>
|
||||
/// <param name="isPrimary">If a primary server is being launched</param>
|
||||
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
|
||||
/// <returns>A <see cref="Task{TResult}"/> resulting in the exit code of DreamDaemon</returns>
|
||||
async Task<int> RunServer(DreamDaemonLaunchParameters launchParameters, TaskCompletionSource<object> onSuccessfulStartup, string accessToken, string dreamDaemonPath, bool isPrimary, CancellationToken cancellationToken)
|
||||
{
|
||||
using (var dmb = await dmbFactory.LockNextDmb(cancellationToken).ConfigureAwait(false))
|
||||
return await dreamDaemonExecutor.RunDreamDaemon(launchParameters, onSuccessfulStartup, dreamDaemonPath, String.Concat(dmb.PrimaryDirectory, dmb.DmbName), accessToken, isPrimary, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Locks in a <see cref="IByond"/> version and runs a server through <see cref="RunServer(DreamDaemonLaunchParameters, TaskCompletionSource{object}, string, string, bool, CancellationToken)"/>
|
||||
/// </summary>
|
||||
/// <param name="launchParameters">The <see cref="DreamDaemonLaunchParameters"/> for the run</param>
|
||||
/// <param name="onSuccessfulStartup">The <see cref="TaskCompletionSource{TResult}"/> to be completed once the server starts if any</param>
|
||||
/// <param name="accessToken">The access token for the server</param>
|
||||
/// <param name="isPrimary">If a primary server is being launched</param>
|
||||
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
|
||||
/// <param name="cancellationTokenSource">A <see cref="CancellationTokenSource"/> tied to the lifetime of the resulting <see cref="Task{TResult}"/></param>
|
||||
/// <returns>A <see cref="Task{TResult}"/> resulting in the exit code of DreamDaemon</returns>
|
||||
Task<int> StartServer(DreamDaemonLaunchParameters launchParameters, TaskCompletionSource<object> onSuccessfulStartup, string accessToken, bool isPrimary, CancellationToken cancellationToken, out CancellationTokenSource cancellationTokenSource)
|
||||
{
|
||||
cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||
try
|
||||
{
|
||||
var ddToken = cancellationTokenSource.Token;
|
||||
var ddTask = byond.UseExecutable(dreamDaemonPath => RunServer(launchParameters, onSuccessfulStartup, accessToken, dreamDaemonPath, isPrimary, ddToken), false, true);
|
||||
interop.SetRun(isPrimary ? launchParameters.PrimaryPort : launchParameters.SecondaryPort, accessToken, isPrimary);
|
||||
return ddTask;
|
||||
}
|
||||
catch
|
||||
{
|
||||
cancellationTokenSource.Dispose();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Handle a crash or exit of a server
|
||||
/// </summary>
|
||||
/// <param name="serverTask">The <see cref="Task{TResult}"/> resulting in the exit code of the server</param>
|
||||
/// <param name="isPrimary">If the ended server was the primary server</param>
|
||||
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
|
||||
/// <returns>A <see cref="Task{TResult}"/> resulting in <see langword="true"/> if the server was ended due to a <see cref="CancellationToken"/>, <see langword="false"/> otherwise</returns>
|
||||
async Task<bool> HandleServerCrashed(Task<int> serverTask, bool isPrimary, CancellationToken cancellationToken)
|
||||
{
|
||||
int exitCode;
|
||||
try
|
||||
{
|
||||
//nothing to do except try and reboot it
|
||||
exitCode = await serverTask.ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
await eventConsumer.HandleEvent(exitCode == 0 ? (isPrimary ? EventType.DDExit : EventType.DDOtherExit) : (isPrimary ? EventType.DDCrash : EventType.DDOtherCrash), null, cancellationToken).ConfigureAwait(false);
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Main <see cref="Watchdog"/> loop
|
||||
/// </summary>
|
||||
/// <param name="launchParametersFactory">The <see cref="ILaunchParametersFactory"/> for the run</param>
|
||||
/// <param name="onSuccessfulStartup">The <see cref="TaskCompletionSource{TResult}"/> to be completed once the first server starts</param>
|
||||
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
|
||||
/// <returns>A <see cref="Task"/> representing the running operation</returns>
|
||||
async Task Run(ILaunchParametersFactory launchParametersFactory, TaskCompletionSource<object> onSuccessfulStartup, CancellationToken cancellationToken)
|
||||
{
|
||||
if (await byond.GetVersion(cancellationToken).ConfigureAwait(false) == null)
|
||||
throw new InvalidOperationException("No byond version installed!");
|
||||
await byond.ClearCache(cancellationToken).ConfigureAwait(false);
|
||||
var accessToken = cryptographySuite.GetSecureString();
|
||||
|
||||
var retries = 0;
|
||||
do
|
||||
{
|
||||
var retryDelay = (int)Math.Min(Math.Pow(2, retries), TimeSpan.FromHours(1).Milliseconds); //max of one hour
|
||||
await Task.Delay(retryDelay, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
//load the event tcs' and get the initial launch parameters
|
||||
var primaryPrimedTcs = new TaskCompletionSource<object>();
|
||||
interop.OnServerPrimed(() => primaryPrimedTcs.SetResult(null));
|
||||
var initialLaunchParameters = await launchParametersFactory.GetLaunchParameters(cancellationToken).ConfigureAwait(false);
|
||||
//start the primary server
|
||||
var ddPrimaryTask = StartServer(initialLaunchParameters, onSuccessfulStartup, accessToken, true, cancellationToken, out CancellationTokenSource primaryCts);
|
||||
try
|
||||
{
|
||||
//wait to make sure we got this far
|
||||
await onSuccessfulStartup.Task.ConfigureAwait(false);
|
||||
onSuccessfulStartup = null;
|
||||
|
||||
//wait for either the server to exit or be primed
|
||||
await Task.WhenAny(ddPrimaryTask, primaryPrimedTcs.Task).ConfigureAwait(false);
|
||||
|
||||
if (ddPrimaryTask.IsCompleted)
|
||||
{
|
||||
if (await HandleServerCrashed(ddPrimaryTask, true, cancellationToken).ConfigureAwait(false))
|
||||
return;
|
||||
++retries;
|
||||
continue;
|
||||
}
|
||||
|
||||
var launchParameters = initialLaunchParameters;
|
||||
Task<int> ddSecondaryTask = null;
|
||||
CancellationTokenSource secondaryCts = null;
|
||||
try
|
||||
{
|
||||
do
|
||||
{
|
||||
if (ddSecondaryTask == null)
|
||||
//start the secondary server
|
||||
ddSecondaryTask = StartServer(initialLaunchParameters, null, accessToken, false, cancellationToken, out secondaryCts);
|
||||
|
||||
var newDmbTask = dmbFactory.OnNewerDmb();
|
||||
|
||||
//now we wait for something to happen
|
||||
await Task.WhenAny(ddSecondaryTask, ddPrimaryTask, newDmbTask).ConfigureAwait(false);
|
||||
|
||||
//some helpers
|
||||
void PrimaryRestart()
|
||||
{
|
||||
primaryCts.Dispose();
|
||||
ddPrimaryTask = StartServer(initialLaunchParameters, null, accessToken, true, cancellationToken, out primaryCts);
|
||||
}
|
||||
void SecondaryRestart()
|
||||
{
|
||||
ddSecondaryTask = null;
|
||||
secondaryCts.Dispose();
|
||||
};
|
||||
Task<bool> PrimaryCrash() => HandleServerCrashed(ddPrimaryTask, interop.SecondaryIsOther, cancellationToken);
|
||||
Task<bool> SecondaryCrash() => HandleServerCrashed(ddSecondaryTask, !interop.SecondaryIsOther, cancellationToken);
|
||||
|
||||
//update available
|
||||
if (newDmbTask.IsCompleted)
|
||||
{
|
||||
//restart the other server but don't treat it as an error
|
||||
launchParameters = await launchParametersFactory.GetLaunchParameters(cancellationToken).ConfigureAwait(false);
|
||||
//restart other server
|
||||
if (interop.SecondaryIsOther)
|
||||
{
|
||||
secondaryCts.Cancel();
|
||||
await ddSecondaryTask.ConfigureAwait(false);
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
SecondaryRestart();
|
||||
}
|
||||
else
|
||||
{
|
||||
primaryCts.Cancel();
|
||||
await ddPrimaryTask.ConfigureAwait(false);
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
PrimaryRestart();
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
//crash of both servers
|
||||
if (ddSecondaryTask.IsCompleted && ddPrimaryTask.IsCompleted)
|
||||
{
|
||||
//catastrophic, start over
|
||||
var t1 = PrimaryCrash();
|
||||
await Task.WhenAll(t1, SecondaryCrash()).ConfigureAwait(false);
|
||||
if (t1.Result)
|
||||
return;
|
||||
++retries;
|
||||
continue;
|
||||
}
|
||||
|
||||
//below this point: crash of single server
|
||||
|
||||
//activate the other server and load new launch params
|
||||
var otherServerActivation = interop.ActivateOtherServer(cancellationToken);
|
||||
launchParameters = await launchParametersFactory.GetLaunchParameters(cancellationToken).ConfigureAwait(false);
|
||||
await otherServerActivation.ConfigureAwait(false);
|
||||
|
||||
//crash of secondary server
|
||||
if (ddSecondaryTask.IsCompleted)
|
||||
{
|
||||
if (await SecondaryCrash().ConfigureAwait(false))
|
||||
return;
|
||||
SecondaryRestart();
|
||||
}
|
||||
//crash of primary server
|
||||
else
|
||||
{
|
||||
if (await PrimaryCrash().ConfigureAwait(false))
|
||||
return;
|
||||
PrimaryRestart();
|
||||
}
|
||||
} while (true);
|
||||
}
|
||||
finally
|
||||
{
|
||||
secondaryCts?.Dispose();
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
primaryCts.Dispose();
|
||||
}
|
||||
} while (true);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task Start(ILaunchParametersFactory launchParametersFactory, CancellationToken cancellationToken)
|
||||
{
|
||||
TaskCompletionSource<object> taskCompletionSource;
|
||||
lock (this)
|
||||
{
|
||||
if (watchdogTask != null)
|
||||
throw new InvalidOperationException("Watchdog already running!");
|
||||
watchdogCancellationTokenSource?.Dispose();
|
||||
watchdogCancellationTokenSource = new CancellationTokenSource();
|
||||
taskCompletionSource = new TaskCompletionSource<object>();
|
||||
watchdogTask = Run(launchParametersFactory, taskCompletionSource, watchdogCancellationTokenSource.Token);
|
||||
}
|
||||
|
||||
using (cancellationToken.Register(() => watchdogCancellationTokenSource.Cancel()))
|
||||
await taskCompletionSource.Task.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task Stop()
|
||||
{
|
||||
lock(this)
|
||||
{
|
||||
if (watchdogTask == null)
|
||||
throw new InvalidOperationException("Watchdog not running!");
|
||||
watchdogCancellationTokenSource.Cancel();
|
||||
var task = watchdogTask;
|
||||
watchdogTask = null;
|
||||
return task;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user