diff --git a/src/Tgstation.Server.Host/Components/DreamDaemon.cs b/src/Tgstation.Server.Host/Components/DreamDaemon.cs index d90137ae82..04055e8757 100644 --- a/src/Tgstation.Server.Host/Components/DreamDaemon.cs +++ b/src/Tgstation.Server.Host/Components/DreamDaemon.cs @@ -3,12 +3,11 @@ using System.Threading; using System.Threading.Tasks; using Tgstation.Server.Api.Models; using Tgstation.Server.Api.Models.Internal; -using Tgstation.Server.Host.Security; namespace Tgstation.Server.Host.Components { /// - sealed class DreamDaemon : IDreamDaemon, IDisposable + sealed class DreamDaemon : IDreamDaemon, ILaunchParametersFactory, IDisposable { /// public bool Running { get; private set; } @@ -33,29 +32,13 @@ namespace Tgstation.Server.Host.Components /// readonly IEventConsumer eventConsumer; /// - /// The for - /// - readonly IByond byond; - /// - /// The for - /// - readonly ICryptographySuite cryptographySuite; - /// /// The for /// readonly IInterop interop; /// - /// The for + /// The for /// - readonly IInstanceShutdownMethod instanceShutdownMethod; - /// - /// The for - /// - readonly IDreamDaemonExecutor dreamDaemonExecutor; - /// - /// The for - /// - readonly IDmbFactory dmbFactory; + readonly IWatchdog watchdog; /// /// Used for write control to class variables @@ -72,39 +55,18 @@ namespace Tgstation.Server.Host.Components /// DreamDaemonLaunchParameters currentLaunchParameters; - /// - /// The for - /// - CancellationTokenSource watchdogCancellationTokenSource; - /// - /// The monitor for the DD process - /// - Task watchdogTask; - /// - /// to complete when the primary server is primed - /// - TaskCompletionSource onPrimaryServerPrimed; - /// /// Construct /// /// The value of - /// The value of - /// The value of /// The value of - /// The value of - /// The value of - /// The value of + /// The value of /// The initial value of and - public DreamDaemon(IEventConsumer eventConsumer, IByond byond, ICryptographySuite cryptographySuite, IInterop interop, IInstanceShutdownMethod instanceShutdownMethod, IDreamDaemonExecutor dreamDaemonExecutor, IDmbFactory dmbFactory, DreamDaemonSettings initialSettings) + public DreamDaemon(IEventConsumer eventConsumer, IInterop interop, IWatchdog watchdog, DreamDaemonSettings initialSettings) { this.eventConsumer = eventConsumer ?? throw new ArgumentNullException(nameof(eventConsumer)); - this.byond = byond ?? throw new ArgumentNullException(nameof(byond)); - this.cryptographySuite = cryptographySuite ?? throw new ArgumentNullException(nameof(cryptographySuite)); this.interop = interop ?? throw new ArgumentNullException(nameof(interop)); - this.instanceShutdownMethod = instanceShutdownMethod ?? throw new ArgumentNullException(nameof(instanceShutdownMethod)); - this.dreamDaemonExecutor = dreamDaemonExecutor ?? throw new ArgumentNullException(nameof(dreamDaemonExecutor)); - this.dmbFactory = dmbFactory ?? throw new ArgumentNullException(nameof(dmbFactory)); + this.watchdog = watchdog ?? throw new ArgumentNullException(nameof(watchdog)); currentLaunchParameters = initialSettings ?? throw new ArgumentNullException(nameof(initialSettings)); interop.SetServerControlHandler(OnServerControl); @@ -115,12 +77,7 @@ namespace Tgstation.Server.Host.Components } /// - public void Dispose() - { - if (watchdogCancellationTokenSource != null) - watchdogCancellationTokenSource.Dispose(); - semaphore.Dispose(); - } + public void Dispose() => semaphore.Dispose(); /// /// Handler for server control events @@ -137,208 +94,6 @@ namespace Tgstation.Server.Host.Components throw new NotImplementedException(); } - /// - /// Main DD execution and monitoring - /// - /// The to be completed when the server initially starts - /// The for the operation - /// A representing the running operation - async Task Watchdog(TaskCompletionSource onSuccessfulStartup, CancellationToken cancellationToken) - { - if (await byond.GetVersion(cancellationToken).ConfigureAwait(false) == null) - throw new InvalidOperationException("No byond version installed!"); - await byond.ClearCache(cancellationToken).ConfigureAwait(false); - var accessToken = cryptographySuite.GetSecureString(); - - //lock the byond executable and run the server - async Task RunServer(DreamDaemonLaunchParameters launchParameters, string dreamDaemonPath, bool isPrimary, CancellationToken serverCancellationToken) - { - using (var dmb = await dmbFactory.LockNextDmb(cancellationToken).ConfigureAwait(false)) - { - return await dreamDaemonExecutor.RunDreamDaemon(launchParameters, onSuccessfulStartup, dreamDaemonPath, String.Concat(dmb.PrimaryDirectory, dmb.DmbName), accessToken, isPrimary, serverCancellationToken).ConfigureAwait(false); - } - }; - - void StartServer(DreamDaemonLaunchParameters launchParameters, bool isPrimary, out Task ddTask, out CancellationTokenSource cancellationTokenSource) - { - cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); - try - { - var ddToken = cancellationTokenSource.Token; - ddTask = byond.UseExecutable(dreamDaemonPath => RunServer(launchParameters, dreamDaemonPath, isPrimary, ddToken), false, true); - interop.SetRun(isPrimary ? launchParameters.PrimaryPort : launchParameters.SecondaryPort, accessToken, isPrimary); - } - catch - { - cancellationTokenSource.Dispose(); - throw; - } - }; - - var retries = 0; - do - { - var retryDelay = (int)Math.Min(Math.Pow(2, retries), TimeSpan.FromHours(1).Milliseconds); //max of one hour - await Task.Delay(retryDelay, cancellationToken).ConfigureAwait(false); - - //load the event tcs' and get the initial launch parameters - var secondaryRebootedTcs = new TaskCompletionSource(); - var primaryPrimedTcs = new TaskCompletionSource(); - DreamDaemonLaunchParameters initialLaunchParameters; - await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false); - try - { - onPrimaryServerPrimed = primaryPrimedTcs; - initialLaunchParameters = currentLaunchParameters; - } - finally - { - semaphore.Release(); - } - - //start the primary server - StartServer(initialLaunchParameters, true, out Task ddPrimaryTask, out CancellationTokenSource primaryCts); - try - { - //wait to make sure we got this far - await onSuccessfulStartup.Task.ConfigureAwait(false); - onSuccessfulStartup = null; - - //wait for either the server to exit or be primed - await Task.WhenAny(ddPrimaryTask, primaryPrimedTcs.Task).ConfigureAwait(false); - - //if the server has exited - async Task HandleServerCrashed(Task serverTask, bool isPrimary) - { - - int exitCode; - try - { - //nothing to do except try and reboot it - exitCode = await serverTask.ConfigureAwait(false); - } - catch (OperationCanceledException) - { - return true; - } - await eventConsumer.HandleEvent(exitCode == 0 ? (isPrimary ? EventType.DDExit : EventType.DDOtherExit) : (isPrimary ? EventType.DDCrash : EventType.DDOtherCrash), null, cancellationToken).ConfigureAwait(false); - return false; - }; - - if (ddPrimaryTask.IsCompleted) - { - if (await HandleServerCrashed(ddPrimaryTask, true).ConfigureAwait(false)) - return; - ++retries; - continue; - } - - var launchParameters = initialLaunchParameters; - Task ddSecondaryTask = null; - CancellationTokenSource secondaryCts = null; - try - { - do - { - if (ddSecondaryTask == null) - //start the secondary server - StartServer(initialLaunchParameters, false, out ddSecondaryTask, out secondaryCts); - - var newDmbTask = dmbFactory.OnNewerDmb(); - - //now we wait for something to happen - await Task.WhenAny(ddSecondaryTask, ddPrimaryTask, newDmbTask).ConfigureAwait(false); - - if (newDmbTask.IsCompleted) - { - //restart the other server but don't treat it as an error - - //load new launch parameters - await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false); - try - { - launchParameters = currentLaunchParameters; - } - finally - { - semaphore.Release(); - } - - //restart other server - if (interop.SecondaryIsOther) - { - secondaryCts.Cancel(); - await ddSecondaryTask.ConfigureAwait(false); - ddSecondaryTask = null; - secondaryCts.Dispose(); - continue; - } - else - { - primaryCts.Cancel(); - await ddPrimaryTask.ConfigureAwait(false); - primaryCts.Dispose(); - StartServer(initialLaunchParameters, true, out ddPrimaryTask, out primaryCts); - continue; - } - } - - //crash of both servers - if (ddSecondaryTask.IsCompleted && ddPrimaryTask.IsCompleted) - { - //catastrophic, start over - var t1 = HandleServerCrashed(ddPrimaryTask, interop.SecondaryIsOther); - var t2 = HandleServerCrashed(ddSecondaryTask, !interop.SecondaryIsOther); - await Task.WhenAll(t1, t2).ConfigureAwait(false); - if (t1.Result) - return; - ++retries; - continue; - } - - //activate the other server - await interop.ActivateOtherServer(cancellationToken).ConfigureAwait(false); - - //load new launch parameters - await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false); - try - { - launchParameters = currentLaunchParameters; - } - finally - { - semaphore.Release(); - } - - //crash of secondary server, just reboot it - if (ddSecondaryTask.IsCompleted) - { - await HandleServerCrashed(ddSecondaryTask, !interop.SecondaryIsOther).ConfigureAwait(false); - secondaryCts.Dispose(); - //restart the server - ddSecondaryTask = null; - } - //crash of primary server, bring it back - else - { - await HandleServerCrashed(ddPrimaryTask, interop.SecondaryIsOther).ConfigureAwait(false); - primaryCts.Dispose(); - StartServer(initialLaunchParameters, true, out ddPrimaryTask, out primaryCts); - } - } while (true); - } - finally - { - secondaryCts?.Dispose(); - } - } - finally - { - primaryCts.Dispose(); - } - } while (true); - } - /// public async Task CancelGracefulActions(CancellationToken cancellationToken) { @@ -381,25 +136,22 @@ namespace Tgstation.Server.Host.Components { if (launchParameters == null) throw new ArgumentNullException(nameof(launchParameters)); - TaskCompletionSource startupTcs; await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false); + Task watchdogStartup; try { if (Running) throw new InvalidOperationException("DreamDaemon already running!"); Running = true; await eventConsumer.HandleEvent(EventType.DDLaunched, null, cancellationToken).ConfigureAwait(false); - watchdogCancellationTokenSource?.Dispose(); - watchdogCancellationTokenSource = new CancellationTokenSource(); - startupTcs = new TaskCompletionSource(); - watchdogTask = Watchdog(startupTcs, watchdogCancellationTokenSource.Token); + watchdogStartup = watchdog.Start(this, cancellationToken); } finally { semaphore.Release(); } //important to leave the lock so the watchdog can enter it - await startupTcs.Task.ConfigureAwait(false); + await watchdogStartup.ConfigureAwait(false); } /// @@ -419,7 +171,7 @@ namespace Tgstation.Server.Host.Components } await eventConsumer.HandleEvent(EventType.DDRestart, null, cancellationToken).ConfigureAwait(false); if (Running) - watchdogCancellationTokenSource.Cancel(); + await watchdog.Stop().ConfigureAwait(false); await Launch(currentLaunchParameters, cancellationToken).ConfigureAwait(false); } finally @@ -449,8 +201,21 @@ namespace Tgstation.Server.Host.Components return; } await eventConsumer.HandleEvent(EventType.DDTerminated, null, cancellationToken).ConfigureAwait(false); - watchdogCancellationTokenSource.Cancel(); - await watchdogTask.ConfigureAwait(false); + await watchdog.Stop().ConfigureAwait(false); + } + finally + { + semaphore.Release(); + } + } + + /// + public async Task GetLaunchParameters(CancellationToken cancellationToken) + { + await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + return currentLaunchParameters; } finally { diff --git a/src/Tgstation.Server.Host/Components/IInterop.cs b/src/Tgstation.Server.Host/Components/IInterop.cs index 3c6ba60b75..32edd2e047 100644 --- a/src/Tgstation.Server.Host/Components/IInterop.cs +++ b/src/Tgstation.Server.Host/Components/IInterop.cs @@ -21,5 +21,6 @@ namespace Tgstation.Server.Host.Components Task ActivateOtherServer(CancellationToken cancellationToken); Task ChatCommand(string command, string arguments, CancellationToken cancellationToken); - } + void OnServerPrimed(Action actionToTake); + } } diff --git a/src/Tgstation.Server.Host/Components/ILaunchParametersFactory.cs b/src/Tgstation.Server.Host/Components/ILaunchParametersFactory.cs new file mode 100644 index 0000000000..ee2a67ec21 --- /dev/null +++ b/src/Tgstation.Server.Host/Components/ILaunchParametersFactory.cs @@ -0,0 +1,19 @@ +using System.Threading; +using System.Threading.Tasks; +using Tgstation.Server.Api.Models.Internal; + +namespace Tgstation.Server.Host.Components +{ + /// + /// For retrieving current + /// + interface ILaunchParametersFactory + { + /// + /// Get the latest + /// + /// The for the operation + /// A resulting in the latest + Task GetLaunchParameters(CancellationToken cancellationToken); + } +} diff --git a/src/Tgstation.Server.Host/Components/IWatchdog.cs b/src/Tgstation.Server.Host/Components/IWatchdog.cs new file mode 100644 index 0000000000..59882f6c89 --- /dev/null +++ b/src/Tgstation.Server.Host/Components/IWatchdog.cs @@ -0,0 +1,25 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace Tgstation.Server.Host.Components +{ + /// + /// For monitoring DreamDaemon uptime + /// + interface IWatchdog + { + /// + /// Start the + /// + /// The for the run + /// The for the operation + /// A representing the running operation + Task Start(ILaunchParametersFactory launchParametersFactory, CancellationToken cancellationToken); + + /// + /// Stop the + /// + /// A representing the running operation + Task Stop(); + } +} diff --git a/src/Tgstation.Server.Host/Components/Watchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog.cs new file mode 100644 index 0000000000..3213641a92 --- /dev/null +++ b/src/Tgstation.Server.Host/Components/Watchdog.cs @@ -0,0 +1,309 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Tgstation.Server.Api.Models.Internal; +using Tgstation.Server.Host.Security; + +namespace Tgstation.Server.Host.Components +{ + /// + sealed class Watchdog : IWatchdog, IDisposable + { + /// + /// The for the + /// + readonly IByond byond; + /// + /// The for the + /// + readonly IDreamDaemonExecutor dreamDaemonExecutor; + /// + /// The for the + /// + readonly IInterop interop; + /// + /// The for the + /// + readonly IDmbFactory dmbFactory; + /// + /// The for the + /// + readonly ICryptographySuite cryptographySuite; + /// + /// The for the + /// + readonly IEventConsumer eventConsumer; + + /// + /// Represents the currently running + /// + Task watchdogTask; + /// + /// for + /// + CancellationTokenSource watchdogCancellationTokenSource; + + /// + /// Construct a + /// + /// The value of + /// The value of + /// The value of + /// The value of + /// The value of + /// The value of + public Watchdog(IByond byond, IDreamDaemonExecutor dreamDaemonExecutor, IInterop interop, IDmbFactory dmbFactory, ICryptographySuite cryptographySuite, IEventConsumer eventConsumer) + { + this.byond = byond ?? throw new ArgumentNullException(nameof(byond)); + this.dreamDaemonExecutor = dreamDaemonExecutor ?? throw new ArgumentNullException(nameof(dreamDaemonExecutor)); + this.interop = interop ?? throw new ArgumentNullException(nameof(interop)); + this.dmbFactory = dmbFactory ?? throw new ArgumentNullException(nameof(dmbFactory)); + this.cryptographySuite = cryptographySuite ?? throw new ArgumentNullException(nameof(cryptographySuite)); + this.eventConsumer = eventConsumer ?? throw new ArgumentNullException(nameof(eventConsumer)); + } + + /// + public void Dispose() => watchdogCancellationTokenSource?.Dispose(); + + /// + /// Loads a dmb and runs it through + /// + /// The for the run + /// The to be completed once the server starts if any + /// The access token for the server + /// The path to the DreamDaemon executable + /// If a primary server is being launched + /// The for the operation + /// A resulting in the exit code of DreamDaemon + async Task RunServer(DreamDaemonLaunchParameters launchParameters, TaskCompletionSource onSuccessfulStartup, string accessToken, string dreamDaemonPath, bool isPrimary, CancellationToken cancellationToken) + { + using (var dmb = await dmbFactory.LockNextDmb(cancellationToken).ConfigureAwait(false)) + return await dreamDaemonExecutor.RunDreamDaemon(launchParameters, onSuccessfulStartup, dreamDaemonPath, String.Concat(dmb.PrimaryDirectory, dmb.DmbName), accessToken, isPrimary, cancellationToken).ConfigureAwait(false); + } + + /// + /// Locks in a version and runs a server through + /// + /// The for the run + /// The to be completed once the server starts if any + /// The access token for the server + /// If a primary server is being launched + /// The for the operation + /// A tied to the lifetime of the resulting + /// A resulting in the exit code of DreamDaemon + Task StartServer(DreamDaemonLaunchParameters launchParameters, TaskCompletionSource onSuccessfulStartup, string accessToken, bool isPrimary, CancellationToken cancellationToken, out CancellationTokenSource cancellationTokenSource) + { + cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + try + { + var ddToken = cancellationTokenSource.Token; + var ddTask = byond.UseExecutable(dreamDaemonPath => RunServer(launchParameters, onSuccessfulStartup, accessToken, dreamDaemonPath, isPrimary, ddToken), false, true); + interop.SetRun(isPrimary ? launchParameters.PrimaryPort : launchParameters.SecondaryPort, accessToken, isPrimary); + return ddTask; + } + catch + { + cancellationTokenSource.Dispose(); + throw; + } + } + + /// + /// Handle a crash or exit of a server + /// + /// The resulting in the exit code of the server + /// If the ended server was the primary server + /// The for the operation + /// A resulting in if the server was ended due to a , otherwise + async Task HandleServerCrashed(Task serverTask, bool isPrimary, CancellationToken cancellationToken) + { + int exitCode; + try + { + //nothing to do except try and reboot it + exitCode = await serverTask.ConfigureAwait(false); + } + catch (OperationCanceledException) + { + return true; + } + await eventConsumer.HandleEvent(exitCode == 0 ? (isPrimary ? EventType.DDExit : EventType.DDOtherExit) : (isPrimary ? EventType.DDCrash : EventType.DDOtherCrash), null, cancellationToken).ConfigureAwait(false); + return false; + } + + /// + /// Main loop + /// + /// The for the run + /// The to be completed once the first server starts + /// The for the operation + /// A representing the running operation + async Task Run(ILaunchParametersFactory launchParametersFactory, TaskCompletionSource onSuccessfulStartup, CancellationToken cancellationToken) + { + if (await byond.GetVersion(cancellationToken).ConfigureAwait(false) == null) + throw new InvalidOperationException("No byond version installed!"); + await byond.ClearCache(cancellationToken).ConfigureAwait(false); + var accessToken = cryptographySuite.GetSecureString(); + + var retries = 0; + do + { + var retryDelay = (int)Math.Min(Math.Pow(2, retries), TimeSpan.FromHours(1).Milliseconds); //max of one hour + await Task.Delay(retryDelay, cancellationToken).ConfigureAwait(false); + + //load the event tcs' and get the initial launch parameters + var primaryPrimedTcs = new TaskCompletionSource(); + interop.OnServerPrimed(() => primaryPrimedTcs.SetResult(null)); + var initialLaunchParameters = await launchParametersFactory.GetLaunchParameters(cancellationToken).ConfigureAwait(false); + //start the primary server + var ddPrimaryTask = StartServer(initialLaunchParameters, onSuccessfulStartup, accessToken, true, cancellationToken, out CancellationTokenSource primaryCts); + try + { + //wait to make sure we got this far + await onSuccessfulStartup.Task.ConfigureAwait(false); + onSuccessfulStartup = null; + + //wait for either the server to exit or be primed + await Task.WhenAny(ddPrimaryTask, primaryPrimedTcs.Task).ConfigureAwait(false); + + if (ddPrimaryTask.IsCompleted) + { + if (await HandleServerCrashed(ddPrimaryTask, true, cancellationToken).ConfigureAwait(false)) + return; + ++retries; + continue; + } + + var launchParameters = initialLaunchParameters; + Task ddSecondaryTask = null; + CancellationTokenSource secondaryCts = null; + try + { + do + { + if (ddSecondaryTask == null) + //start the secondary server + ddSecondaryTask = StartServer(initialLaunchParameters, null, accessToken, false, cancellationToken, out secondaryCts); + + var newDmbTask = dmbFactory.OnNewerDmb(); + + //now we wait for something to happen + await Task.WhenAny(ddSecondaryTask, ddPrimaryTask, newDmbTask).ConfigureAwait(false); + + //some helpers + void PrimaryRestart() + { + primaryCts.Dispose(); + ddPrimaryTask = StartServer(initialLaunchParameters, null, accessToken, true, cancellationToken, out primaryCts); + } + void SecondaryRestart() + { + ddSecondaryTask = null; + secondaryCts.Dispose(); + }; + Task PrimaryCrash() => HandleServerCrashed(ddPrimaryTask, interop.SecondaryIsOther, cancellationToken); + Task SecondaryCrash() => HandleServerCrashed(ddSecondaryTask, !interop.SecondaryIsOther, cancellationToken); + + //update available + if (newDmbTask.IsCompleted) + { + //restart the other server but don't treat it as an error + launchParameters = await launchParametersFactory.GetLaunchParameters(cancellationToken).ConfigureAwait(false); + //restart other server + if (interop.SecondaryIsOther) + { + secondaryCts.Cancel(); + await ddSecondaryTask.ConfigureAwait(false); + cancellationToken.ThrowIfCancellationRequested(); + SecondaryRestart(); + } + else + { + primaryCts.Cancel(); + await ddPrimaryTask.ConfigureAwait(false); + cancellationToken.ThrowIfCancellationRequested(); + PrimaryRestart(); + } + continue; + } + + //crash of both servers + if (ddSecondaryTask.IsCompleted && ddPrimaryTask.IsCompleted) + { + //catastrophic, start over + var t1 = PrimaryCrash(); + await Task.WhenAll(t1, SecondaryCrash()).ConfigureAwait(false); + if (t1.Result) + return; + ++retries; + continue; + } + + //below this point: crash of single server + + //activate the other server and load new launch params + var otherServerActivation = interop.ActivateOtherServer(cancellationToken); + launchParameters = await launchParametersFactory.GetLaunchParameters(cancellationToken).ConfigureAwait(false); + await otherServerActivation.ConfigureAwait(false); + + //crash of secondary server + if (ddSecondaryTask.IsCompleted) + { + if (await SecondaryCrash().ConfigureAwait(false)) + return; + SecondaryRestart(); + } + //crash of primary server + else + { + if (await PrimaryCrash().ConfigureAwait(false)) + return; + PrimaryRestart(); + } + } while (true); + } + finally + { + secondaryCts?.Dispose(); + } + } + finally + { + primaryCts.Dispose(); + } + } while (true); + } + + /// + public async Task Start(ILaunchParametersFactory launchParametersFactory, CancellationToken cancellationToken) + { + TaskCompletionSource taskCompletionSource; + lock (this) + { + if (watchdogTask != null) + throw new InvalidOperationException("Watchdog already running!"); + watchdogCancellationTokenSource?.Dispose(); + watchdogCancellationTokenSource = new CancellationTokenSource(); + taskCompletionSource = new TaskCompletionSource(); + watchdogTask = Run(launchParametersFactory, taskCompletionSource, watchdogCancellationTokenSource.Token); + } + + using (cancellationToken.Register(() => watchdogCancellationTokenSource.Cancel())) + await taskCompletionSource.Task.ConfigureAwait(false); + } + + /// + public Task Stop() + { + lock(this) + { + if (watchdogTask == null) + throw new InvalidOperationException("Watchdog not running!"); + watchdogCancellationTokenSource.Cancel(); + var task = watchdogTask; + watchdogTask = null; + return task; + } + } + } +}