mirror of
https://github.com/tgstation/tgstation-server.git
synced 2026-08-27 07:04:57 +01:00
Watchdog docs
- Cleanup code names/docs - Error code 80 returned when attempting to restart a stopped watchdog
This commit is contained in:
Jordan Brown
+3
-3
@@ -2,9 +2,9 @@
|
||||
<PropertyGroup>
|
||||
<!-- This is the authorative version list -->
|
||||
<!-- Integration tests will ensure they match across the board -->
|
||||
<TgsCoreVersion>4.3.1</TgsCoreVersion>
|
||||
<TgsApiVersion>6.5.1</TgsApiVersion>
|
||||
<TgsClientVersion>7.1.1</TgsClientVersion>
|
||||
<TgsCoreVersion>4.3.2</TgsCoreVersion>
|
||||
<TgsApiVersion>6.6.0</TgsApiVersion>
|
||||
<TgsClientVersion>7.2.0</TgsClientVersion>
|
||||
<TgsDmapiVersion>5.2.2</TgsDmapiVersion>
|
||||
<TgsControlPanelVersion>0.4.0</TgsControlPanelVersion>
|
||||
<TgsHostWatchdogVersion>1.1.0</TgsHostWatchdogVersion>
|
||||
|
||||
@@ -489,5 +489,11 @@ namespace Tgstation.Server.Api.Models
|
||||
/// </summary>
|
||||
[Description("The deployment succeeded but one or more notification events failed!")]
|
||||
PostDeployFailure,
|
||||
|
||||
/// <summary>
|
||||
/// Attempted to restart a stopped watchdog.
|
||||
/// </summary>
|
||||
[Description("Cannot restart the watchdog as it is not running!")]
|
||||
WatchdogNotRunning,
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
"Session" refers to a single invocation of DreamDaemon. The code in here is meant for creating and managing sessions. This is different from the watchdog as it is more low level.
|
||||
|
||||
[ISessionController](./ISessionController.cs) ([implementation](SessionController.cs)) is the representation of DreamDaemon. It handles most of the DMAPI interop, such as DMAPI validation, and raising events for things such as the world rebooting, or the process ending. These are created by the [ISessionControllerFactory](./ISessionControllerFactory.cs) ([implementation](./SessionControllerFactory.cs)) which handles actually launching DreamDaemon and linking together all the components it needs. Sessions give out a [LaunchResult](./LaunchResult.cs) which indicates how long it took for DreamDaemon to become responsive or if it exited before it did.
|
||||
[ISessionController](./ISessionController.cs) ([implementation](SessionController.cs)) is the representation of DreamDaemon. It handles most of the DMAPI interop, such as DMAPI validation, and raising events for things such as the world rebooting, or the process ending. These are created by the [ISessionControllerFactory](./ISessionControllerFactory.cs) ([implementation](./SessionControllerFactory.cs)) which handles actually launching DreamDaemon and linking together all the components it needs. It also runs some sanity checks, such as if the port requested is avaiable or if the BYOND pager is running (which can cause issues). Sessions give out a [LaunchResult](./LaunchResult.cs) which indicates how long it took for DreamDaemon to become responsive or if it exited before it did.
|
||||
|
||||
Sessions can be detached and reattached. When a session is detached, it generates [ReattachInformation](./ReattachInformation.cs). This is stored in and the database as [DualReattachInformation](./DualReattachInformation.cs) by passing it into and out of [IReattachInfoHandler](./IReattachInfoHandler.cs) ([implementation](./ReattachInfoHandler.cs)). This data can be passed back into the `ISessionControllerFactory` to reattach the session.
|
||||
|
||||
|
||||
@@ -328,5 +328,23 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
|
||||
return Restart(true, cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Send a chat message and log about a failed reattach operation and attempts another call to <see cref="WatchdogBase.LaunchNoLock(bool, bool, DualReattachInformation, CancellationToken)"/>.
|
||||
/// </summary>
|
||||
/// <param name="inactiveReattachSuccess">If the inactive server was reattached successfully.</param>
|
||||
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation/</param>
|
||||
/// <returns>A <see cref="Task"/> representing the running operation.</returns>
|
||||
async Task NotifyOfFailedReattach(bool inactiveReattachSuccess, CancellationToken cancellationToken)
|
||||
{
|
||||
// we lost the server, just restart entirely
|
||||
DisposeAndNullControllers();
|
||||
const string FailReattachMessage = "Unable to properly reattach to server! Restarting...";
|
||||
Logger.LogWarning(FailReattachMessage);
|
||||
Logger.LogDebug(inactiveReattachSuccess ? "Also could not reattach to inactive server!" : "Inactive server was reattached successfully!");
|
||||
Task chatTask = Chat.SendWatchdogMessage(FailReattachMessage, false, cancellationToken);
|
||||
await LaunchNoLock(true, false, null, cancellationToken).ConfigureAwait(false);
|
||||
await chatTask.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -512,7 +512,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
Logger.LogDebug(bothServersDead ? "Also could not reattach to inactive server!" : "Inactive server was reattached successfully!");
|
||||
chatTask = Chat.SendWatchdogMessage(FailReattachMessage, false, cancellationToken);
|
||||
callBeforeRecurse();
|
||||
await LaunchImplNoLock(true, false, null, cancellationToken).ConfigureAwait(false);
|
||||
await LaunchNoLock(true, false, null, cancellationToken).ConfigureAwait(false);
|
||||
await chatTask.ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
# The Watchdog
|
||||
|
||||
The watchdog is what keeps DreamDaemon alive and well. It's responsible for starting, stopping, rebooting it, sending TGS events to the DMAPI, running heartbeats, and applying patches.
|
||||
|
||||
There are three main implementations of the interface [IWatchdog](./IWatchdog.cs) that deal with how it handles the last of those jobs.
|
||||
|
||||
- The [BasicWatchdog](./BasicWatchdog.cs) kills DreamDaemon when the world reboots and immediately restarts it with the newly compiled .dmb.
|
||||
- The [WindowsWatchdog](./WindowsWatchdog.cs) uses a quirk of the interaction of DD with the Windows file system. A symlink to the game folder is created. From here, we launch the game. When a new .dmb is available, this symlink is deleted and then immediately recreated pointing to the new game directory. When DreamDaemon reboots it will load the new game.
|
||||
- The [ExperimentalWatchdog](./ExperimentalWatchdog) starts new servers when the new .dmb comes in. But it uses some, rather invasive, port trickery to reroute traffic from the old game server to the new one. The idea for this is to always have two servers running so one can act as a fallback for if the main one fails. This is currently a big work-in-progress as it has issues with BYOND bugs and static file sharing.
|
||||
|
||||
[WatchdogBase](./WatchdogBase.cs) is the parent of all these implementations and contains the bulk of the monitoring code. More on that later.
|
||||
|
||||
`IWatchdog`s are created via the [IWatchdogFactory](./IWatchdogFactory.cs) interface. The two implementations, [WatchdogFactory](./WatchdogFactory.cs) and [WindowsWatchdogFactory](./WindowsWatchdogFactory.cs), differ in that the latter creates `WindowsWatchdog`s as opposed to `BasicWatchdog`s.
|
||||
|
||||
## Startup
|
||||
|
||||
From the perspective of `WatchdogBase`
|
||||
|
||||
- When an instance is onlined, `StartAsync()` is called on `WatchdogBase`. If the instance is configured to autostart the server, or if reattach information is available from the `IReattachInfoHandler` the watchdog will be launched here. Otherwise, it waits to be activated from the controller.
|
||||
- We first do some sanity checks such as if the watchdog is already running and if the `IDmbFactory` has a IDmbProvider for us.
|
||||
- We send out the annoucement of launch/reattach to the chat system.
|
||||
- The `ActiveLaunchParameters` are made the `LastLaunchParameters` for reference.
|
||||
- `InitControllers()` is called, which abstractly creates the `ISessionController`(s) for the watchdog.
|
||||
- For the `BasicWatchdog`, the controller is created normally.
|
||||
- For the `WindowsWatchdog`, we replace the `IDmbProvider` with a `WindowsSwappableDmbProvider` and setup the symlinking before deferring to the `BasicWatchdog`
|
||||
- For the `ExperimentalWatchdog` we launch two servers in tandem.
|
||||
- For reattaching, all three work similarly in that they use `ISessionControllerFactory` to reattach their sessions. The `ExperimentalWatchdog` will insert a `DeadSessionController` in place of one if only one server could not reattach.
|
||||
- `MonitorLifetimes()` is called which is the "main loop" of the watchdog.
|
||||
|
||||
## Monitoring
|
||||
|
||||
Watchdog monitoring takes place in the `MonitorLifetimes()` functiona and is entirely event based. It responds to a set of [MonitorActivationReason](./MonitorActivationReason.cs)s and takes the apporprate action. This includes:
|
||||
- When a server crashes.
|
||||
- When a server calls `/world/prod/Reboot()`.
|
||||
- When a new .dmb is deployed.
|
||||
- When DreamDaemon settings are changed via the API.
|
||||
- When it is time to make a heartbeat.
|
||||
|
||||
Here's how this works.
|
||||
|
||||
- A [MonitorState](./MonitorState.cs) object is created
|
||||
- `Task`s are created for all possible activation reasons and an asynchrounous wait is made on them.
|
||||
- Each activation reason is processed in a particular order via a call to `HandleMonitorWakeup()`. This function is allowed to mutate `MonitorState`.
|
||||
- How each activation reason is handled depends on the watchdog type. For example, the `BasicWatchdog` will queue a graceful reboot when a new .dmb becomes available, wheras the `WindowsWatchdog` will immediately swap the symlink to it.
|
||||
- Some actions are universal. i.e. if the server crashes, a reboot will occur. Or the server will shutdown on reboot if a graceful stop was queued.
|
||||
- If the state's [MonitorAction](./MonitorAction.cs) changes, what happens after each activation may change
|
||||
|
||||
This continues until `StopMonitor` is called, which triggers the `CancellationToken` in `MonitorLifetimes()` and terminates all `ISessionController`s.
|
||||
|
||||
## Detaching
|
||||
|
||||
`WatchdogBase` maintains a `IRestartRegistration` with TGS, meaning that, when TGS is rebooted, `HandleRestart()` will be called before `StopAsync()`. This has the effect of adjusting what happens when the monitor exits. Instead of terminating the server(s) it will instead call `ISessionController.Release()` on them. This generates the `ReattachInformation` which will be saved to the database in `StopAsync()`.
|
||||
@@ -119,7 +119,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
readonly object controllerDisposeLock;
|
||||
|
||||
/// <summary>
|
||||
/// If the <see cref="WatchdogBase"/> should <see cref="LaunchImplNoLock(bool, bool, DualReattachInformation, CancellationToken)"/> in <see cref="StartAsync(CancellationToken)"/>
|
||||
/// If the <see cref="WatchdogBase"/> should <see cref="LaunchNoLock(bool, bool, DualReattachInformation, CancellationToken)"/> in <see cref="StartAsync(CancellationToken)"/>
|
||||
/// </summary>
|
||||
readonly bool autoStart;
|
||||
|
||||
@@ -332,14 +332,14 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
/// <param name="reattachInfo"><see cref="DualReattachInformation"/> to use, if any</param>
|
||||
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
|
||||
/// <returns>A <see cref="Task"/> representing the running operation</returns>
|
||||
protected async Task LaunchImplNoLock(bool startMonitor, bool announce, DualReattachInformation reattachInfo, CancellationToken cancellationToken)
|
||||
protected async Task LaunchNoLock(bool startMonitor, bool announce, DualReattachInformation reattachInfo, CancellationToken cancellationToken)
|
||||
{
|
||||
Logger.LogTrace("Begin LaunchImplNoLock");
|
||||
|
||||
if (Running)
|
||||
throw new JobException(ErrorCode.WatchdogRunning);
|
||||
|
||||
if (!DmbFactory.DmbAvailable)
|
||||
if (reattachInfo == null && !DmbFactory.DmbAvailable)
|
||||
throw new JobException(ErrorCode.WatchdogCompileJobCorrupted);
|
||||
|
||||
// this is necessary, the monitor could be in it's sleep loop trying to restart, if so cancel THAT monitor and start our own with blackjack and hookers
|
||||
@@ -356,10 +356,13 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
heartbeatsMissed = 0;
|
||||
|
||||
// for when we call ourself and want to not catch thrown exceptions
|
||||
var ignoreNestedException = false;
|
||||
var recursiveCallToHappen = false;
|
||||
try
|
||||
{
|
||||
await InitControllers(() => ignoreNestedException = true, chatTask, reattachInfo, cancellationToken).ConfigureAwait(false);
|
||||
await InitControllers(() => recursiveCallToHappen = true, chatTask, reattachInfo, cancellationToken).ConfigureAwait(false);
|
||||
if (recursiveCallToHappen)
|
||||
return;
|
||||
|
||||
await chatTask.ConfigureAwait(false);
|
||||
|
||||
Logger.LogInformation("Launched servers successfully");
|
||||
@@ -367,13 +370,14 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
|
||||
if (startMonitor)
|
||||
{
|
||||
StartMonitor();
|
||||
monitorCts = new CancellationTokenSource();
|
||||
monitorTask = MonitorLifetimes(monitorCts.Token);
|
||||
}
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
// don't try to send chat tasks or warning logs if were suppressing exceptions or cancelled
|
||||
if (!ignoreNestedException && !cancellationToken.IsCancellationRequested)
|
||||
if (!recursiveCallToHappen && !cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
var originalChatTask = chatTask;
|
||||
async Task ChainChatTaskWithErrorMessage()
|
||||
@@ -399,15 +403,6 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Call <see cref="MonitorLifetimes(CancellationToken)"/> and setup <see cref="monitorCts"/> and <see cref="monitorTask"/>.
|
||||
/// </summary>
|
||||
protected void StartMonitor()
|
||||
{
|
||||
monitorCts = new CancellationTokenSource();
|
||||
monitorTask = MonitorLifetimes(monitorCts.Token);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Stops <see cref="MonitorLifetimes(CancellationToken)"/>. Doesn't kill the servers
|
||||
/// </summary>
|
||||
@@ -427,24 +422,6 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
return wasRunning;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Send a chat message and log about a failed reattach operation and attempts another call to <see cref="LaunchImplNoLock(bool, bool, DualReattachInformation, CancellationToken)"/>.
|
||||
/// </summary>
|
||||
/// <param name="inactiveReattachSuccess">If the inactive server was reattached successfully.</param>
|
||||
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation/</param>
|
||||
/// <returns>A <see cref="Task"/> representing the running operation.</returns>
|
||||
protected async Task NotifyOfFailedReattach(bool inactiveReattachSuccess, CancellationToken cancellationToken)
|
||||
{
|
||||
// we lost the server, just restart entirely
|
||||
DisposeAndNullControllers();
|
||||
const string FailReattachMessage = "Unable to properly reattach to server! Restarting...";
|
||||
Logger.LogWarning(FailReattachMessage);
|
||||
Logger.LogDebug(inactiveReattachSuccess ? "Also could not reattach to inactive server!" : "Inactive server was reattached successfully!");
|
||||
Task chatTask = Chat.SendWatchdogMessage(FailReattachMessage, false, cancellationToken);
|
||||
await LaunchImplNoLock(true, false, null, cancellationToken).ConfigureAwait(false);
|
||||
await chatTask.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Check the <see cref="LaunchResult"/> of a given <paramref name="controller"/> for errors and throw a <see cref="JobException"/> if any are detected.
|
||||
/// </summary>
|
||||
@@ -477,6 +454,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
/// </summary>
|
||||
protected void DisposeAndNullControllers()
|
||||
{
|
||||
Logger.LogTrace("DisposeAndNullControllers");
|
||||
lock (controllerDisposeLock)
|
||||
DisposeAndNullControllersImpl();
|
||||
}
|
||||
@@ -513,6 +491,11 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
MonitorState monitorState,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Attempt to restart the monitor from scratch.
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
|
||||
/// <returns>A <see cref="Task{TResult}"/> resulting in the new <see cref="MonitorState"/>.</returns>
|
||||
private async Task<MonitorState> MonitorRestart(CancellationToken cancellationToken)
|
||||
{
|
||||
Logger.LogTrace("Monitor restart!");
|
||||
@@ -526,7 +509,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
try
|
||||
{
|
||||
// use LaunchImplNoLock without announcements or restarting the monitor
|
||||
await LaunchImplNoLock(false, false, null, cancellationToken).ConfigureAwait(false);
|
||||
await LaunchNoLock(false, false, null, cancellationToken).ConfigureAwait(false);
|
||||
if (Running)
|
||||
{
|
||||
Logger.LogDebug("Relaunch successful, resetting monitor state...");
|
||||
@@ -568,7 +551,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The loop that watches the watchdog.
|
||||
/// The main loop of the watchdog. Ayschronously waits for events to occur and then responds to them.
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
|
||||
/// <returns>A <see cref="Task"/> representing the running operation.</returns>
|
||||
@@ -607,35 +590,32 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
|
||||
// cancel waiting if requested
|
||||
var cancelTcs = new TaskCompletionSource<object>();
|
||||
using (cancellationToken.Register(() => cancelTcs.SetCanceled()))
|
||||
{
|
||||
var toWaitOn = Task.WhenAny(
|
||||
activeServerLifetime,
|
||||
activeServerReboot,
|
||||
inactiveServerLifetime,
|
||||
inactiveServerReboot,
|
||||
inactiveStartupComplete,
|
||||
heartbeat,
|
||||
newDmbAvailable,
|
||||
cancelTcs.Task,
|
||||
activeLaunchParametersChanged);
|
||||
var toWaitOn = Task.WhenAny(
|
||||
activeServerLifetime,
|
||||
activeServerReboot,
|
||||
inactiveServerLifetime,
|
||||
inactiveServerReboot,
|
||||
inactiveStartupComplete,
|
||||
heartbeat,
|
||||
newDmbAvailable,
|
||||
cancelTcs.Task,
|
||||
activeLaunchParametersChanged);
|
||||
|
||||
// wait for something to happen
|
||||
// wait for something to happen
|
||||
using (cancellationToken.Register(() => cancelTcs.SetCanceled()))
|
||||
await toWaitOn.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
Logger.LogTrace("Monitor activated");
|
||||
|
||||
// always run HandleMonitorWakeup from the context of the semaphore lock
|
||||
using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
// always run HandleMonitorWakeup from the context of the semaphore lock
|
||||
// multiple things may have happened, handle them one at a time
|
||||
for (var moreActivationsToProcess = true; moreActivationsToProcess && (monitorState.NextAction == MonitorAction.Continue || monitorState.NextAction == MonitorAction.Skip);)
|
||||
{
|
||||
MonitorActivationReason activationReason = default; // this will always be assigned before being used
|
||||
|
||||
// process the tasks in this order and call HandlerMonitorWakup for each
|
||||
bool CheckActivationReason(ref Task task, MonitorActivationReason testActivationReason)
|
||||
{
|
||||
var taskCompleted = task?.IsCompleted == true;
|
||||
@@ -651,6 +631,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
return false;
|
||||
}
|
||||
|
||||
// process the tasks in this order and call HandlerMonitorWakup for each depending on the new monitorState
|
||||
var anyActivation = CheckActivationReason(ref activeServerLifetime, MonitorActivationReason.ActiveServerCrashed)
|
||||
|| CheckActivationReason(ref activeServerReboot, MonitorActivationReason.ActiveServerRebooted)
|
||||
|| CheckActivationReason(ref newDmbAvailable, MonitorActivationReason.NewDmbAvailable)
|
||||
@@ -681,11 +662,14 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
}
|
||||
|
||||
Logger.LogTrace("Next monitor action is to {0}", monitorState.NextAction);
|
||||
|
||||
// Restart if requested
|
||||
if (monitorState.NextAction == MonitorAction.Restart)
|
||||
monitorState = await MonitorRestart(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// let this bubble, other exceptions caught below
|
||||
throw;
|
||||
}
|
||||
catch (Exception e)
|
||||
@@ -715,6 +699,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// stop signal
|
||||
Logger.LogDebug("Monitor cancelled");
|
||||
|
||||
if (releaseServers)
|
||||
@@ -732,7 +717,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
/// <summary>
|
||||
/// Starts all <see cref="ISessionController"/>s.
|
||||
/// </summary>
|
||||
/// <param name="callBeforeRecurse">An <see cref="Action"/> that must be run before making a recursive call to <see cref="LaunchImplNoLock(bool, bool, DualReattachInformation, CancellationToken)"/>.</param>
|
||||
/// <param name="callBeforeRecurse">An <see cref="Action"/> that must be run before making a recursive call to <see cref="LaunchNoLock(bool, bool, DualReattachInformation, CancellationToken)"/>.</param>
|
||||
/// <param name="chatTask">A, possibly active, <see cref="Task"/> for an outgoing chat message.</param>
|
||||
/// <param name="reattachInfo"><see cref="DualReattachInformation"/> to use, if any</param>
|
||||
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
|
||||
@@ -818,7 +803,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
public async Task Launch(CancellationToken cancellationToken)
|
||||
{
|
||||
using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false))
|
||||
await LaunchImplNoLock(true, true, null, cancellationToken).ConfigureAwait(false);
|
||||
await LaunchNoLock(true, true, null, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
@@ -837,10 +822,13 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
/// <inheritdoc />
|
||||
public async Task Restart(bool graceful, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!Running)
|
||||
throw new JobException(ErrorCode.WatchdogRunning);
|
||||
|
||||
Logger.LogTrace("Begin Restart. Graceful: {0}", graceful);
|
||||
using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
if (!graceful || !Running)
|
||||
if (!graceful)
|
||||
{
|
||||
Task chatTask;
|
||||
if (Running)
|
||||
@@ -850,16 +838,14 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
}
|
||||
else
|
||||
chatTask = Task.CompletedTask;
|
||||
await LaunchImplNoLock(true, !Running, null, cancellationToken).ConfigureAwait(false);
|
||||
await LaunchNoLock(true, !Running, null, cancellationToken).ConfigureAwait(false);
|
||||
await chatTask.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var toReboot = GetActiveController();
|
||||
if (toReboot != null)
|
||||
{
|
||||
if (!await toReboot.SetRebootState(Session.RebootState.Restart, cancellationToken).ConfigureAwait(false))
|
||||
Logger.LogWarning("Unable to send reboot state change event!");
|
||||
}
|
||||
if (toReboot != null
|
||||
&& !await toReboot.SetRebootState(Session.RebootState.Restart, cancellationToken).ConfigureAwait(false))
|
||||
Logger.LogWarning("Unable to send reboot state change event!");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -898,7 +884,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
await jobManager.RegisterOperation(job, async (j, databaseContextFactory, progressFunction, ct) =>
|
||||
{
|
||||
using (await SemaphoreSlimContext.Lock(Semaphore, ct).ConfigureAwait(false))
|
||||
await LaunchImplNoLock(true, true, reattachInfo, ct).ConfigureAwait(false);
|
||||
await LaunchNoLock(true, true, reattachInfo, ct).ConfigureAwait(false);
|
||||
}, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
@@ -907,7 +893,11 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
{
|
||||
await TerminateNoLock(false, !releaseServers, cancellationToken).ConfigureAwait(false);
|
||||
if (releasedReattachInformation != null)
|
||||
{
|
||||
await reattachInfoHandler.Save(releasedReattachInformation, cancellationToken).ConfigureAwait(false);
|
||||
releasedReattachInformation = null;
|
||||
releaseServers = false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
|
||||
Reference in New Issue
Block a user