cyberpresidentvanellope/tgstation-server
1
0
Fork 0
mirror of https://github.com/tgstation/tgstation-server.git synced 2026-07-20 12:33:00 +01:00
Code Issues Packages Projects Releases Wiki Activity

Implement watchdog heartbeats

Browse Source 
- Refactor MonitorLifetimes so the code isn't duplicated in the Basic and Experimental watchdogs
- Add IsAdminChannel to ChannelMapping
- Allow watchdog messages to be potentially restricted to admin chats
- Add topic command type for heartbeats
- Change ISessionController.SendCommand return type to also give the Byond.TopicSender response
- Added OnPrime event to ISessionController, this is now used for inactive server startup
- Added TaskExtension for creating infinite tasks.
- Fixed watchdog shutdown crash.

WIP
This commit is contained in:
Jordan Brown
2020-05-11 03:21:52 -04:00
parent 7058e0962e
commit d3132eb528
23 changed files with 636 additions and 397 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Tgstation.Server.Host/Components/Chat/ChannelMapping.cs
+5
View File
@@ -25,6 +25,11 @@
/// </summary>
public bool IsUpdatesChannel { get; set; }
/// <summary>
/// If the <see cref="Channel"/> is an admin channel.
/// </summary>
public bool IsAdminChannel { get; set; }
/// <summary>
/// The <see cref="ChannelRepresentation"/> with the mapped Id.
/// </summary>
src/Tgstation.Server.Host/Components/Chat/ChatManager.cs
+18 -4
View File
@@ -452,6 +452,7 @@ namespace Tgstation.Server.Host.Components.Chat
{
IsWatchdogChannel = x.IsWatchdogChannel == true,
IsUpdatesChannel = x.IsUpdatesChannel == true,
IsAdminChannel = x.IsAdminChannel == true,
ProviderChannelId = y.RealId,
ProviderId = connectionId,
Channel = y
@@ -596,12 +597,25 @@ namespace Tgstation.Server.Host.Components.Chat
}
/// <inheritdoc />
public Task SendWatchdogMessage(string message, CancellationToken cancellationToken)
public Task SendWatchdogMessage(string message, bool adminOnly, CancellationToken cancellationToken)
{
List<ulong> wdChannels;
List<ulong> wdChannels = null;
message = String.Format(CultureInfo.InvariantCulture, "WD: {0}", message);
lock (mappedChannels) // so it doesn't change while we're using it
wdChannels = mappedChannels.Where(x => x.Value.IsWatchdogChannel).Select(x => x.Key).ToList();
// so it doesn't change while we're using it
lock (mappedChannels)
{
if (adminOnly)
{
wdChannels = mappedChannels.Where(x => x.Value.IsAdminChannel).Select(x => x.Key).ToList();
if (wdChannels.Count == 0)
adminOnly = false;
}
if (!adminOnly)
wdChannels = mappedChannels.Where(x => x.Value.IsWatchdogChannel).Select(x => x.Key).ToList();
}
return SendMessage(message, wdChannels, cancellationToken);
}
src/Tgstation.Server.Host/Components/Chat/IChatManager.cs
+2 -1
View File
@@ -56,9 +56,10 @@ namespace Tgstation.Server.Host.Components.Chat
/// Send a chat <paramref name="message"/> to configured watchdog channels
/// </summary>
/// <param name="message">The message being sent</param>
/// <param name="adminOnly">If the message should be sent to admin channels only. If no admin-only channels are configured, it will be sent to the watchdog channels.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
/// <returns>A <see cref="Task"/> representing the running operation</returns>
Task SendWatchdogMessage(string message, CancellationToken cancellationToken);
Task SendWatchdogMessage(string message, bool adminOnly, CancellationToken cancellationToken);
/// <summary>
/// Send a chat <paramref name="message"/> to configured update channels
src/Tgstation.Server.Host/Components/Interop/DMApiConstants.cs
+1 -1
View File
@@ -33,7 +33,7 @@ namespace Tgstation.Server.Host.Components.Interop
/// <summary>
/// The DMAPI <see cref="Version"/> being used.
/// </summary>
public static readonly Version Version = new Version(5, 0, 0);
public static readonly Version Version = new Version(5, 1, 0);
/// <summary>
/// <see cref="JsonSerializerSettings"/> for use when communicating with the DMAPI.
src/Tgstation.Server.Host/Components/Interop/Topic/TopicCommandType.cs
+6 -1
View File
@@ -38,6 +38,11 @@
/// <summary>
/// The server's port was possibly changed.
/// </summary>
ServerPortUpdate
ServerPortUpdate,
/// <summary>
/// Ping to ensure the server is running.
/// </summary>
Heartbeat,
}
}
src/Tgstation.Server.Host/Components/Interop/Topic/TopicParameters.cs
+9
View File
@@ -113,5 +113,14 @@ namespace Tgstation.Server.Host.Components.Interop.Topic
{
ChatUpdate = channelsUpdate ?? throw new ArgumentNullException(nameof(channelsUpdate));
}
/// <summary>
/// Initializes a new instance of the <see cref="TopicParameters"/> <see langword="class"/>.
/// </summary>
/// <remarks>Constructor for <see cref="TopicCommandType.Heartbeat"/>s.</remarks>
public TopicParameters()
: this(TopicCommandType.Heartbeat)
{
}
}
}
src/Tgstation.Server.Host/Components/Interop/Topic/TopicResponse.cs
+2 -2
View File
@@ -10,11 +10,11 @@ namespace Tgstation.Server.Host.Components.Interop.Topic
/// <summary>
/// The text to reply with as the result of a <see cref="TopicCommandType.ChatCommand"/> request, if any.
/// </summary>
public string CommandResponseMessage { get; set; }
public string CommandResponseMessage { get; private set; }
/// <summary>
/// The <see cref="ChatMessage"/>s to send as the result of a <see cref="TopicCommandType.EventNotification"/> request, if any.
/// </summary>
public ICollection<ChatMessage> ChatResponses { get; set; }
public ICollection<ChatMessage> ChatResponses { get; private set; }
}
}
src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs
+59 -143
View File
@@ -1,5 +1,6 @@
using Microsoft.Extensions.Logging;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Threading;
using System.Threading.Tasks;
@@ -79,7 +80,25 @@ namespace Tgstation.Server.Host.Components.Watchdog
autoStart)
{ }
async Task<MonitorAction> HandleMonitorWakeup(MonitorActivationReason reason, CancellationToken cancellationToken)
/// <inheritdoc />
protected override IReadOnlyDictionary<MonitorActivationReason, Task> GetMonitoredServerTasks(MonitorState monitorState)
{
Logger.LogDebug("Server compile job ID is {0}", Server.Dmb.CompileJob.Id);
monitorState.ActiveServer = Server;
return new Dictionary<MonitorActivationReason, Task>
{
{ MonitorActivationReason.ActiveServerCrashed, Server.Lifetime },
{ MonitorActivationReason.ActiveServerRebooted, Server.OnReboot },
{ MonitorActivationReason.InactiveServerCrashed, Extensions.TaskExtensions.InfiniteTask() },
{ MonitorActivationReason.InactiveServerRebooted, Extensions.TaskExtensions.InfiniteTask() },
{ MonitorActivationReason.InactiveServerStartupComplete, Extensions.TaskExtensions.InfiniteTask() }
};
}
/// <inheritdoc />
protected override async Task HandleMonitorWakeup(MonitorActivationReason reason, MonitorState monitorState, CancellationToken cancellationToken)
{
switch (reason)
{
@@ -88,13 +107,31 @@ namespace Tgstation.Server.Host.Components.Watchdog
if (Server.RebootState == Watchdog.RebootState.Shutdown)
{
// the time for graceful shutdown is now
await Chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Server {0}! Exiting due to graceful termination request...", exitWord), cancellationToken).ConfigureAwait(false);
await Chat.SendWatchdogMessage(
String.Format(
CultureInfo.InvariantCulture,
"Server {0}! Stopping due to graceful termination request...",
exitWord),
false,
cancellationToken)
.ConfigureAwait(false);
DisposeAndNullControllers();
return MonitorAction.Exit;
monitorState.NextAction = MonitorAction.Exit;
}
else
{
await Chat.SendWatchdogMessage(
String.Format(
CultureInfo.InvariantCulture,
"Server {0}! Rebooting...",
exitWord),
false,
cancellationToken)
.ConfigureAwait(false);
monitorState.NextAction = MonitorAction.Restart;
}
await Chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Server {0}! Rebooting...", exitWord), cancellationToken).ConfigureAwait(false);
return MonitorAction.Restart;
break;
case MonitorActivationReason.ActiveServerRebooted:
var rebootState = Server.RebootState;
gracefulRebootSetDueToNewDmb = false;
@@ -103,28 +140,39 @@ namespace Tgstation.Server.Host.Components.Watchdog
switch (rebootState)
{
case Watchdog.RebootState.Normal:
return HandleNormalReboot();
monitorState.NextAction = HandleNormalReboot();
break;
case Watchdog.RebootState.Restart:
return MonitorAction.Restart;
monitorState.NextAction = MonitorAction.Restart;
break;
case Watchdog.RebootState.Shutdown:
// graceful shutdown time
await Chat.SendWatchdogMessage("Active server rebooted! Exiting due to graceful termination request...", cancellationToken).ConfigureAwait(false);
await Chat.SendWatchdogMessage(
"Active server rebooted! Stopping due to graceful termination request...",
false,
cancellationToken)
.ConfigureAwait(false);
DisposeAndNullControllers();
return MonitorAction.Exit;
monitorState.NextAction = MonitorAction.Exit;
break;
default:
throw new InvalidOperationException($"Invalid reboot state: {rebootState}");
}
break;
case MonitorActivationReason.ActiveLaunchParametersUpdated:
await Server.SetRebootState(Watchdog.RebootState.Restart, cancellationToken).ConfigureAwait(false);
return MonitorAction.Continue;
monitorState.NextAction = MonitorAction.Continue;
break;
case MonitorActivationReason.NewDmbAvailable:
await HandleNewDmbAvailable(cancellationToken).ConfigureAwait(false);
return MonitorAction.Continue;
monitorState.NextAction = MonitorAction.Continue;
break;
case MonitorActivationReason.InactiveServerCrashed:
case MonitorActivationReason.InactiveServerRebooted:
case MonitorActivationReason.InactiveServerStartupComplete:
throw new NotSupportedException($"Unsupported activation reason: {reason}");
case MonitorActivationReason.Heartbeat:
default:
throw new InvalidOperationException($"Invalid activation reason: {reason}");
}
@@ -238,138 +286,6 @@ namespace Tgstation.Server.Host.Components.Watchdog
}
}
/// <inheritdoc />
protected sealed override async Task MonitorLifetimes(CancellationToken cancellationToken)
{
Logger.LogTrace("Entered MonitorLifetimes");
// this function is responsible for calling HandlerMonitorWakeup when necessary and manitaining the MonitorState
var iteration = 1;
for (MonitorAction nextAction = MonitorAction.Continue; nextAction != MonitorAction.Exit; ++iteration)
{
// always start out with continue
nextAction = MonitorAction.Continue;
// dump some info to the logs
Logger.LogDebug("Iteration {0} of monitor loop", iteration);
try
{
Logger.LogDebug("Server Compile Job ID: {0}", Server.Dmb.CompileJob.Id);
// load the activation tasks into local variables
Task activeServerLifetime = Server.Lifetime;
var activeServerReboot = Server.OnReboot;
Task activeLaunchParametersChanged = ActiveParametersUpdated.Task;
var newDmbAvailable = DmbFactory.OnNewerDmb;
// cancel waiting if requested
var cancelTcs = new TaskCompletionSource<object>();
using (cancellationToken.Register(() => cancelTcs.SetCanceled()))
{
var toWaitOn = Task.WhenAny(activeServerLifetime, activeServerReboot, newDmbAvailable, cancelTcs.Task, activeLaunchParametersChanged);
// wait for something to happen
await toWaitOn.ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested();
}
var chatTask = Task.CompletedTask;
using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false))
{
// always run HandleMonitorWakeup from the context of the semaphore lock
// multiple things may have happened, handle them one at a time
for (var moreActivationsToProcess = true; moreActivationsToProcess && (nextAction == MonitorAction.Continue || nextAction == MonitorAction.Skip);)
{
MonitorActivationReason activationReason = default; // this will always be assigned before being used
// process the tasks in this order and call HandlerMonitorWakup for each
bool CheckActivationReason(ref Task task, MonitorActivationReason testActivationReason)
{
var taskCompleted = task?.IsCompleted == true;
task = null;
if (nextAction == MonitorAction.Skip)
nextAction = MonitorAction.Continue;
else if (taskCompleted)
{
activationReason = testActivationReason;
return true;
}
return false;
}
if (CheckActivationReason(ref activeServerLifetime, MonitorActivationReason.ActiveServerCrashed)
|| CheckActivationReason(ref activeServerReboot, MonitorActivationReason.ActiveServerRebooted)
|| CheckActivationReason(ref newDmbAvailable, MonitorActivationReason.NewDmbAvailable)
|| CheckActivationReason(ref activeLaunchParametersChanged, MonitorActivationReason.ActiveLaunchParametersUpdated))
{
Logger.LogTrace("Monitor activation: {0}", activationReason);
nextAction = await HandleMonitorWakeup(activationReason, cancellationToken).ConfigureAwait(false);
}
else
moreActivationsToProcess = false;
}
}
// full reboot required
if (nextAction == MonitorAction.Restart)
{
Logger.LogDebug("Next state action is to restart");
DisposeAndNullControllers();
for (var retryAttempts = 1; nextAction == MonitorAction.Restart; ++retryAttempts)
{
Exception launchException = null;
using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false))
try
{
// use LaunchImplNoLock without announcements or restarting the monitor
await LaunchImplNoLock(false, false, null, cancellationToken).ConfigureAwait(false);
if (Running)
{
Logger.LogDebug("Relaunch successful, resetting monitor state...");
break; // continue on main loop
}
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception e)
{
launchException = e;
}
await chatTask.ConfigureAwait(false);
if (!Running)
{
if (launchException == null)
Logger.LogWarning("Failed to automatically restart the watchdog! Attempt: {0}", retryAttempts);
else
Logger.LogWarning("Failed to automatically restart the watchdog! Attempt: {0}, Exception: {1}", retryAttempts, launchException);
var retryDelay = Math.Min(Math.Pow(2, retryAttempts), 3600); // max of one hour, increasing by a power of 2 each time
chatTask = Chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Failed to restart watchdog (Attempt: {0}), retrying in {1} seconds...", retryAttempts, retryDelay), cancellationToken);
await Task.WhenAll(AsyncDelayer.Delay(TimeSpan.FromSeconds(retryDelay), cancellationToken), chatTask).ConfigureAwait(false);
}
}
}
}
catch (OperationCanceledException)
{
Logger.LogDebug("Monitor cancelled");
break;
}
catch (Exception e)
{
// really, this should NEVER happen
Logger.LogError("Monitor crashed! Iteration: {0}, NextAction: {1}, Exception: {2}", iteration, nextAction, e);
await Chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Monitor crashed, this should NEVER happen! Please report this, full details in logs! Restarting monitor... Error: {0}", e.Message), cancellationToken).ConfigureAwait(false);
}
}
Logger.LogTrace("Monitor exiting...");
}
/// <summary>
/// Handler for <see cref="MonitorActivationReason.ActiveServerRebooted"/> when the <see cref="RebootState"/> is <see cref="RebootState.Normal"/>.
/// </summary>
src/Tgstation.Server.Host/Components/Watchdog/CombinedTopicResponse.cs
+32
View File
@@ -0,0 +1,32 @@
using System;
using Tgstation.Server.Host.Components.Interop.Topic;
namespace Tgstation.Server.Host.Components.Watchdog
{
/// <summary>
/// Combines a <see cref="global::Byond.TopicSender.TopicResponse"/> with a <see cref="TopicResponse"/>.
/// </summary>
sealed class CombinedTopicResponse
{
/// <summary>
/// The raw <see cref="global::Byond.TopicSender.TopicResponse"/>.
/// </summary>
public global::Byond.TopicSender.TopicResponse ByondTopicResponse { get; }
/// <summary>
/// The interop <see cref="TopicResponse"/>, if any.
/// </summary>
public TopicResponse InteropResponse { get; }
/// <summary>
/// Initializes a new instance of the <see cref="CombinedTopicResponse"/> <see langword="class"/>.
/// </summary>
/// <param name="byondTopicResponse">The value of <see cref="ByondTopicResponse"/>.</param>
/// <param name="interopResponse">The optional value of <see cref="InteropResponse"/>.</param>
public CombinedTopicResponse(global::Byond.TopicSender.TopicResponse byondTopicResponse, TopicResponse interopResponse)
{
ByondTopicResponse = byondTopicResponse ?? throw new ArgumentNullException(nameof(byondTopicResponse));
InteropResponse = interopResponse;
}
}
}
src/Tgstation.Server.Host/Components/Watchdog/DeadSessionController.cs
+6 -2
View File
@@ -42,6 +42,9 @@ namespace Tgstation.Server.Host.Components.Watchdog
/// <inheritdoc />
public Task OnReboot { get; }
/// <inheritdoc />
public Task OnPrime { get; }
/// <inheritdoc />
public Task<int> Lifetime { get; }
@@ -70,7 +73,8 @@ namespace Tgstation.Server.Host.Components.Watchdog
StartupTime = TimeSpan.FromSeconds(0)
});
Lifetime = Task.FromResult(-1);
OnReboot = new TaskCompletionSource<object>().Task;
OnReboot = Extensions.TaskExtensions.InfiniteTask();
OnPrime = Extensions.TaskExtensions.InfiniteTask();
disposeLock = new object();
}
@@ -97,7 +101,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
public void ResetRebootState() => throw new NotSupportedException();
/// <inheritdoc />
public Task<TopicResponse> SendCommand(TopicParameters parameters, CancellationToken cancellationToken) => throw new NotSupportedException();
public Task<CombinedTopicResponse> SendCommand(TopicParameters parameters, CancellationToken cancellationToken) => throw new NotSupportedException();
/// <inheritdoc />
public void SetHighPriority() => throw new NotSupportedException();
src/Tgstation.Server.Host/Components/Watchdog/ExperimentalWatchdog.cs
+82 -180
View File
@@ -1,6 +1,6 @@
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
using System.Threading;
@@ -92,15 +92,9 @@ namespace Tgstation.Server.Host.Components.Watchdog
alphaIsActive = true;
}
/// <summary>
/// Handles the actions to take when the monitor has to "wake up"
/// </summary>
/// <param name="activationReason">The <see cref="MonitorActivationReason"/> that caused the invocation</param>
/// <param name="monitorState">The current <see cref="MonitorState"/>. Will be modified upon retrn</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
/// <returns>A <see cref="Task"/> representing the running operation</returns>
/// <inheritdoc />
#pragma warning disable CA1502 // TODO: Decomplexify
async Task HandlerMonitorWakeup(MonitorActivationReason activationReason, MonitorState monitorState, CancellationToken cancellationToken)
protected override async Task HandleMonitorWakeup(MonitorActivationReason activationReason, MonitorState monitorState, CancellationToken cancellationToken)
{
Logger.LogDebug("Monitor activation. Reason: {0}", activationReason);
@@ -203,7 +197,10 @@ namespace Tgstation.Server.Host.Components.Watchdog
cancellationToken)
.ConfigureAwait(false);
monitorState.InactiveServer.SetHighPriority();
await Chat.SendWatchdogMessage("Staging newest DMB on inactive server failed: {0} Falling back to previous dmb...", cancellationToken).ConfigureAwait(false);
await Chat.SendWatchdogMessage(
"Staging newest DMB on inactive server failed: {0} Falling back to previous dmb...",
false,
cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
@@ -214,7 +211,10 @@ namespace Tgstation.Server.Host.Components.Watchdog
// fuuuuucckkk
Logger.LogError("Backup strategy failed! Monitor will restart when active server reboots! Exception: {0}", e2.ToString());
monitorState.InactiveServerCritFail = true;
await Chat.SendWatchdogMessage("Attempted reboot of inactive server failed. Watchdog will reset when active server fails or exits", cancellationToken).ConfigureAwait(false);
await Chat.SendWatchdogMessage(
"Attempted reboot of inactive server failed. Watchdog will reset when active server fails or exits",
false,
cancellationToken).ConfigureAwait(false);
return;
}
}
@@ -232,7 +232,13 @@ namespace Tgstation.Server.Host.Components.Watchdog
if (monitorState.ActiveServer.RebootState == Watchdog.RebootState.Shutdown)
{
// the time for graceful shutdown is now
await Chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Active server {0}! Exiting due to graceful termination request...", ExitWord(monitorState.ActiveServer)), cancellationToken).ConfigureAwait(false);
await Chat.SendWatchdogMessage(
String.Format(
CultureInfo.InvariantCulture,
"Active server {0}! Exiting due to graceful termination request...",
ExitWord(monitorState.ActiveServer)),
false,
cancellationToken).ConfigureAwait(false);
DisposeAndNullControllers();
monitorState.NextAction = MonitorAction.Exit;
break;
@@ -241,14 +247,27 @@ namespace Tgstation.Server.Host.Components.Watchdog
if (FullRestartDeadInactive())
{
// tell chat about it and go ahead
await Chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Active server {0}! Inactive server unable to online!", ExitWord(monitorState.ActiveServer)), cancellationToken).ConfigureAwait(false);
await Chat.SendWatchdogMessage(
String.Format(
CultureInfo.InvariantCulture,
"Active server {0}! Inactive server unable to online!",
ExitWord(monitorState.ActiveServer)),
false,
cancellationToken).ConfigureAwait(false);
// we've already been set to restart
break;
}
// tell chat about it
await Chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Active server {0}! Onlining inactive server...", ExitWord(monitorState.ActiveServer)), cancellationToken).ConfigureAwait(false);
await Chat.SendWatchdogMessage(
String.Format(
CultureInfo.InvariantCulture,
"Active server {0}! Onlining inactive server...",
ExitWord(monitorState.ActiveServer)),
false,
cancellationToken)
.ConfigureAwait(false);
// try to activate the inactive server
if (!await MakeInactiveActive().ConfigureAwait(false))
@@ -259,7 +278,14 @@ namespace Tgstation.Server.Host.Components.Watchdog
break;
case MonitorActivationReason.InactiveServerCrashed:
// just announce and try to bring it back
await Chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Inactive server {0}! Rebooting...", ExitWord(monitorState.InactiveServer)), cancellationToken).ConfigureAwait(false);
await Chat.SendWatchdogMessage(
String.Format(
CultureInfo.InvariantCulture,
"Inactive server {0}! Rebooting...",
ExitWord(monitorState.InactiveServer)),
false,
cancellationToken)
.ConfigureAwait(false);
await UpdateAndRestartInactiveServer(false).ConfigureAwait(false);
break;
case MonitorActivationReason.ActiveServerRebooted:
@@ -285,7 +311,11 @@ namespace Tgstation.Server.Host.Components.Watchdog
break;
case Watchdog.RebootState.Shutdown:
// graceful shutdown time
await Chat.SendWatchdogMessage("Active server rebooted! Exiting due to graceful termination request...", cancellationToken).ConfigureAwait(false);
await Chat.SendWatchdogMessage(
"Active server rebooted! Stopping due to graceful termination request...",
false,
cancellationToken)
.ConfigureAwait(false);
DisposeAndNullControllers();
monitorState.NextAction = MonitorAction.Exit;
return;
@@ -352,12 +382,16 @@ namespace Tgstation.Server.Host.Components.Watchdog
// just reload the inactive server and wait for a swap to apply the changes
await UpdateAndRestartInactiveServer(true).ConfigureAwait(false);
break;
case MonitorActivationReason.Heartbeat:
default:
Trace.Assert(false, String.Format(CultureInfo.InvariantCulture, "Invalid monitor activation reason: {0}!", activationReason));
break;
throw new InvalidOperationException(
String.Format(
CultureInfo.InvariantCulture,
"Invalid monitor activation reason: {0}!",
activationReason));
}
}
#pragma warning restore CA1502
#pragma warning restore CA1502
/// <summary>
/// Call <see cref="IDisposable.Dispose"/> on <see cref="alphaServer"/> and <see cref="bravoServer"/> and set them to <see langword="null"/>
@@ -372,169 +406,37 @@ namespace Tgstation.Server.Host.Components.Watchdog
}
/// <inheritdoc />
#pragma warning disable CA1502 // TODO: Decomplexify
protected override async Task MonitorLifetimes(CancellationToken cancellationToken)
protected override IReadOnlyDictionary<MonitorActivationReason, Task> GetMonitoredServerTasks(MonitorState monitorState)
{
Logger.LogTrace("Entered MonitorLifetimes");
if (AlphaIsActive)
Logger.LogDebug("Alpha is the active server");
else
Logger.LogDebug("Bravo is the active server");
// this function is responsible for calling HandlerMonitorWakeup when necessary and manitaining the MonitorState
var iteration = 1;
for (var monitorState = new MonitorState(); monitorState.NextAction != MonitorAction.Exit; ++iteration)
if (monitorState.RebootingInactiveServer)
Logger.LogDebug("Inactive server is rebooting");
// update the monitor state with the inactive/active servers
monitorState.ActiveServer = AlphaIsActive ? alphaServer : bravoServer;
monitorState.InactiveServer = AlphaIsActive ? bravoServer : alphaServer;
if (monitorState.ActiveServer.ClosePortOnReboot)
Logger.LogDebug("Active server will close port on reboot");
if (monitorState.InactiveServer.ClosePortOnReboot)
Logger.LogDebug("Inactive server will close port on reboot");
Logger.LogDebug("Active server Compile Job ID: {0}", monitorState.ActiveServer.Dmb.CompileJob.Id);
Logger.LogDebug("Inactive server Compile Job ID: {0}", monitorState.InactiveServer.Dmb.CompileJob.Id);
return new Dictionary<MonitorActivationReason, Task>
{
// always start out with continue
monitorState.NextAction = MonitorAction.Continue;
// dump some info to the logs
Logger.LogDebug("Iteration {0} of monitor loop", iteration);
try
{
if (AlphaIsActive)
Logger.LogDebug("Alpha is the active server");
else
Logger.LogDebug("Bravo is the active server");
if (monitorState.RebootingInactiveServer)
Logger.LogDebug("Inactive server is rebooting");
// update the monitor state with the inactive/active servers
monitorState.ActiveServer = AlphaIsActive ? alphaServer : bravoServer;
monitorState.InactiveServer = AlphaIsActive ? bravoServer : alphaServer;
if (monitorState.ActiveServer.ClosePortOnReboot)
Logger.LogDebug("Active server will close port on reboot");
if (monitorState.InactiveServer.ClosePortOnReboot)
Logger.LogDebug("Inactive server will close port on reboot");
Logger.LogDebug("Active server Compile Job ID: {0}", monitorState.ActiveServer.Dmb.CompileJob.Id);
Logger.LogDebug("Inactive server Compile Job ID: {0}", monitorState.InactiveServer.Dmb.CompileJob.Id);
// load the activation tasks into local variables
Task activeServerLifetime = monitorState.ActiveServer.Lifetime;
Task inactiveServerLifetime = monitorState.InactiveServer.Lifetime;
var activeServerReboot = monitorState.ActiveServer.OnReboot;
var inactiveServerReboot = monitorState.InactiveServer.OnReboot;
Task inactiveServerStartup = monitorState.RebootingInactiveServer ? monitorState.InactiveServer.LaunchResult : null;
Task activeLaunchParametersChanged = ActiveParametersUpdated.Task;
var newDmbAvailable = DmbFactory.OnNewerDmb;
// cancel waiting if requested
var cancelTcs = new TaskCompletionSource<object>();
using (cancellationToken.Register(() => cancelTcs.SetCanceled()))
{
var toWaitOn = Task.WhenAny(activeServerLifetime, inactiveServerLifetime, activeServerReboot, inactiveServerReboot, newDmbAvailable, cancelTcs.Task, activeLaunchParametersChanged);
if (monitorState.RebootingInactiveServer)
toWaitOn = Task.WhenAny(toWaitOn, inactiveServerStartup);
// wait for something to happen
await toWaitOn.ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested();
}
var chatTask = Task.CompletedTask;
using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false))
{
// always run HandleMonitorWakeup from the context of the semaphore lock
// multiple things may have happened, handle them one at a time
for (var moreActivationsToProcess = true; moreActivationsToProcess && (monitorState.NextAction == MonitorAction.Continue || monitorState.NextAction == MonitorAction.Skip);)
{
MonitorActivationReason activationReason = default; // this will always be assigned before being used
// process the tasks in this order and call HandlerMonitorWakup for each
bool CheckActivationReason(ref Task task, MonitorActivationReason testActivationReason)
{
var taskCompleted = task?.IsCompleted == true;
task = null;
if (monitorState.NextAction == MonitorAction.Skip)
monitorState.NextAction = MonitorAction.Continue;
else if (taskCompleted)
{
activationReason = testActivationReason;
return true;
}
return false;
}
if (CheckActivationReason(ref activeServerLifetime, MonitorActivationReason.ActiveServerCrashed)
|| CheckActivationReason(ref inactiveServerLifetime, MonitorActivationReason.InactiveServerCrashed)
|| CheckActivationReason(ref activeServerReboot, MonitorActivationReason.ActiveServerRebooted)
|| CheckActivationReason(ref inactiveServerReboot, MonitorActivationReason.InactiveServerRebooted)
|| CheckActivationReason(ref inactiveServerStartup, MonitorActivationReason.InactiveServerStartupComplete)
|| CheckActivationReason(ref newDmbAvailable, MonitorActivationReason.NewDmbAvailable)
|| CheckActivationReason(ref activeLaunchParametersChanged, MonitorActivationReason.ActiveLaunchParametersUpdated))
{
Logger.LogTrace("Monitor activation: {0}", activationReason);
await HandlerMonitorWakeup(activationReason, monitorState, cancellationToken).ConfigureAwait(false);
}
else
moreActivationsToProcess = false;
}
// writeback alphaServer and bravoServer from monitor state in case they changesd
alphaServer = AlphaIsActive ? monitorState.ActiveServer : monitorState.InactiveServer;
bravoServer = !AlphaIsActive ? monitorState.ActiveServer : monitorState.InactiveServer;
}
// full reboot required
if (monitorState.NextAction == MonitorAction.Restart)
{
Logger.LogDebug("Next state action is to restart");
DisposeAndNullControllers();
chatTask = Chat.SendWatchdogMessage("Restarting entirely due to complications...", cancellationToken);
for (var retryAttempts = 1; monitorState.NextAction == MonitorAction.Restart; ++retryAttempts)
{
Exception launchException = null;
using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false))
try
{
// use LaunchImplNoLock without announcements or restarting the monitor
await LaunchImplNoLock(false, false, null, cancellationToken).ConfigureAwait(false);
if (Running)
{
Logger.LogDebug("Relaunch successful, resetting monitor state...");
monitorState = new MonitorState(); // clean the slate and continue
}
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception e)
{
launchException = e;
}
await chatTask.ConfigureAwait(false);
if (!Running)
{
if (launchException == null)
Logger.LogWarning("Failed to automatically restart the watchdog! Attempt: {0}", retryAttempts);
else
Logger.LogWarning("Failed to automatically restart the watchdog! Attempt: {0}, Exception: {1}", retryAttempts, launchException);
var retryDelay = Math.Min(Math.Pow(2, retryAttempts), 3600); // max of one hour, increasing by a power of 2 each time
chatTask = Chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Failed to restart watchdog (Attempt: {0}), retrying in {1} seconds...", retryAttempts, retryDelay), cancellationToken);
await Task.WhenAll(AsyncDelayer.Delay(TimeSpan.FromSeconds(retryDelay), cancellationToken), chatTask).ConfigureAwait(false);
}
}
}
}
catch (OperationCanceledException)
{
Logger.LogDebug("Monitor cancelled");
break;
}
catch (Exception e)
{
// really, this should NEVER happen
Logger.LogError("Monitor crashed! Iteration: {0}, State: {1}, Exception: {2}", iteration, JsonConvert.SerializeObject(monitorState), e);
await Chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Monitor crashed, this should NEVER happen! Please report this, full details in logs! Restarting monitor... Error: {0}", e.Message), cancellationToken).ConfigureAwait(false);
}
}
Logger.LogTrace("Monitor exiting...");
{ MonitorActivationReason.ActiveServerCrashed, monitorState.ActiveServer.Lifetime },
{ MonitorActivationReason.ActiveServerRebooted, monitorState.ActiveServer.OnReboot },
{ MonitorActivationReason.InactiveServerCrashed, monitorState.InactiveServer.Lifetime },
{ MonitorActivationReason.InactiveServerRebooted, monitorState.InactiveServer.OnReboot },
{ MonitorActivationReason.InactiveServerStartupComplete, monitorState.InactiveServer.OnPrime }
};
}
#pragma warning restore CA1502
/// <inheritdoc />
#pragma warning disable CA1502 // TODO: Decomplexify
@@ -611,7 +513,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
const string FailReattachMessage = "Unable to properly reattach to active server! Restarting...";
Logger.LogWarning(FailReattachMessage);
Logger.LogDebug(bothServersDead ? "Also could not reattach to inactive server!" : "Inactive server was reattached successfully!");
chatTask = Chat.SendWatchdogMessage(FailReattachMessage, cancellationToken);
chatTask = Chat.SendWatchdogMessage(FailReattachMessage, false, cancellationToken);
callBeforeRecurse();
await LaunchImplNoLock(true, false, null, cancellationToken).ConfigureAwait(false);
await chatTask.ConfigureAwait(false);
@@ -620,7 +522,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
// we still have the active server but the other one is dead to us, hand it off to the monitor to restart
const string InactiveReattachFailureMessage = "Unable to reattach to inactive server. Leaving for monitor to reboot...";
chatTask = Chat.SendWatchdogMessage(InactiveReattachFailureMessage, cancellationToken);
chatTask = Chat.SendWatchdogMessage(InactiveReattachFailureMessage, false, cancellationToken);
Logger.LogWarning(InactiveReattachFailureMessage);
if (reattachInfo.AlphaIsActive)
src/Tgstation.Server.Host/Components/Watchdog/ISessionController.cs
+6 -1
View File
@@ -62,6 +62,11 @@ namespace Tgstation.Server.Host.Components.Watchdog
/// </summary>
Task OnReboot { get; }
/// <summary>
/// A <see cref="Task"/> that completes when the server calls /world/TgsInitializationsComplete()
/// </summary>
Task OnPrime { get; }
/// <summary>
/// Releases the <see cref="IProcess"/> without terminating it. Also calls <see cref="IDisposable.Dispose"/>
/// </summary>
@@ -74,7 +79,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
/// <param name="parameters">The <see cref="TopicParameters"/> to send.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
/// <returns>A <see cref="Task{TResult}"/> resulting in the <see cref="TopicResponse"/> of /world/Topic()</returns>
Task<TopicResponse> SendCommand(TopicParameters parameters, CancellationToken cancellationToken);
Task<CombinedTopicResponse> SendCommand(TopicParameters parameters, CancellationToken cancellationToken);
/// <summary>
/// Causes the world to start listening on a <paramref name="newPort"/>
src/Tgstation.Server.Host/Components/Watchdog/MonitorActivationReason.cs
+6 -1
View File
@@ -38,6 +38,11 @@
/// <summary>
/// Server launch parameters were changed
/// </summary>
ActiveLaunchParametersUpdated
ActiveLaunchParametersUpdated,
/// <summary>
/// A heartbeat is required.
/// </summary>
Heartbeat,
}
}
src/Tgstation.Server.Host/Components/Watchdog/MonitorState.cs
+1 -1
View File
@@ -18,7 +18,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
public bool InactiveServerCritFail { get; set; }
/// <summary>
/// The next <see cref="MonitorAction"/> to take in <see cref="ExperimentalWatchdog.MonitorLifetimes(global::System.Threading.CancellationToken)"/>
/// The next <see cref="MonitorAction"/> to take in <see cref="WatchdogBase.MonitorLifetimes(global::System.Threading.CancellationToken)"/>
/// </summary>
public MonitorAction NextAction { get; set; }
src/Tgstation.Server.Host/Components/Watchdog/SessionController.cs
+40 -22
View File
@@ -96,6 +96,9 @@ namespace Tgstation.Server.Host.Components.Watchdog
/// <inheritdoc />
public Task OnReboot => rebootTcs.Task;
/// <inheritdoc />
public Task OnPrime => primeTcs.Task;
/// <summary>
/// The up to date <see cref="ReattachInformation"/>
/// </summary>
@@ -156,6 +159,11 @@ namespace Tgstation.Server.Host.Components.Watchdog
/// </summary>
TaskCompletionSource<object> rebootTcs;
/// <summary>
/// The <see cref="TaskCompletionSource{TResult}"/> that completes when DD tells us it's primed.
/// </summary>
TaskCompletionSource<object> primeTcs;
/// <summary>
/// If we know DreamDaemon currently has it's port closed
/// </summary>
@@ -218,6 +226,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
released = false;
rebootTcs = new TaskCompletionSource<object>();
primeTcs = new TaskCompletionSource<object>();
synchronizationLock = new object();
@@ -365,7 +374,9 @@ namespace Tgstation.Server.Host.Components.Watchdog
cancellationToken).ConfigureAwait(false);
break;
case BridgeCommandType.Prime:
// currently unused, maybe in the future
var oldPrimeTcs = primeTcs;
primeTcs = new TaskCompletionSource<object>();
oldPrimeTcs.SetResult(null);
break;
case BridgeCommandType.Kill:
logger.LogInformation("Bridge requested process termination!");
@@ -451,9 +462,9 @@ namespace Tgstation.Server.Host.Components.Watchdog
portClosedForReboot = true;
}
var oldTcs = rebootTcs;
var oldRebootTcs = rebootTcs;
rebootTcs = new TaskCompletionSource<object>();
oldTcs.SetResult(null);
oldRebootTcs.SetResult(null);
break;
case null:
response.ErrorMessage = "Missing commandType!";
@@ -495,7 +506,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
}
/// <inheritdoc />
public async Task<Interop.Topic.TopicResponse> SendCommand(TopicParameters parameters, CancellationToken cancellationToken)
public async Task<CombinedTopicResponse> SendCommand(TopicParameters parameters, CancellationToken cancellationToken)
{
if (Lifetime.IsCompleted)
{
@@ -509,6 +520,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
var json = JsonConvert.SerializeObject(parameters, DMApiConstants.SerializerSettings);
logger.LogTrace("Topic request: {0}", json);
Exception caughtException;
try
{
var commandString = String.Format(CultureInfo.InvariantCulture,
@@ -524,33 +536,39 @@ namespace Tgstation.Server.Host.Components.Watchdog
cancellationToken).ConfigureAwait(false);
var topicReturn = topicResponse.StringData;
if (topicReturn != null)
logger.LogTrace("Topic response: {0}", topicReturn);
try
{
var result = JsonConvert.DeserializeObject<Interop.Topic.TopicResponse>(topicReturn, DMApiConstants.SerializerSettings);
if (result.ErrorMessage != null)
Interop.Topic.TopicResponse interopResponse = null;
if (topicReturn != null)
try
{
logger.LogWarning("Errored topic response for command {0}: {1}", parameters.CommandType, result.ErrorMessage);
interopResponse = JsonConvert.DeserializeObject<Interop.Topic.TopicResponse>(topicReturn, DMApiConstants.SerializerSettings);
if (interopResponse.ErrorMessage != null)
{
logger.LogWarning("Errored topic response for command {0}: {1}", parameters.CommandType, interopResponse.ErrorMessage);
}
logger.LogTrace("Interop response: {0}", topicReturn);
}
catch
{
logger.LogWarning("Invalid interop response: {0}", topicReturn);
}
return result;
}
catch
{
logger.LogWarning("Invalid topic response: {0}", topicReturn);
}
return new CombinedTopicResponse(topicResponse, interopResponse);
}
catch (OperationCanceledException)
catch (OperationCanceledException e)
{
throw;
cancellationToken.ThrowIfCancellationRequested();
caughtException = e;
}
catch (Exception e)
{
logger.LogWarning("Send command exception:{0}{1}", Environment.NewLine, e.Message);
caughtException = e;
}
if (caughtException == null)
logger.LogWarning("Send command exception:{0}{1}", Environment.NewLine, caughtException.Message);
return null;
}
@@ -569,7 +587,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
cancellationToken)
.ConfigureAwait(false);
if (commandResult.ErrorMessage != null)
if (commandResult.InteropResponse?.ErrorMessage != null)
return false;
reattachInformation.Port = port;
@@ -600,7 +618,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
cancellationToken)
.ConfigureAwait(false);
return result != null && result.ErrorMessage == null;
return result?.InteropResponse != null && result.InteropResponse?.ErrorMessage == null;
}
/// <inheritdoc />
src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs
+274 -20
View File
@@ -1,5 +1,6 @@
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Linq;
@@ -117,6 +118,11 @@ namespace Tgstation.Server.Host.Components.Watchdog
/// </summary>
Task monitorTask;
/// <summary>
/// The number of hearbeats missed.
/// </summary>
int heartbeatsMissed;
/// <summary>
/// If the servers should be released instead of shutdown
/// </summary>
@@ -206,7 +212,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
return;
if (!graceful)
{
var chatTask = announce ? Chat.SendWatchdogMessage("Terminating...", cancellationToken) : Task.CompletedTask;
var chatTask = announce ? Chat.SendWatchdogMessage("Terminating...", false, cancellationToken) : Task.CompletedTask;
await StopMonitor().ConfigureAwait(false);
DisposeAndNullControllers();
@@ -223,6 +229,56 @@ namespace Tgstation.Server.Host.Components.Watchdog
await toKill.SetRebootState(Watchdog.RebootState.Shutdown, cancellationToken).ConfigureAwait(false);
}
/// <summary>
/// Handles a watchdog heartbeat.
/// </summary>
/// <param name="activeServer">The active <see cref="ISessionController"/>.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
/// <returns>A <see cref="Task{TResult}"/> resulting in the next <see cref="MonitorAction"/> to take.</returns>
async Task<MonitorAction> HandleHeartbeat(ISessionController activeServer, CancellationToken cancellationToken)
{
Logger.LogTrace("Sending heartbeat to session controller on :{0}", activeServer.Port);
var response = await activeServer.SendCommand(new TopicParameters(), cancellationToken).ConfigureAwait(false);
if (response == null)
{
switch (++heartbeatsMissed)
{
case 1:
Logger.LogDebug("DEFCON 4: Watchdog missed first heartbeat!");
break;
case 2:
var message2 = "DEFCON 3: Watchdog has missed 2 heartbeats!";
Logger.LogInformation(message2);
await Chat.SendWatchdogMessage(message2, true, cancellationToken).ConfigureAwait(false);
break;
case 3:
var message3 = "DEFCON 2: Watchdog has missed 3 heartbeats! If DreamDaemon does not respond to the next one, the server will be restarted!";
Logger.LogWarning(message3);
await Chat.SendWatchdogMessage(message3, false, cancellationToken).ConfigureAwait(false);
break;
case 4:
var shouldShutdown = activeServer.RebootState == Watchdog.RebootState.Shutdown;
var actionToTake = shouldShutdown
? "Shutting down due to graceful termination request"
: "Restarting";
var message4 = $"DEFCON 1: Four heartbeats have been missed! {actionToTake}...";
Logger.LogWarning(message4);
DisposeAndNullControllers();
await Chat.SendWatchdogMessage(message4, false, cancellationToken).ConfigureAwait(false);
return shouldShutdown ? MonitorAction.Exit : MonitorAction.Restart;
default:
Logger.LogError("Invalid heartbeats missed count: {0}", heartbeatsMissed);
break;
}
}
else
heartbeatsMissed = 0;
return MonitorAction.Continue;
}
/// <summary>
/// Launches the watchdog.
/// </summary>
@@ -244,14 +300,15 @@ namespace Tgstation.Server.Host.Components.Watchdog
// this is necessary, the monitor could be in it's sleep loop trying to restart, if so cancel THAT monitor and start our own with blackjack and hookers
Task chatTask;
if (startMonitor && await StopMonitor().ConfigureAwait(false))
chatTask = Chat.SendWatchdogMessage("Automatic retry sequence cancelled by manual launch. Restarting...", cancellationToken);
chatTask = Chat.SendWatchdogMessage("Automatic retry sequence cancelled by manual launch. Restarting...", false, cancellationToken);
else if (announce)
chatTask = Chat.SendWatchdogMessage(reattachInfo == null ? "Starting..." : "Reattaching...", cancellationToken); // simple announce
chatTask = Chat.SendWatchdogMessage(reattachInfo == null ? "Starting..." : "Reattaching...", false, cancellationToken); // simple announce
else
chatTask = Task.CompletedTask; // no announce
// since neither server is running, this is safe to do
LastLaunchParameters = ActiveLaunchParameters;
heartbeatsMissed = 0;
// for when we call ourself and want to not catch thrown exceptions
var ignoreNestedException = false;
@@ -277,7 +334,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
async Task ChainChatTaskWithErrorMessage()
{
await originalChatTask.ConfigureAwait(false);
await Chat.SendWatchdogMessage("Startup failed!", cancellationToken).ConfigureAwait(false);
await Chat.SendWatchdogMessage("Startup failed!", false, cancellationToken).ConfigureAwait(false);
}
chatTask = ChainChatTaskWithErrorMessage();
@@ -337,7 +394,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
const string FailReattachMessage = "Unable to properly reattach to server! Restarting...";
Logger.LogWarning(FailReattachMessage);
Logger.LogDebug(inactiveReattachSuccess ? "Also could not reattach to inactive server!" : "Inactive server was reattached successfully!");
Task chatTask = Chat.SendWatchdogMessage(FailReattachMessage, cancellationToken);
Task chatTask = Chat.SendWatchdogMessage(FailReattachMessage, false, cancellationToken);
await LaunchImplNoLock(true, false, null, cancellationToken).ConfigureAwait(false);
await chatTask.ConfigureAwait(false);
}
@@ -382,11 +439,201 @@ namespace Tgstation.Server.Host.Components.Watchdog
protected abstract WatchdogReattachInformation CreateReattachInformation();
/// <summary>
/// The loop that watches the watchdog
/// Gets the tasks for the following <see cref="MonitorActivationReason"/>s: <see cref="MonitorActivationReason.ActiveServerCrashed"/>, <see cref="MonitorActivationReason.ActiveServerRebooted"/>, <see cref="MonitorActivationReason.InactiveServerCrashed"/>, <see cref="MonitorActivationReason.InactiveServerRebooted"/>, <see cref="MonitorActivationReason.InactiveServerStartupComplete"/>.
/// </summary>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
/// <returns>A <see cref="Task"/> representing the running operation</returns>
protected abstract Task MonitorLifetimes(CancellationToken cancellationToken);
/// <param name="monitorState">The current <see cref="MonitorState"/>.</param>
/// <returns>A <see cref="IReadOnlyDictionary{TKey, TValue}"/> of the <see cref="Task"/>s keyed by their <see cref="MonitorActivationReason"/>.</returns>
protected abstract IReadOnlyDictionary<MonitorActivationReason, Task> GetMonitoredServerTasks(MonitorState monitorState);
/// <summary>
/// Handles the actions to take when the monitor has to "wake up"
/// </summary>
/// <param name="activationReason">The <see cref="MonitorActivationReason"/> that caused the invocation.</param>
/// <param name="monitorState">The current <see cref="MonitorState"/>.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
/// <returns>A <see cref="Task"/> representing the running operation.</returns>
protected abstract Task HandleMonitorWakeup(
MonitorActivationReason activationReason,
MonitorState monitorState,
CancellationToken cancellationToken);
/// <summary>
/// The loop that watches the watchdog.
/// </summary>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
/// <returns>A <see cref="Task"/> representing the running operation.</returns>
private async Task MonitorLifetimes(CancellationToken cancellationToken)
{
Logger.LogTrace("Entered MonitorLifetimes");
// this function is responsible for calling HandlerMonitorWakeup when necessary and manitaining the MonitorState
var iteration = 1;
for (var monitorState = new MonitorState(); monitorState.NextAction != MonitorAction.Exit; ++iteration)
{
Logger.LogDebug("Iteration {0} of monitor loop", iteration);
try
{
// load the activation tasks into local variables
cancellationToken.ThrowIfCancellationRequested();
var serverTasks = GetMonitoredServerTasks(monitorState);
if (serverTasks.Count != 5)
throw new InvalidOperationException("Expected 5 monitored server tasks!");
var activeServerLifetime = serverTasks[MonitorActivationReason.ActiveServerCrashed];
var activeServerReboot = serverTasks[MonitorActivationReason.ActiveServerRebooted];
var inactiveServerLifetime = serverTasks[MonitorActivationReason.InactiveServerCrashed];
var inactiveServerReboot = serverTasks[MonitorActivationReason.InactiveServerRebooted];
var inactiveStartupComplete = serverTasks[MonitorActivationReason.InactiveServerStartupComplete];
Task activeLaunchParametersChanged = ActiveParametersUpdated.Task;
var newDmbAvailable = DmbFactory.OnNewerDmb;
var heartbeatSeconds = ActiveLaunchParameters.HeartbeatSeconds.Value;
var heartbeat = heartbeatSeconds == 0
? Extensions.TaskExtensions.InfiniteTask()
: Task.Delay(TimeSpan.FromSeconds(heartbeatSeconds));
// cancel waiting if requested
var cancelTcs = new TaskCompletionSource<object>();
using (cancellationToken.Register(() => cancelTcs.SetCanceled()))
{
var toWaitOn = Task.WhenAny(
activeServerLifetime,
activeServerReboot,
inactiveServerLifetime,
inactiveServerReboot,
inactiveStartupComplete,
heartbeat,
newDmbAvailable,
cancelTcs.Task,
activeLaunchParametersChanged);
// wait for something to happen
await toWaitOn.ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested();
}
var chatTask = Task.CompletedTask;
using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false))
{
// always run HandleMonitorWakeup from the context of the semaphore lock
// multiple things may have happened, handle them one at a time
for (var moreActivationsToProcess = true; moreActivationsToProcess && (monitorState.NextAction == MonitorAction.Continue || monitorState.NextAction == MonitorAction.Skip);)
{
MonitorActivationReason activationReason = default; // this will always be assigned before being used
// process the tasks in this order and call HandlerMonitorWakup for each
bool CheckActivationReason(ref Task task, MonitorActivationReason testActivationReason)
{
var taskCompleted = task?.IsCompleted == true;
task = null;
if (monitorState.NextAction == MonitorAction.Skip)
monitorState.NextAction = MonitorAction.Continue;
else if (taskCompleted)
{
activationReason = testActivationReason;
return true;
}
return false;
}
if (CheckActivationReason(ref activeServerLifetime, MonitorActivationReason.ActiveServerCrashed)
|| CheckActivationReason(ref activeServerReboot, MonitorActivationReason.ActiveServerRebooted)
|| CheckActivationReason(ref newDmbAvailable, MonitorActivationReason.NewDmbAvailable)
|| CheckActivationReason(ref inactiveServerLifetime, MonitorActivationReason.InactiveServerCrashed)
|| CheckActivationReason(ref inactiveServerReboot, MonitorActivationReason.InactiveServerRebooted)
|| CheckActivationReason(ref inactiveStartupComplete, MonitorActivationReason.InactiveServerStartupComplete)
|| CheckActivationReason(ref activeLaunchParametersChanged, MonitorActivationReason.ActiveLaunchParametersUpdated))
{
Logger.LogTrace("Monitor activation: {0}", activationReason);
await HandleMonitorWakeup(activationReason, monitorState, cancellationToken).ConfigureAwait(false);
}
else if (CheckActivationReason(ref heartbeat, MonitorActivationReason.Heartbeat))
{
Logger.LogTrace("Monitor activation: {0}", activationReason);
monitorState.NextAction = await HandleHeartbeat(monitorState.ActiveServer, cancellationToken).ConfigureAwait(false);
}
else
moreActivationsToProcess = false;
}
}
// full reboot required
if (monitorState.NextAction == MonitorAction.Restart)
{
Logger.LogDebug("Next state action is to restart");
DisposeAndNullControllers();
for (var retryAttempts = 1; ; ++retryAttempts)
{
Exception launchException = null;
using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false))
try
{
// use LaunchImplNoLock without announcements or restarting the monitor
await LaunchImplNoLock(false, false, null, cancellationToken).ConfigureAwait(false);
if (Running)
{
Logger.LogDebug("Relaunch successful, resetting monitor state...");
break; // continue on main loop
}
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception e)
{
launchException = e;
}
await chatTask.ConfigureAwait(false);
if (!Running)
{
if (launchException == null)
Logger.LogWarning("Failed to automatically restart the watchdog! Attempt: {0}", retryAttempts);
else
Logger.LogWarning("Failed to automatically restart the watchdog! Attempt: {0}, Exception: {1}", retryAttempts, launchException);
var retryDelay = Math.Min(
Convert.ToInt32(
Math.Pow(2, retryAttempts)),
TimeSpan.FromHours(1).Seconds); // max of one hour, increasing by a power of 2 each time
chatTask = Chat.SendWatchdogMessage(
$"Failed to restart (Attempt: {retryAttempts}), retrying in {retryDelay}",
false,
cancellationToken);
await Task.WhenAll(
AsyncDelayer.Delay(
TimeSpan.FromSeconds(retryDelay),
cancellationToken),
chatTask)
.ConfigureAwait(false);
}
}
}
}
catch (OperationCanceledException)
{
Logger.LogDebug("Monitor cancelled");
break;
}
catch (Exception e)
{
// really, this should NEVER happen
Logger.LogError(
"Monitor crashed! Iteration: {0}, Monitor State: {1}, Exception: {2}",
iteration,
JsonConvert.SerializeObject(monitorState),
e);
await Chat.SendWatchdogMessage(
$"Monitor crashed, this should NEVER happen! Please report this, full details in logs! Restarting monitor... Error: {e.Message}",
false,
cancellationToken)
.ConfigureAwait(false);
}
}
Logger.LogTrace("Monitor exiting...");
}
/// <summary>
/// Starts all <see cref="ISessionController"/>s.
@@ -417,7 +664,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
if (!Running)
return true;
TopicResponse result;
CombinedTopicResponse result;
using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false))
{
if (!Running)
@@ -437,11 +684,11 @@ namespace Tgstation.Server.Host.Components.Watchdog
.ConfigureAwait(false);
}
if (result?.ChatResponses == null)
if (result?.InteropResponse?.ChatResponses == null)
return true;
await Task.WhenAll(
result.ChatResponses.Select(
result.InteropResponse.ChatResponses.Select(
x => Chat.SendMessage(
x.Text,
x.ChannelIds
@@ -474,10 +721,14 @@ namespace Tgstation.Server.Host.Components.Watchdog
var activeServer = GetActiveController();
var commandResult = await activeServer.SendCommand(command, cancellationToken).ConfigureAwait(false);
return commandResult?.CommandResponseMessage ??
(commandResult == null
? "ERROR: Bad topic exchange!"
: "TGS: Command processed but no DMAPI response returned!");
if (commandResult == null)
return "TGS: Bad topic exchange!";
if (commandResult.InteropResponse == null)
return "TGS: Bad topic response!";
return commandResult.InteropResponse.CommandResponseMessage ??
"TGS: Command processed but no DMAPI response returned!";
}
}
@@ -512,7 +763,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
Task chatTask;
if (Running)
{
chatTask = Chat.SendWatchdogMessage("Manual restart triggered...", cancellationToken);
chatTask = Chat.SendWatchdogMessage("Manual restart triggered...", false, cancellationToken);
await TerminateNoLock(false, false, cancellationToken).ConfigureAwait(false);
}
else
@@ -569,7 +820,10 @@ namespace Tgstation.Server.Host.Components.Watchdog
{
try
{
if (releaseServers && Running)
if (!Running)
return;
if (releaseServers)
{
await StopMonitor().ConfigureAwait(false);
@@ -577,7 +831,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
await reattachInfoHandler.Save(reattachInformation, cancellationToken).ConfigureAwait(false);
}
await Terminate(false, cancellationToken).ConfigureAwait(false);
await TerminateNoLock(false, !releaseServers, cancellationToken).ConfigureAwait(false);
}
catch
{
@@ -598,7 +852,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
{
releaseServers = true;
if (Running)
await Chat.SendWatchdogMessage("Detaching...", cancellationToken).ConfigureAwait(false);
await Chat.SendWatchdogMessage("Detaching...", false, cancellationToken).ConfigureAwait(false);
}
}
}
src/Tgstation.Server.Host/Controllers/DreamDaemonController.cs
+10 -10
View File
@@ -120,19 +120,19 @@ namespace Tgstation.Server.Host.Controllers
var alphaActive = dd.AlphaIsActive;
var llp = dd.LastLaunchParameters;
var rstate = dd.RebootState;
result.AutoStart = settings.AutoStart;
result.CurrentPort = alphaActive ? llp?.PrimaryPort : llp?.SecondaryPort;
result.CurrentSecurity = llp?.SecurityLevel;
result.CurrentAllowWebclient = llp?.AllowWebClient;
result.PrimaryPort = settings.PrimaryPort;
result.AllowWebClient = settings.AllowWebClient;
result.AutoStart = settings.AutoStart.Value;
result.CurrentPort = alphaActive ? llp?.PrimaryPort.Value : llp?.SecondaryPort.Value;
result.CurrentSecurity = llp?.SecurityLevel.Value;
result.CurrentAllowWebclient = llp?.AllowWebClient.Value;
result.PrimaryPort = settings.PrimaryPort.Value;
result.AllowWebClient = settings.AllowWebClient.Value;
result.Running = dd.Running;
result.SecondaryPort = settings.SecondaryPort;
result.SecurityLevel = settings.SecurityLevel;
result.SecondaryPort = settings.SecondaryPort.Value;
result.SecurityLevel = settings.SecurityLevel.Value;
result.SoftRestart = rstate == RebootState.Restart;
result.SoftShutdown = rstate == RebootState.Shutdown;
result.StartupTimeout = settings.StartupTimeout;
result.HeartbeatSeconds = settings.HeartbeatSeconds;
result.StartupTimeout = settings.StartupTimeout.Value;
result.HeartbeatSeconds = settings.HeartbeatSeconds.Value;
}
if (revision)
src/Tgstation.Server.Host/Core/Application.cs
+4 -2
View File
@@ -276,8 +276,10 @@ namespace Tgstation.Server.Host.Core
services.AddSingleton<ITopicClient, TopicClient>();
services.AddSingleton(new SocketParameters
{
ReceiveTimeout = postSetupServices.GeneralConfiguration.ByondTopicTimeout,
SendTimeout = postSetupServices.GeneralConfiguration.ByondTopicTimeout
ReceiveTimeout = TimeSpan.FromMilliseconds(postSetupServices.GeneralConfiguration.ByondTopicTimeout),
SendTimeout = TimeSpan.FromMilliseconds(postSetupServices.GeneralConfiguration.ByondTopicTimeout),
ConnectTimeout = TimeSpan.FromMilliseconds(postSetupServices.GeneralConfiguration.ByondTopicTimeout),
DisconnectTimeout = TimeSpan.FromMilliseconds(postSetupServices.GeneralConfiguration.ByondTopicTimeout)
});
// configure component services
src/Tgstation.Server.Host/Extensions/TaskExtensions.cs
+6
View File
@@ -48,5 +48,11 @@ namespace Tgstation.Server.Host.Extensions
return await task.ConfigureAwait(false);
}
/// <summary>
/// Creates a <see cref="Task"/> that never completes.
/// </summary>
/// <returns>A never ending <see cref="Task"/>.</returns>
public static Task InfiniteTask() => new TaskCompletionSource<object>().Task;
}
}
src/Tgstation.Server.Host/Tgstation.Server.Host.csproj
+1 -1
View File
@@ -48,7 +48,7 @@
<ItemGroup>
<PackageReference Include="BetterWin32Errors" Version="0.2.0" />
<PackageReference Include="Byond.TopicSender" Version="3.0.1" />
<PackageReference Include="Byond.TopicSender" Version="4.0.0" />
<PackageReference Include="Cyberboss.AspNetCore.AsyncInitializer" Version="1.2.0" />
<PackageReference Include="Cyberboss.SmartIrc4net.Standard" Version="0.4.6" />
<PackageReference Include="Discord.Net.WebSocket" Version="2.2.0" />
tests/Tgstation.Server.Tests/Instance/WatchdogTest.cs
+58 -4
View File
@@ -1,7 +1,10 @@
using Byond.TopicSender;
using Microsoft.Extensions.Logging;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Moq;
using System;
using System.IO;
using System.Linq;
using System.Net;
using System.Threading;
using System.Threading.Tasks;
@@ -24,13 +27,16 @@ namespace Tgstation.Server.Tests.Instance
public async Task Run(CancellationToken cancellationToken)
{
// Increase startup timeout
// Increase startup timeout, disable heartbeats
await instanceClient.DreamDaemon.Update(new DreamDaemon
{
StartupTimeout = 45
StartupTimeout = 45,
HeartbeatSeconds = 0,
}, cancellationToken);
await RunBasicTest(cancellationToken);
await RunHeartbeatTest(cancellationToken);
// await RunLongRunningTestThenUpdate(cancellationToken);
// await RunLongRunningTestThenUpdateWithByondVersionSwitch(cancellationToken);
@@ -64,6 +70,52 @@ namespace Tgstation.Server.Tests.Instance
await CheckDMApiFail(daemonStatus.ActiveCompileJob, cancellationToken);
}
async Task RunHeartbeatTest(CancellationToken cancellationToken)
{
// enable heartbeats
await instanceClient.DreamDaemon.Update(new DreamDaemon
{
HeartbeatSeconds = 1,
}, cancellationToken);
var startJob = await instanceClient.DreamDaemon.Start(cancellationToken).ConfigureAwait(false);
await WaitForJob(startJob, 10, false, cancellationToken);
await instanceClient.DreamDaemon.Update(new DreamDaemon
{
SoftShutdown = true
}, cancellationToken);
// lock on to DD and pause it so it can't heartbeat
var ddProcs = System.Diagnostics.Process.GetProcessesByName("DreamDaemon").ToList();
if (ddProcs.Count != 1)
Assert.Inconclusive($"Incorrect number of DD processes: {ddProcs.Count}");
var pid = ddProcs.Single().Id;
using var ourProcessHandler = new ProcessExecutor(
new PlatformIdentifier().IsWindows ? (IProcessSuspender)new WindowsProcessSuspender(Mock.Of<ILogger<WindowsProcessSuspender>>()) : new PosixProcessSuspender(Mock.Of<ILogger<PosixProcessSuspender>>()),
Mock.Of<ILogger<ProcessExecutor>>(),
LoggerFactory.Create(x => { })).GetProcess(pid);
ourProcessHandler.Suspend();
await Task.WhenAny(ourProcessHandler.Lifetime, Task.Delay(TimeSpan.FromSeconds(20)));
var timeout = 10;
do
{
await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken).ConfigureAwait(false);
var ddStatus = await instanceClient.DreamDaemon.Read(cancellationToken);
Assert.AreEqual(1U, ddStatus.HeartbeatSeconds.Value);
if (!ddStatus.Running.Value)
break;
if (--timeout == 0)
Assert.Fail("DreamDaemon didn't shutdown within the timeout!");
}
while (timeout > 0);
}
async Task RunLongRunningTestThenUpdate(CancellationToken cancellationToken)
{
const string DmeName = "LongRunning/long_running_test";
@@ -163,8 +215,10 @@ namespace Tgstation.Server.Tests.Instance
{
var bts = new TopicClient(new SocketParameters
{
SendTimeout = 5000,
ReceiveTimeout = 5000
SendTimeout = TimeSpan.FromSeconds(5),
ReceiveTimeout = TimeSpan.FromSeconds(5),
ConnectTimeout = TimeSpan.FromSeconds(5),
DisconnectTimeout = TimeSpan.FromSeconds(5)
});
try
tests/Tgstation.Server.Tests/TestingServer.cs
+2 -1
View File
@@ -75,7 +75,8 @@ namespace Tgstation.Server.Tests
String.Format(CultureInfo.InvariantCulture, "General:MinimumPasswordLength={0}", 10),
String.Format(CultureInfo.InvariantCulture, "General:InstanceLimit={0}", 11),
String.Format(CultureInfo.InvariantCulture, "General:UserLimit={0}", 150),
String.Format(CultureInfo.InvariantCulture, "General:ValidInstancePaths:0={0}", Directory)
String.Format(CultureInfo.InvariantCulture, "General:ValidInstancePaths:0={0}", Directory),
"General:ByondTopicTimeout=3000"
};
if (!String.IsNullOrEmpty(gitHubAccessToken))
tests/Tgstation.Server.Tests/Tgstation.Server.Tests.csproj
+6
View File
@@ -20,4 +20,10 @@
<ProjectReference Include="..\..\src\Tgstation.Server.Host\Tgstation.Server.Host.csproj" />
</ItemGroup>
<ItemGroup>
<Reference Include="Byond.TopicSender">
<HintPath>..\..\..\Byond.TopicSender\Byond.TopicSender\bin\Debug\netstandard2.1\Byond.TopicSender.dll</HintPath>
</Reference>
</ItemGroup>
</Project>