cyberpresidentvanellope/tgstation-server
1
0
Fork 0
mirror of https://github.com/tgstation/tgstation-server.git synced 2026-08-26 06:27:19 +01:00
Code Issues Packages Projects Releases Wiki Activity

Do not let chat message queuing hold up the watchdog

Browse Source
This commit is contained in:
Dominion
2023-05-21 00:59:30 -04:00
parent 51eb8348ae
commit ee1d752f4d
4 changed files with 88 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Tgstation.Server.Host/Components/Chat/ChatManager.cs
+40 -15
View File
@@ -325,35 +325,33 @@ namespace Tgstation.Server.Host.Components.Chat
if (channelIds == null)
throw new ArgumentNullException(nameof(channelIds));
var task = SendMessage(
channelIds,
null,
message,
handlerCts.Token);
AddMessageTask(task);
QueueMessageInternal(message, channelIds, false);
}
/// <inheritdoc />
public async Task QueueWatchdogMessage(string message, CancellationToken cancellationToken)
public void QueueWatchdogMessage(string message)
{
if (message == null)
throw new ArgumentNullException(nameof(message));
List<ulong> wdChannels = null;
message = String.Format(CultureInfo.InvariantCulture, "WD: {0}", message);
if (!initialProviderConnectionsTask.IsCompleted)
logger.LogTrace("Waiting for initial provider connections before sending watchdog message...");
await initialProviderConnectionsTask.WithToken(cancellationToken);
// so it doesn't change while we're using it
lock (mappedChannels)
wdChannels = mappedChannels.Where(x => x.Value.IsWatchdogChannel).Select(x => x.Key).ToList();
QueueMessage(
// Reimplementing QueueMessage
QueueMessageInternal(
new MessageContent
{
Text = message,
},
wdChannels);
wdChannels,
true);
}
/// <inheritdoc />
@@ -467,13 +465,16 @@ namespace Tgstation.Server.Host.Components.Chat
/// <inheritdoc />
public Task UpdateTrackingContexts(CancellationToken cancellationToken)
{
async Task UpdateTrackingContext(IChannelSink channelSink, IEnumerable<ChannelRepresentation> channels)
{
await initialProviderConnectionsTask.WithToken(cancellationToken);
await channelSink.UpdateChannels(channels, cancellationToken);
}
lock (mappedChannels)
lock (trackingContexts)
return Task.WhenAll(
trackingContexts.Select(
x => x.UpdateChannels(
mappedChannels.Select(y => y.Value.Channel).ToList(),
cancellationToken)));
trackingContexts.Select(x => UpdateTrackingContext(x, mappedChannels.Select(y => y.Value.Channel).ToList())));
}
/// <inheritdoc />
@@ -1014,5 +1015,29 @@ namespace Tgstation.Server.Host.Components.Chat
lock (handlerCts)
messageSendTask = Wrap(messageSendTask);
}
/// <summary>
/// Adds a given <paramref name="message"/> to the send queue.
/// </summary>
/// <param name="message">The <see cref="MessageContent"/> being sent.</param>
/// <param name="channelIds">The <see cref="Models.ChatChannel.Id"/>s of the <see cref="Models.ChatChannel"/>s to send to.</param>
/// <param name="waitForConnections">If <see langword="true"/>, the message send will wait for <see cref="initialProviderConnectionsTask"/> to complete before running.</param>
void QueueMessageInternal(MessageContent message, IEnumerable<ulong> channelIds, bool waitForConnections)
{
async Task SendMessageTask()
{
var cancellationToken = handlerCts.Token;
if (waitForConnections)
await initialProviderConnectionsTask.WithToken(cancellationToken);
await SendMessage(
channelIds,
null,
message,
cancellationToken);
}
AddMessageTask(SendMessageTask());
}
}
}
src/Tgstation.Server.Host/Components/Chat/IChatManager.cs
+1 -3
View File
@@ -57,9 +57,7 @@ namespace Tgstation.Server.Host.Components.Chat
/// Queue a chat <paramref name="message"/> to configured watchdog channels.
/// </summary>
/// <param name="message">The message being sent.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
/// <returns>A <see cref="Task"/> representing the running operation.</returns>
Task QueueWatchdogMessage(string message, CancellationToken cancellationToken);
void QueueWatchdogMessage(string message);
/// <summary>
/// Send the message for a deployment to configured deployment channels.
src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs
+17 -19
View File
@@ -118,22 +118,19 @@ namespace Tgstation.Server.Host.Components.Watchdog
if (Server.RebootState == Session.RebootState.Shutdown)
{
// the time for graceful shutdown is now
await Chat.QueueWatchdogMessage(
Chat.QueueWatchdogMessage(
String.Format(
CultureInfo.InvariantCulture,
"Server {0}! Shutting down due to graceful termination request...",
exitWord),
cancellationToken)
;
exitWord));
return MonitorAction.Exit;
}
await Chat.QueueWatchdogMessage(
Chat.QueueWatchdogMessage(
String.Format(
CultureInfo.InvariantCulture,
"Server {0}! Rebooting...",
exitWord),
cancellationToken);
exitWord));
return MonitorAction.Restart;
case MonitorActivationReason.ActiveServerRebooted:
var rebootState = Server.RebootState;
@@ -156,10 +153,8 @@ namespace Tgstation.Server.Host.Components.Watchdog
return MonitorAction.Restart;
case Session.RebootState.Shutdown:
// graceful shutdown time
await Chat.QueueWatchdogMessage(
"Active server rebooted! Shutting down due to graceful termination request...",
cancellationToken)
;
Chat.QueueWatchdogMessage(
"Active server rebooted! Shutting down due to graceful termination request...");
return MonitorAction.Exit;
default:
throw new InvalidOperationException($"Invalid reboot state: {rebootState}");
@@ -200,7 +195,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
/// <inheritdoc />
protected override async Task InitController(
Task chatTask,
Task eventTask,
ReattachInformation reattachInfo,
CancellationToken cancellationToken)
{
@@ -221,7 +216,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
await BeforeApplyDmb(dmbToUse.CompileJob, cancellationToken);
dmbToUse = await PrepServerForLaunch(dmbToUse, cancellationToken);
await chatTask;
await eventTask;
serverLaunchTask = SessionControllerFactory.LaunchNew(
dmbToUse,
null,
@@ -231,7 +226,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
}
else
{
await chatTask;
await eventTask;
serverLaunchTask = SessionControllerFactory.Reattach(reattachInfo, cancellationToken);
}
@@ -299,14 +294,17 @@ namespace Tgstation.Server.Host.Components.Watchdog
/// </summary>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
/// <returns>A <see cref="Task"/> representing the running operation.</returns>
protected virtual Task HandleNewDmbAvailable(CancellationToken cancellationToken)
protected virtual async Task HandleNewDmbAvailable(CancellationToken cancellationToken)
{
gracefulRebootRequired = true;
if (Server.CompileJob.DMApiVersion == null)
return Chat.QueueWatchdogMessage(
"A new deployment has been made but cannot be applied automatically as the currently running server has no DMAPI. Please manually reboot the server to apply the update.",
cancellationToken);
return Server.SetRebootState(Session.RebootState.Restart, cancellationToken);
{
Chat.QueueWatchdogMessage(
"A new deployment has been made but cannot be applied automatically as the currently running server has no DMAPI. Please manually reboot the server to apply the update.");
return;
}
await Server.SetRebootState(Session.RebootState.Restart, cancellationToken);
}
/// <summary>
src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs
+30 -51
View File
@@ -347,10 +347,9 @@ namespace Tgstation.Server.Host.Components.Watchdog
{
if (!graceful)
{
var chatTask = Chat.QueueWatchdogMessage("Manual restart triggered...", cancellationToken);
Chat.QueueWatchdogMessage("Manual restart triggered...");
await TerminateNoLock(false, false, cancellationToken);
await LaunchNoLock(true, false, true, null, cancellationToken);
await chatTask;
return;
}
@@ -425,7 +424,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
releaseServers = true;
if (Status == WatchdogStatus.Online)
await Chat.QueueWatchdogMessage("Detaching...", cancellationToken);
Chat.QueueWatchdogMessage("Detaching...");
else
Logger.LogTrace("Not sending detach chat message as status is: {status}", Status);
}
@@ -477,11 +476,11 @@ namespace Tgstation.Server.Host.Components.Watchdog
/// <summary>
/// Starts all <see cref="ISessionController"/>s.
/// </summary>
/// <param name="chatTask">A, possibly active, <see cref="Task"/> for an outgoing chat message.</param>
/// <param name="eventTask">A, possibly active, <see cref="Task"/> for an event that's running.</param>
/// <param name="reattachInfo"><see cref="ReattachInformation"/> to use, if any.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
/// <returns>A <see cref="Task"/> representing the running operation.</returns>
protected abstract Task InitController(Task chatTask, ReattachInformation reattachInfo, CancellationToken cancellationToken);
protected abstract Task InitController(Task eventTask, ReattachInformation reattachInfo, CancellationToken cancellationToken);
/// <summary>
/// Launches the watchdog.
@@ -507,21 +506,16 @@ namespace Tgstation.Server.Host.Components.Watchdog
throw new JobException(ErrorCode.WatchdogCompileJobCorrupted);
// this is necessary, the monitor could be in it's sleep loop trying to restart, if so cancel THAT monitor and start our own with blackjack and hookers
Task announceTask;
var eventTask = Task.CompletedTask;
if (announce)
{
announceTask = Chat.QueueWatchdogMessage(
Chat.QueueWatchdogMessage(
reattachInfo == null
? "Launching..."
: "Reattaching...",
cancellationToken); // simple announce
: "Reattaching..."); // simple announce
if (reattachInfo == null)
announceTask = Task.WhenAll(
HandleEvent(EventType.WatchdogLaunch, Enumerable.Empty<string>(), false, cancellationToken),
announceTask);
eventTask = HandleEvent(EventType.WatchdogLaunch, Enumerable.Empty<string>(), false, cancellationToken);
}
else
announceTask = Task.CompletedTask; // no announce
// since neither server is running, this is safe to do
LastLaunchParameters = ActiveLaunchParameters;
@@ -529,7 +523,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
try
{
await InitController(announceTask, reattachInfo, cancellationToken);
await InitController(eventTask, reattachInfo, cancellationToken);
}
catch (OperationCanceledException ex)
{
@@ -539,15 +533,15 @@ namespace Tgstation.Server.Host.Components.Watchdog
catch (Exception e)
{
Logger.LogWarning(e, "Failed to start watchdog!");
var originalChatTask = announceTask;
async Task ChainChatTaskWithErrorMessage()
var originalChatTask = eventTask;
async Task ChainEventTaskWithErrorMessage()
{
await originalChatTask;
if (announceFailure)
await Chat.QueueWatchdogMessage("Startup failed!", cancellationToken);
Chat.QueueWatchdogMessage("Startup failed!");
}
announceTask = ChainChatTaskWithErrorMessage();
eventTask = ChainEventTaskWithErrorMessage();
throw;
}
finally
@@ -555,7 +549,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
// finish the chat task that's in flight
try
{
await announceTask;
await eventTask;
}
catch (OperationCanceledException ex)
{
@@ -626,8 +620,8 @@ namespace Tgstation.Server.Host.Components.Watchdog
const string FailReattachMessage = "Unable to properly reattach to server! Restarting watchdog...";
Logger.LogWarning(FailReattachMessage);
var chatTask = Chat.QueueWatchdogMessage(FailReattachMessage, cancellationToken);
await InitController(chatTask, null, cancellationToken);
Chat.QueueWatchdogMessage(FailReattachMessage);
await InitController(Task.CompletedTask, null, cancellationToken);
}
/// <summary>
@@ -730,7 +724,6 @@ namespace Tgstation.Server.Host.Components.Watchdog
await DisposeAndNullControllers(cancellationToken);
var chatTask = Task.CompletedTask;
for (var retryAttempts = 1; ; ++retryAttempts)
{
Status = WatchdogStatus.Restoring;
@@ -748,10 +741,6 @@ namespace Tgstation.Server.Host.Components.Watchdog
{
launchException = e;
}
finally
{
await chatTask;
}
Logger.LogWarning(launchException, "Failed to automatically restart the watchdog! Attempt: {attemptNumber}", retryAttempts);
Status = WatchdogStatus.DelayedRestart;
@@ -761,16 +750,12 @@ namespace Tgstation.Server.Host.Components.Watchdog
Math.Pow(2, retryAttempts)),
TimeSpan.FromHours(1).TotalSeconds); // max of one hour, increasing by a power of 2 each time
chatTask = Chat.QueueWatchdogMessage(
$"Failed to restart (Attempt: {retryAttempts}), retrying in {retryDelay}s...",
cancellationToken);
Chat.QueueWatchdogMessage(
$"Failed to restart (Attempt: {retryAttempts}), retrying in {retryDelay}s...");
await Task.WhenAll(
AsyncDelayer.Delay(
TimeSpan.FromSeconds(retryDelay),
cancellationToken),
chatTask)
;
await AsyncDelayer.Delay(
TimeSpan.FromSeconds(retryDelay),
cancellationToken);
}
}
@@ -966,9 +951,8 @@ namespace Tgstation.Server.Host.Components.Watchdog
var nextActionMessage = nextAction != MonitorAction.Exit
? "Recovering"
: "Shutting down";
var chatTask = Chat.QueueWatchdogMessage(
$"Monitor crashed, this should NEVER happen! Please report this, full details in logs! {nextActionMessage}. Error: {e.Message}",
cancellationToken);
Chat.QueueWatchdogMessage(
$"Monitor crashed, this should NEVER happen! Please report this, full details in logs! {nextActionMessage}. Error: {e.Message}");
if (disposed)
nextAction = MonitorAction.Exit;
@@ -980,8 +964,6 @@ namespace Tgstation.Server.Host.Components.Watchdog
Logger.LogDebug("Server seems to be okay, not restarting");
nextAction = MonitorAction.Continue;
}
await chatTask;
}
}
catch (OperationCanceledException)
@@ -1026,15 +1008,14 @@ namespace Tgstation.Server.Host.Components.Watchdog
releaseServers,
cancellationToken);
var chatTask = announce ? Chat.QueueWatchdogMessage("Shutting down...", cancellationToken) : Task.CompletedTask;
if (announce)
Chat.QueueWatchdogMessage("Shutting down...");
await eventTask;
await StopMonitor();
LastLaunchParameters = null;
await chatTask;
return;
}
@@ -1071,7 +1052,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
case 2:
const string message2 = "DEFCON 3: DreamDaemon has missed 2 heartbeats!";
Logger.LogInformation(message2);
await Chat.QueueWatchdogMessage(message2, cancellationToken);
Chat.QueueWatchdogMessage(message2);
break;
case 3:
var actionToTake = shouldShutdown
@@ -1079,12 +1060,11 @@ namespace Tgstation.Server.Host.Components.Watchdog
: "be restarted";
const string logTemplate1 = "DEFCON 2: DreamDaemon has missed 3 heartbeats! If it does not respond to the next one, the watchdog will {actionToTake}!";
Logger.LogWarning(logTemplate1, actionToTake);
await Chat.QueueWatchdogMessage(
Chat.QueueWatchdogMessage(
logTemplate1.Replace(
"{actionToTake}",
actionToTake,
StringComparison.Ordinal),
cancellationToken);
StringComparison.Ordinal));
break;
case 4:
var actionTaken = shouldShutdown
@@ -1092,12 +1072,11 @@ namespace Tgstation.Server.Host.Components.Watchdog
: "Restarting";
const string logTemplate2 = "DEFCON 1: Four heartbeats have been missed! {actionTaken}...";
Logger.LogWarning(logTemplate2, actionTaken);
await Chat.QueueWatchdogMessage(
Chat.QueueWatchdogMessage(
logTemplate2.Replace(
"{actionTaken}",
actionTaken,
StringComparison.Ordinal),
cancellationToken);
StringComparison.Ordinal));
if (ActiveLaunchParameters.DumpOnHeartbeatRestart.Value)
{