diff --git a/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs b/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs
index a1623b312c..1869a4bb95 100644
--- a/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs
+++ b/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs
@@ -325,35 +325,33 @@ namespace Tgstation.Server.Host.Components.Chat
if (channelIds == null)
throw new ArgumentNullException(nameof(channelIds));
- var task = SendMessage(
- channelIds,
- null,
- message,
- handlerCts.Token);
- AddMessageTask(task);
+ QueueMessageInternal(message, channelIds, false);
}
///
- public async Task QueueWatchdogMessage(string message, CancellationToken cancellationToken)
+ public void QueueWatchdogMessage(string message)
{
+ if (message == null)
+ throw new ArgumentNullException(nameof(message));
+
List wdChannels = null;
message = String.Format(CultureInfo.InvariantCulture, "WD: {0}", message);
if (!initialProviderConnectionsTask.IsCompleted)
logger.LogTrace("Waiting for initial provider connections before sending watchdog message...");
- await initialProviderConnectionsTask.WithToken(cancellationToken);
-
// so it doesn't change while we're using it
lock (mappedChannels)
wdChannels = mappedChannels.Where(x => x.Value.IsWatchdogChannel).Select(x => x.Key).ToList();
- QueueMessage(
+ // Reimplementing QueueMessage
+ QueueMessageInternal(
new MessageContent
{
Text = message,
},
- wdChannels);
+ wdChannels,
+ true);
}
///
@@ -467,13 +465,16 @@ namespace Tgstation.Server.Host.Components.Chat
///
public Task UpdateTrackingContexts(CancellationToken cancellationToken)
{
+ async Task UpdateTrackingContext(IChannelSink channelSink, IEnumerable channels)
+ {
+ await initialProviderConnectionsTask.WithToken(cancellationToken);
+ await channelSink.UpdateChannels(channels, cancellationToken);
+ }
+
lock (mappedChannels)
lock (trackingContexts)
return Task.WhenAll(
- trackingContexts.Select(
- x => x.UpdateChannels(
- mappedChannels.Select(y => y.Value.Channel).ToList(),
- cancellationToken)));
+ trackingContexts.Select(x => UpdateTrackingContext(x, mappedChannels.Select(y => y.Value.Channel).ToList())));
}
///
@@ -1014,5 +1015,29 @@ namespace Tgstation.Server.Host.Components.Chat
lock (handlerCts)
messageSendTask = Wrap(messageSendTask);
}
+
+ ///
+ /// Adds a given to the send queue.
+ ///
+ /// The being sent.
+ /// The s of the s to send to.
+ /// If , the message send will wait for to complete before running.
+ void QueueMessageInternal(MessageContent message, IEnumerable channelIds, bool waitForConnections)
+ {
+ async Task SendMessageTask()
+ {
+ var cancellationToken = handlerCts.Token;
+ if (waitForConnections)
+ await initialProviderConnectionsTask.WithToken(cancellationToken);
+
+ await SendMessage(
+ channelIds,
+ null,
+ message,
+ cancellationToken);
+ }
+
+ AddMessageTask(SendMessageTask());
+ }
}
}
diff --git a/src/Tgstation.Server.Host/Components/Chat/IChatManager.cs b/src/Tgstation.Server.Host/Components/Chat/IChatManager.cs
index 6518f1c894..59f3e2b1c5 100644
--- a/src/Tgstation.Server.Host/Components/Chat/IChatManager.cs
+++ b/src/Tgstation.Server.Host/Components/Chat/IChatManager.cs
@@ -57,9 +57,7 @@ namespace Tgstation.Server.Host.Components.Chat
/// Queue a chat to configured watchdog channels.
///
/// The message being sent.
- /// The for the operation.
- /// A representing the running operation.
- Task QueueWatchdogMessage(string message, CancellationToken cancellationToken);
+ void QueueWatchdogMessage(string message);
///
/// Send the message for a deployment to configured deployment channels.
diff --git a/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs
index 5445ae311c..b8f23b2339 100644
--- a/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs
+++ b/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs
@@ -118,22 +118,19 @@ namespace Tgstation.Server.Host.Components.Watchdog
if (Server.RebootState == Session.RebootState.Shutdown)
{
// the time for graceful shutdown is now
- await Chat.QueueWatchdogMessage(
+ Chat.QueueWatchdogMessage(
String.Format(
CultureInfo.InvariantCulture,
"Server {0}! Shutting down due to graceful termination request...",
- exitWord),
- cancellationToken)
- ;
+ exitWord));
return MonitorAction.Exit;
}
- await Chat.QueueWatchdogMessage(
+ Chat.QueueWatchdogMessage(
String.Format(
CultureInfo.InvariantCulture,
"Server {0}! Rebooting...",
- exitWord),
- cancellationToken);
+ exitWord));
return MonitorAction.Restart;
case MonitorActivationReason.ActiveServerRebooted:
var rebootState = Server.RebootState;
@@ -156,10 +153,8 @@ namespace Tgstation.Server.Host.Components.Watchdog
return MonitorAction.Restart;
case Session.RebootState.Shutdown:
// graceful shutdown time
- await Chat.QueueWatchdogMessage(
- "Active server rebooted! Shutting down due to graceful termination request...",
- cancellationToken)
- ;
+ Chat.QueueWatchdogMessage(
+ "Active server rebooted! Shutting down due to graceful termination request...");
return MonitorAction.Exit;
default:
throw new InvalidOperationException($"Invalid reboot state: {rebootState}");
@@ -200,7 +195,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
///
protected override async Task InitController(
- Task chatTask,
+ Task eventTask,
ReattachInformation reattachInfo,
CancellationToken cancellationToken)
{
@@ -221,7 +216,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
await BeforeApplyDmb(dmbToUse.CompileJob, cancellationToken);
dmbToUse = await PrepServerForLaunch(dmbToUse, cancellationToken);
- await chatTask;
+ await eventTask;
serverLaunchTask = SessionControllerFactory.LaunchNew(
dmbToUse,
null,
@@ -231,7 +226,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
}
else
{
- await chatTask;
+ await eventTask;
serverLaunchTask = SessionControllerFactory.Reattach(reattachInfo, cancellationToken);
}
@@ -299,14 +294,17 @@ namespace Tgstation.Server.Host.Components.Watchdog
///
/// The for the operation.
/// A representing the running operation.
- protected virtual Task HandleNewDmbAvailable(CancellationToken cancellationToken)
+ protected virtual async Task HandleNewDmbAvailable(CancellationToken cancellationToken)
{
gracefulRebootRequired = true;
if (Server.CompileJob.DMApiVersion == null)
- return Chat.QueueWatchdogMessage(
- "A new deployment has been made but cannot be applied automatically as the currently running server has no DMAPI. Please manually reboot the server to apply the update.",
- cancellationToken);
- return Server.SetRebootState(Session.RebootState.Restart, cancellationToken);
+ {
+ Chat.QueueWatchdogMessage(
+ "A new deployment has been made but cannot be applied automatically as the currently running server has no DMAPI. Please manually reboot the server to apply the update.");
+ return;
+ }
+
+ await Server.SetRebootState(Session.RebootState.Restart, cancellationToken);
}
///
diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs
index 5188d636b3..aceb6c0fcf 100644
--- a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs
+++ b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs
@@ -347,10 +347,9 @@ namespace Tgstation.Server.Host.Components.Watchdog
{
if (!graceful)
{
- var chatTask = Chat.QueueWatchdogMessage("Manual restart triggered...", cancellationToken);
+ Chat.QueueWatchdogMessage("Manual restart triggered...");
await TerminateNoLock(false, false, cancellationToken);
await LaunchNoLock(true, false, true, null, cancellationToken);
- await chatTask;
return;
}
@@ -425,7 +424,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
releaseServers = true;
if (Status == WatchdogStatus.Online)
- await Chat.QueueWatchdogMessage("Detaching...", cancellationToken);
+ Chat.QueueWatchdogMessage("Detaching...");
else
Logger.LogTrace("Not sending detach chat message as status is: {status}", Status);
}
@@ -477,11 +476,11 @@ namespace Tgstation.Server.Host.Components.Watchdog
///
/// Starts all s.
///
- /// A, possibly active, for an outgoing chat message.
+ /// A, possibly active, for an event that's running.
/// to use, if any.
/// The for the operation.
/// A representing the running operation.
- protected abstract Task InitController(Task chatTask, ReattachInformation reattachInfo, CancellationToken cancellationToken);
+ protected abstract Task InitController(Task eventTask, ReattachInformation reattachInfo, CancellationToken cancellationToken);
///
/// Launches the watchdog.
@@ -507,21 +506,16 @@ namespace Tgstation.Server.Host.Components.Watchdog
throw new JobException(ErrorCode.WatchdogCompileJobCorrupted);
// this is necessary, the monitor could be in it's sleep loop trying to restart, if so cancel THAT monitor and start our own with blackjack and hookers
- Task announceTask;
+ var eventTask = Task.CompletedTask;
if (announce)
{
- announceTask = Chat.QueueWatchdogMessage(
+ Chat.QueueWatchdogMessage(
reattachInfo == null
? "Launching..."
- : "Reattaching...",
- cancellationToken); // simple announce
+ : "Reattaching..."); // simple announce
if (reattachInfo == null)
- announceTask = Task.WhenAll(
- HandleEvent(EventType.WatchdogLaunch, Enumerable.Empty(), false, cancellationToken),
- announceTask);
+ eventTask = HandleEvent(EventType.WatchdogLaunch, Enumerable.Empty(), false, cancellationToken);
}
- else
- announceTask = Task.CompletedTask; // no announce
// since neither server is running, this is safe to do
LastLaunchParameters = ActiveLaunchParameters;
@@ -529,7 +523,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
try
{
- await InitController(announceTask, reattachInfo, cancellationToken);
+ await InitController(eventTask, reattachInfo, cancellationToken);
}
catch (OperationCanceledException ex)
{
@@ -539,15 +533,15 @@ namespace Tgstation.Server.Host.Components.Watchdog
catch (Exception e)
{
Logger.LogWarning(e, "Failed to start watchdog!");
- var originalChatTask = announceTask;
- async Task ChainChatTaskWithErrorMessage()
+ var originalChatTask = eventTask;
+ async Task ChainEventTaskWithErrorMessage()
{
await originalChatTask;
if (announceFailure)
- await Chat.QueueWatchdogMessage("Startup failed!", cancellationToken);
+ Chat.QueueWatchdogMessage("Startup failed!");
}
- announceTask = ChainChatTaskWithErrorMessage();
+ eventTask = ChainEventTaskWithErrorMessage();
throw;
}
finally
@@ -555,7 +549,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
// finish the chat task that's in flight
try
{
- await announceTask;
+ await eventTask;
}
catch (OperationCanceledException ex)
{
@@ -626,8 +620,8 @@ namespace Tgstation.Server.Host.Components.Watchdog
const string FailReattachMessage = "Unable to properly reattach to server! Restarting watchdog...";
Logger.LogWarning(FailReattachMessage);
- var chatTask = Chat.QueueWatchdogMessage(FailReattachMessage, cancellationToken);
- await InitController(chatTask, null, cancellationToken);
+ Chat.QueueWatchdogMessage(FailReattachMessage);
+ await InitController(Task.CompletedTask, null, cancellationToken);
}
///
@@ -730,7 +724,6 @@ namespace Tgstation.Server.Host.Components.Watchdog
await DisposeAndNullControllers(cancellationToken);
- var chatTask = Task.CompletedTask;
for (var retryAttempts = 1; ; ++retryAttempts)
{
Status = WatchdogStatus.Restoring;
@@ -748,10 +741,6 @@ namespace Tgstation.Server.Host.Components.Watchdog
{
launchException = e;
}
- finally
- {
- await chatTask;
- }
Logger.LogWarning(launchException, "Failed to automatically restart the watchdog! Attempt: {attemptNumber}", retryAttempts);
Status = WatchdogStatus.DelayedRestart;
@@ -761,16 +750,12 @@ namespace Tgstation.Server.Host.Components.Watchdog
Math.Pow(2, retryAttempts)),
TimeSpan.FromHours(1).TotalSeconds); // max of one hour, increasing by a power of 2 each time
- chatTask = Chat.QueueWatchdogMessage(
- $"Failed to restart (Attempt: {retryAttempts}), retrying in {retryDelay}s...",
- cancellationToken);
+ Chat.QueueWatchdogMessage(
+ $"Failed to restart (Attempt: {retryAttempts}), retrying in {retryDelay}s...");
- await Task.WhenAll(
- AsyncDelayer.Delay(
- TimeSpan.FromSeconds(retryDelay),
- cancellationToken),
- chatTask)
- ;
+ await AsyncDelayer.Delay(
+ TimeSpan.FromSeconds(retryDelay),
+ cancellationToken);
}
}
@@ -966,9 +951,8 @@ namespace Tgstation.Server.Host.Components.Watchdog
var nextActionMessage = nextAction != MonitorAction.Exit
? "Recovering"
: "Shutting down";
- var chatTask = Chat.QueueWatchdogMessage(
- $"Monitor crashed, this should NEVER happen! Please report this, full details in logs! {nextActionMessage}. Error: {e.Message}",
- cancellationToken);
+ Chat.QueueWatchdogMessage(
+ $"Monitor crashed, this should NEVER happen! Please report this, full details in logs! {nextActionMessage}. Error: {e.Message}");
if (disposed)
nextAction = MonitorAction.Exit;
@@ -980,8 +964,6 @@ namespace Tgstation.Server.Host.Components.Watchdog
Logger.LogDebug("Server seems to be okay, not restarting");
nextAction = MonitorAction.Continue;
}
-
- await chatTask;
}
}
catch (OperationCanceledException)
@@ -1026,15 +1008,14 @@ namespace Tgstation.Server.Host.Components.Watchdog
releaseServers,
cancellationToken);
- var chatTask = announce ? Chat.QueueWatchdogMessage("Shutting down...", cancellationToken) : Task.CompletedTask;
+ if (announce)
+ Chat.QueueWatchdogMessage("Shutting down...");
await eventTask;
await StopMonitor();
LastLaunchParameters = null;
-
- await chatTask;
return;
}
@@ -1071,7 +1052,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
case 2:
const string message2 = "DEFCON 3: DreamDaemon has missed 2 heartbeats!";
Logger.LogInformation(message2);
- await Chat.QueueWatchdogMessage(message2, cancellationToken);
+ Chat.QueueWatchdogMessage(message2);
break;
case 3:
var actionToTake = shouldShutdown
@@ -1079,12 +1060,11 @@ namespace Tgstation.Server.Host.Components.Watchdog
: "be restarted";
const string logTemplate1 = "DEFCON 2: DreamDaemon has missed 3 heartbeats! If it does not respond to the next one, the watchdog will {actionToTake}!";
Logger.LogWarning(logTemplate1, actionToTake);
- await Chat.QueueWatchdogMessage(
+ Chat.QueueWatchdogMessage(
logTemplate1.Replace(
"{actionToTake}",
actionToTake,
- StringComparison.Ordinal),
- cancellationToken);
+ StringComparison.Ordinal));
break;
case 4:
var actionTaken = shouldShutdown
@@ -1092,12 +1072,11 @@ namespace Tgstation.Server.Host.Components.Watchdog
: "Restarting";
const string logTemplate2 = "DEFCON 1: Four heartbeats have been missed! {actionTaken}...";
Logger.LogWarning(logTemplate2, actionTaken);
- await Chat.QueueWatchdogMessage(
+ Chat.QueueWatchdogMessage(
logTemplate2.Replace(
"{actionTaken}",
actionTaken,
- StringComparison.Ordinal),
- cancellationToken);
+ StringComparison.Ordinal));
if (ActiveLaunchParameters.DumpOnHeartbeatRestart.Value)
{