diff --git a/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs b/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs index a1623b312c..1869a4bb95 100644 --- a/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs +++ b/src/Tgstation.Server.Host/Components/Chat/ChatManager.cs @@ -325,35 +325,33 @@ namespace Tgstation.Server.Host.Components.Chat if (channelIds == null) throw new ArgumentNullException(nameof(channelIds)); - var task = SendMessage( - channelIds, - null, - message, - handlerCts.Token); - AddMessageTask(task); + QueueMessageInternal(message, channelIds, false); } /// - public async Task QueueWatchdogMessage(string message, CancellationToken cancellationToken) + public void QueueWatchdogMessage(string message) { + if (message == null) + throw new ArgumentNullException(nameof(message)); + List wdChannels = null; message = String.Format(CultureInfo.InvariantCulture, "WD: {0}", message); if (!initialProviderConnectionsTask.IsCompleted) logger.LogTrace("Waiting for initial provider connections before sending watchdog message..."); - await initialProviderConnectionsTask.WithToken(cancellationToken); - // so it doesn't change while we're using it lock (mappedChannels) wdChannels = mappedChannels.Where(x => x.Value.IsWatchdogChannel).Select(x => x.Key).ToList(); - QueueMessage( + // Reimplementing QueueMessage + QueueMessageInternal( new MessageContent { Text = message, }, - wdChannels); + wdChannels, + true); } /// @@ -467,13 +465,16 @@ namespace Tgstation.Server.Host.Components.Chat /// public Task UpdateTrackingContexts(CancellationToken cancellationToken) { + async Task UpdateTrackingContext(IChannelSink channelSink, IEnumerable channels) + { + await initialProviderConnectionsTask.WithToken(cancellationToken); + await channelSink.UpdateChannels(channels, cancellationToken); + } + lock (mappedChannels) lock (trackingContexts) return Task.WhenAll( - trackingContexts.Select( - x => x.UpdateChannels( - mappedChannels.Select(y => y.Value.Channel).ToList(), - cancellationToken))); + trackingContexts.Select(x => UpdateTrackingContext(x, mappedChannels.Select(y => y.Value.Channel).ToList()))); } /// @@ -1014,5 +1015,29 @@ namespace Tgstation.Server.Host.Components.Chat lock (handlerCts) messageSendTask = Wrap(messageSendTask); } + + /// + /// Adds a given to the send queue. + /// + /// The being sent. + /// The s of the s to send to. + /// If , the message send will wait for to complete before running. + void QueueMessageInternal(MessageContent message, IEnumerable channelIds, bool waitForConnections) + { + async Task SendMessageTask() + { + var cancellationToken = handlerCts.Token; + if (waitForConnections) + await initialProviderConnectionsTask.WithToken(cancellationToken); + + await SendMessage( + channelIds, + null, + message, + cancellationToken); + } + + AddMessageTask(SendMessageTask()); + } } } diff --git a/src/Tgstation.Server.Host/Components/Chat/IChatManager.cs b/src/Tgstation.Server.Host/Components/Chat/IChatManager.cs index 6518f1c894..59f3e2b1c5 100644 --- a/src/Tgstation.Server.Host/Components/Chat/IChatManager.cs +++ b/src/Tgstation.Server.Host/Components/Chat/IChatManager.cs @@ -57,9 +57,7 @@ namespace Tgstation.Server.Host.Components.Chat /// Queue a chat to configured watchdog channels. /// /// The message being sent. - /// The for the operation. - /// A representing the running operation. - Task QueueWatchdogMessage(string message, CancellationToken cancellationToken); + void QueueWatchdogMessage(string message); /// /// Send the message for a deployment to configured deployment channels. diff --git a/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs index 5445ae311c..b8f23b2339 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs @@ -118,22 +118,19 @@ namespace Tgstation.Server.Host.Components.Watchdog if (Server.RebootState == Session.RebootState.Shutdown) { // the time for graceful shutdown is now - await Chat.QueueWatchdogMessage( + Chat.QueueWatchdogMessage( String.Format( CultureInfo.InvariantCulture, "Server {0}! Shutting down due to graceful termination request...", - exitWord), - cancellationToken) - ; + exitWord)); return MonitorAction.Exit; } - await Chat.QueueWatchdogMessage( + Chat.QueueWatchdogMessage( String.Format( CultureInfo.InvariantCulture, "Server {0}! Rebooting...", - exitWord), - cancellationToken); + exitWord)); return MonitorAction.Restart; case MonitorActivationReason.ActiveServerRebooted: var rebootState = Server.RebootState; @@ -156,10 +153,8 @@ namespace Tgstation.Server.Host.Components.Watchdog return MonitorAction.Restart; case Session.RebootState.Shutdown: // graceful shutdown time - await Chat.QueueWatchdogMessage( - "Active server rebooted! Shutting down due to graceful termination request...", - cancellationToken) - ; + Chat.QueueWatchdogMessage( + "Active server rebooted! Shutting down due to graceful termination request..."); return MonitorAction.Exit; default: throw new InvalidOperationException($"Invalid reboot state: {rebootState}"); @@ -200,7 +195,7 @@ namespace Tgstation.Server.Host.Components.Watchdog /// protected override async Task InitController( - Task chatTask, + Task eventTask, ReattachInformation reattachInfo, CancellationToken cancellationToken) { @@ -221,7 +216,7 @@ namespace Tgstation.Server.Host.Components.Watchdog await BeforeApplyDmb(dmbToUse.CompileJob, cancellationToken); dmbToUse = await PrepServerForLaunch(dmbToUse, cancellationToken); - await chatTask; + await eventTask; serverLaunchTask = SessionControllerFactory.LaunchNew( dmbToUse, null, @@ -231,7 +226,7 @@ namespace Tgstation.Server.Host.Components.Watchdog } else { - await chatTask; + await eventTask; serverLaunchTask = SessionControllerFactory.Reattach(reattachInfo, cancellationToken); } @@ -299,14 +294,17 @@ namespace Tgstation.Server.Host.Components.Watchdog /// /// The for the operation. /// A representing the running operation. - protected virtual Task HandleNewDmbAvailable(CancellationToken cancellationToken) + protected virtual async Task HandleNewDmbAvailable(CancellationToken cancellationToken) { gracefulRebootRequired = true; if (Server.CompileJob.DMApiVersion == null) - return Chat.QueueWatchdogMessage( - "A new deployment has been made but cannot be applied automatically as the currently running server has no DMAPI. Please manually reboot the server to apply the update.", - cancellationToken); - return Server.SetRebootState(Session.RebootState.Restart, cancellationToken); + { + Chat.QueueWatchdogMessage( + "A new deployment has been made but cannot be applied automatically as the currently running server has no DMAPI. Please manually reboot the server to apply the update."); + return; + } + + await Server.SetRebootState(Session.RebootState.Restart, cancellationToken); } /// diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs index 5188d636b3..aceb6c0fcf 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs @@ -347,10 +347,9 @@ namespace Tgstation.Server.Host.Components.Watchdog { if (!graceful) { - var chatTask = Chat.QueueWatchdogMessage("Manual restart triggered...", cancellationToken); + Chat.QueueWatchdogMessage("Manual restart triggered..."); await TerminateNoLock(false, false, cancellationToken); await LaunchNoLock(true, false, true, null, cancellationToken); - await chatTask; return; } @@ -425,7 +424,7 @@ namespace Tgstation.Server.Host.Components.Watchdog releaseServers = true; if (Status == WatchdogStatus.Online) - await Chat.QueueWatchdogMessage("Detaching...", cancellationToken); + Chat.QueueWatchdogMessage("Detaching..."); else Logger.LogTrace("Not sending detach chat message as status is: {status}", Status); } @@ -477,11 +476,11 @@ namespace Tgstation.Server.Host.Components.Watchdog /// /// Starts all s. /// - /// A, possibly active, for an outgoing chat message. + /// A, possibly active, for an event that's running. /// to use, if any. /// The for the operation. /// A representing the running operation. - protected abstract Task InitController(Task chatTask, ReattachInformation reattachInfo, CancellationToken cancellationToken); + protected abstract Task InitController(Task eventTask, ReattachInformation reattachInfo, CancellationToken cancellationToken); /// /// Launches the watchdog. @@ -507,21 +506,16 @@ namespace Tgstation.Server.Host.Components.Watchdog throw new JobException(ErrorCode.WatchdogCompileJobCorrupted); // this is necessary, the monitor could be in it's sleep loop trying to restart, if so cancel THAT monitor and start our own with blackjack and hookers - Task announceTask; + var eventTask = Task.CompletedTask; if (announce) { - announceTask = Chat.QueueWatchdogMessage( + Chat.QueueWatchdogMessage( reattachInfo == null ? "Launching..." - : "Reattaching...", - cancellationToken); // simple announce + : "Reattaching..."); // simple announce if (reattachInfo == null) - announceTask = Task.WhenAll( - HandleEvent(EventType.WatchdogLaunch, Enumerable.Empty(), false, cancellationToken), - announceTask); + eventTask = HandleEvent(EventType.WatchdogLaunch, Enumerable.Empty(), false, cancellationToken); } - else - announceTask = Task.CompletedTask; // no announce // since neither server is running, this is safe to do LastLaunchParameters = ActiveLaunchParameters; @@ -529,7 +523,7 @@ namespace Tgstation.Server.Host.Components.Watchdog try { - await InitController(announceTask, reattachInfo, cancellationToken); + await InitController(eventTask, reattachInfo, cancellationToken); } catch (OperationCanceledException ex) { @@ -539,15 +533,15 @@ namespace Tgstation.Server.Host.Components.Watchdog catch (Exception e) { Logger.LogWarning(e, "Failed to start watchdog!"); - var originalChatTask = announceTask; - async Task ChainChatTaskWithErrorMessage() + var originalChatTask = eventTask; + async Task ChainEventTaskWithErrorMessage() { await originalChatTask; if (announceFailure) - await Chat.QueueWatchdogMessage("Startup failed!", cancellationToken); + Chat.QueueWatchdogMessage("Startup failed!"); } - announceTask = ChainChatTaskWithErrorMessage(); + eventTask = ChainEventTaskWithErrorMessage(); throw; } finally @@ -555,7 +549,7 @@ namespace Tgstation.Server.Host.Components.Watchdog // finish the chat task that's in flight try { - await announceTask; + await eventTask; } catch (OperationCanceledException ex) { @@ -626,8 +620,8 @@ namespace Tgstation.Server.Host.Components.Watchdog const string FailReattachMessage = "Unable to properly reattach to server! Restarting watchdog..."; Logger.LogWarning(FailReattachMessage); - var chatTask = Chat.QueueWatchdogMessage(FailReattachMessage, cancellationToken); - await InitController(chatTask, null, cancellationToken); + Chat.QueueWatchdogMessage(FailReattachMessage); + await InitController(Task.CompletedTask, null, cancellationToken); } /// @@ -730,7 +724,6 @@ namespace Tgstation.Server.Host.Components.Watchdog await DisposeAndNullControllers(cancellationToken); - var chatTask = Task.CompletedTask; for (var retryAttempts = 1; ; ++retryAttempts) { Status = WatchdogStatus.Restoring; @@ -748,10 +741,6 @@ namespace Tgstation.Server.Host.Components.Watchdog { launchException = e; } - finally - { - await chatTask; - } Logger.LogWarning(launchException, "Failed to automatically restart the watchdog! Attempt: {attemptNumber}", retryAttempts); Status = WatchdogStatus.DelayedRestart; @@ -761,16 +750,12 @@ namespace Tgstation.Server.Host.Components.Watchdog Math.Pow(2, retryAttempts)), TimeSpan.FromHours(1).TotalSeconds); // max of one hour, increasing by a power of 2 each time - chatTask = Chat.QueueWatchdogMessage( - $"Failed to restart (Attempt: {retryAttempts}), retrying in {retryDelay}s...", - cancellationToken); + Chat.QueueWatchdogMessage( + $"Failed to restart (Attempt: {retryAttempts}), retrying in {retryDelay}s..."); - await Task.WhenAll( - AsyncDelayer.Delay( - TimeSpan.FromSeconds(retryDelay), - cancellationToken), - chatTask) - ; + await AsyncDelayer.Delay( + TimeSpan.FromSeconds(retryDelay), + cancellationToken); } } @@ -966,9 +951,8 @@ namespace Tgstation.Server.Host.Components.Watchdog var nextActionMessage = nextAction != MonitorAction.Exit ? "Recovering" : "Shutting down"; - var chatTask = Chat.QueueWatchdogMessage( - $"Monitor crashed, this should NEVER happen! Please report this, full details in logs! {nextActionMessage}. Error: {e.Message}", - cancellationToken); + Chat.QueueWatchdogMessage( + $"Monitor crashed, this should NEVER happen! Please report this, full details in logs! {nextActionMessage}. Error: {e.Message}"); if (disposed) nextAction = MonitorAction.Exit; @@ -980,8 +964,6 @@ namespace Tgstation.Server.Host.Components.Watchdog Logger.LogDebug("Server seems to be okay, not restarting"); nextAction = MonitorAction.Continue; } - - await chatTask; } } catch (OperationCanceledException) @@ -1026,15 +1008,14 @@ namespace Tgstation.Server.Host.Components.Watchdog releaseServers, cancellationToken); - var chatTask = announce ? Chat.QueueWatchdogMessage("Shutting down...", cancellationToken) : Task.CompletedTask; + if (announce) + Chat.QueueWatchdogMessage("Shutting down..."); await eventTask; await StopMonitor(); LastLaunchParameters = null; - - await chatTask; return; } @@ -1071,7 +1052,7 @@ namespace Tgstation.Server.Host.Components.Watchdog case 2: const string message2 = "DEFCON 3: DreamDaemon has missed 2 heartbeats!"; Logger.LogInformation(message2); - await Chat.QueueWatchdogMessage(message2, cancellationToken); + Chat.QueueWatchdogMessage(message2); break; case 3: var actionToTake = shouldShutdown @@ -1079,12 +1060,11 @@ namespace Tgstation.Server.Host.Components.Watchdog : "be restarted"; const string logTemplate1 = "DEFCON 2: DreamDaemon has missed 3 heartbeats! If it does not respond to the next one, the watchdog will {actionToTake}!"; Logger.LogWarning(logTemplate1, actionToTake); - await Chat.QueueWatchdogMessage( + Chat.QueueWatchdogMessage( logTemplate1.Replace( "{actionToTake}", actionToTake, - StringComparison.Ordinal), - cancellationToken); + StringComparison.Ordinal)); break; case 4: var actionTaken = shouldShutdown @@ -1092,12 +1072,11 @@ namespace Tgstation.Server.Host.Components.Watchdog : "Restarting"; const string logTemplate2 = "DEFCON 1: Four heartbeats have been missed! {actionTaken}..."; Logger.LogWarning(logTemplate2, actionTaken); - await Chat.QueueWatchdogMessage( + Chat.QueueWatchdogMessage( logTemplate2.Replace( "{actionTaken}", actionTaken, - StringComparison.Ordinal), - cancellationToken); + StringComparison.Ordinal)); if (ActiveLaunchParameters.DumpOnHeartbeatRestart.Value) {