From 0be37fb6b48b96fab71499c1e5e38b2ec7937a84 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Mon, 6 Jul 2020 12:32:14 -0400 Subject: [PATCH] Improve monitor crash handling - Fix Status never getting set to DelayedRestart --- .../Components/Watchdog/WatchdogBase.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs index e24c1d595a..34f250cee7 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs @@ -559,6 +559,7 @@ namespace Tgstation.Server.Host.Components.Watchdog } Logger.LogWarning("Failed to automatically restart the watchdog! Attempt: {0}, Exception: {1}", retryAttempts, launchException); + Status = WatchdogStatus.DelayedRestart; var retryDelay = Math.Min( Convert.ToInt32( @@ -702,7 +703,7 @@ namespace Tgstation.Server.Host.Components.Watchdog e); var nextActionMessage = nextAction != MonitorAction.Exit - ? "Restarting" + ? "Recovering" : "Shutting down"; var chatTask = Chat.SendWatchdogMessage( $"Monitor crashed, this should NEVER happen! Please report this, full details in logs! {nextActionMessage}. Error: {e.Message}", @@ -713,7 +714,10 @@ namespace Tgstation.Server.Host.Components.Watchdog nextAction = MonitorAction.Exit; else if (nextAction != MonitorAction.Exit) { - await MonitorRestart(cancellationToken).ConfigureAwait(false); + if (GetActiveController()?.Lifetime.IsCompleted != true) + await MonitorRestart(cancellationToken).ConfigureAwait(false); + else + Logger.LogDebug("Server seems to be okay, not restarting"); nextAction = MonitorAction.Continue; }