From a6caed51e88d02581c9d89c91d08c48bed1d8ab5 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 14 Jan 2021 11:30:21 -0500 Subject: [PATCH 1/6] Re-add tgs.dm to .sln --- tgstation-server.sln | 1 + 1 file changed, 1 insertion(+) diff --git a/tgstation-server.sln b/tgstation-server.sln index 7e771b4b01..32dabe36ba 100644 --- a/tgstation-server.sln +++ b/tgstation-server.sln @@ -65,6 +65,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tgs", "tgs", "{F7765A4B-021 src\DMAPI\tgs\includes.dm = src\DMAPI\tgs\includes.dm src\DMAPI\tgs\LICENSE = src\DMAPI\tgs\LICENSE src\DMAPI\tgs\README.md = src\DMAPI\tgs\README.md + src\DMAPI\tgs.dm = src\DMAPI\tgs.dm EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "core", "core", "{DCCBA9DA-47BA-4C70-823B-E99A3ACA0377}" From e9e4439a861290eee2918bf5c03e6de5b85d3b18 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 14 Jan 2021 11:38:19 -0500 Subject: [PATCH 2/6] Adds a few event types - Improves watchdog monitoring logic so events that happen while one is already processing are no longer dropped. --- src/DMAPI/tgs.dm | 7 +-- .../Components/Events/EventType.cs | 14 ++++- .../Components/Session/ISessionController.cs | 4 +- .../Components/Watchdog/BasicWatchdog.cs | 8 ++- .../Watchdog/MonitorActivationReason.cs | 7 ++- .../Components/Watchdog/WatchdogBase.cs | 53 ++++++++++++++++--- 6 files changed, 78 insertions(+), 15 deletions(-) diff --git a/src/DMAPI/tgs.dm b/src/DMAPI/tgs.dm index fb6261a9bc..f54d519b9e 100644 --- a/src/DMAPI/tgs.dm +++ b/src/DMAPI/tgs.dm @@ -95,10 +95,12 @@ #define TGS_EVENT_WATCHDOG_SHUTDOWN 15 /// Before the watchdog detaches for a TGS update/restart. No parameters. #define TGS_EVENT_WATCHDOG_DETACH 16 -// We don't actually implement this value as the DMAPI can never receive it +// We don't actually implement these 3 events as the DMAPI can never receive them. // #define TGS_EVENT_WATCHDOG_LAUNCH 17 -// Same here. We don't actually implement this value as the DMAPI can never receive it // #define TGS_EVENT_WORLD_END_PROCESS 18 +// #define TGS_EVENT_WORLD_DEL 19 +/// Watchdog event when TgsInitializationComplete() is called. No parameters. + #define TGS_EVENT_WORLD_PRIME 20 // OTHER ENUMS @@ -132,7 +134,6 @@ * * This may use [/world/var/sleep_offline] to make this happen so ensure no changes are made to it while this call is running. * Afterwards, consider explicitly setting it to what you want to avoid this BYOND bug: http://www.byond.com/forum/post/2575184 - * Before this point, note that any static files or directories may be in use by another server. Your code should account for this. * This function should not be called before ..() in [/world/proc/New]. */ /world/proc/TgsInitializationComplete() diff --git a/src/Tgstation.Server.Host/Components/Events/EventType.cs b/src/Tgstation.Server.Host/Components/Events/EventType.cs index f22c1db8e0..d99ee079e2 100644 --- a/src/Tgstation.Server.Host/Components/Events/EventType.cs +++ b/src/Tgstation.Server.Host/Components/Events/EventType.cs @@ -114,9 +114,21 @@ namespace Tgstation.Server.Host.Components.Events WatchdogLaunch, /// - /// In between DD restarts if the process has been force-ended by the DMAPI (TgsEndProcess()) + /// In between watchdog DreamDaemon restarts if the process has been force-ended by the DMAPI (TgsEndProcess()). No parameters. /// [EventScript("WorldEndProcess")] WorldEndProcess, + + /// + /// After any sort of watchdog DreamDaemon closure or reboot. Reboots only supported with DMAPI. Not synchronous. Called after . No parameters. + /// + [EventScript("WorldDel")] + WorldDel, + + /// + /// Watchdog event when TgsInitializationComplete() is called. No parameters. + /// + [EventScript("WorldPrime")] + WorldPrime, } } diff --git a/src/Tgstation.Server.Host/Components/Session/ISessionController.cs b/src/Tgstation.Server.Host/Components/Session/ISessionController.cs index 778c39ba38..9c7a43cb3a 100644 --- a/src/Tgstation.Server.Host/Components/Session/ISessionController.cs +++ b/src/Tgstation.Server.Host/Components/Session/ISessionController.cs @@ -49,12 +49,12 @@ namespace Tgstation.Server.Host.Components.Session RebootState RebootState { get; } /// - /// A that completes when the server calls /world/Reboot() + /// A that completes when the server calls /world/TgsReboot(). /// Task OnReboot { get; } /// - /// A that completes when the server calls /world/TgsInitializationsComplete() + /// A that completes when the server calls /world/TgsInitializationComplete() /// Task OnPrime { get; } diff --git a/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs index 433680a941..d8db5b4163 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs @@ -95,6 +95,8 @@ namespace Tgstation.Server.Host.Components.Watchdog if (Server.TerminationWasRequested) await EventConsumer.HandleEvent(EventType.WorldEndProcess, Enumerable.Empty(), cancellationToken).ConfigureAwait(false); + await EventConsumer.HandleEvent(EventType.WorldDel, Enumerable.Empty(), cancellationToken).ConfigureAwait(false); + string exitWord = Server.TerminationWasRequested ? "exited" : "crashed"; if (Server.RebootState == Session.RebootState.Shutdown) { @@ -128,6 +130,8 @@ namespace Tgstation.Server.Host.Components.Watchdog gracefulRebootRequired = false; Server.ResetRebootState(); + await EventConsumer.HandleEvent(EventType.WorldDel, Enumerable.Empty(), cancellationToken).ConfigureAwait(false); + switch (rebootState) { case Session.RebootState.Normal: @@ -144,7 +148,6 @@ namespace Tgstation.Server.Host.Components.Watchdog default: throw new InvalidOperationException($"Invalid reboot state: {rebootState}"); } - case MonitorActivationReason.ActiveLaunchParametersUpdated: await Server.SetRebootState(Session.RebootState.Restart, cancellationToken).ConfigureAwait(false); gracefulRebootRequired = true; @@ -152,6 +155,9 @@ namespace Tgstation.Server.Host.Components.Watchdog case MonitorActivationReason.NewDmbAvailable: await HandleNewDmbAvailable(cancellationToken).ConfigureAwait(false); break; + case MonitorActivationReason.ActiveServerPrimed: + await EventConsumer.HandleEvent(EventType.WorldPrime, Enumerable.Empty(), cancellationToken).ConfigureAwait(false); + break; case MonitorActivationReason.Heartbeat: default: throw new InvalidOperationException($"Invalid activation reason: {reason}"); diff --git a/src/Tgstation.Server.Host/Components/Watchdog/MonitorActivationReason.cs b/src/Tgstation.Server.Host/Components/Watchdog/MonitorActivationReason.cs index 4b0d046561..4f1073e742 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/MonitorActivationReason.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/MonitorActivationReason.cs @@ -1,4 +1,4 @@ -namespace Tgstation.Server.Host.Components.Watchdog +namespace Tgstation.Server.Host.Components.Watchdog { /// /// Reasons for the monitor to wake up @@ -29,5 +29,10 @@ /// A heartbeat is required. /// Heartbeat, + + /// + /// Server primed. + /// + ActiveServerPrimed, } } diff --git a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs index f6a7ed6160..3e89293cb3 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/WatchdogBase.cs @@ -609,6 +609,7 @@ namespace Tgstation.Server.Host.Components.Watchdog /// /// The for the operation. /// A representing the running operation. + #pragma warning disable CA1502 private async Task MonitorLifetimes(CancellationToken cancellationToken) { Logger.LogTrace("Entered MonitorLifetimes"); @@ -619,6 +620,12 @@ namespace Tgstation.Server.Host.Components.Watchdog try { MonitorAction nextAction = MonitorAction.Continue; + Task activeServerLifetime = null, + activeServerReboot = null, + serverPrimed = null, + activeLaunchParametersChanged = null, + newDmbAvailable = null; + ISessionController lastController = null; for (ulong iteration = 1; nextAction != MonitorAction.Exit; ++iteration) using (LogContext.PushProperty("Monitor", iteration)) try @@ -627,17 +634,44 @@ namespace Tgstation.Server.Host.Components.Watchdog nextAction = MonitorAction.Continue; var controller = GetActiveController(); - Task activeServerLifetime = controller.Lifetime; - var activeServerReboot = controller.OnReboot; - Task activeLaunchParametersChanged = ActiveParametersUpdated.Task; - var newDmbAvailable = DmbFactory.OnNewerDmb; + void UpdateMonitoredTasks() + { + static void TryUpdateTask(ref Task oldTask, Task newTask) + { + if (oldTask?.IsCompleted == true) + return; + + oldTask = newTask; + } + + if (lastController == controller) + { + TryUpdateTask(ref activeServerLifetime, controller.Lifetime); + TryUpdateTask(ref activeServerReboot, controller.OnReboot); + TryUpdateTask(ref serverPrimed, controller.OnPrime); + } + else + { + activeServerLifetime = controller.Lifetime; + activeServerReboot = controller.OnReboot; + serverPrimed = controller.OnPrime; + lastController = controller; + } + + TryUpdateTask(ref activeLaunchParametersChanged, ActiveParametersUpdated.Task); + TryUpdateTask(ref newDmbAvailable, DmbFactory.OnNewerDmb); + } + + UpdateMonitoredTasks(); var heartbeatSeconds = ActiveLaunchParameters.HeartbeatSeconds.Value; var heartbeat = heartbeatSeconds == 0 || !controller.DMApiAvailable ? Extensions.TaskExtensions.InfiniteTask() - : Task.Delay(TimeSpan.FromSeconds(heartbeatSeconds), cancellationToken); + : Task.Delay( + TimeSpan.FromSeconds(heartbeatSeconds), + cancellationToken); // cancel waiting if requested var cancelTcs = new TaskCompletionSource(); @@ -647,7 +681,8 @@ namespace Tgstation.Server.Host.Components.Watchdog heartbeat, newDmbAvailable, cancelTcs.Task, - activeLaunchParametersChanged); + activeLaunchParametersChanged, + serverPrimed); // wait for something to happen using (cancellationToken.Register(() => cancelTcs.SetCanceled())) @@ -688,7 +723,10 @@ namespace Tgstation.Server.Host.Components.Watchdog || CheckActivationReason(ref activeServerReboot, MonitorActivationReason.ActiveServerRebooted) || CheckActivationReason(ref newDmbAvailable, MonitorActivationReason.NewDmbAvailable) || CheckActivationReason(ref activeLaunchParametersChanged, MonitorActivationReason.ActiveLaunchParametersUpdated) - || CheckActivationReason(ref heartbeat, MonitorActivationReason.Heartbeat); + || CheckActivationReason(ref heartbeat, MonitorActivationReason.Heartbeat) + || CheckActivationReason(ref serverPrimed, MonitorActivationReason.ActiveServerPrimed); + + UpdateMonitoredTasks(); if (!anyActivation) moreActivationsToProcess = false; @@ -769,6 +807,7 @@ namespace Tgstation.Server.Host.Components.Watchdog Logger.LogTrace("Monitor exiting..."); } + #pragma warning restore CA1502 /// /// Starts all s. From a04b1cccf7048855392dca8e65d5b395c21b8177 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 14 Jan 2021 13:32:24 -0500 Subject: [PATCH 3/6] Do not call WorldDel on crashes --- src/Tgstation.Server.Host/Components/Events/EventType.cs | 4 ++-- .../Components/Watchdog/BasicWatchdog.cs | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/Tgstation.Server.Host/Components/Events/EventType.cs b/src/Tgstation.Server.Host/Components/Events/EventType.cs index d99ee079e2..38aed033d9 100644 --- a/src/Tgstation.Server.Host/Components/Events/EventType.cs +++ b/src/Tgstation.Server.Host/Components/Events/EventType.cs @@ -120,13 +120,13 @@ namespace Tgstation.Server.Host.Components.Events WorldEndProcess, /// - /// After any sort of watchdog DreamDaemon closure or reboot. Reboots only supported with DMAPI. Not synchronous. Called after . No parameters. + /// Watchdog event when TgsReboot() is called. Not synchronous. Called after . No parameters. /// [EventScript("WorldDel")] WorldDel, /// - /// Watchdog event when TgsInitializationComplete() is called. No parameters. + /// Watchdog event when TgsInitializationsComplete() is called. No parameters. /// [EventScript("WorldPrime")] WorldPrime, diff --git a/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs index d8db5b4163..176ed5b106 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs @@ -95,8 +95,6 @@ namespace Tgstation.Server.Host.Components.Watchdog if (Server.TerminationWasRequested) await EventConsumer.HandleEvent(EventType.WorldEndProcess, Enumerable.Empty(), cancellationToken).ConfigureAwait(false); - await EventConsumer.HandleEvent(EventType.WorldDel, Enumerable.Empty(), cancellationToken).ConfigureAwait(false); - string exitWord = Server.TerminationWasRequested ? "exited" : "crashed"; if (Server.RebootState == Session.RebootState.Shutdown) { From afcdafa564f1bcf3d002c518f45ea926be71ac2b Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 14 Jan 2021 13:35:47 -0500 Subject: [PATCH 4/6] Add a seperate event for unexpected DreamDaemon exits --- src/DMAPI/tgs.dm | 7 ++++--- src/Tgstation.Server.Host/Components/Events/EventType.cs | 6 ++++++ .../Components/Watchdog/BasicWatchdog.cs | 8 +++++--- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/DMAPI/tgs.dm b/src/DMAPI/tgs.dm index f54d519b9e..c08b5d1ae2 100644 --- a/src/DMAPI/tgs.dm +++ b/src/DMAPI/tgs.dm @@ -97,10 +97,11 @@ #define TGS_EVENT_WATCHDOG_DETACH 16 // We don't actually implement these 3 events as the DMAPI can never receive them. // #define TGS_EVENT_WATCHDOG_LAUNCH 17 -// #define TGS_EVENT_WORLD_END_PROCESS 18 -// #define TGS_EVENT_WORLD_DEL 19 +// #define TGS_EVENT_WATCHDOG_CRASH 18 +// #define TGS_EVENT_WORLD_END_PROCESS 19 +// #define TGS_EVENT_WORLD_DEL 20 /// Watchdog event when TgsInitializationComplete() is called. No parameters. - #define TGS_EVENT_WORLD_PRIME 20 + #define TGS_EVENT_WORLD_PRIME 21 // OTHER ENUMS diff --git a/src/Tgstation.Server.Host/Components/Events/EventType.cs b/src/Tgstation.Server.Host/Components/Events/EventType.cs index 38aed033d9..d0a6b2b4be 100644 --- a/src/Tgstation.Server.Host/Components/Events/EventType.cs +++ b/src/Tgstation.Server.Host/Components/Events/EventType.cs @@ -113,6 +113,12 @@ namespace Tgstation.Server.Host.Components.Events [EventScript("WatchdogLaunch")] WatchdogLaunch, + /// + /// Watchdog event when DreamDaemon exits unexpectedly. No parameters. + /// + [EventScript("WatchdogLaunch")] + WatchdogCrash, + /// /// In between watchdog DreamDaemon restarts if the process has been force-ended by the DMAPI (TgsEndProcess()). No parameters. /// diff --git a/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs index 176ed5b106..e334bdc63b 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs @@ -92,10 +92,12 @@ namespace Tgstation.Server.Host.Components.Watchdog switch (reason) { case MonitorActivationReason.ActiveServerCrashed: - if (Server.TerminationWasRequested) - await EventConsumer.HandleEvent(EventType.WorldEndProcess, Enumerable.Empty(), cancellationToken).ConfigureAwait(false); + var eventType = Server.TerminationWasRequested + ? EventType.WorldEndProcess + : EventType.WatchdogCrash; + await EventConsumer.HandleEvent(eventType, Enumerable.Empty(), cancellationToken).ConfigureAwait(false); - string exitWord = Server.TerminationWasRequested ? "exited" : "crashed"; + var exitWord = Server.TerminationWasRequested ? "exited" : "crashed"; if (Server.RebootState == Session.RebootState.Shutdown) { // the time for graceful shutdown is now From 944a6a4f2f53163fc80de743cf1fa9242ed94fdf Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 14 Jan 2021 14:00:13 -0500 Subject: [PATCH 5/6] Why am I calling it Del? --- src/DMAPI/tgs.dm | 2 +- src/Tgstation.Server.Host/Components/Events/EventType.cs | 6 +++--- .../Components/Watchdog/BasicWatchdog.cs | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/DMAPI/tgs.dm b/src/DMAPI/tgs.dm index c08b5d1ae2..7ef3268322 100644 --- a/src/DMAPI/tgs.dm +++ b/src/DMAPI/tgs.dm @@ -99,7 +99,7 @@ // #define TGS_EVENT_WATCHDOG_LAUNCH 17 // #define TGS_EVENT_WATCHDOG_CRASH 18 // #define TGS_EVENT_WORLD_END_PROCESS 19 -// #define TGS_EVENT_WORLD_DEL 20 +// #define TGS_EVENT_WORLD_REBOOT 20 /// Watchdog event when TgsInitializationComplete() is called. No parameters. #define TGS_EVENT_WORLD_PRIME 21 diff --git a/src/Tgstation.Server.Host/Components/Events/EventType.cs b/src/Tgstation.Server.Host/Components/Events/EventType.cs index d0a6b2b4be..090c5c6982 100644 --- a/src/Tgstation.Server.Host/Components/Events/EventType.cs +++ b/src/Tgstation.Server.Host/Components/Events/EventType.cs @@ -96,7 +96,7 @@ namespace Tgstation.Server.Host.Components.Events DeploymentComplete, /// - /// Before the watchdog shutsdown. Not sent for graceful shutdowns. No parameters. + /// Before the watchdog shuts down. Not sent for graceful shutdowns. No parameters. /// [EventScript("WatchdogShutdown")] WatchdogShutdown, @@ -128,8 +128,8 @@ namespace Tgstation.Server.Host.Components.Events /// /// Watchdog event when TgsReboot() is called. Not synchronous. Called after . No parameters. /// - [EventScript("WorldDel")] - WorldDel, + [EventScript("WorldReboot")] + WorldReboot, /// /// Watchdog event when TgsInitializationsComplete() is called. No parameters. diff --git a/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs b/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs index e334bdc63b..3cddd03a02 100644 --- a/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs +++ b/src/Tgstation.Server.Host/Components/Watchdog/BasicWatchdog.cs @@ -130,7 +130,7 @@ namespace Tgstation.Server.Host.Components.Watchdog gracefulRebootRequired = false; Server.ResetRebootState(); - await EventConsumer.HandleEvent(EventType.WorldDel, Enumerable.Empty(), cancellationToken).ConfigureAwait(false); + await EventConsumer.HandleEvent(EventType.WorldReboot, Enumerable.Empty(), cancellationToken).ConfigureAwait(false); switch (rebootState) { From 453ddacd0c169381af290ad7e569f7a4a9a4eb84 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Thu, 14 Jan 2021 14:24:23 -0500 Subject: [PATCH 6/6] Fix duplicate script name --- src/Tgstation.Server.Host/Components/Events/EventType.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tgstation.Server.Host/Components/Events/EventType.cs b/src/Tgstation.Server.Host/Components/Events/EventType.cs index 090c5c6982..dabb822de7 100644 --- a/src/Tgstation.Server.Host/Components/Events/EventType.cs +++ b/src/Tgstation.Server.Host/Components/Events/EventType.cs @@ -116,7 +116,7 @@ namespace Tgstation.Server.Host.Components.Events /// /// Watchdog event when DreamDaemon exits unexpectedly. No parameters. /// - [EventScript("WatchdogLaunch")] + [EventScript("WatchdogCrash")] WatchdogCrash, ///