mirror of
https://github.com/tgstation/tgstation-server.git
synced 2026-07-17 19:12:52 +01:00
Merge pull request #1516 from tgstation/DMAPIHeartbeatEvent [DMDeploy]
DMAPI Health Check Event
This commit is contained in:
Jordan Dominion
+3
-3
@@ -6,9 +6,9 @@
|
||||
<TgsCoreVersion>5.12.4</TgsCoreVersion>
|
||||
<TgsConfigVersion>4.6.0</TgsConfigVersion>
|
||||
<TgsApiVersion>9.10.2</TgsApiVersion>
|
||||
<TgsApiLibraryVersion>10.4.1</TgsApiLibraryVersion>
|
||||
<TgsClientVersion>11.4.3</TgsClientVersion>
|
||||
<TgsDmapiVersion>6.4.5</TgsDmapiVersion>
|
||||
<TgsApiLibraryVersion>11.0.0</TgsApiLibraryVersion>
|
||||
<TgsClientVersion>12.0.0</TgsClientVersion>
|
||||
<TgsDmapiVersion>6.5.0</TgsDmapiVersion>
|
||||
<TgsInteropVersion>5.6.1</TgsInteropVersion>
|
||||
<TgsHostWatchdogVersion>1.2.2</TgsHostWatchdogVersion>
|
||||
<TgsContainerScriptVersion>1.2.1</TgsContainerScriptVersion>
|
||||
|
||||
+2
-2
@@ -25,7 +25,7 @@
|
||||
- Multiple chat bots per instance: Up to 65535 as a matter of fact (who knows why I chose that number?)!
|
||||
- Automatic Chat Bot Reconnection Intervals: Set in minutes.
|
||||
- Better Error State Handling: The Server and Watchdog aren't your momma's boys anymore. Every error state will be automatically resolved or reported with recommended actions.
|
||||
- Watchdog Heartbeats: An interval in seconds can now be set at which TGS will send /world/Topic() packets to DreamDaemon. If four of these are missed, the server will be rebooted. No more endless @Key Holder pings in discord (and I can finally unmute the /tg/ guild)! This feature can be disabled.
|
||||
- Watchdog Health Checks: An interval in seconds can now be set at which TGS will send /world/Topic() packets to DreamDaemon. If four of these are missed, the server will be rebooted. No more endless @Key Holder pings in discord (and I can finally unmute the /tg/ guild)! This feature can be disabled.
|
||||
- Better DMAPI: No longer requires injecting a .NET runtime .dll into the DreamDaemon process. DD -> TGS communication is now handled securely via BYOND's native /world/Export() API ("But Cyberboss, BYOND only supports GET requests." Who said anything about respecting HTTP standards when dealing with BYOND?).
|
||||
- Safe/Ultrasafe Security Support: Thanks to the new DMAPI, the ultrasafe and safe security levels may be used without running into BYOND's limitations. But no one really cares...
|
||||
- Private/Invisible visibility Support: Stored per instance.
|
||||
@@ -86,7 +86,7 @@ tgstation-server is a BYOND server managment suite. It includes all the followin
|
||||
- GitHub Deployment Environments support
|
||||
- The Watchdog, a DreamDaemon uptime monitor
|
||||
- Automatically restarts the server when crashed until told to stop
|
||||
- "Heartbeat" system for checking DreamDaemon isn't hung
|
||||
- "Health Check" system for checking DreamDaemon isn't hung
|
||||
- Seamless automatic application of new deployments on /world/Reboot
|
||||
- Automatic startup at server boot time
|
||||
- Automatically handles Windows prompts related to trusted mode
|
||||
|
||||
+65
-60
@@ -1,6 +1,6 @@
|
||||
// tgstation-server DMAPI
|
||||
|
||||
#define TGS_DMAPI_VERSION "6.4.5"
|
||||
#define TGS_DMAPI_VERSION "6.5.0"
|
||||
|
||||
// All functions and datums outside this document are subject to change with any version and should not be relied on.
|
||||
|
||||
@@ -12,8 +12,8 @@
|
||||
// Comment this out once you've filled in the below.
|
||||
#error TGS API unconfigured
|
||||
|
||||
// Uncomment this if you wish to allow the game to interact with TGS 3.
|
||||
// This will raise the minimum required security level of your game to TGS_SECURITY_TRUSTED due to it utilizing call()()
|
||||
// Uncomment this if you wish to allow the game to interact with TGS 3..
|
||||
// This will raise the minimum required security level of your game to TGS_SECURITY_TRUSTED due to it utilizing call()().
|
||||
//#define TGS_V3_API
|
||||
|
||||
// Required interfaces (fill in with your codebase equivalent):
|
||||
@@ -52,44 +52,46 @@
|
||||
|
||||
// EVENT CODES
|
||||
|
||||
/// Before a reboot mode change, extras parameters are the current and new reboot mode enums
|
||||
/// Before a reboot mode change, extras parameters are the current and new reboot mode enums.
|
||||
#define TGS_EVENT_REBOOT_MODE_CHANGE -1
|
||||
/// Before a port change is about to happen, extra parameters is new port
|
||||
/// Before a port change is about to happen, extra parameters is new port.
|
||||
#define TGS_EVENT_PORT_SWAP -2
|
||||
/// Before the instance is renamed, extra parameter is the new name
|
||||
/// Before the instance is renamed, extra parameter is the new name.
|
||||
#define TGS_EVENT_INSTANCE_RENAMED -3
|
||||
/// After the watchdog reattaches to DD, extra parameter is the new [/datum/tgs_version] of the server
|
||||
/// After the watchdog reattaches to DD, extra parameter is the new [/datum/tgs_version] of the server.
|
||||
#define TGS_EVENT_WATCHDOG_REATTACH -4
|
||||
/// When the watchdog sends a health check to DD. No parameters.
|
||||
#define TGS_EVENT_HEALTH_CHECK -5
|
||||
|
||||
/// When the repository is reset to its origin reference. Parameters: Reference name, Commit SHA
|
||||
/// When the repository is reset to its origin reference. Parameters: Reference name, Commit SHA.
|
||||
#define TGS_EVENT_REPO_RESET_ORIGIN 0
|
||||
/// When the repository performs a checkout. Parameters: Checkout git object
|
||||
/// When the repository performs a checkout. Parameters: Checkout git object.
|
||||
#define TGS_EVENT_REPO_CHECKOUT 1
|
||||
/// When the repository performs a fetch operation. No parameters
|
||||
/// When the repository performs a fetch operation. No parameters.
|
||||
#define TGS_EVENT_REPO_FETCH 2
|
||||
/// When the repository test merges. Parameters: PR Number, PR Sha, (Nullable) Comment made by TGS user
|
||||
/// When the repository test merges. Parameters: PR Number, PR Sha, (Nullable) Comment made by TGS user.
|
||||
#define TGS_EVENT_REPO_MERGE_PULL_REQUEST 3
|
||||
/// Before the repository makes a sychronize operation. Parameters: Absolute repostiory path
|
||||
/// Before the repository makes a sychronize operation. Parameters: Absolute repostiory path.
|
||||
#define TGS_EVENT_REPO_PRE_SYNCHRONIZE 4
|
||||
/// Before a BYOND install operation begins. Parameters: [/datum/tgs_version] of the installing BYOND
|
||||
/// Before a BYOND install operation begins. Parameters: [/datum/tgs_version] of the installing BYOND.
|
||||
#define TGS_EVENT_BYOND_INSTALL_START 5
|
||||
/// When a BYOND install operation fails. Parameters: Error message
|
||||
#define TGS_EVENT_BYOND_INSTALL_FAIL 6
|
||||
/// When the active BYOND version changes. Parameters: (Nullable) [/datum/tgs_version] of the current BYOND, [/datum/tgs_version] of the new BYOND
|
||||
/// When the active BYOND version changes. Parameters: (Nullable) [/datum/tgs_version] of the current BYOND, [/datum/tgs_version] of the new BYOND.
|
||||
#define TGS_EVENT_BYOND_ACTIVE_VERSION_CHANGE 7
|
||||
/// When the compiler starts running. Parameters: Game directory path, origin commit SHA
|
||||
/// When the compiler starts running. Parameters: Game directory path, origin commit SHA.
|
||||
#define TGS_EVENT_COMPILE_START 8
|
||||
/// When a compile is cancelled. No parameters
|
||||
/// When a compile is cancelled. No parameters.
|
||||
#define TGS_EVENT_COMPILE_CANCELLED 9
|
||||
/// When a compile fails. Parameters: Game directory path, [TRUE]/[FALSE] based on if the cause for failure was DMAPI validation
|
||||
/// When a compile fails. Parameters: Game directory path, [TRUE]/[FALSE] based on if the cause for failure was DMAPI validation.
|
||||
#define TGS_EVENT_COMPILE_FAILURE 10
|
||||
/// When a compile operation completes. Note, this event fires before the new .dmb is loaded into the watchdog. Consider using the [TGS_EVENT_DEPLOYMENT_COMPLETE] instead. Parameters: Game directory path
|
||||
/// When a compile operation completes. Note, this event fires before the new .dmb is loaded into the watchdog. Consider using the [TGS_EVENT_DEPLOYMENT_COMPLETE] instead. Parameters: Game directory path.
|
||||
#define TGS_EVENT_COMPILE_COMPLETE 11
|
||||
/// When an automatic update for the current instance begins. No parameters
|
||||
/// When an automatic update for the current instance begins. No parameters.
|
||||
#define TGS_EVENT_INSTANCE_AUTO_UPDATE_START 12
|
||||
/// When the repository encounters a merge conflict: Parameters: Base SHA, target SHA, base reference, target reference
|
||||
/// When the repository encounters a merge conflict: Parameters: Base SHA, target SHA, base reference, target reference.
|
||||
#define TGS_EVENT_REPO_MERGE_CONFLICT 13
|
||||
/// When a deployment completes. No Parameters
|
||||
/// When a deployment completes. No Parameters.
|
||||
#define TGS_EVENT_DEPLOYMENT_COMPLETE 14
|
||||
/// Before the watchdog shuts down. Not sent for graceful shutdowns. No parameters.
|
||||
#define TGS_EVENT_WATCHDOG_SHUTDOWN 15
|
||||
@@ -104,11 +106,11 @@
|
||||
#define TGS_EVENT_WORLD_PRIME 21
|
||||
// DMAPI also doesnt implement this
|
||||
// #define TGS_EVENT_DREAM_DAEMON_LAUNCH 22
|
||||
/// After a single submodule update is performed. Parameters: Updated submodule name
|
||||
/// After a single submodule update is performed. Parameters: Updated submodule name.
|
||||
#define TGS_EVENT_REPO_SUBMODULE_UPDATE 23
|
||||
/// After CodeModifications are applied, before DreamMaker is run. Parameters: Game directory path, origin commit sha, byond version
|
||||
/// After CodeModifications are applied, before DreamMaker is run. Parameters: Game directory path, origin commit sha, byond version.
|
||||
#define TGS_EVENT_PRE_DREAM_MAKER 24
|
||||
/// Whenever a deployment folder is deleted from disk. Parameters: Game directory path
|
||||
/// Whenever a deployment folder is deleted from disk. Parameters: Game directory path.
|
||||
#define TGS_EVENT_DEPLOYMENT_CLEANUP 25
|
||||
|
||||
// OTHER ENUMS
|
||||
@@ -164,28 +166,28 @@
|
||||
/datum/tgs_revision_information
|
||||
/// Full SHA of the commit.
|
||||
var/commit
|
||||
/// ISO 8601 timestamp of when the commit was created
|
||||
/// ISO 8601 timestamp of when the commit was created.
|
||||
var/timestamp
|
||||
/// Full sha of last known remote commit. This may be null if the TGS repository is not currently tracking a remote branch.
|
||||
var/origin_commit
|
||||
|
||||
/// Represents a version.
|
||||
/datum/tgs_version
|
||||
/// The suite/major version number
|
||||
/// The suite/major version number.
|
||||
var/suite
|
||||
|
||||
// This group of variables can be null to represent a wild card
|
||||
/// The minor version number. null for wildcards
|
||||
// This group of variables can be null to represent a wild card.
|
||||
/// The minor version number. null for wildcards.
|
||||
var/minor
|
||||
/// The patch version number. null for wildcards
|
||||
/// The patch version number. null for wildcards.
|
||||
var/patch
|
||||
|
||||
/// Legacy version number. Generally null
|
||||
/// Legacy version number. Generally null.
|
||||
var/deprecated_patch
|
||||
|
||||
/// Unparsed string value
|
||||
/// Unparsed string value.
|
||||
var/raw_parameter
|
||||
/// String value minus prefix
|
||||
/// String value minus prefix.
|
||||
var/deprefixed_parameter
|
||||
|
||||
/**
|
||||
@@ -231,38 +233,43 @@
|
||||
var/is_admin_channel
|
||||
/// [TRUE]/[FALSE] if the channel is a private message channel for a [/datum/tgs_chat_user].
|
||||
var/is_private_channel
|
||||
/// Tag string associated with the channel in TGS
|
||||
/// Tag string associated with the channel in TGS.
|
||||
var/custom_tag
|
||||
/// [TRUE]/[FALSE] if the channel supports embeds
|
||||
/// [TRUE]/[FALSE] if the channel supports embeds.
|
||||
var/embeds_supported
|
||||
|
||||
// Represents a chat user
|
||||
/datum/tgs_chat_user
|
||||
/// TGS internal user ID.
|
||||
var/id
|
||||
// The user's display name.
|
||||
/// The user's display name.
|
||||
var/friendly_name
|
||||
// The string to use to ping this user in a message.
|
||||
/// The string to use to ping this user in a message.
|
||||
var/mention
|
||||
/// The [/datum/tgs_chat_channel] the user was from
|
||||
/// The [/datum/tgs_chat_channel] the user was from.
|
||||
var/datum/tgs_chat_channel/channel
|
||||
|
||||
/// User definable handler for TGS events.
|
||||
/datum/tgs_event_handler
|
||||
/// If the handler receieves [TGS_EVENT_HEALTH_CHECK] events.
|
||||
var/receive_health_checks = FALSE
|
||||
|
||||
/**
|
||||
* User definable callback for handling TGS events.
|
||||
*
|
||||
* event_code - One of the TGS_EVENT_ defines. Extra parameters will be documented in each
|
||||
* event_code - One of the TGS_EVENT_ defines. Extra parameters will be documented in each.
|
||||
*/
|
||||
/datum/tgs_event_handler/proc/HandleEvent(event_code, ...)
|
||||
set waitfor = FALSE
|
||||
return
|
||||
|
||||
/// User definable chat command
|
||||
/// User definable chat command.
|
||||
/datum/tgs_chat_command
|
||||
/// The string to trigger this command on a chat bot. e.g `@bot name ...` or `!tgs name ...`
|
||||
/// The string to trigger this command on a chat bot. e.g `@bot name ...` or `!tgs name ...`.
|
||||
var/name = ""
|
||||
/// The help text displayed for this command
|
||||
/// The help text displayed for this command.
|
||||
var/help_text = ""
|
||||
/// If this command should be available to game administrators only
|
||||
/// If this command should be available to game administrators only.
|
||||
var/admin_only = FALSE
|
||||
/// A subtype of [/datum/tgs_chat_command] that is ignored when enumerating available commands. Use this to create shared base /datums for commands.
|
||||
var/ignore_type
|
||||
@@ -276,7 +283,7 @@
|
||||
/datum/tgs_chat_command/proc/Run(datum/tgs_chat_user/sender, params)
|
||||
CRASH("[type] has no implementation for Run()")
|
||||
|
||||
/// User definable chat message
|
||||
/// User definable chat message.
|
||||
/datum/tgs_message_content
|
||||
/// The tring content of the message. Must be provided in New().
|
||||
var/text
|
||||
@@ -300,7 +307,7 @@
|
||||
/// Timestamp must be encoded as: time2text(world.timeofday, "YYYY-MM-DD hh:mm:ss"). Use the active timezone.
|
||||
var/timestamp
|
||||
|
||||
/// Colour must be #AARRGGBB or #RRGGBB hex string
|
||||
/// Colour must be #AARRGGBB or #RRGGBB hex string.
|
||||
var/colour
|
||||
|
||||
/// See https://discord.com/developers/docs/resources/channel#embed-object-embed-image-structure for details.
|
||||
@@ -318,7 +325,7 @@
|
||||
|
||||
var/list/datum/tgs_chat_embed/field/fields
|
||||
|
||||
/// Common datum for similar discord embed medias
|
||||
/// Common datum for similar discord embed medias.
|
||||
/datum/tgs_chat_embed/media
|
||||
/// Must be set in New().
|
||||
var/url
|
||||
@@ -396,16 +403,16 @@
|
||||
// No function below this succeeds if it TgsAvailable() returns FALSE or if TgsNew() has yet to be called.
|
||||
|
||||
/**
|
||||
* Forces a hard reboot of DreamDaemon by ending the process.
|
||||
* Forces a hard reboot of DreamDaemon by ending the process. This function may sleep!
|
||||
*
|
||||
* Unlike del(world) clients will try to reconnect.
|
||||
* If TGS has not requested a [TGS_REBOOT_MODE_SHUTDOWN] DreamDaemon will be launched again
|
||||
* If TGS has not requested a [TGS_REBOOT_MODE_SHUTDOWN] DreamDaemon will be launched again.
|
||||
*/
|
||||
/world/proc/TgsEndProcess()
|
||||
return
|
||||
|
||||
/**
|
||||
* Send a message to connected chats.
|
||||
* Send a message to connected chats. This function may sleep!
|
||||
*
|
||||
* message - The [/datum/tgs_message_content] to send.
|
||||
* admin_only: If [TRUE], message will be sent to admin connected chats. Vice-versa applies.
|
||||
@@ -414,7 +421,7 @@
|
||||
return
|
||||
|
||||
/**
|
||||
* Send a private message to a specific user.
|
||||
* Send a private message to a specific user. This function may sleep!
|
||||
*
|
||||
* message - The [/datum/tgs_message_content] to send.
|
||||
* user: The [/datum/tgs_chat_user] to PM.
|
||||
@@ -422,10 +429,8 @@
|
||||
/world/proc/TgsChatPrivateMessage(datum/tgs_message_content/message, datum/tgs_chat_user/user)
|
||||
return
|
||||
|
||||
// The following functions will sleep if a call to TgsNew() is sleeping
|
||||
|
||||
/**
|
||||
* Send a message to connected chats that are flagged as game-related in TGS.
|
||||
* Send a message to connected chats that are flagged as game-related in TGS. This function may sleep!
|
||||
*
|
||||
* message - The [/datum/tgs_message_content] to send.
|
||||
* channels - Optional list of [/datum/tgs_chat_channel]s to restrict the message to.
|
||||
@@ -433,38 +438,38 @@
|
||||
/world/proc/TgsChatBroadcast(datum/tgs_message_content/message, list/channels = null)
|
||||
return
|
||||
|
||||
/// Returns the current [/datum/tgs_version] of TGS if it is running the server, null otherwise.
|
||||
/// Returns the current [/datum/tgs_version] of TGS if it is running the server, null otherwise. This function may sleep if the call to [/world/proc/TgsNew] is sleeping!
|
||||
/world/proc/TgsVersion()
|
||||
return
|
||||
|
||||
/// Returns the current [/datum/tgs_version] of the DMAPI being used if it was activated, null otherwise.
|
||||
/// Returns the current [/datum/tgs_version] of the DMAPI being used if it was activated, null otherwise. This function may sleep if the call to [/world/proc/TgsNew] is sleeping!
|
||||
/world/proc/TgsApiVersion()
|
||||
return
|
||||
|
||||
/// Returns the name of the TGS instance running the game if TGS is present, null otherwise.
|
||||
/// Returns the name of the TGS instance running the game if TGS is present, null otherwise. This function may sleep if the call to [/world/proc/TgsNew] is sleeping!
|
||||
/world/proc/TgsInstanceName()
|
||||
return
|
||||
|
||||
/// Return the current [/datum/tgs_revision_information] of the running server if TGS is present, null otherwise.
|
||||
/// Return the current [/datum/tgs_revision_information] of the running server if TGS is present, null otherwise. This function may sleep if the call to [/world/proc/TgsNew] is sleeping!
|
||||
/world/proc/TgsRevision()
|
||||
return
|
||||
|
||||
/// Returns the current BYOND security level as a TGS_SECURITY_ define if TGS is present, null otherwise.
|
||||
/// Returns the current BYOND security level as a TGS_SECURITY_ define if TGS is present, null otherwise. This function may sleep if the call to [/world/proc/TgsNew] is sleeping!
|
||||
/world/proc/TgsSecurityLevel()
|
||||
return
|
||||
|
||||
/// Returns a list of active [/datum/tgs_revision_information/test_merge]s if TGS is present, null otherwise.
|
||||
/// Returns a list of active [/datum/tgs_revision_information/test_merge]s if TGS is present, null otherwise. This function may sleep if the call to [/world/proc/TgsNew] is sleeping!
|
||||
/world/proc/TgsTestMerges()
|
||||
return
|
||||
|
||||
/// Returns a list of connected [/datum/tgs_chat_channel]s if TGS is present, null otherwise.
|
||||
/// Returns a list of connected [/datum/tgs_chat_channel]s if TGS is present, null otherwise. This function may sleep if the call to [/world/proc/TgsNew] is sleeping!
|
||||
/world/proc/TgsChatChannelInfo()
|
||||
return
|
||||
|
||||
/*
|
||||
The MIT License
|
||||
|
||||
Copyright (c) 2017 Jordan Brown
|
||||
Copyright (c) 2017-2023 Jordan Brown
|
||||
|
||||
Permission is hereby granted, free of charge,
|
||||
to any person obtaining a copy of this software and
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
The MIT License
|
||||
|
||||
Copyright (c) 2017 Jordan Brown
|
||||
Copyright (c) 2017-2023 Jordan Brown
|
||||
|
||||
Permission is hereby granted, free of charge,
|
||||
to any person obtaining a copy of this software and
|
||||
|
||||
@@ -75,7 +75,7 @@
|
||||
#define DMAPI5_TOPIC_COMMAND_INSTANCE_RENAMED 4
|
||||
#define DMAPI5_TOPIC_COMMAND_CHAT_CHANNELS_UPDATE 5
|
||||
#define DMAPI5_TOPIC_COMMAND_SERVER_PORT_UPDATE 6
|
||||
#define DMAPI5_TOPIC_COMMAND_HEARTBEAT 7
|
||||
#define DMAPI5_TOPIC_COMMAND_HEALTHCHECK 7
|
||||
#define DMAPI5_TOPIC_COMMAND_WATCHDOG_REATTACH 8
|
||||
#define DMAPI5_TOPIC_COMMAND_SEND_CHUNK 9
|
||||
#define DMAPI5_TOPIC_COMMAND_RECEIVE_CHUNK 10
|
||||
|
||||
@@ -137,7 +137,9 @@
|
||||
server_port = new_port
|
||||
return TopicResponse()
|
||||
|
||||
if(DMAPI5_TOPIC_COMMAND_HEARTBEAT)
|
||||
if(DMAPI5_TOPIC_COMMAND_HEALTHCHECK)
|
||||
if(event_handler?.receive_health_checks)
|
||||
event_handler.HandleEvent(TGS_EVENT_HEALTH_CHECK)
|
||||
return TopicResponse()
|
||||
|
||||
if(DMAPI5_TOPIC_COMMAND_WATCHDOG_REATTACH)
|
||||
|
||||
@@ -75,7 +75,7 @@
|
||||
#undef DMAPI5_TOPIC_COMMAND_INSTANCE_RENAMED
|
||||
#undef DMAPI5_TOPIC_COMMAND_CHAT_CHANNELS_UPDATE
|
||||
#undef DMAPI5_TOPIC_COMMAND_SERVER_PORT_UPDATE
|
||||
#undef DMAPI5_TOPIC_COMMAND_HEARTBEAT
|
||||
#undef DMAPI5_TOPIC_COMMAND_HEALTHCHECK
|
||||
#undef DMAPI5_TOPIC_COMMAND_WATCHDOG_REATTACH
|
||||
|
||||
#undef DMAPI5_TOPIC_PARAMETER_COMMAND_TYPE
|
||||
|
||||
@@ -55,14 +55,14 @@ namespace Tgstation.Server.Api.Models.Internal
|
||||
public uint? StartupTimeout { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The number of seconds between each watchdog heartbeat. 0 disables.
|
||||
/// The number of seconds between each watchdog health check. 0 disables.
|
||||
/// </summary>
|
||||
[Required]
|
||||
[ResponseOptions]
|
||||
public uint? HeartbeatSeconds { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// If a process core dump should be created prior to restarting the watchdog due to heartbeat failure.
|
||||
/// If a process core dump should be created prior to restarting the watchdog due to health check failure.
|
||||
/// </summary>
|
||||
[Required]
|
||||
[ResponseOptions]
|
||||
@@ -108,7 +108,7 @@ namespace Tgstation.Server.Api.Models.Internal
|
||||
&& TopicRequestTimeout == otherParameters.TopicRequestTimeout
|
||||
&& AdditionalParameters == otherParameters.AdditionalParameters
|
||||
&& StartProfiler == otherParameters.StartProfiler
|
||||
&& LogOutput == otherParameters.LogOutput; // We intentionally don't check StartupTimeout, heartbeat seconds, or heartbeat dump as they don't matter in terms of the watchdog
|
||||
&& LogOutput == otherParameters.LogOutput; // We intentionally don't check StartupTimeout, health check seconds, or health check dump as they don't matter in terms of the watchdog
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -76,7 +76,7 @@ namespace Tgstation.Server.Api.Rights
|
||||
/// <summary>
|
||||
/// User can change <see cref="Models.Internal.DreamDaemonLaunchParameters.HeartbeatSeconds"/>
|
||||
/// </summary>
|
||||
SetHeartbeatInterval = 1 << 12,
|
||||
SetHealthCheckInterval = 1 << 12,
|
||||
|
||||
/// <summary>
|
||||
/// User can create DreamDaemon process dumps or change <see cref="Models.Internal.DreamDaemonLaunchParameters.DumpOnHeartbeatRestart"/>.
|
||||
|
||||
@@ -46,7 +46,7 @@ namespace Tgstation.Server.Host.Components.Interop.Topic
|
||||
/// <summary>
|
||||
/// Ping to ensure the server is running.
|
||||
/// </summary>
|
||||
Heartbeat,
|
||||
HealthCheck,
|
||||
|
||||
/// <summary>
|
||||
/// Notify the server of a reattach and potentially new version.
|
||||
|
||||
@@ -70,7 +70,7 @@ namespace Tgstation.Server.Host.Components.Interop.Topic
|
||||
or TopicCommandType.ChatChannelsUpdate
|
||||
or TopicCommandType.ServerRestarted => true,
|
||||
TopicCommandType.ChatCommand
|
||||
or TopicCommandType.Heartbeat
|
||||
or TopicCommandType.HealthCheck
|
||||
or TopicCommandType.ReceiveChunk => false,
|
||||
TopicCommandType.SendChunk => throw new InvalidOperationException("SendChunk topic priority should be based on the original TopicParameters!"),
|
||||
_ => throw new InvalidOperationException($"Invalid value for {nameof(CommandType)}: {CommandType}"),
|
||||
@@ -161,9 +161,9 @@ namespace Tgstation.Server.Host.Components.Interop.Topic
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="TopicParameters"/> class.
|
||||
/// </summary>
|
||||
/// <remarks>Constructor for <see cref="TopicCommandType.Heartbeat"/>s.</remarks>
|
||||
/// <remarks>Constructor for <see cref="TopicCommandType.HealthCheck"/>s.</remarks>
|
||||
public TopicParameters()
|
||||
: this(TopicCommandType.Heartbeat)
|
||||
: this(TopicCommandType.HealthCheck)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
@@ -170,7 +170,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
case MonitorActivationReason.ActiveServerPrimed:
|
||||
await HandleEvent(EventType.WorldPrime, Enumerable.Empty<string>(), false, cancellationToken);
|
||||
break;
|
||||
case MonitorActivationReason.Heartbeat:
|
||||
case MonitorActivationReason.HealthCheck:
|
||||
default:
|
||||
throw new InvalidOperationException($"Invalid activation reason: {reason}");
|
||||
}
|
||||
|
||||
@@ -26,9 +26,9 @@
|
||||
ActiveLaunchParametersUpdated,
|
||||
|
||||
/// <summary>
|
||||
/// A heartbeat is required.
|
||||
/// A health check is required.
|
||||
/// </summary>
|
||||
Heartbeat,
|
||||
HealthCheck,
|
||||
|
||||
/// <summary>
|
||||
/// Server primed.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# The Watchdog
|
||||
|
||||
The watchdog is what keeps DreamDaemon alive and well. It's responsible for starting, stopping, rebooting it, sending TGS events to the DMAPI, running heartbeats, and applying patches.
|
||||
The watchdog is what keeps DreamDaemon alive and well. It's responsible for starting, stopping, rebooting it, sending TGS events to the DMAPI, running health checks, and applying patches.
|
||||
|
||||
There are three main implementations of the interface [IWatchdog](./IWatchdog.cs) that deal with how it handles the last of those jobs.
|
||||
|
||||
@@ -32,7 +32,7 @@ Watchdog monitoring takes place in the `MonitorLifetimes()` functiona and is ent
|
||||
- When a server calls `/world/prod/Reboot()`.
|
||||
- When a new .dmb is deployed.
|
||||
- When DreamDaemon settings are changed via the API.
|
||||
- When it is time to make a heartbeat.
|
||||
- When it is time to make a health check.
|
||||
|
||||
Here's how this works.
|
||||
|
||||
|
||||
@@ -155,7 +155,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
/// <summary>
|
||||
/// The number of hearbeats missed.
|
||||
/// </summary>
|
||||
int heartbeatsMissed;
|
||||
int healthChecksMissed;
|
||||
|
||||
/// <summary>
|
||||
/// If the servers should be released instead of shutdown.
|
||||
@@ -517,7 +517,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
|
||||
// since neither server is running, this is safe to do
|
||||
LastLaunchParameters = ActiveLaunchParameters;
|
||||
heartbeatsMissed = 0;
|
||||
healthChecksMissed = 0;
|
||||
|
||||
try
|
||||
{
|
||||
@@ -653,7 +653,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
/// <summary>
|
||||
/// Handles the actions to take when the monitor has to "wake up".
|
||||
/// </summary>
|
||||
/// <param name="activationReason">The <see cref="MonitorActivationReason"/> that caused the invocation. Will never be <see cref="MonitorActivationReason.Heartbeat"/>.</param>
|
||||
/// <param name="activationReason">The <see cref="MonitorActivationReason"/> that caused the invocation. Will never be <see cref="MonitorActivationReason.HealthCheck"/>.</param>
|
||||
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
|
||||
/// <returns>A <see cref="Task{TResult}"/> resulting in the <see cref="MonitorAction"/> to take.</returns>
|
||||
protected abstract Task<MonitorAction> HandleMonitorWakeup(
|
||||
@@ -850,12 +850,12 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
|
||||
UpdateMonitoredTasks();
|
||||
|
||||
var heartbeatSeconds = ActiveLaunchParameters.HeartbeatSeconds.Value;
|
||||
var heartbeat = heartbeatSeconds == 0
|
||||
var healthCheckSeconds = ActiveLaunchParameters.HeartbeatSeconds.Value;
|
||||
var healthCheck = healthCheckSeconds == 0
|
||||
|| !controller.DMApiAvailable
|
||||
? Extensions.TaskExtensions.InfiniteTask()
|
||||
: Task.Delay(
|
||||
TimeSpan.FromSeconds(heartbeatSeconds),
|
||||
TimeSpan.FromSeconds(healthCheckSeconds),
|
||||
cancellationToken);
|
||||
|
||||
// cancel waiting if requested
|
||||
@@ -863,7 +863,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
var toWaitOn = Task.WhenAny(
|
||||
activeServerLifetime,
|
||||
activeServerReboot,
|
||||
heartbeat,
|
||||
healthCheck,
|
||||
newDmbAvailable,
|
||||
cancelTcs.Task,
|
||||
activeLaunchParametersChanged,
|
||||
@@ -908,7 +908,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
|| CheckActivationReason(ref activeServerReboot, MonitorActivationReason.ActiveServerRebooted)
|
||||
|| CheckActivationReason(ref newDmbAvailable, MonitorActivationReason.NewDmbAvailable)
|
||||
|| CheckActivationReason(ref activeLaunchParametersChanged, MonitorActivationReason.ActiveLaunchParametersUpdated)
|
||||
|| CheckActivationReason(ref heartbeat, MonitorActivationReason.Heartbeat)
|
||||
|| CheckActivationReason(ref healthCheck, MonitorActivationReason.HealthCheck)
|
||||
|| CheckActivationReason(ref serverPrimed, MonitorActivationReason.ActiveServerPrimed);
|
||||
|
||||
UpdateMonitoredTasks();
|
||||
@@ -918,8 +918,8 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
else
|
||||
{
|
||||
Logger.LogTrace("Reason: {activationReason}", activationReason);
|
||||
if (activationReason == MonitorActivationReason.Heartbeat)
|
||||
nextAction = await HandleHeartbeat(
|
||||
if (activationReason == MonitorActivationReason.HealthCheck)
|
||||
nextAction = await HandleHealthCheck(
|
||||
cancellationToken);
|
||||
else
|
||||
nextAction = await HandleMonitorWakeup(
|
||||
@@ -1029,26 +1029,26 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Handles a watchdog heartbeat.
|
||||
/// Handles a watchdog health check.
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
|
||||
/// <returns>A <see cref="Task{TResult}"/> resulting in the next <see cref="MonitorAction"/> to take.</returns>
|
||||
async Task<MonitorAction> HandleHeartbeat(CancellationToken cancellationToken)
|
||||
async Task<MonitorAction> HandleHealthCheck(CancellationToken cancellationToken)
|
||||
{
|
||||
Logger.LogTrace("Sending heartbeat to active server...");
|
||||
Logger.LogTrace("Sending health check to active server...");
|
||||
var activeServer = GetActiveController();
|
||||
var response = await activeServer.SendCommand(new TopicParameters(), cancellationToken);
|
||||
|
||||
var shouldShutdown = activeServer.RebootState == Session.RebootState.Shutdown;
|
||||
if (response == null)
|
||||
{
|
||||
switch (++heartbeatsMissed)
|
||||
switch (++healthChecksMissed)
|
||||
{
|
||||
case 1:
|
||||
Logger.LogDebug("DEFCON 4: DreamDaemon missed first heartbeat!");
|
||||
Logger.LogDebug("DEFCON 4: DreamDaemon missed first health check!");
|
||||
break;
|
||||
case 2:
|
||||
const string message2 = "DEFCON 3: DreamDaemon has missed 2 heartbeats!";
|
||||
const string message2 = "DEFCON 3: DreamDaemon has missed 2 health checks!";
|
||||
Logger.LogInformation(message2);
|
||||
Chat.QueueWatchdogMessage(message2);
|
||||
break;
|
||||
@@ -1056,7 +1056,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
var actionToTake = shouldShutdown
|
||||
? "shutdown"
|
||||
: "be restarted";
|
||||
const string logTemplate1 = "DEFCON 2: DreamDaemon has missed 3 heartbeats! If it does not respond to the next one, the watchdog will {actionToTake}!";
|
||||
const string logTemplate1 = "DEFCON 2: DreamDaemon has missed 3 health checks! If it does not respond to the next one, the watchdog will {actionToTake}!";
|
||||
Logger.LogWarning(logTemplate1, actionToTake);
|
||||
Chat.QueueWatchdogMessage(
|
||||
logTemplate1.Replace(
|
||||
@@ -1068,7 +1068,7 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
var actionTaken = shouldShutdown
|
||||
? "Shutting down due to graceful termination request"
|
||||
: "Restarting";
|
||||
const string logTemplate2 = "DEFCON 1: Four heartbeats have been missed! {actionTaken}...";
|
||||
const string logTemplate2 = "DEFCON 1: Four health checks have been missed! {actionTaken}...";
|
||||
Logger.LogWarning(logTemplate2, actionTaken);
|
||||
Chat.QueueWatchdogMessage(
|
||||
logTemplate2.Replace(
|
||||
@@ -1087,12 +1087,12 @@ namespace Tgstation.Server.Host.Components.Watchdog
|
||||
await DisposeAndNullControllers(cancellationToken);
|
||||
return shouldShutdown ? MonitorAction.Exit : MonitorAction.Restart;
|
||||
default:
|
||||
Logger.LogError("Invalid heartbeats missed count: {heartbeatsMissed}", heartbeatsMissed);
|
||||
Logger.LogError("Invalid health checks missed count: {healthChecksMissed}", healthChecksMissed);
|
||||
break;
|
||||
}
|
||||
}
|
||||
else
|
||||
heartbeatsMissed = 0;
|
||||
healthChecksMissed = 0;
|
||||
|
||||
return MonitorAction.Continue;
|
||||
}
|
||||
|
||||
@@ -146,7 +146,7 @@ namespace Tgstation.Server.Host.Controllers
|
||||
| DreamDaemonRights.SoftShutdown
|
||||
| DreamDaemonRights.Start
|
||||
| DreamDaemonRights.SetStartupTimeout
|
||||
| DreamDaemonRights.SetHeartbeatInterval
|
||||
| DreamDaemonRights.SetHealthCheckInterval
|
||||
| DreamDaemonRights.SetTopicTimeout
|
||||
| DreamDaemonRights.SetAdditionalParameters
|
||||
| DreamDaemonRights.SetVisibility
|
||||
@@ -220,7 +220,7 @@ namespace Tgstation.Server.Host.Controllers
|
||||
|| (model.SoftRestart.HasValue && !AuthenticationContext.InstancePermissionSet.DreamDaemonRights.Value.HasFlag(DreamDaemonRights.SoftRestart))
|
||||
|| (model.SoftShutdown.HasValue && !AuthenticationContext.InstancePermissionSet.DreamDaemonRights.Value.HasFlag(DreamDaemonRights.SoftShutdown))
|
||||
|| CheckModified(x => x.StartupTimeout, DreamDaemonRights.SetStartupTimeout)
|
||||
|| CheckModified(x => x.HeartbeatSeconds, DreamDaemonRights.SetHeartbeatInterval)
|
||||
|| CheckModified(x => x.HeartbeatSeconds, DreamDaemonRights.SetHealthCheckInterval)
|
||||
|| CheckModified(x => x.DumpOnHeartbeatRestart, DreamDaemonRights.CreateDump)
|
||||
|| CheckModified(x => x.TopicRequestTimeout, DreamDaemonRights.SetTopicTimeout)
|
||||
|| CheckModified(x => x.AdditionalParameters, DreamDaemonRights.SetAdditionalParameters)
|
||||
|
||||
@@ -128,6 +128,10 @@ var/run_bridge_test
|
||||
var/list/channels = TgsChatChannelInfo()
|
||||
return "[length(channels)]"
|
||||
|
||||
var/tactics8 = data["tgs_integration_test_tactics8"]
|
||||
if(tactics8)
|
||||
return received_health_check ? "received health check" : "did not receive health check"
|
||||
|
||||
TgsChatBroadcast(new /datum/tgs_message_content("Recieved non-tgs topic: `[T]`"))
|
||||
|
||||
return "feck"
|
||||
@@ -151,10 +155,17 @@ var/run_bridge_test
|
||||
TgsChatBroadcast("World Rebooting")
|
||||
TgsReboot()
|
||||
|
||||
var/received_health_check = FALSE
|
||||
|
||||
/datum/tgs_event_handler/impl
|
||||
receive_health_checks = TRUE
|
||||
|
||||
/datum/tgs_event_handler/impl/HandleEvent(event_code, ...)
|
||||
set waitfor = FALSE
|
||||
|
||||
if(event_code == TGS_EVENT_WATCHDOG_DETACH)
|
||||
if(event_code == TGS_EVENT_HEALTH_CHECK)
|
||||
received_health_check = TRUE
|
||||
else if(event_code == TGS_EVENT_WATCHDOG_DETACH)
|
||||
// hack hack, calling world.TgsChatChannelInfo() will try to delay until the channels come back
|
||||
var/datum/tgs_api/v5/api = TGS_READ_GLOBAL(tgs)
|
||||
if(length(api.chat_channels))
|
||||
|
||||
@@ -90,8 +90,8 @@ namespace Tgstation.Server.Tests.Live.Instance
|
||||
await RunLongRunningTestThenUpdateWithNewDme(cancellationToken);
|
||||
await RunLongRunningTestThenUpdateWithByondVersionSwitch(cancellationToken);
|
||||
|
||||
await RunHeartbeatTest(true, cancellationToken);
|
||||
await RunHeartbeatTest(false, cancellationToken);
|
||||
await RunHealthCheckTest(true, cancellationToken);
|
||||
await RunHealthCheckTest(false, cancellationToken);
|
||||
|
||||
await InteropTestsForLongRunningDme(cancellationToken);
|
||||
|
||||
@@ -349,10 +349,10 @@ namespace Tgstation.Server.Tests.Live.Instance
|
||||
Assert.AreEqual(string.Empty, daemonStatus.AdditionalParameters);
|
||||
}
|
||||
|
||||
async Task RunHeartbeatTest(bool checkDump, CancellationToken cancellationToken)
|
||||
async Task RunHealthCheckTest(bool checkDump, CancellationToken cancellationToken)
|
||||
{
|
||||
System.Console.WriteLine("TEST: WATCHDOG HEARTBEAT TEST");
|
||||
// enable heartbeats
|
||||
System.Console.WriteLine("TEST: WATCHDOG HEALTH CHECK TEST");
|
||||
// enable health checks
|
||||
await instanceClient.DreamDaemon.Update(new DreamDaemonRequest
|
||||
{
|
||||
HeartbeatSeconds = 1,
|
||||
@@ -365,7 +365,7 @@ namespace Tgstation.Server.Tests.Live.Instance
|
||||
|
||||
CheckDDPriority(cancellationToken);
|
||||
|
||||
// lock on to DD and pause it so it can't heartbeat
|
||||
// lock on to DD and pause it so it can't health check
|
||||
var ddProcs = System.Diagnostics.Process.GetProcessesByName("DreamDaemon").Where(x => !x.HasExited).ToList();
|
||||
if (ddProcs.Count != 1)
|
||||
Assert.Fail($"Incorrect number of DD processes: {ddProcs.Count}");
|
||||
@@ -382,10 +382,22 @@ namespace Tgstation.Server.Tests.Live.Instance
|
||||
await using var ourProcessHandler = executor
|
||||
.GetProcess(ddProc.Id);
|
||||
|
||||
// Ensure it's responding to heartbeats
|
||||
// Ensure it's responding to health checks
|
||||
await Task.WhenAny(Task.Delay(20000, cancellationToken), ourProcessHandler.Lifetime);
|
||||
Assert.IsFalse(ddProc.HasExited);
|
||||
|
||||
// check DD agrees
|
||||
var topicRequestResult = await TopicClient.SendTopic(
|
||||
IPAddress.Loopback,
|
||||
$"tgs_integration_test_tactics8=1",
|
||||
TestLiveServer.DDPort,
|
||||
cancellationToken);
|
||||
|
||||
Assert.IsNotNull(topicRequestResult);
|
||||
Assert.AreEqual(TopicResponseType.StringResponse, topicRequestResult.ResponseType);
|
||||
Assert.IsNotNull(topicRequestResult.StringData);
|
||||
Assert.AreEqual(topicRequestResult.StringData, "received health check");
|
||||
|
||||
await instanceClient.DreamDaemon.Update(new DreamDaemonRequest
|
||||
{
|
||||
SoftShutdown = true
|
||||
@@ -412,7 +424,7 @@ namespace Tgstation.Server.Tests.Live.Instance
|
||||
}
|
||||
while (timeout > 0);
|
||||
|
||||
// disable heartbeats
|
||||
// disable health checks
|
||||
await instanceClient.DreamDaemon.Update(new DreamDaemonRequest
|
||||
{
|
||||
HeartbeatSeconds = 0,
|
||||
|
||||
Reference in New Issue
Block a user