mirror of
https://github.com/VOREStation/VOREStation.git
synced 2026-08-30 15:38:02 +01:00
Failsafe & Stack End Detector
This commit is contained in:
ItsSelis
File diff suppressed because it is too large
Load Diff
@@ -1,64 +0,0 @@
|
||||
//
|
||||
// Lets read our settings from the configuration file on startup too!
|
||||
//
|
||||
|
||||
/datum/configuration
|
||||
var/static/time_off = FALSE
|
||||
var/static/pto_job_change = FALSE
|
||||
var/static/limit_interns = -1 //Unlimited by default
|
||||
var/static/limit_visitors = -1 //Unlimited by default
|
||||
var/static/pto_cap = 100 //Hours
|
||||
var/static/require_flavor = FALSE
|
||||
var/static/ipqualityscore_apikey //API key for ipqualityscore.com
|
||||
var/static/use_playtime_restriction_for_jobs = FALSE
|
||||
|
||||
/hook/startup/proc/read_vs_config()
|
||||
var/list/Lines = file2list("config/config.txt")
|
||||
for(var/t in Lines)
|
||||
if(!t) continue
|
||||
|
||||
t = trim(t)
|
||||
if (length(t) == 0)
|
||||
continue
|
||||
else if (copytext(t, 1, 2) == "#")
|
||||
continue
|
||||
|
||||
var/pos = findtext(t, " ")
|
||||
var/name = null
|
||||
var/value = null
|
||||
|
||||
if (pos)
|
||||
name = lowertext(copytext(t, 1, pos))
|
||||
value = copytext(t, pos + 1)
|
||||
else
|
||||
name = lowertext(t)
|
||||
|
||||
if (!name)
|
||||
continue
|
||||
|
||||
switch (name)
|
||||
if ("chat_webhook_url")
|
||||
config.chat_webhook_url = value
|
||||
if ("chat_webhook_key")
|
||||
config.chat_webhook_key = value
|
||||
if ("fax_export_dir")
|
||||
config.fax_export_dir = value
|
||||
if ("items_survive_digestion")
|
||||
config.items_survive_digestion = 1
|
||||
if ("limit_interns")
|
||||
config.limit_interns = text2num(value)
|
||||
if ("limit_visitors")
|
||||
config.limit_visitors = text2num(value)
|
||||
if ("pto_cap")
|
||||
config.pto_cap = text2num(value)
|
||||
if ("time_off")
|
||||
config.time_off = TRUE
|
||||
if ("pto_job_change")
|
||||
config.pto_job_change = TRUE
|
||||
if ("require_flavor")
|
||||
config.require_flavor = TRUE
|
||||
if ("ipqualityscore_apikey")
|
||||
config.ipqualityscore_apikey = value
|
||||
if ("use_playtime_restriction_for_jobs")
|
||||
config.use_playtime_restriction_for_jobs = TRUE
|
||||
return 1
|
||||
+104
-26
@@ -1,10 +1,11 @@
|
||||
/**
|
||||
* Failsafe
|
||||
*
|
||||
* Pretty much pokes the MC to make sure it's still alive.
|
||||
/**
|
||||
* Failsafe
|
||||
*
|
||||
* Pretty much pokes the MC to make sure it's still alive.
|
||||
**/
|
||||
|
||||
var/datum/controller/failsafe/Failsafe
|
||||
// See initialization order in /code/game/world.dm
|
||||
GLOBAL_REAL(Failsafe, /datum/controller/failsafe)
|
||||
|
||||
/datum/controller/failsafe // This thing pretty much just keeps poking the master controller
|
||||
name = "Failsafe"
|
||||
@@ -15,7 +16,7 @@ var/datum/controller/failsafe/Failsafe
|
||||
// The alert level. For every failed poke, we drop a DEFCON level. Once we hit DEFCON 1, restart the MC.
|
||||
var/defcon = 5
|
||||
//the world.time of the last check, so the mc can restart US if we hang.
|
||||
// (Real friends look out for *eachother*)
|
||||
// (Real friends look out for *each other*)
|
||||
var/lasttick = 0
|
||||
|
||||
// Track the MC iteration to make sure its still on track.
|
||||
@@ -31,8 +32,24 @@ var/datum/controller/failsafe/Failsafe
|
||||
Initialize()
|
||||
|
||||
/datum/controller/failsafe/Initialize()
|
||||
set waitfor = 0
|
||||
set waitfor = FALSE
|
||||
Failsafe.Loop()
|
||||
if (!Master || defcon == 0) //Master is gone/not responding and Failsafe just exited its loop
|
||||
defcon = 3 //Reset defcon level as its used inside the emergency loop
|
||||
while (defcon > 0)
|
||||
var/recovery_result = emergency_loop()
|
||||
if (recovery_result == 1) //Exit emergency loop and delete self if it was able to recover MC
|
||||
break
|
||||
else if (defcon == 1) //Exit Failsafe if we weren't able to recover the MC in the last stage
|
||||
log_game("FailSafe: Failed to recover MC while in emergency state. Failsafe exiting.")
|
||||
message_admins(span_boldannounce("Failsafe failed critically while trying to recreate broken MC. Please manually fix the MC or reboot the server. Failsafe exiting now."))
|
||||
message_admins(span_boldannounce("You can try manually calling these two procs:."))
|
||||
message_admins(span_boldannounce("/proc/recover_all_SS_and_recreate_master: Most stuff should still function but expect instability/runtimes/broken stuff."))
|
||||
message_admins(span_boldannounce("/proc/delete_all_SS_and_recreate_master: Most stuff will be broken but basic stuff like movement and chat should still work."))
|
||||
else if (recovery_result == -1) //Failed to recreate MC
|
||||
defcon--
|
||||
sleep(initial(processing_interval)) //Wait a bit until the next try
|
||||
|
||||
if(!QDELETED(src))
|
||||
qdel(src) //when Loop() returns, we delete ourselves and let the mc recreate us
|
||||
|
||||
@@ -45,43 +62,56 @@ var/datum/controller/failsafe/Failsafe
|
||||
while(running)
|
||||
lasttick = world.time
|
||||
if(!Master)
|
||||
// Replace the missing Master! This should never, ever happen.
|
||||
new /datum/controller/master()
|
||||
// Break out of the main loop so we go into emergency state
|
||||
break
|
||||
// Only poke it if overrides are not in effect.
|
||||
if(processing_interval > 0)
|
||||
if(Master.processing && Master.iteration)
|
||||
if (defcon > 1 && (!Master.stack_end_detector || !Master.stack_end_detector.check()))
|
||||
|
||||
to_chat(GLOB.admins, span_boldannounce("ERROR: The Master Controller code stack has exited unexpectedly, Restarting..."))
|
||||
defcon = 0
|
||||
var/rtn = Recreate_MC()
|
||||
if(rtn > 0)
|
||||
master_iteration = 0
|
||||
to_chat(GLOB.admins, span_adminnotice("MC restarted successfully"))
|
||||
else if(rtn < 0)
|
||||
log_game("FailSafe: Could not restart MC, runtime encountered. Entering defcon 0")
|
||||
to_chat(GLOB.admins, span_boldannounce("ERROR: DEFCON [defcon_pretty()]. Could not restart MC, runtime encountered. I will silently keep retrying."))
|
||||
// Check if processing is done yet.
|
||||
if(Master.iteration == master_iteration)
|
||||
log_debug("DEFCON [defcon]: Master.iteration=[Master.iteration] Master.last_run=[Master.last_run] world.time=[world.time]")
|
||||
switch(defcon)
|
||||
if(4,5)
|
||||
--defcon
|
||||
if(3)
|
||||
log_and_message_admins(span_adminnotice("SSfailsafe Notice: DEFCON [defcon_pretty()]. The Master Controller (\ref[Master]) has not fired in the last [(5-defcon) * processing_interval] ticks."))
|
||||
--defcon
|
||||
if(2)
|
||||
log_and_message_admins(span_boldannounce("SSfailsafe Warning: DEFCON [defcon_pretty()]. The Master Controller (\ref[Master]) has not fired in the last [(5-defcon) * processing_interval] ticks. Automatic restart in [processing_interval] ticks."))
|
||||
--defcon
|
||||
if(1)
|
||||
|
||||
log_and_message_admins(span_boldannounce("SSfailsafe Warning: DEFCON [defcon_pretty()]. The Master Controller (\ref[Master]) has still not fired within the last [(5-defcon) * processing_interval] ticks. Killing and restarting..."))
|
||||
if(3)
|
||||
message_admins(span_adminnotice("Notice: DEFCON [defcon_pretty()]. The Master Controller has not fired in the last [(5-defcon) * processing_interval] ticks."))
|
||||
--defcon
|
||||
|
||||
if(2)
|
||||
to_chat(GLOB.admins, span_boldannounce("Warning: DEFCON [defcon_pretty()]. The Master Controller has not fired in the last [(5-defcon) * processing_interval] ticks. Automatic restart in [processing_interval] ticks."))
|
||||
--defcon
|
||||
|
||||
if(1)
|
||||
to_chat(GLOB.admins, span_boldannounce("Warning: DEFCON [defcon_pretty()]. The Master Controller has still not fired within the last [(5-defcon) * processing_interval] ticks. Killing and restarting..."))
|
||||
--defcon
|
||||
var/rtn = Recreate_MC()
|
||||
if(rtn > 0)
|
||||
defcon = 4
|
||||
master_iteration = 0
|
||||
log_and_message_admins(span_adminnotice("SSfailsafe Notice: MC (New:\ref[Master]) restarted successfully"))
|
||||
to_chat(GLOB.admins, span_adminnotice("MC restarted successfully"))
|
||||
else if(rtn < 0)
|
||||
log_game("SSfailsafe Notice: Could not restart MC (\ref[Master]), runtime encountered. Entering defcon 0")
|
||||
log_and_message_admins(span_boldannounce("SSFAILSAFE ERROR: DEFCON [defcon_pretty()]. Could not restart MC (\ref[Master]), runtime encountered. I will silently keep retrying."))
|
||||
log_game("FailSafe: Could not restart MC, runtime encountered. Entering defcon 0")
|
||||
to_chat(GLOB.admins, span_boldannounce("ERROR: DEFCON [defcon_pretty()]. Could not restart MC, runtime encountered. I will silently keep retrying."))
|
||||
//if the return number was 0, it just means the mc was restarted too recently, and it just needs some time before we try again
|
||||
//no need to handle that specially when defcon 0 can handle it
|
||||
|
||||
if(0) //DEFCON 0! (mc failed to restart)
|
||||
var/rtn = Recreate_MC()
|
||||
if(rtn > 0)
|
||||
defcon = 4
|
||||
master_iteration = 0
|
||||
log_and_message_admins(span_adminnotice("SSfailsafe Notice: MC (New:\ref[Master]) restarted successfully"))
|
||||
to_chat(GLOB.admins, span_adminnotice("MC restarted successfully"))
|
||||
else
|
||||
defcon = min(defcon + 1,5)
|
||||
master_iteration = Master.iteration
|
||||
@@ -93,12 +123,60 @@ var/datum/controller/failsafe/Failsafe
|
||||
defcon = 5
|
||||
sleep(initial(processing_interval))
|
||||
|
||||
//Emergency loop used when Master got deleted or the main loop exited while Defcon == 0
|
||||
//Loop is driven externally so runtimes only cancel the current recovery attempt
|
||||
/datum/controller/failsafe/proc/emergency_loop()
|
||||
//The code in this proc should be kept as simple as possible, anything complicated like to_chat might rely on master existing and runtime
|
||||
//The goal should always be to get a new Master up and running before anything else
|
||||
. = -1
|
||||
switch (defcon) //The lower defcon goes the harder we try to fix the MC
|
||||
if (2 to 3) //Try to normally recreate the MC two times
|
||||
. = Recreate_MC()
|
||||
if (1) //Delete the old MC first so we don't transfer any info, in case that caused any issues
|
||||
del(Master)
|
||||
. = Recreate_MC()
|
||||
|
||||
if (. == 1) //We were able to create a new master
|
||||
master_iteration = 0
|
||||
SSticker.Recover(); //Recover the ticket system so the Masters runlevel gets set
|
||||
Master.Initialize(10, FALSE, FALSE) //Need to manually start the MC, normally world.new would do this
|
||||
to_chat(GLOB.admins, span_adminnotice("Failsafe recovered MC while in emergency state [defcon_pretty()]"))
|
||||
else
|
||||
log_game("FailSafe: Failsafe in emergency state and was unable to recreate MC while in defcon state [defcon_pretty()].")
|
||||
message_admins(span_boldannounce("Failsafe in emergency state and master down, trying to recreate MC while in defcon level [defcon_pretty()] failed."))
|
||||
|
||||
///Recreate all SSs which will still cause data survive due to Recover(), the new Master will then find and take them from global.vars
|
||||
/proc/recover_all_SS_and_recreate_master()
|
||||
del(Master)
|
||||
var/list/subsytem_types = subtypesof(/datum/controller/subsystem)
|
||||
sortTim(subsytem_types, GLOBAL_PROC_REF(cmp_subsystem_init))
|
||||
for(var/I in subsytem_types)
|
||||
new I
|
||||
. = Recreate_MC()
|
||||
if (. == 1) //We were able to create a new master
|
||||
SSticker.Recover(); //Recover the ticket system so the Masters runlevel gets set
|
||||
Master.Initialize(10, FALSE, FALSE) //Need to manually start the MC, normally world.new would do this
|
||||
to_chat(GLOB.admins, span_adminnotice("MC successfully recreated after recovering all subsystems!"))
|
||||
else
|
||||
message_admins(span_boldannounce("Failed to create new MC!"))
|
||||
|
||||
///Delete all existing SS to basically start over
|
||||
/proc/delete_all_SS_and_recreate_master()
|
||||
del(Master)
|
||||
for(var/global_var in global.vars)
|
||||
if (istype(global.vars[global_var], /datum/controller/subsystem))
|
||||
del(global.vars[global_var])
|
||||
. = Recreate_MC()
|
||||
if (. == 1) //We were able to create a new master
|
||||
SSticker.Recover(); //Recover the ticket system so the Masters runlevel gets set
|
||||
Master.Initialize(10, FALSE, FALSE) //Need to manually start the MC, normally world.new would do this
|
||||
to_chat(GLOB.admins, span_adminnotice("MC successfully recreated after deleting and recreating all subsystems!"))
|
||||
else
|
||||
message_admins(span_boldannounce("Failed to create new MC!"))
|
||||
|
||||
/datum/controller/failsafe/proc/defcon_pretty()
|
||||
return defcon
|
||||
|
||||
/datum/controller/failsafe/stat_entry(msg)
|
||||
if(!statclick)
|
||||
statclick = new/obj/effect/statclick/debug(null, "Initializing...", src)
|
||||
|
||||
msg = "Failsafe Controller: [statclick.update("Defcon: [defcon_pretty()] (Interval: [Failsafe.processing_interval] | Iteration: [Failsafe.master_iteration])")]"
|
||||
msg = "Defcon: [defcon_pretty()] (Interval: [Failsafe.processing_interval] | Iteration: [Failsafe.master_iteration])"
|
||||
return msg
|
||||
|
||||
+17
-16
@@ -1,26 +1,23 @@
|
||||
/**
|
||||
* StonedMC
|
||||
*
|
||||
* Designed to properly split up a given tick among subsystems
|
||||
* Note: if you read parts of this code and think "why is it doing it that way"
|
||||
* Odds are, there is a reason
|
||||
*
|
||||
/**
|
||||
* StonedMC
|
||||
*
|
||||
* Designed to properly split up a given tick among subsystems
|
||||
* Note: if you read parts of this code and think "why is it doing it that way"
|
||||
* Odds are, there is a reason
|
||||
*
|
||||
**/
|
||||
|
||||
//This is the ABSOLUTE ONLY THING that should init globally like this
|
||||
// See initialization order in /code/game/world.dm
|
||||
GLOBAL_REAL(Master, /datum/controller/master) = new
|
||||
|
||||
//THIS IS THE INIT ORDER
|
||||
//Master -> SSPreInit -> GLOB -> world -> config -> SSInit -> Failsafe
|
||||
//GOT IT MEMORIZED?
|
||||
|
||||
/datum/controller/master
|
||||
name = "Master"
|
||||
|
||||
// Are we processing (higher values increase the processing delay by n ticks)
|
||||
/// Are we processing (higher values increase the processing delay by n ticks)
|
||||
var/processing = TRUE
|
||||
// How many times have we ran
|
||||
/// How many times have we ran
|
||||
var/iteration = 0
|
||||
/// Stack end detector to detect stack overflows that kill the mc's main loop
|
||||
var/datum/stack_end_detector/stack_end_detector
|
||||
|
||||
// world.time of last fire, for tracking lag outside of the mc
|
||||
var/last_run
|
||||
@@ -291,8 +288,12 @@ GLOBAL_REAL(Master, /datum/controller/master) = new
|
||||
var/error_level = 0
|
||||
var/sleep_delta = 1
|
||||
var/list/subsystems_to_check
|
||||
//the actual loop.
|
||||
|
||||
//setup the stack overflow detector
|
||||
stack_end_detector = new()
|
||||
var/datum/stack_canary/canary = stack_end_detector.prime_canary()
|
||||
canary.use_variable()
|
||||
//the actual loop.
|
||||
while (1)
|
||||
tickdrift = max(0, MC_AVERAGE_FAST(tickdrift, (((REALTIMEOFDAY - init_timeofday) - (world.time - init_time)) / world.tick_lag)))
|
||||
var/starting_tick_usage = TICK_USAGE
|
||||
|
||||
Reference in New Issue
Block a user