From 9dd5cf8e351e6a0c298b7008435cb6d283fdfc23 Mon Sep 17 00:00:00 2001 From: Kyle Spier-Swenson Date: Mon, 21 Feb 2022 22:20:33 -0800 Subject: [PATCH] The mc now detects if its priority budget gets out of sync with reality and soft resets. (#64498) Co-authored-by: LemonInTheDark <58055496+LemonInTheDark@users.noreply.github.com> --- code/controllers/master.dm | 81 +++++++++++++++++++++++++------------- 1 file changed, 54 insertions(+), 27 deletions(-) diff --git a/code/controllers/master.dm b/code/controllers/master.dm index 8508a3259fa..88dadb8b52e 100644 --- a/code/controllers/master.dm +++ b/code/controllers/master.dm @@ -299,7 +299,7 @@ GLOBAL_REAL(Master, /datum/controller/master) = new SS.queue_next = null SS.queue_prev = null SS.state = SS_IDLE - if (SS.flags & SS_TICKER) + if ((SS.flags & (SS_TICKER|SS_BACKGROUND)) == SS_TICKER) tickersubsystems += SS timer += world.tick_lag * rand(1, 5) SS.next_fire = timer @@ -393,29 +393,40 @@ GLOBAL_REAL(Master, /datum/controller/master) = new else subsystems_to_check = tickersubsystems - if (CheckQueue(subsystems_to_check) <= 0) + if (CheckQueue(subsystems_to_check) <= 0) //error processing queue + stack_trace("MC: CheckQueue failed. Current error_level is [round(error_level, 0.25)]") if (!SoftReset(tickersubsystems, runlevel_sorted_subsystems)) - log_world("MC: SoftReset() failed, crashing") - return - if (!error_level) + error_level++ + CRASH("MC: SoftReset() failed, exiting loop()") + + if (error_level < 2) //except for the first strike, stop incrmenting our iteration so failsafe enters defcon iteration++ - error_level++ + else + cached_runlevel = null //3 strikes, Lets reset the runlevel lists current_ticklimit = TICK_LIMIT_RUNNING - sleep(10) + sleep((1 SECONDS) * error_level) + error_level++ continue if (queue_head) - if (RunQueue() <= 0) - if (!SoftReset(tickersubsystems, runlevel_sorted_subsystems)) - log_world("MC: SoftReset() failed, crashing") - return - if (!error_level) - iteration++ + if (RunQueue() <= 0) //error running queue + stack_trace("MC: RunQueue failed. Current error_level is [round(error_level, 0.25)]") + if (error_level > 1) //skip the first error, + if (!SoftReset(tickersubsystems, runlevel_sorted_subsystems)) + error_level++ + CRASH("MC: SoftReset() failed, exiting loop()") + + if (error_level <= 2) //after 3 strikes stop incrmenting our iteration so failsafe enters defcon + iteration++ + else + cached_runlevel = null //3 strikes, Lets also reset the runlevel lists + current_ticklimit = TICK_LIMIT_RUNNING + sleep((1 SECONDS) * error_level) + error_level++ + continue error_level++ - current_ticklimit = TICK_LIMIT_RUNNING - sleep(10) - continue - error_level-- + if (error_level > 0) + error_level = max(MC_AVERAGE_SLOW(error_level-1, error_level), 0) if (!queue_head) //reset the counts if the queue is empty, in the off chance they get out of sync queue_priority_count = 0 queue_priority_count_bg = 0 @@ -465,7 +476,8 @@ GLOBAL_REAL(Master, /datum/controller/master) = new . = 1 -/// Run thru the queue of subsystems to run, running them while balancing out their allocated tick precentage +/// RunQueue - Run thru the queue of subsystems to run, running them while balancing out their allocated tick precentage +/// Returns 0 if runtimed, a negitive number for logic errors, and a positive number if the operation completed without errors /datum/controller/master/proc/RunQueue() . = 0 var/datum/controller/subsystem/queue_node @@ -497,17 +509,30 @@ GLOBAL_REAL(Master, /datum/controller/master) = new queue_node = queue_node.queue_next continue - if (!bg_calc && (queue_node_flags & SS_BACKGROUND)) - current_tick_budget = queue_priority_count_bg - bg_calc = TRUE + if ((queue_node_flags & SS_BACKGROUND)) + if (!bg_calc) + current_tick_budget = queue_priority_count_bg + bg_calc = TRUE + else if (bg_calc) + //error state, do sane fallback behavior + if (. == 0) + log_world("MC: Queue logic failure, non-background subsystem queued to run after a background subsystem: [queue_node] queue_prev:[queue_node.queue_prev]") + . = -1 + current_tick_budget = queue_priority_count //this won't even be right, but is the best we have. + bg_calc = FALSE + tick_remaining = TICK_LIMIT_RUNNING - TICK_USAGE - if (current_tick_budget > 0 && queue_node_priority > 0) - //Give the subsystem a precentage of the remaining tick based on the remaning priority + if (queue_node_priority >= 0 && current_tick_budget > 0 && current_tick_budget >= queue_node_priority) + //Give the subsystem a precentage of the remaining tick based on the remaining priority tick_precentage = tick_remaining * (queue_node_priority / current_tick_budget) else - tick_precentage = tick_remaining + //error state + if (. == 0) + log_world("MC: tick_budget sync error. [json_encode(list(current_tick_budget, queue_priority_count, queue_priority_count_bg, bg_calc, queue_node, queue_node_priority))]") + . = -1 + tick_precentage = tick_remaining //just because we lost track of priority calculations doesn't mean we can't try to finish off the run, if the error state persists, we don't want to stop ticks from happening tick_precentage = max(tick_precentage*0.5, tick_precentage-queue_node.tick_overrun) @@ -566,17 +591,19 @@ GLOBAL_REAL(Master, /datum/controller/master) = new queue_node = queue_node.queue_next - . = 1 + if (. == 0) + . = 1 //resets the queue, and all subsystems, while filtering out the subsystem lists // called if any mc's queue procs runtime or exit improperly. /datum/controller/master/proc/SoftReset(list/ticker_SS, list/runlevel_SS) . = 0 - log_world("MC: SoftReset called, resetting MC queue state.") + stack_trace("MC: SoftReset called, resetting MC queue state.") + if (!istype(subsystems) || !istype(ticker_SS) || !istype(runlevel_SS)) log_world("MC: SoftReset: Bad list contents: '[subsystems]' '[ticker_SS]' '[runlevel_SS]'") return - var/subsystemstocheck = subsystems + ticker_SS + var/subsystemstocheck = subsystems | ticker_SS for(var/I in runlevel_SS) subsystemstocheck |= I