Upgrades the MC (#18769)

* Upgrades the MC * More tweaks
2026-01-04 14:42:56 +00:00 · 2022-08-14 14:55:31 +01:00
parent a1f296f2d8
commit d688300c3a
7 changed files with 388 additions and 170 deletions
--- a/code/controllers/configuration/sections/mc_configuration.dm
+++ b/code/controllers/configuration/sections/mc_configuration.dm
@@ -3,7 +3,7 @@
 	/// Server ticklag
 	var/ticklag = 0.5
 	/// Tick limit % during world Init
-	var/world_init_tick_limit = TICK_LIMIT_MC_INIT_DEFAULT
+	var/world_init_tick_limit = 98
 	/// Base MC tick rate
 	var/base_tickrate = 1
 	/// Highpop MC tickrate
--- a/code/controllers/controller.dm
+++ b/code/controllers/controller.dm
@@ -22,6 +22,6 @@
 * Standardized method for tracking startup times.
 */
 /datum/controller/proc/log_startup_progress(message)
-	Master.current_init_stage = "([name]): [message]"
+	Master.last_init_info = "([name]): [message]"
 	to_chat(world, "<span class='danger'><small>\[[name]]</small> [message]</span>")
 	log_world("\[[name]] [message]")
--- a/code/controllers/failsafe.dm
+++ b/code/controllers/failsafe.dm
@@ -1,7 +1,7 @@
- /**
-  * Failsafe
-  *
-  * Pretty much pokes the MC to make sure it's still alive.
+/**
+ * Failsafe
+ *
+ * Pretty much pokes the MC to make sure it's still alive.
 **/

 GLOBAL_REAL(Failsafe, /datum/controller/failsafe)
@@ -15,7 +15,7 @@ GLOBAL_REAL(Failsafe, /datum/controller/failsafe)
 	// The alert level. For every failed poke, we drop a DEFCON level. Once we hit DEFCON 1, restart the MC.
 	var/defcon = 5
 	//the world.time of the last check, so the mc can restart US if we hang.
-	//	(Real friends look out for *eachother*)
+	// (Real friends look out for *eachother*)
 	var/lasttick = 0

 	// Track the MC iteration to make sure its still on track.
@@ -31,8 +31,24 @@ GLOBAL_REAL(Failsafe, /datum/controller/failsafe)
 	Initialize()

 /datum/controller/failsafe/Initialize()
-	set waitfor = 0
+	set waitfor = FALSE
 	Failsafe.Loop()
+	if(!Master || defcon == 0) //Master is gone/not responding and Failsafe just exited its loop
+		defcon = 3 //Reset defcon level as its used inside the emergency loop
+		while(defcon > 0)
+			var/recovery_result = emergency_loop()
+			if(recovery_result == 1) //Exit emergency loop and delete self if it was able to recover MC
+				break
+			else if(defcon == 1) //Exit Failsafe if we weren't able to recover the MC in the last stage
+				log_game("FailSafe: Failed to recover MC while in emergency state. Failsafe exiting.")
+				message_admins("<span class='boldannounce'>Failsafe failed criticaly while trying to recreate broken MC. Please manually fix the MC or reboot the server. Failsafe exiting now.</span>")
+				message_admins("<span class='boldannounce'>You can try manually calling these two procs:.</span>")
+				message_admins("<span class='boldannounce'><code>/proc/recover_all_SS_and_recreate_master</code>: Most stuff should still function but expect instability/runtimes/broken stuff.</span>")
+				message_admins("<span class='boldannounce'><code>/proc/delete_all_SS_and_recreate_master</code>: Most stuff will be broken but basic stuff like movement and chat should still work.</span>")
+			else if(recovery_result == -1) //Failed to recreate MC
+				defcon--
+			sleep(initial(processing_interval)) //Wait a bit until the next try
+
 	if(!QDELETED(src))
 		qdel(src) //when Loop() returns, we delete ourselves and let the mc recreate us

@@ -45,8 +61,8 @@ GLOBAL_REAL(Failsafe, /datum/controller/failsafe)
 	while(running)
 		lasttick = world.time
 		if(!Master)
-			// Replace the missing Master! This should never, ever happen.
-			new /datum/controller/master()
+			// Break out of the main loop so we go into emergency state
+			break
 		// Only poke it if overrides are not in effect.
 		if(processing_interval > 0)
 			if(Master.processing && Master.iteration)
@@ -55,32 +71,35 @@ GLOBAL_REAL(Failsafe, /datum/controller/failsafe)
 					switch(defcon)
 						if(4,5)
 							--defcon
-						if(3)
-							message_admins("<span class='adminnotice'>Notice: DEFCON [defcon_pretty()]. The Master Controller has not fired in the last [(5 - defcon) * processing_interval] ticks.</span>")
-							--defcon
-						if(2)
-							to_chat(GLOB.admins, "<span class='boldannounce'>Warning: DEFCON [defcon_pretty()]. The Master Controller has not fired in the last [(5 - defcon) * processing_interval] ticks. Automatic restart in [processing_interval] ticks.</span>")
-							--defcon
-						if(1)

-							to_chat(GLOB.admins, "<span class='boldannounce'>Warning: DEFCON [defcon_pretty()]. The Master Controller has still not fired within the last [(5 - defcon) * processing_interval] ticks. Killing and restarting...</span>")
+						if(3)
+							message_admins("<span class='boldnotice'>Notice: DEFCON [defcon_pretty()]. The Master Controller has not fired in the last [(5-defcon) * processing_interval] ticks.</span>")
+							--defcon
+
+						if(2)
+							to_chat(GLOB.admins, "<span class='boldannounce'>Warning: DEFCON [defcon_pretty()]. The Master Controller has not fired in the last [(5-defcon) * processing_interval] ticks. Automatic restart in [processing_interval] ticks.</span>")
+							--defcon
+
+						if(1)
+							to_chat(GLOB.admins, "<span class='boldannounce'>Warning: DEFCON [defcon_pretty()]. The Master Controller has still not fired within the last [(5-defcon) * processing_interval] ticks. Killing and restarting...</span>")
 							--defcon
 							var/rtn = Recreate_MC()
 							if(rtn > 0)
 								defcon = 4
 								master_iteration = 0
-								to_chat(GLOB.admins, "<span class='adminnotice'>MC restarted successfully</span>")
+								to_chat(GLOB.admins, "<span class='boldnotice'>MC restarted successfully</span>")
 							else if(rtn < 0)
 								log_game("FailSafe: Could not restart MC, runtime encountered. Entering defcon 0")
 								to_chat(GLOB.admins, "<span class='boldannounce'>ERROR: DEFCON [defcon_pretty()]. Could not restart MC, runtime encountered. I will silently keep retrying.</span>")
 							//if the return number was 0, it just means the mc was restarted too recently, and it just needs some time before we try again
 							//no need to handle that specially when defcon 0 can handle it
+
 						if(0) //DEFCON 0! (mc failed to restart)
 							var/rtn = Recreate_MC()
 							if(rtn > 0)
 								defcon = 4
 								master_iteration = 0
-								to_chat(GLOB.admins, "<span class='adminnotice'>MC restarted successfully</span>")
+								to_chat(GLOB.admins, "<span class='boldnotice'>MC restarted successfully</span>")
 				else
 					defcon = min(defcon + 1,5)
 					master_iteration = Master.iteration
@@ -92,6 +111,63 @@ GLOBAL_REAL(Failsafe, /datum/controller/failsafe)
 			defcon = 5
 			sleep(initial(processing_interval))

+//Emergency loop used when Master got deleted or the main loop exited while Defcon == 0
+//Loop is driven externally so runtimes only cancel the current recovery attempt
+/datum/controller/failsafe/proc/emergency_loop()
+	//The code in this proc should be kept as simple as possible, anything complicated like to_chat might rely on master existing and runtime
+	//The goal should always be to get a new Master up and running before anything else
+	. = -1
+	switch(defcon) //The lower defcon goes the harder we try to fix the MC
+		if(2 to 3) //Try to normally recreate the MC two times
+			. = Recreate_MC()
+		if(1) //Delete the old MC first so we don't transfer any info, in case that caused any issues
+			del(Master)
+			. = Recreate_MC()
+
+	if(. == 1) //We were able to create a new master
+		master_iteration = 0
+		SSticker.Recover(); //Recover the ticket system so the Masters runlevel gets set
+		Master.Initialize(10, FALSE, TRUE) //Need to manually start the MC, normally world.new would do this
+		to_chat(GLOB.admins, "<span class='boldnotice'>Failsafe recovered MC while in emergency state [defcon_pretty()]</span>")
+	else
+		log_game("FailSafe: Failsafe in emergency state and was unable to recreate MC while in defcon state [defcon_pretty()].")
+		message_admins("<span class='boldannounce'>Failsafe in emergency state and master down, trying to recreate MC while in defcon level [defcon_pretty()] failed.</span>")
+
+///Recreate all SSs which will still cause data survive due to Recover(), the new Master will then find and take them from global.vars
+/proc/recover_all_SS_and_recreate_master()
+	// You can break EVERYTHING with this
+	if(usr && !check_rights(R_MAINTAINER))
+		return
+	del(Master)
+	var/list/subsytem_types = subtypesof(/datum/controller/subsystem)
+	sortTim(subsytem_types, /proc/cmp_subsystem_init)
+	for(var/I in subsytem_types)
+		new I
+	. = Recreate_MC()
+	if(. == 1) //We were able to create a new master
+		SSticker.Recover(); //Recover the ticket system so the Masters runlevel gets set
+		Master.Initialize(10, FALSE, TRUE) //Need to manually start the MC, normally world.new would do this
+		to_chat(GLOB.admins, "<span class='boldnotice'>MC successfully recreated after recovering all subsystems!</span>")
+	else
+		message_admins("<span class='boldannounce'>Failed to create new MC!</span>")
+
+///Delete all existing SS to basically start over
+/proc/delete_all_SS_and_recreate_master()
+	// You can break EVERYTHING with this
+	if(usr && !check_rights(R_MAINTAINER))
+		return
+	del(Master)
+	for(var/global_var in global.vars)
+		if(istype(global.vars[global_var], /datum/controller/subsystem))
+			del(global.vars[global_var])
+	. = Recreate_MC()
+	if(. == 1) //We were able to create a new master
+		SSticker.Recover(); //Recover the ticket system so the Masters runlevel gets set
+		Master.Initialize(10, FALSE, TRUE) //Need to manually start the MC, normally world.new would do this
+		to_chat(GLOB.admins, "<span class=\"boldnotice\">MC successfully recreated after deleting and recreating all subsystems!</span>")
+	else
+		message_admins("<span class=\"boldnotice\">Failed to create new MC!</span>")
+
 /datum/controller/failsafe/proc/defcon_pretty()
 	return defcon

--- a/code/controllers/master.dm
+++ b/code/controllers/master.dm
@@ -1,13 +1,14 @@
- /**
-  * StonedMC
-  *
-  * Designed to properly split up a given tick among subsystems
-  * Note: if you read parts of this code and think "why is it doing it that way"
-  * Odds are, there is a reason
-  *
+/**
+ * StonedMC
+ *
+ * Designed to properly split up a given tick among subsystems
+ * Note: if you read parts of this code and think "why is it doing it that way"
+ * Odds are, there is a reason
+ *
 **/

 //This is the ABSOLUTE ONLY THING that should init globally like this
+//2019 update: the failsafe,config and Global controllers also do it
 GLOBAL_REAL(Master, /datum/controller/master) = new

 //THIS IS THE INIT ORDER
@@ -18,7 +19,7 @@ GLOBAL_REAL(Master, /datum/controller/master) = new
 	name = "Master"

 	/// Are we processing (higher values increase the processing delay by n ticks)
-	var/processing = 1
+	var/processing = TRUE
 	/// How many times have we ran
 	var/iteration = 0

@@ -28,8 +29,8 @@ GLOBAL_REAL(Master, /datum/controller/master) = new
 	/// List of subsystems to fire().
 	var/list/subsystems

-	/// Current init stage
-	var/current_init_stage
+	/// Last reported init info
+	var/last_init_info

 	// Vars for keeping track of tick drift.
 	var/init_timeofday
@@ -42,10 +43,13 @@ GLOBAL_REAL(Master, /datum/controller/master) = new
 	/// Set this to 1 to debug the MC with a detailed stack trace. Do not set on a production server.
 	var/make_runtime = 0

+	/// Only run ticker subsystems for the next n ticks.
+	var/skip_ticks = 0
+
 	/// Did inits finish with no one logged in
 	var/initializations_finished_with_no_players_logged_in

-	// The type of the last subsystem to be fire()'d.
+	/// The type of the last subsystem to be fire()'d.
 	var/last_type_processed

 	/// Start of queue linked list
@@ -75,24 +79,41 @@ GLOBAL_REAL(Master, /datum/controller/master) = new
 	var/static/current_ticklimit = TICK_LIMIT_RUNNING

 /datum/controller/master/New()
-
 	if(!random_seed)
+		#ifdef UNIT_TESTS
+		random_seed = 29051994
+		#else
 		random_seed = rand(1, 1e9)
+		#endif
 		rand_seed(random_seed)

 	var/list/_subsystems = list()
 	subsystems = _subsystems
 	// Highlander-style: there can only be one! Kill off the old and replace it with the new.
 	if(Master != src)
-		if(istype(Master))
+		if(istype(Master)) //If there is an existing MC take over his stuff and delete it
 			Recover()
 			qdel(Master)
+			Master = src
 		else
-			var/list/subsytem_types = subtypesof(/datum/controller/subsystem)
-			sortTim(subsytem_types, /proc/cmp_subsystem_init)
-			for(var/I in subsytem_types)
-				_subsystems += new I
-		Master = src
+			//Code used for first master on game boot or if existing master got deleted
+			Master = src
+			var/list/subsystem_types = subtypesof(/datum/controller/subsystem)
+			sortTim(subsystem_types, /proc/cmp_subsystem_init)
+
+			//Find any abandoned subsystem from the previous master (if there was any)
+			var/list/existing_subsystems = list()
+			for(var/global_var in global.vars)
+				if(istype(global.vars[global_var], /datum/controller/subsystem))
+					existing_subsystems += global.vars[global_var]
+
+			//Either init a new SS or if an existing one was found use that
+			for(var/I in subsystem_types)
+				var/ss_idx = existing_subsystems.Find(I)
+				if(ss_idx)
+					_subsystems += existing_subsystems[ss_idx]
+				else
+					_subsystems += new I

 	if(!GLOB)
 		new /datum/controller/global_vars
@@ -112,7 +133,7 @@ GLOBAL_REAL(Master, /datum/controller/master) = new
 	log_world("Shutdown complete")

 // Returns 1 if we created a new mc, 0 if we couldn't due to a recent restart,
-//	-1 if we encountered a runtime trying to recreate it
+// -1 if we encountered a runtime trying to recreate it
 /proc/Recreate_MC()
 	. = -1 //so if we runtime, things know we failed
 	if(world.time < Master.restart_timeout)
@@ -123,7 +144,9 @@ GLOBAL_REAL(Master, /datum/controller/master) = new
 	var/delay = 50 * ++Master.restart_count
 	Master.restart_timeout = world.time + delay
 	Master.restart_clear = world.time + (delay * 2)
-	Master.processing = FALSE //stop ticking this one
+	if(Master) //Can only do this if master hasn't been deleted
+		Master.processing = FALSE //stop ticking this one
+
 	try
 		new/datum/controller/master()
 	catch
@@ -174,7 +197,7 @@ GLOBAL_REAL(Master, /datum/controller/master) = new


 // Please don't stuff random bullshit here,
-// 	Make a subsystem, give it the SS_NO_FIRE flag, and do your work in it's Initialize()
+// Make a subsystem, give it the SS_NO_FIRE flag, and do your work in it's Initialize()
 /datum/controller/master/Initialize(delay, init_sss, tgs_prime)
 	set waitfor = 0

@@ -210,12 +233,11 @@ GLOBAL_REAL(Master, /datum/controller/master) = new
 		SSticker.force_start = TRUE

 	if(!current_runlevel)
-		SetRunLevel(1)
+		SetRunLevel(RUNLEVEL_LOBBY)

 	// Sort subsystems by display setting for easy access.
 	sortTim(subsystems, /proc/cmp_subsystem_display)
 	// Set world options.
-	// world.fps = CONFIG_GET(number/fps) // TIGER TODO
 	world.tick_lag = GLOB.configuration.mc.ticklag
 	var/initialized_tod = REALTIMEOFDAY

@@ -259,11 +281,11 @@ GLOBAL_REAL(Master, /datum/controller/master) = new
 /datum/controller/master/proc/Loop()
 	. = -1
 	//Prep the loop (most of this is because we want MC restarts to reset as much state as we can, and because
-	//	local vars rock
+	// local vars rock

 	//all this shit is here so that flag edits can be refreshed by restarting the MC. (and for speed)
 	var/list/tickersubsystems = list()
-	var/list/runlevel_sorted_subsystems = list(list())	//ensure we always have at least one runlevel
+	var/list/runlevel_sorted_subsystems = list(list()) //ensure we always have at least one runlevel
 	var/timer = world.time
 	for(var/thing in subsystems)
 		var/datum/controller/subsystem/SS = thing
@@ -273,7 +295,7 @@ GLOBAL_REAL(Master, /datum/controller/master) = new
 		SS.queue_next = null
 		SS.queue_prev = null
 		SS.state = SS_IDLE
-		if(SS.flags & SS_TICKER)
+		if((SS.flags & (SS_TICKER|SS_BACKGROUND)) == SS_TICKER)
 			tickersubsystems += SS
 			timer += world.tick_lag * rand(1, 5)
 			SS.next_fire = timer
@@ -309,8 +331,8 @@ GLOBAL_REAL(Master, /datum/controller/master) = new
 	var/error_level = 0
 	var/sleep_delta = 1
 	var/list/subsystems_to_check
-	//the actual loop.

+	//the actual loop.
 	while(1)
 		tickdrift = max(0, MC_AVERAGE_FAST(tickdrift, (((REALTIMEOFDAY - init_timeofday) - (world.time - init_time)) / world.tick_lag)))
 		var/starting_tick_usage = TICK_USAGE
@@ -319,9 +341,9 @@ GLOBAL_REAL(Master, /datum/controller/master) = new
 			sleep(10)
 			continue

-		//Anti-tick-contention heuristics:
-		//if there are mutiple sleeping procs running before us hogging the cpu, we have to run later.
-		//	(because sleeps are processed in the order received, longer sleeps are more likely to run first)
+		// Anti-tick-contention heuristics:
+		// if there are mutiple sleeping procs running before us hogging the cpu, we have to run later.
+		// (because sleeps are processed in the order received, longer sleeps are more likely to run first)
 		if(starting_tick_usage > TICK_LIMIT_MC) //if there isn't enough time to bother doing anything this tick, sleep a bit.
 			sleep_delta *= 2
 			current_ticklimit = TICK_LIMIT_RUNNING * 0.5
@@ -346,7 +368,7 @@ GLOBAL_REAL(Master, /datum/controller/master) = new
 			new/datum/controller/failsafe() // (re)Start the failsafe.

 		//now do the actual stuff
-		if(!queue_head || !(iteration % 3))
+		if(!skip_ticks)
 			var/checking_runlevel = current_runlevel
 			if(cached_runlevel != checking_runlevel)
 				//resechedule subsystems
@@ -363,35 +385,48 @@ GLOBAL_REAL(Master, /datum/controller/master) = new
 		else
 			subsystems_to_check = tickersubsystems

-		if(CheckQueue(subsystems_to_check) <= 0)
+		if(CheckQueue(subsystems_to_check) <= 0) //error processing queue
+			stack_trace("MC: CheckQueue failed. Current error_level is [round(error_level, 0.25)]")
 			if(!SoftReset(tickersubsystems, runlevel_sorted_subsystems))
-				log_world("MC: SoftReset() failed, crashing")
-				return
-			if(!error_level)
+				error_level++
+				CRASH("MC: SoftReset() failed, exiting loop()")
+
+			if(error_level < 2) //except for the first strike, stop incrmenting our iteration so failsafe enters defcon
 				iteration++
-			error_level++
+			else
+				cached_runlevel = null //3 strikes, Lets reset the runlevel lists
 			current_ticklimit = TICK_LIMIT_RUNNING
-			sleep(10)
+			sleep((1 SECONDS) * error_level)
+			error_level++
 			continue

 		if(queue_head)
-			if(RunQueue() <= 0)
-				if(!SoftReset(tickersubsystems, runlevel_sorted_subsystems))
-					log_world("MC: SoftReset() failed, crashing")
-					return
-				if(!error_level)
-					iteration++
+			if(RunQueue() <= 0) //error running queue
+				stack_trace("MC: RunQueue failed. Current error_level is [round(error_level, 0.25)]")
+				if(error_level > 1) //skip the first error,
+					if(!SoftReset(tickersubsystems, runlevel_sorted_subsystems))
+						error_level++
+						CRASH("MC: SoftReset() failed, exiting loop()")
+
+					if(error_level <= 2) //after 3 strikes stop incrmenting our iteration so failsafe enters defcon
+						iteration++
+					else
+						cached_runlevel = null //3 strikes, Lets also reset the runlevel lists
+					current_ticklimit = TICK_LIMIT_RUNNING
+					sleep((1 SECONDS) * error_level)
+					error_level++
+					continue
 				error_level++
-				current_ticklimit = TICK_LIMIT_RUNNING
-				sleep(10)
-				continue
-		error_level--
+		if(error_level > 0)
+			error_level = max(MC_AVERAGE_SLOW(error_level-1, error_level), 0)
 		if(!queue_head) //reset the counts if the queue is empty, in the off chance they get out of sync
 			queue_priority_count = 0
 			queue_priority_count_bg = 0

 		iteration++
 		last_run = world.time
+		if(skip_ticks)
+			skip_ticks--
 		src.sleep_delta = MC_AVERAGE_FAST(src.sleep_delta, sleep_delta)
 		current_ticklimit = TICK_LIMIT_RUNNING
 		if(processing * sleep_delta <= world.tick_lag)
@@ -425,11 +460,16 @@ GLOBAL_REAL(Master, /datum/controller/master) = new
 			continue
 		if((SS_flags & (SS_TICKER|SS_KEEP_TIMING)) == SS_KEEP_TIMING && SS.last_fire + (SS.wait * 0.75) > world.time)
 			continue
+		if(SS.postponed_fires >= 1)
+			SS.postponed_fires--
+			SS.update_nextfire()
+			continue
 		SS.enqueue()
 	. = 1


-// Run thru the queue of subsystems to run, running them while balancing out their allocated tick precentage
+/// RunQueue - Run thru the queue of subsystems to run, running them while balancing out their allocated tick precentage
+/// Returns 0 if runtimed, a negitive number for logic errors, and a positive number if the operation completed without errors
 /datum/controller/master/proc/RunQueue()
 	. = 0
 	var/datum/controller/subsystem/queue_node
@@ -441,12 +481,11 @@ GLOBAL_REAL(Master, /datum/controller/master) = new
 	var/tick_precentage
 	var/tick_remaining
 	var/ran = TRUE //this is right
-	var/ran_non_ticker = FALSE
 	var/bg_calc //have we swtiched current_tick_budget to background mode yet?
 	var/tick_usage

 	//keep running while we have stuff to run and we haven't gone over a tick
-	//	this is so subsystems paused eariler can use tick time that later subsystems never used
+	// this is so subsystems paused eariler can use tick time that later subsystems never used
 	while(ran && queue_head && TICK_USAGE < TICK_LIMIT_MC)
 		ran = FALSE
 		bg_calc = FALSE
@@ -455,42 +494,42 @@ GLOBAL_REAL(Master, /datum/controller/master) = new
 		while(queue_node)
 			if(ran && TICK_USAGE > TICK_LIMIT_RUNNING)
 				break
-
 			queue_node_flags = queue_node.flags
 			queue_node_priority = queue_node.queued_priority

-			//super special case, subsystems where we can't make them pause mid way through
-			//if we can't run them this tick (without going over a tick)
-			//we bump up their priority and attempt to run them next tick
-			//(unless we haven't even ran anything this tick, since its unlikely they will ever be able run
-			//	in those cases, so we just let them run)
-			if(queue_node_flags & SS_NO_TICK_CHECK)
-				if(queue_node.tick_usage > TICK_LIMIT_RUNNING - TICK_USAGE && ran_non_ticker)
-					if(!(queue_node_flags & SS_BACKGROUND))
-						queue_node.queued_priority += queue_priority_count * 0.1
-						queue_priority_count -= queue_node_priority
-						queue_priority_count += queue_node.queued_priority
-						current_tick_budget -= queue_node_priority
-						queue_node = queue_node.queue_next
-					continue
+			if(!(queue_node_flags & SS_TICKER) && skip_ticks)
+				queue_node = queue_node.queue_next
+				continue
+
+			if((queue_node_flags & SS_BACKGROUND))
+				if(!bg_calc)
+					current_tick_budget = queue_priority_count_bg
+					bg_calc = TRUE
+			else if(bg_calc)
+				//error state, do sane fallback behavior
+				if(. == 0)
+					log_world("MC: Queue logic failure, non-background subsystem queued to run after a background subsystem: [queue_node] queue_prev:[queue_node.queue_prev]")
+				. = -1
+				current_tick_budget = queue_priority_count //this won't even be right, but is the best we have.
+				bg_calc = FALSE

-			if(!bg_calc && (queue_node_flags & SS_BACKGROUND))
-				current_tick_budget = queue_priority_count_bg
-				bg_calc = TRUE

 			tick_remaining = TICK_LIMIT_RUNNING - TICK_USAGE

-			if(current_tick_budget > 0 && queue_node_priority > 0)
-				tick_precentage = tick_remaining / (current_tick_budget / queue_node_priority)
+			if(queue_node_priority >= 0 && current_tick_budget > 0 && current_tick_budget >= queue_node_priority)
+				//Give the subsystem a precentage of the remaining tick based on the remaining priority
+				tick_precentage = tick_remaining * (queue_node_priority / current_tick_budget)
 			else
-				tick_precentage = tick_remaining
+				//error state
+				if(. == 0)
+					log_world("MC: tick_budget sync error. [json_encode(list(current_tick_budget, queue_priority_count, queue_priority_count_bg, bg_calc, queue_node, queue_node_priority))]")
+				. = -1
+				tick_precentage = tick_remaining //just because we lost track of priority calculations doesn't mean we can't try to finish off the run, if the error state persists, we don't want to stop ticks from happening

-			tick_precentage = max(tick_precentage * 0.5, tick_precentage - queue_node.tick_overrun)
+			tick_precentage = max(tick_precentage*0.5, tick_precentage-queue_node.tick_overrun)

 			current_ticklimit = round(TICK_USAGE + tick_precentage)

-			if(!(queue_node_flags & SS_TICKER))
-				ran_non_ticker = TRUE
 			ran = TRUE

 			queue_node_paused = (queue_node.state == SS_PAUSED || queue_node.state == SS_PAUSING)
@@ -535,14 +574,7 @@ GLOBAL_REAL(Master, /datum/controller/master) = new
 			queue_node.last_fire = world.time
 			queue_node.times_fired++

-			if(queue_node_flags & SS_TICKER)
-				queue_node.next_fire = world.time + (world.tick_lag * queue_node.wait)
-			else if(queue_node_flags & SS_POST_FIRE_TIMING)
-				queue_node.next_fire = world.time + queue_node.wait + (world.tick_lag * (queue_node.tick_overrun / 100))
-			else if(queue_node_flags & SS_KEEP_TIMING)
-				queue_node.next_fire += queue_node.wait
-			else
-				queue_node.next_fire = queue_node.queued_time + queue_node.wait + (world.tick_lag * (queue_node.tick_overrun / 100))
+			queue_node.update_nextfire()

 			queue_node.queued_time = 0

@@ -551,17 +583,19 @@ GLOBAL_REAL(Master, /datum/controller/master) = new

 			queue_node = queue_node.queue_next

-	. = 1
+	if(. == 0)
+		. = 1

 //resets the queue, and all subsystems, while filtering out the subsystem lists
-//	called if any mc's queue procs runtime or exit improperly.
+// called if any mc's queue procs runtime or exit improperly.
 /datum/controller/master/proc/SoftReset(list/ticker_SS, list/runlevel_SS)
 	. = 0
-	log_world("MC: SoftReset called, resetting MC queue state.")
+	stack_trace("MC: SoftReset called, resetting MC queue state.")
+
 	if(!istype(subsystems) || !istype(ticker_SS) || !istype(runlevel_SS))
 		log_world("MC: SoftReset: Bad list contents: '[subsystems]' '[ticker_SS]' '[runlevel_SS]'")
 		return
-	var/subsystemstocheck = subsystems + ticker_SS
+	var/subsystemstocheck = subsystems | ticker_SS
 	for(var/I in runlevel_SS)
 		subsystemstocheck |= I

@@ -595,15 +629,13 @@ GLOBAL_REAL(Master, /datum/controller/master) = new
 	log_world("MC: SoftReset: Finished.")
 	. = 1

-
-
 /datum/controller/master/stat_entry()
 	if(!statclick)
 		statclick = new/obj/effect/statclick/debug(null, "Initializing...", src)
-	if(current_init_stage)
-		stat("Init Stage", current_init_stage)
+	if(last_init_info)
+		stat("Last Init Info", last_init_info)
 	stat("Byond:", "(FPS:[world.fps]) (TickCount:[world.time / world.tick_lag]) (TickDrift:[round(Master.tickdrift, 1)]([round((Master.tickdrift / (world.time / world.tick_lag)) * 100, 0.1)]%))")
-	stat("Master Controller:", statclick.update("(TickRate:[Master.processing]) (Iteration:[Master.iteration])"))
+	stat("Master Controller:", statclick.update("(TickRate:[Master.processing]) (Iteration:[Master.iteration]) (TickLimit: [round(Master.current_ticklimit, 0.1)])"))

 // Currently unimplemented
 /datum/controller/master/StartLoadingMap()
@@ -631,13 +663,13 @@ GLOBAL_REAL(Master, /datum/controller/master) = new
 	else if(client_count > GLOB.configuration.mc.highpop_enable_threshold)
 		processing = GLOB.configuration.mc.highpop_tickrate

-/datum/controller/master/proc/formatcpu()
-	switch(world.cpu)
+/datum/controller/master/proc/formatcpu(cpu_var)
+	switch(cpu_var)
 		if(0 to 80) // 0-80 = green
-			. = "<font color='#32a852'>[world.cpu]</font>"
+			. = "<font color='#32a852'>[cpu_var]</font>"
 		if(80 to 90) // 80-90 = orange
-			. = "<font color='#fcba03'>[world.cpu]</font>"
+			. = "<font color='#fcba03'>[cpu_var]</font>"
 		if(90 to 100) // 90-100 = red
-			. = "<font color='#eb4034'>[world.cpu]</font>"
+			. = "<font color='#eb4034'>[cpu_var]</font>"
 		if(100 to INFINITY) // >100 = bold red
-			. = "<font color='#eb4034'><b>[world.cpu]</b></font>"
+			. = "<font color='#eb4034'><b>[cpu_var]</b></font>"
--- a/code/controllers/subsystem.dm
+++ b/code/controllers/subsystem.dm
@@ -1,43 +1,107 @@
+/**
+	* # Subsystem base class
+	*
+	* Defines a subsystem to be managed by the [Master Controller][/datum/controller/master]
+	*
+	* Simply define a child of this subsystem, using the [SUBSYSTEM_DEF] macro, and the MC will handle registration.
+	* Changing the name is required
+**/

 /datum/controller/subsystem
 	// Metadata; you should define these.
-	name = "fire codertrain" //name of the subsystem
-	/// Subsystem ID. Used for when we need a technical name for the SS
+
+	/// Name of the subsystem - you must change this
+	name = "fire codertrain"
+
+	/// SS ID - Again, change this but keep it snake_case
 	var/ss_id = "fire_codertrain_again"
-	var/init_order = INIT_ORDER_DEFAULT		//order of initialization. Higher numbers are initialized first, lower numbers later. Use defines in __DEFINES/subsystems.dm for easy understanding of order.
-	var/wait = 20			//time to wait (in deciseconds) between each call to fire(). Must be a positive integer.
-	var/priority = FIRE_PRIORITY_DEFAULT	//When mutiple subsystems need to run in the same tick, higher priority subsystems will run first and be given a higher share of the tick before MC_TICK_CHECK triggers a sleep

-	var/flags = 0			//see MC.dm in __DEFINES Most flags must be set on world start to take full effect. (You can also restart the mc to force them to process again)
+	/// What are the implications of this SS being offlined?
+	var/offline_implications = "None. No immediate action is needed."

-	var/initialized = FALSE	//set to TRUE after it has been initialized, will obviously never be set if the subsystem doesn't initialize
+	/// Order of initialization. Higher numbers are initialized first, lower numbers later. Use or create defines such as [INIT_ORDER_DEFAULT] so we can see the order in one file.
+	var/init_order = INIT_ORDER_DEFAULT

-	//set to 0 to prevent fire() calls, mostly for admin use or subsystems that may be resumed later
-	//	use the SS_NO_FIRE flag instead for systems that never fire to keep it from even being added to the list
+	/// Time to wait (in deciseconds) between each call to fire(). Must be a positive integer.
+	var/wait = 20
+
+	/// Priority Weight: When mutiple subsystems need to run in the same tick, higher priority subsystems will be given a higher share of the tick before MC_TICK_CHECK triggers a sleep, higher priority subsystems also run before lower priority subsystems
+	var/priority = FIRE_PRIORITY_DEFAULT
+
+	/// [Subsystem Flags][SS_NO_INIT] to control binary behavior. Flags must be set at compile time or before preinit finishes to take full effect. (You can also restart the mc to force them to process again)
+	var/flags = NONE
+
+	/// Set to 0 to prevent fire() calls, mostly for admin use or subsystems that may be resumed later
+	/// use the [SS_NO_FIRE] flag instead for systems that never fire to keep it from even being added to list that is checked every tick
 	var/can_fire = TRUE

-	// Bookkeeping variables; probably shouldn't mess with these.
-	var/last_fire = 0		//last world.time we called fire()
-	var/next_fire = 0		//scheduled world.time for next fire()
-	var/cost = 0			//average time to execute
-	var/tick_usage = 0		//average tick usage
-	var/tick_overrun = 0	//average tick overrun
-	var/state = SS_IDLE		//tracks the current state of the ss, running, paused, etc.
-	var/paused_ticks = 0	//ticks this ss is taking to run right now.
-	var/paused_tick_usage	//total tick_usage of all of our runs while pausing this run
-	var/ticks = 1			//how many ticks does this ss take to run on avg.
-	var/times_fired = 0		//number of times we have called fire()
-	var/queued_time = 0		//time we entered the queue, (for timing and priority reasons)
-	var/queued_priority 	//we keep a running total to make the math easier, if priority changes mid-fire that would break our running total, so we store it here
-	//linked list stuff for the queue
+	///Bitmap of what game states can this subsystem fire at. See [RUNLEVELS_DEFAULT] for more details.
+	var/runlevels = RUNLEVELS_DEFAULT //points of the game at which the SS can fire
+
+	/*
+	 * The following variables are managed by the MC and should not be modified directly.
+	 */
+
+	/// This var is set to TRUE after the subsystem has been initialized.
+	var/initialized = FALSE
+
+
+	/// Last world.time the subsystem completed a run (as in wasn't paused by [MC_TICK_CHECK])
+	var/last_fire = 0
+
+	/// Scheduled world.time for next fire()
+	var/next_fire = 0
+
+	/// Running average of the amount of milliseconds it takes the subsystem to complete a run (including all resumes but not the time spent paused)
+	var/cost = 0
+
+	/// Running average of the amount of tick usage in percents of a tick it takes the subsystem to complete a run
+	var/tick_usage = 0
+
+	/// Running average of the amount of tick usage (in percents of a game tick) the subsystem has spent past its allocated time without pausing
+	var/tick_overrun = 0
+
+	/// How much of a tick (in percents of a tick) were we allocated last fire.
+	var/tick_allocation_last = 0
+
+	/// How much of a tick (in percents of a tick) do we get allocated by the mc on avg.
+	var/tick_allocation_avg = 0
+
+	/// Tracks the current execution state of the subsystem. Used to handle subsystems that sleep in fire so the mc doesn't run them again while they are sleeping
+	var/state = SS_IDLE
+
+	/// Tracks how many fires the subsystem has consecutively paused on in the current run
+	var/paused_ticks = 0
+
+	/// Tracks how much of a tick the subsystem has consumed in the current run
+	var/paused_tick_usage
+
+	/// Tracks how many fires the subsystem takes to complete a run on average.
+	var/ticks = 1
+
+	/// Tracks the amount of completed runs for the subsystem
+	var/times_fired = 0
+
+	/// How many fires have we been requested to postpone
+	var/postponed_fires = 0
+
+	/// Time the subsystem entered the queue, (for timing and priority reasons)
+	var/queued_time = 0
+
+	/// Priority at the time the subsystem entered the queue. Needed to avoid changes in priority (by admins and the like) from breaking things.
+	var/queued_priority
+
+	/// How many times we suspect a subsystem type has crashed the MC, 3 strikes and you're out!
+	var/static/list/failure_strikes
+
+	/// Next subsystem in the queue of subsystems to run this tick
 	var/datum/controller/subsystem/queue_next
+	/// Previous subsystem in the queue of subsystems to run this tick
 	var/datum/controller/subsystem/queue_prev

-	var/runlevels = RUNLEVELS_DEFAULT	//points of the game at which the SS can fire
+	//Do not blindly add vars here to the bottom, put it where it goes above
+	//If your var only has two values, put it in as a flag.

-	var/static/list/failure_strikes //How many times we suspect a subsystem type has crashed the MC, 3 strikes and you're out!
-
-	var/offline_implications = "None. No immediate action is needed." // What are the implications of this SS being offlined?

 //Do not override
 ///datum/controller/subsystem/New()
@@ -48,9 +112,15 @@
 /datum/controller/subsystem/proc/PreInit()
 	return

-//This is used so the mc knows when the subsystem sleeps. do not override.
-/datum/controller/subsystem/proc/ignite(resumed = 0)
-	set waitfor = 0
+///This is used so the mc knows when the subsystem sleeps. do not override.
+/datum/controller/subsystem/proc/ignite(resumed = FALSE)
+	SHOULD_NOT_OVERRIDE(TRUE)
+	set waitfor = FALSE
+	. = SS_IDLE
+
+	tick_allocation_last = Master.current_ticklimit-(TICK_USAGE)
+	tick_allocation_avg = MC_AVERAGE(tick_allocation_avg, tick_allocation_last)
+
 	. = SS_SLEEPING
 	fire(resumed)
 	. = state
@@ -62,10 +132,10 @@
 		state = SS_PAUSED
 		queued_time = QT

-//previously, this would have been named 'process()' but that name is used everywhere for different things!
-//fire() seems more suitable. This is the procedure that gets called every 'wait' deciseconds.
-//Sleeping in here prevents future fires until returned.
-/datum/controller/subsystem/proc/fire(resumed = 0)
+///previously, this would have been named 'process()' but that name is used everywhere for different things!
+///fire() seems more suitable. This is the procedure that gets called every 'wait' deciseconds.
+///Sleeping in here prevents future fires until returned.
+/datum/controller/subsystem/proc/fire(resumed = FALSE)
 	flags |= SS_NO_FIRE
 	CRASH("Subsystem [src]([type]) does not fire() but did not set the SS_NO_FIRE flag. Please add the SS_NO_FIRE flag to any subsystem that doesn't fire so it doesn't get added to the processing list and waste cpu.")

@@ -73,12 +143,38 @@
 	dequeue()
 	can_fire = 0
 	flags |= SS_NO_FIRE
-	Master.subsystems -= src
+	if(Master)
+		Master.subsystems -= src
 	return ..()

-//Queue it to run.
-//	(we loop thru a linked list until we get to the end or find the right point)
-//	(this lets us sort our run order correctly without having to re-sort the entire already sorted list)
+
+/** Update next_fire for the next run.
+ *  reset_time (bool) - Ignore things that would normally alter the next fire, like tick_overrun, and last_fire. (also resets postpone)
+ */
+/datum/controller/subsystem/proc/update_nextfire(reset_time = FALSE)
+	var/queue_node_flags = flags
+
+	if(reset_time)
+		postponed_fires = 0
+		if(queue_node_flags & SS_TICKER)
+			next_fire = world.time + (world.tick_lag * wait)
+		else
+			next_fire = world.time + wait
+		return
+
+	if(queue_node_flags & SS_TICKER)
+		next_fire = world.time + (world.tick_lag * wait)
+	else if(queue_node_flags & SS_POST_FIRE_TIMING)
+		next_fire = world.time + wait + (world.tick_lag * (tick_overrun/100))
+	else if(queue_node_flags & SS_KEEP_TIMING)
+		next_fire += wait
+	else
+		next_fire = queued_time + wait + (world.tick_lag * (tick_overrun/100))
+
+
+///Queue it to run.
+/// (we loop thru a linked list until we get to the end or find the right point)
+/// (this lets us sort our run order correctly without having to re-sort the entire already sorted list)
 /datum/controller/subsystem/proc/enqueue()
 	var/SS_priority = priority
 	var/SS_flags = flags
@@ -142,9 +238,9 @@
 		queue_next.queue_prev = queue_prev
 	if(queue_prev)
 		queue_prev.queue_next = queue_next
-	if(src == Master.queue_tail)
+	if(Master && (src == Master.queue_tail))
 		Master.queue_tail = queue_prev
-	if(src == Master.queue_head)
+	if(Master && (src == Master.queue_head))
 		Master.queue_head = queue_next
 	queued_time = 0
 	if(state == SS_QUEUED)
@@ -215,11 +311,11 @@
 			. = "<font color='#4287f5'>"
 		if(SS_IDLE) // Leave it default if the SS is idle
 			. = "<font>"
-//could be used to postpone a costly subsystem for (default one) var/cycles, cycles
-//for instance, during cpu intensive operations like explosions
+
+/// Causes the next "cycle" fires to be missed. Effect is accumulative but can reset by calling update_nextfire(reset_time = TRUE)
 /datum/controller/subsystem/proc/postpone(cycles = 1)
-	if(next_fire - world.time < wait)
-		next_fire += (wait*cycles)
+	if(can_fire && cycles >= 1)
+		postponed_fires += cycles

 //usually called via datum/controller/subsystem/New() when replacing a subsystem (i.e. due to a recurring crash)
 //should attempt to salvage what it can from the old instance of subsystem
@@ -227,12 +323,12 @@

 /datum/controller/subsystem/vv_edit_var(var_name, var_value)
 	switch(var_name)
-		if("can_fire")
+		if(NAMEOF(src, can_fire))
 			//this is so the subsystem doesn't rapid fire to make up missed ticks causing more lag
 			if(var_value)
-				next_fire = world.time + wait
-		if("queued_priority") //editing this breaks things.
-			return 0
+				update_nextfire(reset_time = TRUE)
+		if(NAMEOF(src, queued_priority)) //editing this breaks things.
+			return FALSE
 	. = ..()

 /**