diff --git a/code/__DEFINES/spaceman_dmm.dm b/code/__DEFINES/spaceman_dmm.dm index c2ecc6a5e66..7b31c5134e5 100644 --- a/code/__DEFINES/spaceman_dmm.dm +++ b/code/__DEFINES/spaceman_dmm.dm @@ -37,12 +37,6 @@ /proc/enable_debugging(mode, port) CRASH("auxtools not loaded") -/world/proc/enable_debugger() - var/dll = world.GetConfig("env", "AUXTOOLS_DEBUG_DLL") - if (dll) - call(dll, "auxtools_init")() - enable_debugging() - /world/Del() var/debug_server = world.GetConfig("env", "AUXTOOLS_DEBUG_DLL") if (debug_server) diff --git a/code/_debugger.dm b/code/_debugger.dm new file mode 100644 index 00000000000..dafc759ec56 --- /dev/null +++ b/code/_debugger.dm @@ -0,0 +1,13 @@ +//Datum used to init Auxtools debugging as early as possible +//Datum gets created in master.dm because for whatever reason global code in there gets runs first +//In case we ever figure out how to manipulate global init order please move the datum creation into this file +/datum/debugger + +/datum/debugger/New() + enable_debugger() + +/datum/debugger/proc/enable_debugger() + var/dll = world.GetConfig("env", "AUXTOOLS_DEBUG_DLL") + if (dll) + call(dll, "auxtools_init")() + enable_debugging() diff --git a/code/controllers/failsafe.dm b/code/controllers/failsafe.dm index 8fc589b0b90..04b88eb3be2 100644 --- a/code/controllers/failsafe.dm +++ b/code/controllers/failsafe.dm @@ -33,6 +33,22 @@ GLOBAL_REAL(Failsafe, /datum/controller/failsafe) /datum/controller/failsafe/Initialize() set waitfor = FALSE Failsafe.Loop() + if (!Master || defcon == 0) //Master is gone/not responding and Failsafe just exited its loop + defcon = 3 //Reset defcon level as its used inside the emergency loop + while (defcon > 0) + var/recovery_result = emergency_loop() + if (recovery_result == 1) //Exit emergency loop and delete self if it was able to recover MC + break + else if (defcon == 1) //Exit Failsafe if we weren't able to recover the MC in the last stage + log_game("FailSafe: Failed to recover MC while in emergency state. Failsafe exiting.") + message_admins(span_boldannounce("Failsafe failed criticaly while trying to recreate broken MC. Please manually fix the MC or reboot the server. Failsafe exiting now.")) + message_admins(span_boldannounce("You can try manually calling these two procs:.")) + message_admins(span_boldannounce("/proc/recover_all_SS_and_recreate_master: Most stuff should still function but expect instability/runtimes/broken stuff.")) + message_admins(span_boldannounce("/proc/delete_all_SS_and_recreate_master: Most stuff will be broken but basic stuff like movement and chat should still work.")) + else if (recovery_result == -1) //Failed to recreate MC + defcon-- + sleep(initial(processing_interval)) //Wait a bit until the next try + if(!QDELETED(src)) qdel(src) //when Loop() returns, we delete ourselves and let the mc recreate us @@ -45,8 +61,8 @@ GLOBAL_REAL(Failsafe, /datum/controller/failsafe) while(running) lasttick = world.time if(!Master) - // Replace the missing Master! This should never, ever happen. - new /datum/controller/master() + // Break out of the main loop so we go into emergency state + break // Only poke it if overrides are not in effect. if(processing_interval > 0) if(Master.processing && Master.iteration) @@ -106,6 +122,57 @@ GLOBAL_REAL(Failsafe, /datum/controller/failsafe) defcon = 5 sleep(initial(processing_interval)) +//Emergency loop used when Master got deleted or the main loop exited while Defcon == 0 +//Loop is driven externally so runtimes only cancel the current recovery attempt +/datum/controller/failsafe/proc/emergency_loop() + //The code in this proc should be kept as simple as possible, anything complicated like to_chat might rely on master existing and runtime + //The goal should always be to get a new Master up and running before anything else + . = -1 + switch (defcon) //The lower defcon goes the harder we try to fix the MC + if (2 to 3) //Try to normally recreate the MC two times + . = Recreate_MC() + if (1) //Delete the old MC first so we don't transfer any info, in case that caused any issues + del(Master) + . = Recreate_MC() + + if (. == 1) //We were able to create a new master + master_iteration = 0 + SSticker.Recover(); //Recover the ticket system so the Masters runlevel gets set + Master.Initialize(10, FALSE, TRUE) //Need to manually start the MC, normally world.new would do this + to_chat(GLOB.admins, span_adminnotice("Failsafe recovered MC while in emergency state [defcon_pretty()]")) + else + log_game("FailSafe: Failsafe in emergency state and was unable to recreate MC while in defcon state [defcon_pretty()].") + message_admins(span_boldannounce("Failsafe in emergency state and master down, trying to recreate MC while in defcon level [defcon_pretty()] failed.")) + +///Recreate all SSs which will still cause data survive due to Recover(), the new Master will then find and take them from global.vars +/proc/recover_all_SS_and_recreate_master() + del(Master) + var/list/subsytem_types = subtypesof(/datum/controller/subsystem) + sortTim(subsytem_types, /proc/cmp_subsystem_init) + for(var/I in subsytem_types) + new I + . = Recreate_MC() + if (. == 1) //We were able to create a new master + SSticker.Recover(); //Recover the ticket system so the Masters runlevel gets set + Master.Initialize(10, FALSE, TRUE) //Need to manually start the MC, normally world.new would do this + to_chat(GLOB.admins, span_adminnotice("MC successfully recreated after recovering all subsystems!")) + else + message_admins(span_boldannounce("Failed to create new MC!")) + +///Delete all existing SS to basically start over +/proc/delete_all_SS_and_recreate_master() + del(Master) + for(var/global_var in global.vars) + if (istype(global.vars[global_var], /datum/controller/subsystem)) + del(global.vars[global_var]) + . = Recreate_MC() + if (. == 1) //We were able to create a new master + SSticker.Recover(); //Recover the ticket system so the Masters runlevel gets set + Master.Initialize(10, FALSE, TRUE) //Need to manually start the MC, normally world.new would do this + to_chat(GLOB.admins, span_adminnotice("MC successfully recreated after deleting and recreating all subsystems!")) + else + message_admins(span_boldannounce("Failed to create new MC!")) + /datum/controller/failsafe/proc/defcon_pretty() return defcon diff --git a/code/controllers/master.dm b/code/controllers/master.dm index fcb4a50bedf..e7ddca62933 100644 --- a/code/controllers/master.dm +++ b/code/controllers/master.dm @@ -7,6 +7,11 @@ * **/ +//Init the debugger datum first so we can debug Master +//You might wonder why not just create the debugger datum global in its own file, since its loaded way earlier than this DM file +//Well for whatever reason then the Master gets created first and then the debugger when doing that +//So thats why this code lives here now, until someone finds out how Byond inits globals +GLOBAL_REAL(Debugger, /datum/debugger) = new //This is the ABSOLUTE ONLY THING that should init globally like this //2019 update: the failsafe,config and Global controllers also do it GLOBAL_REAL(Master, /datum/controller/master) = new @@ -85,15 +90,27 @@ GLOBAL_REAL(Master, /datum/controller/master) = new var/list/_subsystems = list() subsystems = _subsystems if (Master != src) - if (istype(Master)) + if (istype(Master)) //If there is an existing MC take over his stuff and delete it Recover() qdel(Master) + Master = src else + //Code used for first master on game boot or if existing master got deleted + Master = src var/list/subsytem_types = subtypesof(/datum/controller/subsystem) sortTim(subsytem_types, /proc/cmp_subsystem_init) + //Find any abandoned subsystem from the previous master (if there was any) + var/list/existing_subsystems = list() + for(var/global_var in global.vars) + if (istype(global.vars[global_var], /datum/controller/subsystem)) + existing_subsystems += global.vars[global_var] + //Either init a new SS or if an existing one was found use that for(var/I in subsytem_types) - _subsystems += new I - Master = src + var/datum/controller/subsystem/existing_subsystem = locate(I) in existing_subsystems + if (istype(existing_subsystem)) + _subsystems += existing_subsystem + else + _subsystems += new I if(!GLOB) new /datum/controller/global_vars @@ -124,7 +141,8 @@ GLOBAL_REAL(Master, /datum/controller/master) = new var/delay = 50 * ++Master.restart_count Master.restart_timeout = world.time + delay Master.restart_clear = world.time + (delay * 2) - Master.processing = FALSE //stop ticking this one + if (Master) //Can only do this if master hasn't been deleted + Master.processing = FALSE //stop ticking this one try new/datum/controller/master() catch @@ -194,7 +212,7 @@ GLOBAL_REAL(Master, /datum/controller/master) = new // Initialize subsystems. current_ticklimit = CONFIG_GET(number/tick_limit_mc_init) for (var/datum/controller/subsystem/SS in subsystems) - if (SS.flags & SS_NO_INIT) + if (SS.flags & SS_NO_INIT || SS.initialized) //Don't init SSs with the correspondig flag or if they already are initialzized continue SS.Initialize(REALTIMEOFDAY) CHECK_TICK diff --git a/code/controllers/subsystem.dm b/code/controllers/subsystem.dm index a1315c484bd..7b47daa2a9a 100644 --- a/code/controllers/subsystem.dm +++ b/code/controllers/subsystem.dm @@ -125,7 +125,8 @@ dequeue() can_fire = 0 flags |= SS_NO_FIRE - Master.subsystems -= src + if (Master) + Master.subsystems -= src return ..() @@ -219,9 +220,9 @@ queue_next.queue_prev = queue_prev if (queue_prev) queue_prev.queue_next = queue_next - if (src == Master.queue_tail) + if (Master && (src == Master.queue_tail)) Master.queue_tail = queue_prev - if (src == Master.queue_head) + if (Master && (src == Master.queue_head)) Master.queue_head = queue_next queued_time = 0 if (state == SS_QUEUED) diff --git a/code/controllers/subsystem/air.dm b/code/controllers/subsystem/air.dm index 6a0a1251ee5..a1508b58dfa 100644 --- a/code/controllers/subsystem/air.dm +++ b/code/controllers/subsystem/air.dm @@ -199,6 +199,23 @@ SUBSYSTEM_DEF(air) currentpart = SSAIR_PIPENETS SStgui.update_uis(SSair) //Lightning fast debugging motherfucker +/datum/controller/subsystem/air/Recover() + excited_groups = SSair.excited_groups + active_turfs = SSair.active_turfs + hotspots = SSair.hotspots + networks = SSair.networks + rebuild_queue = SSair.rebuild_queue + expansion_queue = SSair.expansion_queue + atmos_machinery = SSair.atmos_machinery + pipe_init_dirs_cache = SSair.pipe_init_dirs_cache + gas_reactions = SSair.gas_reactions + atmos_gen = SSair.atmos_gen + planetary = SSair.planetary + active_super_conductivity = SSair.active_super_conductivity + high_pressure_delta = SSair.high_pressure_delta + atom_process = SSair.atom_process + currentrun = SSair.currentrun + queued_for_activation = SSair.queued_for_activation /datum/controller/subsystem/air/proc/process_pipenets(resumed = FALSE) if (!resumed) diff --git a/code/controllers/subsystem/assets.dm b/code/controllers/subsystem/assets.dm index 4f02d32ad0f..ef79e55dbe5 100644 --- a/code/controllers/subsystem/assets.dm +++ b/code/controllers/subsystem/assets.dm @@ -11,7 +11,7 @@ SUBSYSTEM_DEF(assets) switch (CONFIG_GET(string/asset_transport)) if ("webroot") newtransporttype = /datum/asset_transport/webroot - + if (newtransporttype == transport.type) return @@ -31,3 +31,7 @@ SUBSYSTEM_DEF(assets) transport.Initialize(cache) ..() + +/datum/controller/subsystem/assets/Recover() + cache = SSassets.cache + preload = SSassets.preload diff --git a/code/controllers/subsystem/economy.dm b/code/controllers/subsystem/economy.dm index 903bfe1130e..42bb7e0c184 100644 --- a/code/controllers/subsystem/economy.dm +++ b/code/controllers/subsystem/economy.dm @@ -58,6 +58,11 @@ SUBSYSTEM_DEF(economy) new /datum/bank_account/department(A, budget_to_hand_out) return ..() +/datum/controller/subsystem/economy/Recover() + generated_accounts = SSeconomy.generated_accounts + bank_accounts_by_id = SSeconomy.bank_accounts_by_id + dep_cards = SSeconomy.dep_cards + /datum/controller/subsystem/economy/fire(resumed = 0) var/temporary_total = 0 var/delta_time = wait / (5 MINUTES) diff --git a/code/controllers/subsystem/garbage.dm b/code/controllers/subsystem/garbage.dm index f53b40bc96f..e9b69d5da1a 100644 --- a/code/controllers/subsystem/garbage.dm +++ b/code/controllers/subsystem/garbage.dm @@ -57,13 +57,7 @@ SUBSYSTEM_DEF(garbage) /datum/controller/subsystem/garbage/PreInit() - queues = new(GC_QUEUE_COUNT) - pass_counts = new(GC_QUEUE_COUNT) - fail_counts = new(GC_QUEUE_COUNT) - for(var/i in 1 to GC_QUEUE_COUNT) - queues[i] = list() - pass_counts[i] = 0 - fail_counts[i] = 0 + InitQueues() /datum/controller/subsystem/garbage/stat_entry(msg) var/list/counts = list() @@ -131,6 +125,15 @@ SUBSYSTEM_DEF(garbage) +/datum/controller/subsystem/garbage/proc/InitQueues() + if (isnull(queues)) // Only init the queues if they don't already exist, prevents overriding of recovered lists + queues = new(GC_QUEUE_COUNT) + pass_counts = new(GC_QUEUE_COUNT) + fail_counts = new(GC_QUEUE_COUNT) + for(var/i in 1 to GC_QUEUE_COUNT) + queues[i] = list() + pass_counts[i] = 0 + fail_counts[i] = 0 /datum/controller/subsystem/garbage/proc/HandleQueue(level = GC_QUEUE_CHECK) if (level == GC_QUEUE_CHECK) @@ -285,6 +288,7 @@ SUBSYSTEM_DEF(garbage) I.qdel_flags |= QDEL_ITEM_SUSPENDED_FOR_LAG /datum/controller/subsystem/garbage/Recover() + InitQueues() //We first need to create the queues before recovering data if (istype(SSgarbage.queues)) for (var/i in 1 to SSgarbage.queues.len) queues[i] |= SSgarbage.queues[i] diff --git a/code/controllers/subsystem/mapping.dm b/code/controllers/subsystem/mapping.dm index a1c041b8c8a..c5524292745 100644 --- a/code/controllers/subsystem/mapping.dm +++ b/code/controllers/subsystem/mapping.dm @@ -195,6 +195,8 @@ Used by the AI doomsday and the self-destruct nuke. turf_reservations = SSmapping.turf_reservations used_turfs = SSmapping.used_turfs holodeck_templates = SSmapping.holodeck_templates + transit = SSmapping.transit + areas_in_z = SSmapping.areas_in_z config = SSmapping.config next_map_config = SSmapping.next_map_config diff --git a/code/controllers/subsystem/shuttle.dm b/code/controllers/subsystem/shuttle.dm index a140b72619a..145a75546f1 100644 --- a/code/controllers/subsystem/shuttle.dm +++ b/code/controllers/subsystem/shuttle.dm @@ -548,6 +548,7 @@ SUBSYSTEM_DEF(shuttle) return new_transit_dock /datum/controller/subsystem/shuttle/Recover() + initialized = SSshuttle.initialized if (istype(SSshuttle.mobile)) mobile = SSshuttle.mobile if (istype(SSshuttle.stationary)) diff --git a/code/controllers/subsystem/ticker.dm b/code/controllers/subsystem/ticker.dm index 6bab9f99801..c1d45452653 100755 --- a/code/controllers/subsystem/ticker.dm +++ b/code/controllers/subsystem/ticker.dm @@ -549,13 +549,14 @@ SUBSYSTEM_DEF(ticker) queue_delay = SSticker.queue_delay queued_players = SSticker.queued_players - switch (current_state) - if(GAME_STATE_SETTING_UP) - Master.SetRunLevel(RUNLEVEL_SETUP) - if(GAME_STATE_PLAYING) - Master.SetRunLevel(RUNLEVEL_GAME) - if(GAME_STATE_FINISHED) - Master.SetRunLevel(RUNLEVEL_POSTGAME) + if (Master) //Set Masters run level if it exists + switch (current_state) + if(GAME_STATE_SETTING_UP) + Master.SetRunLevel(RUNLEVEL_SETUP) + if(GAME_STATE_PLAYING) + Master.SetRunLevel(RUNLEVEL_GAME) + if(GAME_STATE_FINISHED) + Master.SetRunLevel(RUNLEVEL_POSTGAME) /datum/controller/subsystem/ticker/proc/send_news_report() var/news_message diff --git a/code/controllers/subsystem/timer.dm b/code/controllers/subsystem/timer.dm index 09dede4e01e..c5e5f99f2d4 100644 --- a/code/controllers/subsystem/timer.dm +++ b/code/controllers/subsystem/timer.dm @@ -324,10 +324,15 @@ SUBSYSTEM_DEF(timer) /datum/controller/subsystem/timer/Recover() - second_queue |= SStimer.second_queue - hashes |= SStimer.hashes - timer_id_dict |= SStimer.timer_id_dict - bucket_list |= SStimer.bucket_list + //Find the current timer sub-subsystem in global and recover its buckets etc + var/datum/controller/subsystem/timer/timerSS = null + for(var/global_var in global.vars) + if (istype(global.vars[global_var],src.type)) + timerSS = global.vars[global_var] + second_queue |= timerSS.second_queue + hashes |= timerSS.hashes + timer_id_dict |= timerSS.timer_id_dict + bucket_list |= timerSS.bucket_list /** * # Timed Event diff --git a/code/game/world.dm b/code/game/world.dm index 986c55d45ed..df335f86819 100644 --- a/code/game/world.dm +++ b/code/game/world.dm @@ -29,7 +29,6 @@ GLOBAL_VAR(restart_counter) * All atoms in both compiled and uncompiled maps are initialized() */ /world/New() - enable_debugger() log_world("World loaded at [time_stamp()]!") diff --git a/code/modules/tgchat/to_chat.dm b/code/modules/tgchat/to_chat.dm index f74e0d65197..c0e27a2ba37 100644 --- a/code/modules/tgchat/to_chat.dm +++ b/code/modules/tgchat/to_chat.dm @@ -74,7 +74,7 @@ trailing_newline = TRUE, confidential = FALSE ) - if(Master.current_runlevel == RUNLEVEL_INIT || !SSchat?.initialized) + if(isnull(Master) || Master.current_runlevel == RUNLEVEL_INIT || !SSchat?.initialized) to_chat_immediate(target, html, type, text, avoid_highlighting) return diff --git a/tgstation.dme b/tgstation.dme index 28c7b4528db..5c8b359344c 100644 --- a/tgstation.dme +++ b/tgstation.dme @@ -15,6 +15,7 @@ // BEGIN_INCLUDE #include "_maps\_basemap.dm" #include "code\_compile_options.dm" +#include "code\_debugger.dm" #include "code\world.dm" #include "code\__DEFINES\_click.dm" #include "code\__DEFINES\_globals.dm"