[MIRROR] Adds lazyloading to the asset subsystems [MDB IGNORE] (#15960)

* Adds lazyloading to the asset subsystems (#69454)

* Adds lazyloading to the asset subsystems

This currently applies only to spritesheets because of how monumentally
expensive they are.
If an asset is requested it will immediately be fully loaded, but
otherwise we slowly load them in with a separate subsystem.

This allows us to not hold up initialize with hair stuff. Saves roughly
33% (16 seconds with LOW_MEMORY_MODE) of initialize on my machine

My target is something closer to the 9 second init that had back in
2019, this is a good first step. Lets see how much more we can do yeah
lads?

Co-authored-by: san7890 <the@ san7890.com>

* Adds lazyloading to the asset subsystems

Co-authored-by: LemonInTheDark <58055496+LemonInTheDark@users.noreply.github.com>
Co-authored-by: san7890 <the@ san7890.com>

code/__DEFINES/subsystems.dm

@@ -198,6 +198,7 @@
 #define FIRE_PRIORITY_PARALLAX 65
 #define FIRE_PRIORITY_INSTRUMENTS 80
 #define FIRE_PRIORITY_MOBS 100
+#define FIRE_PRIORITY_ASSETS 105
 #define FIRE_PRIORITY_TGUI 110
 #define FIRE_PRIORITY_TICKER 200
 #define FIRE_PRIORITY_STATPANEL 390

code/controllers/subsystem/asset_loading.dm

@@ -0,0 +1,19 @@
+/// Allows us to lazyload asset datums
+/// Anything inserted here will fully load if directly gotten
+/// So this just serves to remove the requirement to load assets fully during init
+SUBSYSTEM_DEF(asset_loading)
+	name = "Asset Loading"
+	priority = FIRE_PRIORITY_ASSETS
+	flags = SS_NO_INIT
+	runlevels = RUNLEVEL_LOBBY|RUNLEVELS_DEFAULT
+	var/list/datum/asset/generate_queue = list()
+
+/datum/controller/subsystem/asset_loading/fire(resumed)
+	while(length(generate_queue))
+		var/datum/asset/to_load = generate_queue[generate_queue.len]
+
+		to_load.queued_generation()
+
+		if(MC_TICK_CHECK)
+			return
+		generate_queue.len--

code/controllers/subsystem/assets.dm

@@ -26,7 +26,7 @@ SUBSYSTEM_DEF(assets)
 	for(var/type in typesof(/datum/asset))
 		var/datum/asset/A = type
 		if (type != initial(A._abstract))
-			get_asset_datum(type)
+			load_asset_datum(type)
 
 	transport.Initialize(cache)
 

code/controllers/subsystem/early_assets.dm

@@ -16,7 +16,7 @@ SUBSYSTEM_DEF(early_assets)
 		if (!initial(asset_type.early))
 			continue
 
-		if (!get_asset_datum(asset_type))
+		if (!load_asset_datum(asset_type))
 			stack_trace("Could not initialize early asset [asset_type]!")
 
 		CHECK_TICK

code/modules/asset_cache/asset_list.dm

@@ -7,9 +7,14 @@
 GLOBAL_LIST_EMPTY(asset_datums)
 
 //get an assetdatum or make a new one
-/proc/get_asset_datum(type)
+//does NOT ensure it's filled, if you want that use get_asset_datum()
+/proc/load_asset_datum(type)
 	return GLOB.asset_datums[type] || new type()
 
+/proc/get_asset_datum(type)
+	var/datum/asset/loaded_asset = GLOB.asset_datums[type] || new type()
+	return loaded_asset.ensure_ready()
+
 /datum/asset
 	var/_abstract = /datum/asset
 	var/cached_serialized_url_mappings
@@ -27,6 +32,15 @@ GLOBAL_LIST_EMPTY(asset_datums)
 	GLOB.asset_datums[type] = src
 	register()
 
+/// Stub that allows us to react to something trying to get us
+/// Not useful here, more handy for sprite sheets
+/datum/asset/proc/ensure_ready()
+	return src
+
+/// Stub to hook into if your asset is having its generation queued by SSasset_loading
+/datum/asset/proc/queued_generation()
+	CRASH("[type] inserted into SSasset_loading despite not implementing /proc/queued_generation")
+
 /datum/asset/proc/get_url_mappings()
 	return list()
 
@@ -88,7 +102,7 @@ GLOBAL_LIST_EMPTY(asset_datums)
 
 /datum/asset/group/register()
 	for(var/type in children)
-		get_asset_datum(type)
+		load_asset_datum(type)
 
 /datum/asset/group/send(client/C)
 	for(var/type in children)
@@ -113,10 +127,17 @@ GLOBAL_LIST_EMPTY(asset_datums)
 /datum/asset/spritesheet
 	_abstract = /datum/asset/spritesheet
 	var/name
+	/// List of arguments to pass into queuedInsert
+	/// Exists so we can queue icon insertion, mostly for stuff like preferences
+	var/list/to_generate = list()
 	var/list/sizes = list()    // "32x32" -> list(10, icon/normal, icon/stripped)
 	var/list/sprites = list()  // "foo_bar" -> list("32x32", 5)
 	var/list/cached_spritesheets_needed
 	var/generating_cache = FALSE
+	var/fully_generated = FALSE
+	/// If this asset should be fully loaded on new
+	/// Defaults to false so we can process this stuff nicely
+	var/load_immediately = FALSE
 
 /datum/asset/spritesheet/should_refresh()
 	if (..())
@@ -140,7 +161,25 @@ GLOBAL_LIST_EMPTY(asset_datums)
 	if (!should_refresh() && read_from_cache())
 		return
 
+	// If it's cached, may as well load it now, while the loading is cheap
+	if(CONFIG_GET(flag/cache_assets) && cross_round_cachable)
+		load_immediately = TRUE
+
 	create_spritesheets()
+	if(load_immediately)
+		realize_spritesheets(yield = FALSE)
+	else
+		SSasset_loading.generate_queue += src
+
+/datum/asset/spritesheet/proc/realize_spritesheets(yield)
+	if(fully_generated)
+		return
+	while(length(to_generate))
+		var/list/stored_args = to_generate[to_generate.len]
+		to_generate.len--
+		queuedInsert(arglist(stored_args))
+		if(yield && TICK_CHECK)
+			return
 
 	ensure_stripped()
 	for(var/size_id in sizes)
@@ -155,6 +194,17 @@ GLOBAL_LIST_EMPTY(asset_datums)
 
 	if (CONFIG_GET(flag/cache_assets) && cross_round_cachable)
 		write_to_cache()
+	fully_generated = TRUE
+	// If we were ever in there, remove ourselves
+	SSasset_loading.generate_queue -= src
+
+/datum/asset/spritesheet/queued_generation()
+	realize_spritesheets(yield = TRUE)
+
+/datum/asset/spritesheet/ensure_ready()
+	if(!fully_generated)
+		realize_spritesheets(yield = FALSE)
+	return ..()
 
 /datum/asset/spritesheet/send(client/client)
 	if (!name)
@@ -277,6 +327,16 @@ GLOBAL_LIST_EMPTY(asset_datums)
 	CRASH("create_spritesheets() not implemented for [type]!")
 
 /datum/asset/spritesheet/proc/Insert(sprite_name, icon/I, icon_state="", dir=SOUTH, frame=1, moving=FALSE)
+	if(load_immediately)
+		queuedInsert(sprite_name, I, icon_state, dir, frame, moving)
+	else
+		to_generate += list(args.Copy())
+
+// LEMON NOTE
+// A GOON CODER SAYS BAD ICON ERRORS CAN BE THROWN BY THE "ICON CACHE"
+// APPARENTLY IT MAKES ICONS IMMUTABLE
+// LOOK INTO USING THE MUTABLE APPEARANCE PATTERN HERE
+/datum/asset/spritesheet/proc/queuedInsert(sprite_name, icon/I, icon_state="", dir=SOUTH, frame=1, moving=FALSE)
 	I = icon(I, icon_state=icon_state, dir=dir, frame=frame, moving=moving)
 	if (!I || !length(icon_states(I)))  // that direction or state doesn't exist
 		return
@@ -291,8 +351,11 @@ GLOBAL_LIST_EMPTY(asset_datums)
 	if (size)
 		var/position = size[SPRSZ_COUNT]++
 		var/icon/sheet = size[SPRSZ_ICON]
+		var/icon/sheet_copy = icon(sheet)
 		size[SPRSZ_STRIPPED] = null
-		sheet.Insert(I, icon_state=sprite_name)
+		sheet_copy.Insert(I, icon_state=sprite_name)
+		size[SPRSZ_ICON] = sheet_copy
+		
 		sprites[sprite_name] = list(size_id, position)
 	else
 		sizes[size_id] = size = list(1, I, null)

code/modules/client/preferences/assets.dm

@@ -32,7 +32,7 @@
 				CRASH("[create_icon_of] is an invalid preference value (from [preference_key]:[preference_value]).")
 
 			to_insert[preference.get_spritesheet_key(preference_value)] = list(icon, icon_state)
-
+	
 	for (var/spritesheet_key in to_insert)
 		var/list/inserting = to_insert[spritesheet_key]
 		Insert(spritesheet_key, inserting[1], inserting[2])

code/modules/plumbing/plumbers/pill_press.dm

@@ -45,23 +45,6 @@
 
 	AddComponent(/datum/component/plumbing/simple_demand, bolt, layer)
 
-	//expertly copypasted from chemmasters
-	var/datum/asset/spritesheet/simple/assets = get_asset_datum(/datum/asset/spritesheet/simple/pills)
-	pill_styles = list()
-	for (var/x in 1 to PILL_STYLE_COUNT)
-		var/list/SL = list()
-		SL["id"] = x
-		SL["class_name"] = assets.icon_class_name("pill[x]")
-		pill_styles += list(SL)
-	var/datum/asset/spritesheet/simple/patches_assets = get_asset_datum(/datum/asset/spritesheet/simple/patches)
-	patch_styles = list()
-	for (var/raw_patch_style in PATCH_STYLE_LIST)
-		//adding class_name for use in UI
-		var/list/patch_style = list()
-		patch_style["style"] = raw_patch_style
-		patch_style["class_name"] = patches_assets.icon_class_name(raw_patch_style)
-		patch_styles += list(patch_style)
-
 /obj/machinery/plumbing/pill_press/process(delta_time)
 	if(machine_stat & NOPOWER)
 		return
@@ -110,6 +93,24 @@
 
 	use_power(active_power_usage * delta_time)
 
+/obj/machinery/plumbing/pill_press/proc/load_styles()
+	//expertly copypasted from chemmasters
+	var/datum/asset/spritesheet/simple/assets = get_asset_datum(/datum/asset/spritesheet/simple/pills)
+	pill_styles = list()
+	for (var/x in 1 to PILL_STYLE_COUNT)
+		var/list/SL = list()
+		SL["id"] = x
+		SL["class_name"] = assets.icon_class_name("pill[x]")
+		pill_styles += list(SL)
+	var/datum/asset/spritesheet/simple/patches_assets = get_asset_datum(/datum/asset/spritesheet/simple/patches)
+	patch_styles = list()
+	for (var/raw_patch_style in PATCH_STYLE_LIST)
+		//adding class_name for use in UI
+		var/list/patch_style = list()
+		patch_style["style"] = raw_patch_style
+		patch_style["class_name"] = patches_assets.icon_class_name(raw_patch_style)
+		patch_styles += list(patch_style)
+
 /obj/machinery/plumbing/pill_press/ui_assets(mob/user)
 	return list(
 		get_asset_datum(/datum/asset/spritesheet/simple/pills),
@@ -123,6 +124,8 @@
 		ui.open()
 
 /obj/machinery/plumbing/pill_press/ui_data(mob/user)
+	if(!pill_styles || !patch_styles)
+		load_styles()
 	var/list/data = list()
 	data["pill_style"] = pill_number
 	data["current_volume"] = current_volume

code/modules/reagents/chemistry/machinery/chem_master.dm

@@ -43,27 +43,6 @@
 
 /obj/machinery/chem_master/Initialize(mapload)
 	create_reagents(100)
-
-	//Calculate the span tags and ids fo all the available pill icons
-	var/datum/asset/spritesheet/simple/assets = get_asset_datum(/datum/asset/spritesheet/simple/pills)
-	pill_styles = list()
-	for (var/x in 1 to PILL_STYLE_COUNT)
-		var/list/SL = list()
-		SL["id"] = x
-		SL["className"] = assets.icon_class_name("pill[x]")
-		pill_styles += list(SL)
-
-	var/datum/asset/spritesheet/simple/patches_assets = get_asset_datum(/datum/asset/spritesheet/simple/patches)
-	patch_styles = list()
-	for (var/raw_patch_style in PATCH_STYLE_LIST)
-		//adding class_name for use in UI
-		var/list/patch_style = list()
-		patch_style["style"] = raw_patch_style
-		patch_style["class_name"] = patches_assets.icon_class_name(raw_patch_style)
-		patch_styles += list(patch_style)
-
-	condi_styles = strip_condi_styles_to_icons(get_condi_styles())
-
 	. = ..()
 
 /obj/machinery/chem_master/Destroy()
@@ -206,6 +185,27 @@
 		bottle = null
 	return ..()
 
+/obj/machinery/chem_master/proc/load_styles()
+	//Calculate the span tags and ids fo all the available pill icons
+	var/datum/asset/spritesheet/simple/assets = get_asset_datum(/datum/asset/spritesheet/simple/pills)
+	pill_styles = list()
+	for (var/x in 1 to PILL_STYLE_COUNT)
+		var/list/SL = list()
+		SL["id"] = x
+		SL["className"] = assets.icon_class_name("pill[x]")
+		pill_styles += list(SL)
+
+	var/datum/asset/spritesheet/simple/patches_assets = get_asset_datum(/datum/asset/spritesheet/simple/patches)
+	patch_styles = list()
+	for (var/raw_patch_style in PATCH_STYLE_LIST)
+		//adding class_name for use in UI
+		var/list/patch_style = list()
+		patch_style["style"] = raw_patch_style
+		patch_style["class_name"] = patches_assets.icon_class_name(raw_patch_style)
+		patch_styles += list(patch_style)
+
+	condi_styles = strip_condi_styles_to_icons(get_condi_styles())
+
 /obj/machinery/chem_master/ui_assets(mob/user)
 	return list(
 		get_asset_datum(/datum/asset/spritesheet/simple/pills),
@@ -248,7 +248,9 @@
 			buffer_contents.Add(list(list("name" = N.name, "id" = ckey(N.name), "volume" = round(N.volume, 0.01)))) // ^
 	data["bufferContents"] = buffer_contents
 
-	//Calculated at init time as it never changes
+	//Calculated once since it'll never change
+	if(!pill_styles || !condi_styles || !patch_style || !patch_styles)
+		load_styles()
 	data["pillStyles"] = pill_styles
 	data["condiStyles"] = condi_styles
 	data["patch_style"] = patch_style

tgstation.dme

@@ -539,6 +539,7 @@
 #include "code\controllers\subsystem\ai_controllers.dm"
 #include "code\controllers\subsystem\air.dm"
 #include "code\controllers\subsystem\ambience.dm"
+#include "code\controllers\subsystem\asset_loading.dm"
 #include "code\controllers\subsystem\assets.dm"
 #include "code\controllers\subsystem\atoms.dm"
 #include "code\controllers\subsystem\augury.dm"