Asset cache updates. (#50759)

Asset cache now caches the asset's md5 This should solve the high cpu usage issues with it. line by line profile suggests that the next hotspot is the asset log.

Simplied asset cache code, moved verify functionality to a flush_assets proc that blocks until a client has all currently sending assets. (sidenote: there is an argument for moving most of asset_cache's global procs onto the client, get_asset_datum and register_asset are really the only valid global procs.)

Re-added batched passive sends since it does speed up how quickly the asset cache pre-caches to clients.

code/__HELPERS/icons.dm

@@ -1134,7 +1134,7 @@ GLOBAL_LIST_INIT(freon_color_matrix, list("#2E5E69", "#60A2A8", "#A1AFB1", rgb(0
 			var/name = sanitize_filename("[generate_asset_name(thing)].png")
 			register_asset(name, thing)
 			for (var/thing2 in targets)
-				send_asset(thing2, key, FALSE)
+				send_asset(thing2, key)
 			return "<img class='icon icon-misc' src=\"[url_encode(name)]\">"
 		var/atom/A = thing
 		if (isnull(dir))
@@ -1158,7 +1158,7 @@ GLOBAL_LIST_INIT(freon_color_matrix, list("#2E5E69", "#60A2A8", "#A1AFB1", rgb(0
 	key = "[generate_asset_name(I)].png"
 	register_asset(key, I)
 	for (var/thing2 in targets)
-		send_asset(thing2, key, FALSE)
+		send_asset(thing2, key)
 
 	return "<img class='icon icon-[icon_state]' src=\"[url_encode(key)]\">"
 

code/datums/browser.dm

@@ -108,9 +108,9 @@
 	if (width && height)
 		window_size = "size=[width]x[height];"
 	if (stylesheets.len)
-		send_asset_list(user, stylesheets, verify=FALSE)
+		send_asset_list(user, stylesheets)
 	if (scripts.len)
-		send_asset_list(user, scripts, verify=FALSE)
+		send_asset_list(user, scripts)
 	user << browse(get_content(), "window=[window_id];[window_size][window_options]")
 	if (use_onclose)
 		setup_onclose()

code/modules/asset_cache/asset_cache.dm

@@ -1,113 +1,73 @@
 /*
 Asset cache quick users guide:
 
-Make a datum at the bottom of this file with your assets for your thing.
-The simple subsystem will most like be of use for most cases.
+Make a datum in asset_list_items.dm with your assets for your thing.
+Checkout asset_list.dm for the helper subclasses
+The simple subclass will most like be of use for most cases.
 Then call get_asset_datum() with the type of the datum you created and store the return
-Then call .send(client) on that stored return value.
+Then call .send(client) on that stored return value. 
 
-You can set verify to TRUE if you want send() to sleep until the client has the assets.
+Note: If your code uses output() with assets you will need to call asset_flush on the client and wait for it to return before calling output(). You only need do this if .send(client) returned TRUE
 */
 
-
-// Amount of time(ds) MAX to send per asset, if this get exceeded we cancel the sleeping.
-// This is doubled for the first asset, then added per asset after
-#define ASSET_CACHE_SEND_TIMEOUT 7
-
 //When sending mutiple assets, how many before we give the client a quaint little sending resources message
 #define ASSET_CACHE_TELL_CLIENT_AMOUNT 8
 
 //This proc sends the asset to the client, but only if it needs it.
 //This proc blocks(sleeps) unless verify is set to false
-/proc/send_asset(client/client, asset_name, verify = TRUE)
-	return send_asset_list(client, list(asset_name), verify)
+/proc/send_asset(client/client, asset_name)
+	return send_asset_list(client, list(asset_name))
 
-//This proc blocks(sleeps) unless verify is set to false
-/proc/send_asset_list(client/client, list/asset_list, verify = TRUE)
+/// Sends a list of assets to a client
+/// This proc will no longer block, use client.asset_flush() if you to need know when the client has all assets (such as for output()). (This is not required for browse() calls as they use the same message queue as asset sends)
+/// client - a client or mob
+/// asset_list - A list of asset filenames to be sent to the client.
+/// Returns TRUE if any assets were sent.
+/proc/send_asset_list(client/client, list/asset_list)
 	if(!istype(client))
 		if(ismob(client))
 			var/mob/M = client
 			if(M.client)
 				client = M.client
-
 			else
-				return FALSE
-
+				return
 		else
-			return FALSE
+			return
 	
 	var/list/unreceived = list()
-	var/list/sending = list()
 
 	for (var/asset_name in asset_list)
-		var/asset_file = SSassets.cache[asset_name]
+		var/datum/asset_cache_item/asset = SSassets.cache[asset_name]
+		if (!asset)
+			continue
+		var/asset_file = asset.resource
 		if (!asset_file)
 			continue
 		
-		var/asset_md5 = md5(asset_file) || md5(fcopy_rsc(asset_file))
-		
+		var/asset_md5 = asset.md5
 		if (client.sent_assets[asset_name] == asset_md5)
 			continue
-		if (client.sending_assets.Find(asset_name))
-			if (!verify)
-				continue
-			sending += asset_name
-
-		
 		unreceived[asset_name] = asset_md5
 
-	var/t = 0
-	var/timeout_time = DS2TICKS(ASSET_CACHE_SEND_TIMEOUT * client.sending_assets.len)
-
 	if (unreceived.len)
 		if (unreceived.len >= ASSET_CACHE_TELL_CLIENT_AMOUNT)
 			to_chat(client, "Sending Resources...")
-		var/job
-		if (verify)
-			job = ++client.last_asset_job
+
 		for(var/asset in unreceived)
-			if (SSassets.cache[asset])
+			var/datum/asset_cache_item/ACI
+			if ((ACI = SSassets.cache[asset]))
 				log_asset("Sending asset [asset] to client [client]")
-				client << browse_rsc(SSassets.cache[asset], asset)
-				if(verify)
-					client.sending_assets[unreceived] = job
+				client << browse_rsc(ACI.resource, asset)
 
-		if(!verify)
-			client.sent_assets |= unreceived
-			addtimer(CALLBACK(client, /client/proc/asset_cache_update_json), 1 SECONDS, TIMER_UNIQUE|TIMER_OVERRIDE)
-		else
-
-			client.sending_assets |= unreceived
-
-			client << browse({"<script>window.location.href="?asset_cache_confirm_arrival=[job]"</script>"}, "window=asset_cache_browser&file=asset_cache_send_verify.htm")
-
-			while(client && !client.completed_asset_jobs.Find(job) && t < timeout_time) // Reception is handled in Topic()
-				stoplag(1) // Lock up the caller until this is received.
-				t++
-
-			if(client)
-				client.sending_assets -= unreceived
-				client.sent_assets = unreceived | client.sent_assets //if we sent an updated version of an asset, we would want to replace the md5 in the client's list of sent assets
-				client.completed_asset_jobs -= job
-				addtimer(CALLBACK(client, /client/proc/asset_cache_update_json), 1 SECONDS, TIMER_UNIQUE|TIMER_OVERRIDE)
-
-		. = TRUE
-
-	else if (sending.len) //else if because these things are ordered enough to trust that assets sent later on would have arrived after ones that were already in the queue.
-		for (var/sending_asset in sending)
-			var/sending_asset_jobid = client?.sending_assets[sending_asset]
-			if (!sending_asset_jobid)
-				continue
-
-			while(client && client.last_completed_asset_job < sending_asset_jobid && t < timeout_time) // Reception is handled in Topic()
-				stoplag(1) // Lock up the caller until this is received.
-				t++
-
-		. = TRUE
+		client.sent_assets |= unreceived
+		addtimer(CALLBACK(client, /client/proc/asset_cache_update_json), 1 SECONDS, TIMER_UNIQUE|TIMER_OVERRIDE)
+		return TRUE
+	return FALSE
 
 //This proc will download the files without clogging up the browse() queue, used for passively sending files on connection start.
 //The proc calls procs that sleep for long times.
-/proc/getFilesSlow(client/client, list/files, register_asset = TRUE)
+/proc/getFilesSlow(client/client, list/files, register_asset = TRUE, filerate = 3)
+	var/startingfilerate = filerate
 	for(var/file in files)
 		if (!client)
 			break
@@ -115,12 +75,19 @@ You can set verify to TRUE if you want send() to sleep until the client has the
 			register_asset(file, files[file])
 
 		if (send_asset(client, file))
+			if (!(--filerate))
+				filerate = startingfilerate
+				client.asset_flush()
 			stoplag(0) //queuing calls like this too quickly can cause issues in some client versions
 
 //This proc "registers" an asset, it adds it to the cache for further use, you cannot touch it from this point on or you'll fuck things up.
-//if it's an icon or something be careful, you'll have to copy it before further use.
+//icons and virtual assets get copied to the dyn rsc before use
 /proc/register_asset(asset_name, asset)
-	SSassets.cache[asset_name] = asset
+	var/datum/asset_cache_item/ACI = new(asset_name, asset)
+	if (SSassets.cache[asset_name])
+		var/datum/asset_cache_item/OACI = SSassets.cache[asset_name]
+		stack_trace("WARNING: dupe asset added to the asset cache: [asset_name] existing asset md5: [OACI.md5] new asset md5:[ACI.md5]") //this is technically never something that was supported and i want metrics on how often it happens if at all.
+	SSassets.cache[asset_name] = ACI
 
 //Generated names do not include file extention.
 //Used mainly for code that deals with assets in a generic way

code/modules/asset_cache/asset_cache_client.dm

@@ -2,7 +2,7 @@
 /client
 	var/list/sent_assets = list() // List of all asset filenames sent to this client by the asset cache, along with their assoicated md5s
 	var/list/completed_asset_jobs = list() /// List of all completed blocking send jobs awaiting acknowledgement by send_asset
-	var/list/sending_assets = list() /// List of all assets currently being sent in blocking mode
+
 	var/last_asset_job = 0 /// Last asset send job id.
 	var/last_completed_asset_job = 0
 
@@ -10,8 +10,8 @@
 /client/proc/asset_cache_confirm_arrival(job_id)
 	var/asset_cache_job = round(text2num(job_id))
 		//because we skip the limiter, we have to make sure this is a valid arrival and not somebody tricking us into letting them append to a list without limit.
-	if (asset_cache_job > 0 && asset_cache_job <= last_asset_job && !(asset_cache_job in completed_asset_jobs))
-		completed_asset_jobs += asset_cache_job
+	if (asset_cache_job > 0 && asset_cache_job <= last_asset_job && !(completed_asset_jobs["[asset_cache_job]"]))
+		completed_asset_jobs["[asset_cache_job]"] = TRUE
 		last_completed_asset_job = max(last_completed_asset_job, asset_cache_job)
 	else
 		return asset_cache_job || TRUE
@@ -32,8 +32,8 @@
 			continue
 	sent_assets |= preloaded_assets
 
-//Updates the client side stored html/json combo file used to keep track of what assets the client has between restarts/reconnects.
 
+/// Updates the client side stored html/json combo file used to keep track of what assets the client has between restarts/reconnects.
 /client/proc/asset_cache_update_json(verify = FALSE, list/new_assets = list())
 	if (world.time - connection_time < 10 SECONDS) //don't override the existing data file on a new connection
 		return
@@ -41,3 +41,18 @@
 		new_assets = list("[new_assets]" = md5(SSassets.cache[new_assets]))
 
 	src << browse(json_encode(new_assets|sent_assets), "file=asset_data.json&display=0")
+
+/// Blocks until all currently sending browser assets have been sent.
+/// Due to byond limitations, this proc will sleep for 1 client round trip even if the client has no pending asset sends.
+/// This proc will return an untrue value if it had to return before confirming the send, such as timeout or the client going away.
+/client/proc/asset_flush(timeout = 50)
+	var/job = ++last_asset_job
+	var/t = 0
+	var/timeout_time = timeout
+	src << browse({"<script>window.location.href="?asset_cache_confirm_arrival=[job]"</script>"}, "window=asset_cache_browser&file=asset_cache_send_verify.htm")
+	
+	while(!completed_asset_jobs["[job]"] && t < timeout_time) // Reception is handled in Topic()
+		stoplag(1) // Lock up the caller until this is received.
+		t++
+	if (t < timeout_time)
+		return TRUE

code/modules/asset_cache/asset_cache_item.dm

@@ -0,0 +1,21 @@
+/**
+	* # asset_cache_item
+	* 
+	* An internal datum containing info on items in the asset cache. Mainly used to cache md5 info for speed.
+**/
+/datum/asset_cache_item
+	var/name
+	var/md5
+	var/resource
+
+/datum/asset_cache_item/New(name, file)
+	if (!isfile(file))
+		file = fcopy_rsc(file)
+	md5 = md5(file)
+	if (!md5)
+		md5 = md5(fcopy_rsc(file))
+		if (!md5)
+			CRASH("invalid asset sent to asset cache")
+		debug_world_log("asset cache unexpected success of second fcopy_rsc")
+	src.name = name
+	resource = file

code/modules/asset_cache/asset_list.dm

@@ -27,14 +27,13 @@ GLOBAL_LIST_EMPTY(asset_datums)
 /datum/asset/simple
 	_abstract = /datum/asset/simple
 	var/assets = list()
-	var/verify = FALSE
 
 /datum/asset/simple/register()
 	for(var/asset_name in assets)
 		register_asset(asset_name, assets[asset_name])
 
 /datum/asset/simple/send(client)
-	send_asset_list(client,assets,verify)
+	. = send_asset_list(client, assets)
 
 
 // For registering or sending multiple others at once
@@ -49,7 +48,7 @@ GLOBAL_LIST_EMPTY(asset_datums)
 /datum/asset/group/send(client/C)
 	for(var/type in children)
 		var/datum/asset/A = get_asset_datum(type)
-		A.send(C)
+		. = A.send(C) || .
 
 
 // spritesheet implementation - coalesces various icons into a single .png file
@@ -66,7 +65,6 @@ GLOBAL_LIST_EMPTY(asset_datums)
 	var/name
 	var/list/sizes = list()    // "32x32" -> list(10, icon/normal, icon/stripped)
 	var/list/sprites = list()  // "foo_bar" -> list("32x32", 5)
-	var/verify = FALSE
 
 /datum/asset/spritesheet/register()
 	if (!name)
@@ -90,7 +88,7 @@ GLOBAL_LIST_EMPTY(asset_datums)
 	var/all = list("spritesheet_[name].css")
 	for(var/size_id in sizes)
 		all += "[name]_[size_id].png"
-	send_asset_list(C, all, verify)
+	. = send_asset_list(C, all)
 
 /datum/asset/spritesheet/proc/ensure_stripped(sizes_to_strip = sizes)
 	for(var/size_id in sizes_to_strip)
@@ -207,8 +205,6 @@ GLOBAL_LIST_EMPTY(asset_datums)
 	var/prefix = "default" //asset_name = "[prefix].[icon_state_name].png"
 	var/generic_icon_names = FALSE //generate icon filenames using generate_asset_name() instead the above format
 
-	verify = FALSE
-
 /datum/asset/simple/icon_states/register(_icon = icon)
 	for(var/icon_state_name in icon_states(_icon))
 		for(var/direction in directions)

code/modules/asset_cache/asset_list_items.dm

@@ -140,13 +140,11 @@
 	)
 
 /datum/asset/simple/jquery
-	verify = FALSE
 	assets = list(
 		"jquery.min.js"            = 'code/modules/goonchat/browserassets/js/jquery.min.js',
 	)
 
 /datum/asset/simple/goonchat
-	verify = FALSE
 	assets = list(
 		"json2.min.js"             = 'code/modules/goonchat/browserassets/js/json2.min.js',
 		"browserOutput.js"         = 'code/modules/goonchat/browserassets/js/browserOutput.js',
@@ -155,7 +153,6 @@
 	)
 
 /datum/asset/simple/fontawesome
-	verify = FALSE
 	assets = list(
 		"fa-regular-400.eot"  = 'html/font-awesome/webfonts/fa-regular-400.eot',
 		"fa-regular-400.woff" = 'html/font-awesome/webfonts/fa-regular-400.woff',

tgstation.dme

@@ -1517,6 +1517,7 @@
 #include "code\modules\assembly\voice.dm"
 #include "code\modules\asset_cache\asset_cache.dm"
 #include "code\modules\asset_cache\asset_cache_client.dm"
+#include "code\modules\asset_cache\asset_cache_item.dm"
 #include "code\modules\asset_cache\asset_list.dm"
 #include "code\modules\asset_cache\asset_list_items.dm"
 #include "code\modules\atmospherics\multiz.dm"