From c0881fb222f53acb218ce4c4699b4ada49ab43dd Mon Sep 17 00:00:00 2001 From: Artur Date: Tue, 19 May 2020 18:43:41 +0300 Subject: [PATCH] Ports https://github.com/tgstation/tgstation/pull/50759 --- code/__HELPERS/icons.dm | 4 +- code/datums/browser.dm | 4 +- code/modules/asset_cache/asset_cache.dm | 111 ++++++------------ .../modules/asset_cache/asset_cache_client.dm | 23 +++- code/modules/asset_cache/asset_cache_item.dm | 21 ++++ code/modules/asset_cache/asset_list.dm | 10 +- code/modules/asset_cache/asset_list_items.dm | 3 - tgstation.dme | 1 + 8 files changed, 87 insertions(+), 90 deletions(-) create mode 100644 code/modules/asset_cache/asset_cache_item.dm diff --git a/code/__HELPERS/icons.dm b/code/__HELPERS/icons.dm index aee0f3d7f7..62876316f9 100644 --- a/code/__HELPERS/icons.dm +++ b/code/__HELPERS/icons.dm @@ -1138,7 +1138,7 @@ GLOBAL_LIST_INIT(freon_color_matrix, list("#2E5E69", "#60A2A8", "#A1AFB1", rgb(0 var/name = sanitize_filename("[generate_asset_name(thing)].png") register_asset(name, thing) for (var/thing2 in targets) - send_asset(thing2, key, FALSE) + send_asset(thing2, key) return "" var/atom/A = thing if (isnull(dir)) @@ -1162,7 +1162,7 @@ GLOBAL_LIST_INIT(freon_color_matrix, list("#2E5E69", "#60A2A8", "#A1AFB1", rgb(0 key = "[generate_asset_name(I)].png" register_asset(key, I) for (var/thing2 in targets) - send_asset(thing2, key, FALSE) + send_asset(thing2, key) return "" diff --git a/code/datums/browser.dm b/code/datums/browser.dm index 5b1df6cb68..97563b9e1a 100644 --- a/code/datums/browser.dm +++ b/code/datums/browser.dm @@ -106,9 +106,9 @@ if (width && height) window_size = "size=[width]x[height];" if (stylesheets.len) - send_asset_list(user, stylesheets, verify=FALSE) + send_asset_list(user, stylesheets) if (scripts.len) - send_asset_list(user, scripts, verify=FALSE) + send_asset_list(user, scripts) user << browse(get_content(), "window=[window_id];[window_size][window_options]") if (use_onclose) setup_onclose() diff --git a/code/modules/asset_cache/asset_cache.dm b/code/modules/asset_cache/asset_cache.dm index 3f6b342a5d..2f1d3cfb90 100644 --- a/code/modules/asset_cache/asset_cache.dm +++ b/code/modules/asset_cache/asset_cache.dm @@ -1,113 +1,73 @@ /* Asset cache quick users guide: -Make a datum at the bottom of this file with your assets for your thing. -The simple subsystem will most like be of use for most cases. +Make a datum in asset_list_items.dm with your assets for your thing. +Checkout asset_list.dm for the helper subclasses +The simple subclass will most like be of use for most cases. Then call get_asset_datum() with the type of the datum you created and store the return -Then call .send(client) on that stored return value. +Then call .send(client) on that stored return value. -You can set verify to TRUE if you want send() to sleep until the client has the assets. +Note: If your code uses output() with assets you will need to call asset_flush on the client and wait for it to return before calling output(). You only need do this if .send(client) returned TRUE */ - -// Amount of time(ds) MAX to send per asset, if this get exceeded we cancel the sleeping. -// This is doubled for the first asset, then added per asset after -#define ASSET_CACHE_SEND_TIMEOUT 7 - //When sending mutiple assets, how many before we give the client a quaint little sending resources message #define ASSET_CACHE_TELL_CLIENT_AMOUNT 8 //This proc sends the asset to the client, but only if it needs it. //This proc blocks(sleeps) unless verify is set to false -/proc/send_asset(client/client, asset_name, verify = TRUE) - return send_asset_list(client, list(asset_name), verify) +/proc/send_asset(client/client, asset_name) + return send_asset_list(client, list(asset_name)) -//This proc blocks(sleeps) unless verify is set to false -/proc/send_asset_list(client/client, list/asset_list, verify = TRUE) +/// Sends a list of assets to a client +/// This proc will no longer block, use client.asset_flush() if you to need know when the client has all assets (such as for output()). (This is not required for browse() calls as they use the same message queue as asset sends) +/// client - a client or mob +/// asset_list - A list of asset filenames to be sent to the client. +/// Returns TRUE if any assets were sent. +/proc/send_asset_list(client/client, list/asset_list) if(!istype(client)) if(ismob(client)) var/mob/M = client if(M.client) client = M.client - else - return FALSE - + return else - return FALSE + return var/list/unreceived = list() - var/list/sending = list() for (var/asset_name in asset_list) - var/asset_file = SSassets.cache[asset_name] + var/datum/asset_cache_item/asset = SSassets.cache[asset_name] + if (!asset) + continue + var/asset_file = asset.resource if (!asset_file) continue - var/asset_md5 = md5(asset_file) || md5(fcopy_rsc(asset_file)) - + var/asset_md5 = asset.md5 if (client.sent_assets[asset_name] == asset_md5) continue - if (client.sending_assets.Find(asset_name)) - if (!verify) - continue - sending += asset_name - - unreceived[asset_name] = asset_md5 - var/t = 0 - var/timeout_time = DS2TICKS(ASSET_CACHE_SEND_TIMEOUT * client.sending_assets.len) - if (unreceived.len) if (unreceived.len >= ASSET_CACHE_TELL_CLIENT_AMOUNT) to_chat(client, "Sending Resources...") - var/job - if (verify) - job = ++client.last_asset_job + for(var/asset in unreceived) - if (SSassets.cache[asset]) + var/datum/asset_cache_item/ACI + if ((ACI = SSassets.cache[asset])) log_asset("Sending asset [asset] to client [client]") - client << browse_rsc(SSassets.cache[asset], asset) - if(verify) - client.sending_assets[unreceived] = job + client << browse_rsc(ACI.resource, asset) - if(!verify) - client.sent_assets |= unreceived - addtimer(CALLBACK(client, /client/proc/asset_cache_update_json), 1 SECONDS, TIMER_UNIQUE|TIMER_OVERRIDE) - else - - client.sending_assets |= unreceived - - client << browse({""}, "window=asset_cache_browser&file=asset_cache_send_verify.htm") - - while(client && !client.completed_asset_jobs.Find(job) && t < timeout_time) // Reception is handled in Topic() - stoplag(1) // Lock up the caller until this is received. - t++ - - if(client) - client.sending_assets -= unreceived - client.sent_assets = unreceived | client.sent_assets //if we sent an updated version of an asset, we would want to replace the md5 in the client's list of sent assets - client.completed_asset_jobs -= job - addtimer(CALLBACK(client, /client/proc/asset_cache_update_json), 1 SECONDS, TIMER_UNIQUE|TIMER_OVERRIDE) - - . = TRUE - - else if (sending.len) //else if because these things are ordered enough to trust that assets sent later on would have arrived after ones that were already in the queue. - for (var/sending_asset in sending) - var/sending_asset_jobid = client?.sending_assets[sending_asset] - if (!sending_asset_jobid) - continue - - while(client && client.last_completed_asset_job < sending_asset_jobid && t < timeout_time) // Reception is handled in Topic() - stoplag(1) // Lock up the caller until this is received. - t++ - - . = TRUE + client.sent_assets |= unreceived + addtimer(CALLBACK(client, /client/proc/asset_cache_update_json), 1 SECONDS, TIMER_UNIQUE|TIMER_OVERRIDE) + return TRUE + return FALSE //This proc will download the files without clogging up the browse() queue, used for passively sending files on connection start. //The proc calls procs that sleep for long times. -/proc/getFilesSlow(client/client, list/files, register_asset = TRUE) +/proc/getFilesSlow(client/client, list/files, register_asset = TRUE, filerate = 3) + var/startingfilerate = filerate for(var/file in files) if (!client) break @@ -115,12 +75,19 @@ You can set verify to TRUE if you want send() to sleep until the client has the register_asset(file, files[file]) if (send_asset(client, file)) + if (!(--filerate)) + filerate = startingfilerate + client.asset_flush() stoplag(0) //queuing calls like this too quickly can cause issues in some client versions //This proc "registers" an asset, it adds it to the cache for further use, you cannot touch it from this point on or you'll fuck things up. -//if it's an icon or something be careful, you'll have to copy it before further use. +//icons and virtual assets get copied to the dyn rsc before use /proc/register_asset(asset_name, asset) - SSassets.cache[asset_name] = asset + var/datum/asset_cache_item/ACI = new(asset_name, asset) + if (SSassets.cache[asset_name]) + var/datum/asset_cache_item/OACI = SSassets.cache[asset_name] + stack_trace("WARNING: dupe asset added to the asset cache: [asset_name] existing asset md5: [OACI.md5] new asset md5:[ACI.md5]") //this is technically never something that was supported and i want metrics on how often it happens if at all. + SSassets.cache[asset_name] = ACI //Generated names do not include file extention. //Used mainly for code that deals with assets in a generic way diff --git a/code/modules/asset_cache/asset_cache_client.dm b/code/modules/asset_cache/asset_cache_client.dm index 150e062b46..50a5685ded 100644 --- a/code/modules/asset_cache/asset_cache_client.dm +++ b/code/modules/asset_cache/asset_cache_client.dm @@ -2,7 +2,7 @@ /client var/list/sent_assets = list() // List of all asset filenames sent to this client by the asset cache, along with their assoicated md5s var/list/completed_asset_jobs = list() /// List of all completed blocking send jobs awaiting acknowledgement by send_asset - var/list/sending_assets = list() /// List of all assets currently being sent in blocking mode + var/last_asset_job = 0 /// Last asset send job id. var/last_completed_asset_job = 0 @@ -10,8 +10,8 @@ /client/proc/asset_cache_confirm_arrival(job_id) var/asset_cache_job = round(text2num(job_id)) //because we skip the limiter, we have to make sure this is a valid arrival and not somebody tricking us into letting them append to a list without limit. - if (asset_cache_job > 0 && asset_cache_job <= last_asset_job && !(asset_cache_job in completed_asset_jobs)) - completed_asset_jobs += asset_cache_job + if (asset_cache_job > 0 && asset_cache_job <= last_asset_job && !(completed_asset_jobs["[asset_cache_job]"])) + completed_asset_jobs["[asset_cache_job]"] = TRUE last_completed_asset_job = max(last_completed_asset_job, asset_cache_job) else return asset_cache_job || TRUE @@ -32,8 +32,8 @@ continue sent_assets |= preloaded_assets -//Updates the client side stored html/json combo file used to keep track of what assets the client has between restarts/reconnects. +/// Updates the client side stored html/json combo file used to keep track of what assets the client has between restarts/reconnects. /client/proc/asset_cache_update_json(verify = FALSE, list/new_assets = list()) if (world.time - connection_time < 10 SECONDS) //don't override the existing data file on a new connection return @@ -41,3 +41,18 @@ new_assets = list("[new_assets]" = md5(SSassets.cache[new_assets])) src << browse(json_encode(new_assets|sent_assets), "file=asset_data.json&display=0") + +/// Blocks until all currently sending browser assets have been sent. +/// Due to byond limitations, this proc will sleep for 1 client round trip even if the client has no pending asset sends. +/// This proc will return an untrue value if it had to return before confirming the send, such as timeout or the client going away. +/client/proc/asset_flush(timeout = 50) + var/job = ++last_asset_job + var/t = 0 + var/timeout_time = timeout + src << browse({""}, "window=asset_cache_browser&file=asset_cache_send_verify.htm") + + while(!completed_asset_jobs["[job]"] && t < timeout_time) // Reception is handled in Topic() + stoplag(1) // Lock up the caller until this is received. + t++ + if (t < timeout_time) + return TRUE diff --git a/code/modules/asset_cache/asset_cache_item.dm b/code/modules/asset_cache/asset_cache_item.dm new file mode 100644 index 0000000000..c705a17333 --- /dev/null +++ b/code/modules/asset_cache/asset_cache_item.dm @@ -0,0 +1,21 @@ +/** + * # asset_cache_item + * + * An internal datum containing info on items in the asset cache. Mainly used to cache md5 info for speed. +**/ +/datum/asset_cache_item + var/name + var/md5 + var/resource + +/datum/asset_cache_item/New(name, file) + if (!isfile(file)) + file = fcopy_rsc(file) + md5 = md5(file) + if (!md5) + md5 = md5(fcopy_rsc(file)) + if (!md5) + CRASH("invalid asset sent to asset cache") + debug_world_log("asset cache unexpected success of second fcopy_rsc") + src.name = name + resource = file diff --git a/code/modules/asset_cache/asset_list.dm b/code/modules/asset_cache/asset_list.dm index 717a94c3b0..fa8b72a425 100644 --- a/code/modules/asset_cache/asset_list.dm +++ b/code/modules/asset_cache/asset_list.dm @@ -27,14 +27,13 @@ GLOBAL_LIST_EMPTY(asset_datums) /datum/asset/simple _abstract = /datum/asset/simple var/assets = list() - var/verify = FALSE /datum/asset/simple/register() for(var/asset_name in assets) register_asset(asset_name, assets[asset_name]) /datum/asset/simple/send(client) - send_asset_list(client,assets,verify) + . = send_asset_list(client, assets) // For registering or sending multiple others at once @@ -49,7 +48,7 @@ GLOBAL_LIST_EMPTY(asset_datums) /datum/asset/group/send(client/C) for(var/type in children) var/datum/asset/A = get_asset_datum(type) - A.send(C) + . = A.send(C) || . // spritesheet implementation - coalesces various icons into a single .png file @@ -66,7 +65,6 @@ GLOBAL_LIST_EMPTY(asset_datums) var/name var/list/sizes = list() // "32x32" -> list(10, icon/normal, icon/stripped) var/list/sprites = list() // "foo_bar" -> list("32x32", 5) - var/verify = FALSE /datum/asset/spritesheet/register() if (!name) @@ -90,7 +88,7 @@ GLOBAL_LIST_EMPTY(asset_datums) var/all = list("spritesheet_[name].css") for(var/size_id in sizes) all += "[name]_[size_id].png" - send_asset_list(C, all, verify) + . = send_asset_list(C, all) /datum/asset/spritesheet/proc/ensure_stripped(sizes_to_strip = sizes) for(var/size_id in sizes_to_strip) @@ -207,8 +205,6 @@ GLOBAL_LIST_EMPTY(asset_datums) var/prefix = "default" //asset_name = "[prefix].[icon_state_name].png" var/generic_icon_names = FALSE //generate icon filenames using generate_asset_name() instead the above format - verify = FALSE - /datum/asset/simple/icon_states/register(_icon = icon) for(var/icon_state_name in icon_states(_icon)) for(var/direction in directions) diff --git a/code/modules/asset_cache/asset_list_items.dm b/code/modules/asset_cache/asset_list_items.dm index 856f28117e..10d7113139 100644 --- a/code/modules/asset_cache/asset_list_items.dm +++ b/code/modules/asset_cache/asset_list_items.dm @@ -136,13 +136,11 @@ ) /datum/asset/simple/jquery - verify = FALSE assets = list( "jquery.min.js" = 'code/modules/goonchat/browserassets/js/jquery.min.js', ) /datum/asset/simple/goonchat - verify = FALSE assets = list( "json2.min.js" = 'code/modules/goonchat/browserassets/js/json2.min.js', "browserOutput.js" = 'code/modules/goonchat/browserassets/js/browserOutput.js', @@ -152,7 +150,6 @@ ) /datum/asset/simple/fontawesome - verify = FALSE assets = list( "fa-regular-400.eot" = 'html/font-awesome/webfonts/fa-regular-400.eot', "fa-regular-400.woff" = 'html/font-awesome/webfonts/fa-regular-400.woff', diff --git a/tgstation.dme b/tgstation.dme index 96f0c9bcb0..703f62cd40 100755 --- a/tgstation.dme +++ b/tgstation.dme @@ -1597,6 +1597,7 @@ #include "code\modules\assembly\voice.dm" #include "code\modules\asset_cache\asset_cache.dm" #include "code\modules\asset_cache\asset_cache_client.dm" +#include "code\modules\asset_cache\asset_cache_item.dm" #include "code\modules\asset_cache\asset_list.dm" #include "code\modules\asset_cache\asset_list_items.dm" #include "code\modules\atmospherics\multiz.dm"