From af8902331d84180aba360fcd6a87407a43e20579 Mon Sep 17 00:00:00 2001 From: Mothblocks <35135081+Mothblocks@users.noreply.github.com> Date: Fri, 31 Dec 2021 20:36:30 -0800 Subject: [PATCH] Amortize the cost of creating preference assets by caching them per git revision on production, reducing best case init times by ~20 seconds (#63503) Preference asset creation, which while consistently created in early assets, can be requested at any time before then and often is, currently takes about 15 to 25 seconds to produce. Because of extremely hard to reproduce BYOND icon bugs, most of this is done on the same tick. Lowering the cost of initialization itself is very tricky. Some of it we can theoretically optimize, such as creating humans for antagonists, others we can't, such as the raw cost of icon blending. Furthermore, adding new icons later down the line would just increase this initialization time even more. Instead of optimizing the asset creation, which is an uphill battle, this instead chooses to amortize the cost by caching preference assets created per git revision. This means that preference assets will be created, with their long delay, only once whenever the code changes. This is done on a config, defaulting to on so that production needs no changes, as the whole point of these being made at runtime at all is that it keeps assets/art styles consistent, and PRs making subtle bugs that break preference generation in some way is not uncommon. On development, your git revision will stay the same until you commit, no matter what code changes you make. --- .../configuration/entries/general.dm | 3 + code/controllers/subsystem/assets.dm | 2 +- code/modules/asset_cache/asset_list.dm | 114 ++++++++++++++++-- code/modules/asset_cache/asset_list_items.dm | 30 ++--- code/modules/client/preferences/assets.dm | 5 +- .../client/preferences/middleware/antags.dm | 5 +- .../client/preferences/middleware/species.dm | 5 +- config/config.txt | 7 ++ 8 files changed, 134 insertions(+), 37 deletions(-) diff --git a/code/controllers/configuration/entries/general.dm b/code/controllers/configuration/entries/general.dm index fabf4aa5cce..f805312d2b4 100644 --- a/code/controllers/configuration/entries/general.dm +++ b/code/controllers/configuration/entries/general.dm @@ -558,3 +558,6 @@ /datum/config_entry/string/adminhelp_webhook_pfp /datum/config_entry/string/adminhelp_webhook_name + +/datum/config_entry/flag/cache_assets + default = TRUE diff --git a/code/controllers/subsystem/assets.dm b/code/controllers/subsystem/assets.dm index ef79e55dbe5..56314c74963 100644 --- a/code/controllers/subsystem/assets.dm +++ b/code/controllers/subsystem/assets.dm @@ -2,7 +2,7 @@ SUBSYSTEM_DEF(assets) name = "Assets" init_order = INIT_ORDER_ASSETS flags = SS_NO_FIRE - var/list/cache = list() + var/list/datum/asset_cache_item/cache = list() var/list/preload = list() var/datum/asset_transport/transport = new() diff --git a/code/modules/asset_cache/asset_list.dm b/code/modules/asset_cache/asset_list.dm index de2bf531544..372207fe1b5 100644 --- a/code/modules/asset_cache/asset_list.dm +++ b/code/modules/asset_cache/asset_list.dm @@ -1,3 +1,4 @@ +#define ASSET_CROSS_ROUND_CACHE_DIRECTORY "tmp/assets" //These datums are used to populate the asset cache, the proc "register()" does this. //Place any asset datums you create in asset_list_items.dm @@ -16,6 +17,11 @@ GLOBAL_LIST_EMPTY(asset_datums) /// Whether or not this asset should be loaded in the "early assets" SS var/early = FALSE + /// Whether or not this asset can be cached across rounds of the same commit under the `CACHE_ASSETS` config. + /// This is not a *guarantee* the asset will be cached. Not all asset subtypes respect this field, and the + /// config can, of course, be disabled. + var/cross_round_cachable = FALSE + /datum/asset/New() GLOB.asset_datums[type] = src register() @@ -36,6 +42,9 @@ GLOBAL_LIST_EMPTY(asset_datums) /datum/asset/proc/send(client) return +/// Returns whether or not the asset should attempt to read from cache +/datum/asset/proc/should_refresh() + return !cross_round_cachable || !CONFIG_GET(flag/cache_assets) /// If you don't need anything complicated. /datum/asset/simple @@ -105,10 +114,33 @@ GLOBAL_LIST_EMPTY(asset_datums) var/name var/list/sizes = list() // "32x32" -> list(10, icon/normal, icon/stripped) var/list/sprites = list() // "foo_bar" -> list("32x32", 5) + var/list/cached_spritesheets_needed + var/generating_cache = FALSE + +/datum/asset/spritesheet/should_refresh() + if (..()) + return TRUE + + // Static so that the result is the same, even when the files are created, for this run + var/static/should_refresh = null + + if (isnull(should_refresh)) + // `fexists` seems to always fail on static-time + should_refresh = !fexists("[ASSET_CROSS_ROUND_CACHE_DIRECTORY]/spritesheet.[name].css") + + return should_refresh /datum/asset/spritesheet/register() + SHOULD_NOT_OVERRIDE(TRUE) + if (!name) CRASH("spritesheet [type] cannot register without a name") + + if (!should_refresh() && read_from_cache()) + return + + create_spritesheets() + ensure_stripped() for(var/size_id in sizes) var/size = sizes[size_id] @@ -120,23 +152,32 @@ GLOBAL_LIST_EMPTY(asset_datums) SSassets.transport.register_asset(res_name, fcopy_rsc(fname)) fdel(fname) -/datum/asset/spritesheet/send(client/C) + if (CONFIG_GET(flag/cache_assets) && cross_round_cachable) + write_to_cache() + +/datum/asset/spritesheet/send(client/client) if (!name) return + + if (!should_refresh()) + return send_from_cache(client) + var/all = list("spritesheet_[name].css") for(var/size_id in sizes) all += "[name]_[size_id].png" - . = SSassets.transport.send_assets(C, all) + . = SSassets.transport.send_assets(client, all) /datum/asset/spritesheet/get_url_mappings() if (!name) return + + if (!should_refresh()) + return get_cached_url_mappings() + . = list("spritesheet_[name].css" = SSassets.transport.get_asset_url("spritesheet_[name].css")) for(var/size_id in sizes) .["[name]_[size_id].png"] = SSassets.transport.get_asset_url("[name]_[size_id].png") - - /datum/asset/spritesheet/proc/ensure_stripped(sizes_to_strip = sizes) for(var/size_id in sizes_to_strip) var/size = sizes[size_id] @@ -158,7 +199,7 @@ GLOBAL_LIST_EMPTY(asset_datums) for (var/size_id in sizes) var/size = sizes[size_id] var/icon/tiny = size[SPRSZ_ICON] - out += ".[name][size_id]{display:inline-block;width:[tiny.Width()]px;height:[tiny.Height()]px;background:url('[SSassets.transport.get_asset_url("[name]_[size_id].png")]') no-repeat;}" + out += ".[name][size_id]{display:inline-block;width:[tiny.Width()]px;height:[tiny.Height()]px;background:url('[get_background_url("[name]_[size_id].png")]') no-repeat;}" for (var/sprite_id in sprites) var/sprite = sprites[sprite_id] @@ -176,6 +217,64 @@ GLOBAL_LIST_EMPTY(asset_datums) return out.Join("\n") +/datum/asset/spritesheet/proc/read_from_cache() + var/replaced_css = file2text("[ASSET_CROSS_ROUND_CACHE_DIRECTORY]/spritesheet.[name].css") + + var/regex/find_background_urls = regex(@"background:url\('%(.+?)%'\)", "g") + while (find_background_urls.Find(replaced_css)) + var/asset_id = find_background_urls.group[1] + var/asset_cache_item = SSassets.transport.register_asset(asset_id, "[ASSET_CROSS_ROUND_CACHE_DIRECTORY]/spritesheet.[asset_id]") + var/asset_url = SSassets.transport.get_asset_url(asset_cache_item = asset_cache_item) + replaced_css = replacetext(replaced_css, find_background_urls.match, "background:url('[asset_url]')") + LAZYADD(cached_spritesheets_needed, asset_id) + + var/replaced_css_filename = "data/spritesheets/spritesheet_[name].css" + rustg_file_write(replaced_css, replaced_css_filename) + SSassets.transport.register_asset("spritesheet_[name].css", replaced_css_filename) + + fdel(replaced_css_filename) + + return TRUE + +/datum/asset/spritesheet/proc/send_from_cache(client/client) + if (isnull(cached_spritesheets_needed)) + stack_trace("cached_spritesheets_needed was null when sending assets from [type] from cache") + cached_spritesheets_needed = list() + + return SSassets.transport.send_assets(client, cached_spritesheets_needed + "spritesheet_[name].css") + +/// Returns the URL to put in the background:url of the CSS asset +/datum/asset/spritesheet/proc/get_background_url(asset) + if (generating_cache) + return "%[asset]%" + else + return SSassets.transport.get_asset_url(asset) + +/datum/asset/spritesheet/proc/write_to_cache() + for (var/size_id in sizes) + fcopy(SSassets.cache["[name]_[size_id].png"].resource, "[ASSET_CROSS_ROUND_CACHE_DIRECTORY]/spritesheet.[name]_[size_id].png") + + generating_cache = TRUE + var/mock_css = generate_css() + generating_cache = FALSE + + rustg_file_write(mock_css, "[ASSET_CROSS_ROUND_CACHE_DIRECTORY]/spritesheet.[name].css") + +/datum/asset/spritesheet/proc/get_cached_url_mappings() + var/list/mappings = list() + mappings["spritesheet_[name].css"] = SSassets.transport.get_asset_url("spritesheet_[name].css") + + for (var/asset_name in cached_spritesheets_needed) + mappings[asset_name] = SSassets.transport.get_asset_url(asset_name) + + return mappings + +/// Override this in order to start the creation of the spritehseet. +/// This is where all your Insert, InsertAll, etc calls should be inside. +/datum/asset/spritesheet/proc/create_spritesheets() + SHOULD_CALL_PARENT(FALSE) + CRASH("create_spritesheets() not implemented for [type]!") + /datum/asset/spritesheet/proc/Insert(sprite_name, icon/I, icon_state="", dir=SOUTH, frame=1, moving=FALSE) I = icon(I, icon_state=icon_state, dir=dir, frame=frame, moving=moving) if (!I || !length(icon_states(I))) // that direction or state doesn't exist @@ -281,10 +380,9 @@ GLOBAL_LIST_EMPTY(asset_datums) _abstract = /datum/asset/spritesheet/simple var/list/assets -/datum/asset/spritesheet/simple/register() +/datum/asset/spritesheet/simple/create_spritesheets() for (var/key in assets) Insert(key, assets[key]) - ..() //Generates assets based on iconstates of a single icon /datum/asset/simple/icon_states @@ -393,3 +491,5 @@ GLOBAL_LIST_EMPTY(asset_datums) /datum/asset/json/proc/generate() SHOULD_CALL_PARENT(FALSE) CRASH("generate() not implemented for [type]!") + +#undef ASSET_CROSS_ROUND_CACHE_DIRECTORY diff --git a/code/modules/asset_cache/asset_list_items.dm b/code/modules/asset_cache/asset_list_items.dm index 4b1543a3fdd..eac432b21da 100644 --- a/code/modules/asset_cache/asset_list_items.dm +++ b/code/modules/asset_cache/asset_list_items.dm @@ -153,7 +153,7 @@ /datum/asset/spritesheet/chat name = "chat" -/datum/asset/spritesheet/chat/register() +/datum/asset/spritesheet/chat/create_spritesheets() InsertAll("emoji", EMOJI_SET) // pre-loading all lanugage icons also helps to avoid meta InsertAll("language", 'icons/misc/language.dmi') @@ -164,7 +164,6 @@ if (icon != 'icons/misc/language.dmi') var/icon_state = initial(L.icon_state) Insert("language-[icon_state]", icon, icon_state=icon_state) - ..() /datum/asset/simple/lobby assets = list( @@ -208,9 +207,8 @@ /datum/asset/spritesheet/simple/achievements name ="achievements" -/datum/asset/spritesheet/simple/achievements/register() +/datum/asset/spritesheet/simple/achievements/create_spritesheets() InsertAll("", ACHIEVEMENTS_SET) - return ..() /datum/asset/spritesheet/simple/pills name = "pills" @@ -270,15 +268,14 @@ /datum/asset/spritesheet/pipes name = "pipes" -/datum/asset/spritesheet/pipes/register() +/datum/asset/spritesheet/pipes/create_spritesheets() for (var/each in list('icons/obj/atmospherics/pipes/pipe_item.dmi', 'icons/obj/atmospherics/pipes/disposal.dmi', 'icons/obj/atmospherics/pipes/transit_tube.dmi', 'icons/obj/plumbing/fluid_ducts.dmi')) InsertAll("", each, GLOB.alldirs) - ..() /datum/asset/spritesheet/supplypods name = "supplypods" -/datum/asset/spritesheet/supplypods/register() +/datum/asset/spritesheet/supplypods/create_spritesheets() for (var/style in 1 to length(GLOB.podstyles)) if (style == STYLE_SEETHROUGH) Insert("pod_asset[style]", icon('icons/obj/supplypods.dmi' , "seethrough-icon")) @@ -302,13 +299,12 @@ glow = "pod_glow_[glow]" podIcon.Blend(icon('icons/obj/supplypods.dmi', glow), ICON_OVERLAY) Insert("pod_asset[style]", podIcon) - return ..() // Representative icons for each research design /datum/asset/spritesheet/research_designs name = "design" -/datum/asset/spritesheet/research_designs/register() +/datum/asset/spritesheet/research_designs/create_spritesheets() for (var/path in subtypesof(/datum/design)) var/datum/design/D = path @@ -367,12 +363,11 @@ I.Blend(icon(icon_file, keyboard, SOUTH), ICON_OVERLAY) Insert(initial(D.id), I) - return ..() /datum/asset/spritesheet/vending name = "vending" -/datum/asset/spritesheet/vending/register() +/datum/asset/spritesheet/vending/create_spritesheets() for (var/k in GLOB.vending_products) var/atom/item = k if (!ispath(item, /atom)) @@ -405,7 +400,6 @@ var/imgid = replacetext(replacetext("[item]", "/obj/item/", ""), "/", "-") Insert(imgid, I) - return ..() /datum/asset/simple/genetics assets = list( @@ -427,19 +421,17 @@ /datum/asset/spritesheet/sheetmaterials name = "sheetmaterials" -/datum/asset/spritesheet/sheetmaterials/register() +/datum/asset/spritesheet/sheetmaterials/create_spritesheets() InsertAll("", 'icons/obj/stack_objects.dmi') // Special case to handle Bluespace Crystals Insert("polycrystal", 'icons/obj/telescience.dmi', "polycrystal") - ..() /datum/asset/spritesheet/mafia name = "mafia" -/datum/asset/spritesheet/mafia/register() +/datum/asset/spritesheet/mafia/create_spritesheets() InsertAll("", 'icons/obj/mafia.dmi') - ..() /datum/asset/simple/portraits assets = list() @@ -470,7 +462,7 @@ /datum/asset/spritesheet/fish name = "fish" -/datum/asset/spritesheet/fish/register() +/datum/asset/spritesheet/fish/create_spritesheets() for (var/path in subtypesof(/datum/aquarium_behaviour/fish)) var/datum/aquarium_behaviour/fish/fish_type = path var/fish_icon = initial(fish_type.icon) @@ -479,7 +471,6 @@ if(sprites[id]) //no dupes continue Insert(id, fish_icon, fish_icon_state) - ..() /datum/asset/simple/adventure assets = list( @@ -524,12 +515,11 @@ name = "moods" var/iconinserted = 1 -/datum/asset/spritesheet/moods/register() +/datum/asset/spritesheet/moods/create_spritesheets() for(var/i in 1 to 9) var/target_to_insert = "mood"+"[iconinserted]" Insert(target_to_insert, 'icons/hud/screen_gen.dmi', target_to_insert) iconinserted++ - ..() /datum/asset/spritesheet/moods/ModifyInserted(icon/pre_asset) var/blended_color diff --git a/code/modules/client/preferences/assets.dm b/code/modules/client/preferences/assets.dm index 88d4313cae7..de29b6e4fbd 100644 --- a/code/modules/client/preferences/assets.dm +++ b/code/modules/client/preferences/assets.dm @@ -2,8 +2,9 @@ /datum/asset/spritesheet/preferences name = "preferences" early = TRUE + cross_round_cachable = TRUE -/datum/asset/spritesheet/preferences/register() +/datum/asset/spritesheet/preferences/create_spritesheets() var/list/to_insert = list() for (var/preference_key in GLOB.preference_entries_by_key) @@ -36,8 +37,6 @@ var/list/inserting = to_insert[spritesheet_key] Insert(spritesheet_key, inserting[1], inserting[2]) - return ..() - /// Returns the key that will be used in the spritesheet for a given value. /datum/preference/proc/get_spritesheet_key(value) return "[savefile_key]___[sanitize_css_class_name(value)]" diff --git a/code/modules/client/preferences/middleware/antags.dm b/code/modules/client/preferences/middleware/antags.dm index 37a0f7eb471..b782397fd5b 100644 --- a/code/modules/client/preferences/middleware/antags.dm +++ b/code/modules/client/preferences/middleware/antags.dm @@ -108,8 +108,9 @@ /datum/asset/spritesheet/antagonists name = "antagonists" early = TRUE + cross_round_cachable = TRUE -/datum/asset/spritesheet/antagonists/register() +/datum/asset/spritesheet/antagonists/create_spritesheets() // Antagonists that don't have a dynamic ruleset, but do have a preference var/static/list/non_ruleset_antagonists = list( ROLE_FUGITIVE = /datum/antagonist/fugitive, @@ -156,8 +157,6 @@ for (var/spritesheet_key in to_insert) Insert(spritesheet_key, to_insert[spritesheet_key]) - return ..() - /// Serializes an antag name to be used for preferences UI /proc/serialize_antag_name(antag_name) // These are sent through CSS, so they need to be safe to use as class names. diff --git a/code/modules/client/preferences/middleware/species.dm b/code/modules/client/preferences/middleware/species.dm index c6017d9f393..02efe1e223a 100644 --- a/code/modules/client/preferences/middleware/species.dm +++ b/code/modules/client/preferences/middleware/species.dm @@ -9,8 +9,9 @@ /datum/asset/spritesheet/species name = "species" early = TRUE + cross_round_cachable = TRUE -/datum/asset/spritesheet/species/register() +/datum/asset/spritesheet/species/create_spritesheets() var/list/to_insert = list() for (var/species_id in get_selectable_species()) @@ -32,5 +33,3 @@ for (var/spritesheet_key in to_insert) Insert(spritesheet_key, to_insert[spritesheet_key]) - - return ..() diff --git a/config/config.txt b/config/config.txt index ea51ed77042..3e36f266db8 100644 --- a/config/config.txt +++ b/config/config.txt @@ -583,3 +583,10 @@ URGENT_AHELP_COOLDOWN 300 ## Based on config directory, so "motd.txt" points to "config/motd.txt" MOTD motd.txt #MOTD motd_extra.txt + +## Assets can opt-in to caching their results into `tmp`. +## This is important, as preferences assets take upwards of 30 seconds (without sleeps) to collect. +## The cache is assumed to be cleared by TGS recompiling, which deletes `tmp`. +## This should be disabled (through `CACHE_ASSETS 0`) on development, +## but enabled on production (the default). +CACHE_ASSETS 0