diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2e60daf63b..f8a76501b6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -113,7 +113,7 @@ jobs: run: | sudo dpkg --add-architecture i386 sudo apt update || true - sudo apt install zlib1g-dev:i386 libssl-dev:i386 + sudo apt install zlib1g-dev:i386 ldd librust_g.so ldd libbapi_dmm_reader.so ldd libverdigris.so diff --git a/code/__defines/assets.dm b/code/__defines/assets.dm new file mode 100644 index 0000000000..bec54e1e54 --- /dev/null +++ b/code/__defines/assets.dm @@ -0,0 +1,16 @@ +#define ASSET_CROSS_ROUND_CACHE_DIRECTORY "data/spritesheets/legacy_cache" +#define ASSET_CROSS_ROUND_SMART_CACHE_DIRECTORY "data/spritesheets/smart_cache" + +/// When sending mutiple assets, how many before we give the client a quaint little sending resources message +#define ASSET_CACHE_TELL_CLIENT_AMOUNT 8 + +/// How many assets can be sent at once during legacy asset transport +#define SLOW_ASSET_SEND_RATE 6 + +/// Constructs a universal icon. This is done in the same manner as the icon() BYOND proc. +/// "color" will not do anything if a transform is provided. Blend it yourself or use color_transform(). +/// Do note that transforms are NOT COPIED, and are internally lists. So take care not to re-use transforms. +/// This is a DEFINE for performance reasons. +/// Parameters (in order): +/// icon_file, icon_state, dir, frame, transform, color +#define uni_icon(I, icon_state, rest...) new /datum/universal_icon(I, icon_state, ##rest) diff --git a/code/__defines/is_helpers.dm b/code/__defines/is_helpers.dm index fff94f906e..f56b590be9 100644 --- a/code/__defines/is_helpers.dm +++ b/code/__defines/is_helpers.dm @@ -71,3 +71,6 @@ // Diveable water #define isdiveablewater(A) istype(A, /turf/simulated/floor/water/deep/ocean/diving) + +/// NaN isn't a number, damn it. Infinity is a problem too. +#define isnum_safe(x) ( isnum((x)) && !isnan((x)) && !isinf((x)) ) diff --git a/code/_helpers/_lists.dm b/code/_helpers/_lists.dm index b71b20e037..babe0b58d2 100644 --- a/code/_helpers/_lists.dm +++ b/code/_helpers/_lists.dm @@ -960,3 +960,18 @@ var/global/list/json_cache = list() return_list += bit return return_list + +/// A version of deep_copy_list that actually supports associative list nesting: list(list(list("a" = "b"))) will actually copy correctly. +/proc/deep_copy_list_alt(list/inserted_list) + if(!islist(inserted_list)) + return inserted_list + var/copied_list = inserted_list.Copy() + . = copied_list + for(var/key_or_value in inserted_list) + if(isnum_safe(key_or_value) || !inserted_list[key_or_value]) + continue + var/value = inserted_list[key_or_value] + var/new_value = value + if(islist(value)) + new_value = deep_copy_list_alt(value) + copied_list[key_or_value] = new_value diff --git a/code/controllers/configuration/entries/resources.dm b/code/controllers/configuration/entries/resources.dm index 475cb8ca70..8623f4d75c 100644 --- a/code/controllers/configuration/entries/resources.dm +++ b/code/controllers/configuration/entries/resources.dm @@ -8,4 +8,6 @@ /datum/config_entry/flag/cache_assets +/datum/config_entry/flag/smart_cache_assets + /datum/config_entry/flag/save_spritesheets diff --git a/code/controllers/subsystems/asset_loading.dm b/code/controllers/subsystems/asset_loading.dm index 2d2939de4b..ecdf406894 100644 --- a/code/controllers/subsystems/asset_loading.dm +++ b/code/controllers/subsystems/asset_loading.dm @@ -7,6 +7,7 @@ SUBSYSTEM_DEF(asset_loading) flags = SS_NO_INIT runlevels = RUNLEVEL_LOBBY|RUNLEVELS_DEFAULT var/list/datum/asset/generate_queue = list() + var/last_queue_len = 0 /datum/controller/subsystem/asset_loading/fire(resumed) while(length(generate_queue)) @@ -16,7 +17,12 @@ SUBSYSTEM_DEF(asset_loading) if(MC_TICK_CHECK) return + last_queue_len = length(generate_queue) generate_queue.len-- + // We just emptied the queue + if(last_queue_len && !length(generate_queue)) + // Clean up cached icons, freeing memory. + rustg_iconforge_cleanup() /datum/controller/subsystem/asset_loading/proc/queue_asset(datum/asset/queue) #ifdef DO_NOT_DEFER_ASSETS diff --git a/code/modules/admin/admin_verb_lists.dm b/code/modules/admin/admin_verb_lists.dm index 44f5c6e1ca..854f1d0d99 100644 --- a/code/modules/admin/admin_verb_lists.dm +++ b/code/modules/admin/admin_verb_lists.dm @@ -236,6 +236,8 @@ var/list/admin_verbs_debug = list( /client/proc/show_gm_status, /datum/admins/proc/change_weather, /datum/admins/proc/change_time, + /client/proc/cmd_regenerate_asset_cache, + /client/proc/cmd_clear_smart_asset_cache, /client/proc/admin_give_modifier, /client/proc/simple_DPS, /datum/admins/proc/view_feedback, @@ -390,6 +392,8 @@ var/list/admin_verbs_event_manager = list( /proc/release, /datum/admins/proc/change_weather, /datum/admins/proc/change_time, + /client/proc/cmd_regenerate_asset_cache, + /client/proc/cmd_clear_smart_asset_cache, /client/proc/admin_give_modifier, /client/proc/Jump, /client/proc/jumptomob, @@ -417,6 +421,8 @@ var/list/admin_verbs_event_manager = list( /client/proc/show_gm_status, /datum/admins/proc/change_weather, /datum/admins/proc/change_time, + /client/proc/cmd_regenerate_asset_cache, + /client/proc/cmd_clear_smart_asset_cache, /client/proc/admin_give_modifier, /datum/admins/proc/cmd_admin_dress, /client/proc/cmd_admin_gib_self, diff --git a/code/modules/admin/admin_verb_lists_vr.dm b/code/modules/admin/admin_verb_lists_vr.dm index 04831c5bf3..bebed4393d 100644 --- a/code/modules/admin/admin_verb_lists_vr.dm +++ b/code/modules/admin/admin_verb_lists_vr.dm @@ -281,6 +281,8 @@ var/list/admin_verbs_debug = list( /datum/admins/proc/set_uplink, //VOREStation Add, /datum/admins/proc/change_weather, /datum/admins/proc/change_time, + /client/proc/cmd_regenerate_asset_cache, + /client/proc/cmd_clear_smart_asset_cache, /client/proc/admin_give_modifier, /client/proc/simple_DPS, /datum/admins/proc/view_feedback, @@ -453,6 +455,8 @@ var/list/admin_verbs_event_manager = list( /proc/release, /datum/admins/proc/change_weather, /datum/admins/proc/change_time, + /client/proc/cmd_regenerate_asset_cache, + /client/proc/cmd_clear_smart_asset_cache, /client/proc/admin_give_modifier, /client/proc/Jump, /client/proc/jumptomob, @@ -480,6 +484,8 @@ var/list/admin_verbs_event_manager = list( // /client/proc/show_gm_status, // VOREStation Edit - We don't use SSgame_master yet. /datum/admins/proc/change_weather, /datum/admins/proc/change_time, + /client/proc/cmd_regenerate_asset_cache, + /client/proc/cmd_clear_smart_asset_cache, /client/proc/admin_give_modifier, /datum/admins/proc/cmd_admin_dress, /client/proc/cmd_admin_gib_self, diff --git a/code/modules/admin/verbs/debug.dm b/code/modules/admin/verbs/debug.dm index 6033bfc205..31328025b5 100644 --- a/code/modules/admin/verbs/debug.dm +++ b/code/modules/admin/verbs/debug.dm @@ -695,3 +695,36 @@ var/log = "[key_name(src)] changed [planet.name]'s time to [planet.current_time.show_time("hh:mm")]." message_admins(log) log_admin(log) + +/client/proc/cmd_regenerate_asset_cache() + set category = "Debug.Assets" + set name = "Regenerate Asset Cache" + set desc = "Clears the asset cache and regenerates it immediately." + if(!CONFIG_GET(flag/cache_assets)) + to_chat(usr, span_warning("Asset caching is disabled in the config!")) + return + var/regenerated = 0 + for(var/datum/asset/A as() in subtypesof(/datum/asset)) + if(!initial(A.cross_round_cachable)) + continue + if(A == initial(A._abstract)) + continue + var/datum/asset/asset_datum = GLOB.asset_datums[A] + asset_datum.regenerate() + regenerated++ + to_chat(usr, span_notice("Regenerated [regenerated] asset\s.")) + +/client/proc/cmd_clear_smart_asset_cache() + set category = "Debug.Assets" + set name = "Clear Smart Asset Cache" + set desc = "Clears the smart asset cache." + if(!CONFIG_GET(flag/smart_cache_assets)) + to_chat(usr, span_warning("Smart asset caching is disabled in the config!")) + return + var/cleared = 0 + for(var/datum/asset/spritesheet_batched/A as() in subtypesof(/datum/asset/spritesheet_batched)) + if(A == initial(A._abstract)) + continue + fdel("[ASSET_CROSS_ROUND_SMART_CACHE_DIRECTORY]/spritesheet_cache.[initial(A.name)].json") + cleared++ + to_chat(usr, span_notice("Cleared [cleared] asset\s.")) diff --git a/code/modules/asset_cache/asset_cache_item.dm b/code/modules/asset_cache/asset_cache_item.dm index ad4596bded..dbee71b525 100644 --- a/code/modules/asset_cache/asset_cache_item.dm +++ b/code/modules/asset_cache/asset_cache_item.dm @@ -28,22 +28,28 @@ ///pass in a valid file_hash if you have one to save it from needing to do it again. ///pass in a valid dmi file path string e.g. "icons/path/to/dmi_file.dmi" to make generating the hash less expensive /datum/asset_cache_item/New(name, file, file_hash, dmi_file_path) - if (!isfile(file)) + if(!isfile(file)) file = fcopy_rsc(file) - hash = file_hash + if(length(file) == 0) + log_asset("WARNING: [name] is an empty file, this is almost certainly not intended and could indicate a bad boot!") + message_admins("ASSET WARNING: [name] is an empty file, this is almost certainly not intended and could indicate a bad boot!") + hash = file_hash //the given file is directly from a dmi file and is thus in the rsc already, we know that its file_hash will be correct if(!hash) if(dmi_file_path) hash = md5(file) else hash = md5asfile(file) //icons sent to the rsc md5 incorrectly when theyre given incorrect data - if (!hash) - CRASH("invalid asset sent to asset cache") + if(!hash) + hash = md5(fcopy_rsc(file)) + if (!hash) + CRASH("invalid asset sent to asset cache") + log_debug("asset cache unexpected success of second fcopy_rsc") src.name = name var/extstart = findlasttext(name, ".") - if (extstart) + if(extstart) ext = ".[copytext(name, extstart+1)]" resource = file diff --git a/code/modules/asset_cache/asset_list.dm b/code/modules/asset_cache/asset_list.dm index bdc062e066..d3c75b77e6 100644 --- a/code/modules/asset_cache/asset_list.dm +++ b/code/modules/asset_cache/asset_list.dm @@ -1,5 +1,3 @@ -#define ASSET_CROSS_ROUND_CACHE_DIRECTORY "cache/assets" - //These datums are used to populate the asset cache, the proc "register()" does this. //Place any asset datums you create in asset_list_items.dm @@ -20,15 +18,15 @@ GLOBAL_LIST_EMPTY(asset_datums) var/cached_serialized_url_mappings var/cached_serialized_url_mappings_transport_type - /// Whether or not this asset should be loaded in the "early assets" SS - var/early = FALSE - /// Whether or not this asset can be cached across rounds of the same commit under the `CACHE_ASSETS` config. /// This is not a *guarantee* the asset will be cached. Not all asset subtypes respect this field, and the /// config can, of course, be disabled. /// Disable this if your asset can change between rounds on the same exact version of the code. var/cross_round_cachable = FALSE + /// Whether or not this asset should be loaded in the "early assets" SS + var/early = FALSE + /datum/asset/New() GLOB.asset_datums[type] = src register() @@ -63,6 +61,19 @@ GLOBAL_LIST_EMPTY(asset_datums) /datum/asset/proc/should_refresh() return !cross_round_cachable || !CONFIG_GET(flag/cache_assets) +/// Immediately regenerate the asset, overwriting any cache. +/datum/asset/proc/regenerate() + SHOULD_CALL_PARENT(FALSE) + unregister() + cached_serialized_url_mappings = null + cached_serialized_url_mappings_transport_type = null + register() + +/// Unregisters any assets from the transport. +/datum/asset/proc/unregister() + SHOULD_CALL_PARENT(FALSE) + CRASH("unregister() not implemented for asset [type]!") + /// Simply takes any generated file and saves it to the round-specific /logs folder. Useful for debugging potential issues with spritesheet generation/display. /// Only called when the SAVE_SPRITESHEETS config option is uncommented. /datum/asset/proc/save_to_logs(file_name, file_location) @@ -86,12 +97,12 @@ GLOBAL_LIST_EMPTY(asset_datums) /datum/asset/simple/register() for(var/asset_name in assets) var/datum/asset_cache_item/ACI = SSassets.transport.register_asset(asset_name, assets[asset_name]) - if (!ACI) + if(!istype(ACI)) log_asset("ERROR: Invalid asset: [type]:[asset_name]:[ACI]") continue - if (legacy) + if(legacy) ACI.legacy = legacy - if (keep_local_name) + if(keep_local_name) ACI.keep_local_name = keep_local_name assets[asset_name] = ACI @@ -103,6 +114,7 @@ GLOBAL_LIST_EMPTY(asset_datums) for (var/asset_name in assets) .[asset_name] = SSassets.transport.get_asset_url(asset_name, assets[asset_name]) + // For registering or sending multiple others at once /datum/asset/group _abstract = /datum/asset/group @@ -123,6 +135,11 @@ GLOBAL_LIST_EMPTY(asset_datums) var/datum/asset/A = get_asset_datum(type) . += A.get_url_mappings() +/datum/asset/group/unregister() + for(var/type in children) + var/datum/asset/A = get_asset_datum(type) + A.unregister() + // spritesheet implementation - coalesces various icons into a single .png file // and uses CSS to select icons out of that file - saves on transferring some // 1400-odd individual PNG files @@ -147,9 +164,10 @@ GLOBAL_LIST_EMPTY(asset_datums) /// If this asset should be fully loaded on new /// Defaults to false so we can process this stuff nicely var/load_immediately = FALSE - VAR_PRIVATE - // Kept in state so that the result is the same, even when the files are created, for this run - should_refresh = null + /// Allows resizing all icons it comes across by a multiplier (32x32 * 2 = 64x64) + var/resize = 1 + /// If this asset should CRASH or ignore when duplicate sprite keys are added + var/duplicates_allowed = FALSE /datum/asset/spritesheet/proc/should_load_immediately() #ifdef DO_NOT_DEFER_ASSETS @@ -158,28 +176,58 @@ GLOBAL_LIST_EMPTY(asset_datums) return load_immediately #endif - /datum/asset/spritesheet/should_refresh() if (..()) return TRUE + // Static so that the result is the same, even when the files are created, for this run + var/static/should_refresh = null + if (isnull(should_refresh)) // `fexists` seems to always fail on static-time - should_refresh = !fexists(css_cache_filename()) || !fexists(data_cache_filename()) + should_refresh = !fexists("[ASSET_CROSS_ROUND_CACHE_DIRECTORY]/spritesheet.[name].css") return should_refresh +/datum/asset/spritesheet/unregister() + SSassets.transport.unregister_asset("spritesheet_[name].css") + if(length(sizes)) + for(var/size_id in sizes) + SSassets.transport.unregister_asset("[name]_[size_id].png") + else + for(var/sheet in cached_spritesheets_needed) + SSassets.transport.unregister_asset(sheet) + +/datum/asset/spritesheet/regenerate() + unregister() + sprites = list() + fdel("[ASSET_CROSS_ROUND_CACHE_DIRECTORY]/spritesheet.[name].css") + for(var/sheet in cached_spritesheets_needed) + fdel("[ASSET_CROSS_ROUND_CACHE_DIRECTORY]/spritesheet.[sheet].png") + fdel("data/spritesheets/spritesheet_[name].css") + for(var/size_id in sizes) + fdel("data/spritesheets/[name]_[size_id].png") + sizes = list() + to_generate = list() + cached_serialized_url_mappings = null + cached_serialized_url_mappings_transport_type = null + fully_generated = FALSE + var/old_load = load_immediately + load_immediately = TRUE + create_spritesheets() + realize_spritesheets(yield = FALSE) + load_immediately = old_load + /datum/asset/spritesheet/register() SHOULD_NOT_OVERRIDE(TRUE) - if (!name) + if(!name) CRASH("spritesheet [type] cannot register without a name") - if (!should_refresh() && read_from_cache()) + if(!should_refresh() && read_from_cache()) fully_generated = TRUE return - // If it's cached, may as well load it now, while the loading is cheap if(CONFIG_GET(flag/cache_assets) && cross_round_cachable) load_immediately = TRUE @@ -202,17 +250,19 @@ GLOBAL_LIST_EMPTY(asset_datums) ensure_stripped() for(var/size_id in sizes) var/size = sizes[size_id] - SSassets.transport.register_asset("[name]_[size_id].png", size[SPRSZ_STRIPPED]) - var/css_name = "spritesheet_[name].css" - var/file_directory = "data/spritesheets/[css_name]" - fdel(file_directory) - text2file(generate_css(), file_directory) - SSassets.transport.register_asset(css_name, fcopy_rsc(file_directory)) - + var/file_path = size[SPRSZ_STRIPPED] + var/file_hash = rustg_hash_file("md5", file_path) + SSassets.transport.register_asset("[name]_[size_id].png", file_path, file_hash=file_hash) + var/res_name = "spritesheet_[name].css" + var/fname = "data/spritesheets/[res_name]" + fdel(fname) + var/css = generate_css() + rustg_file_write(css, fname) + var/css_hash = rustg_hash_string("md5", css) + SSassets.transport.register_asset(res_name, fcopy_rsc(fname), file_hash=css_hash) if(CONFIG_GET(flag/save_spritesheets)) - save_to_logs(file_name = css_name, file_location = file_directory) - - fdel(file_directory) + save_to_logs(file_name = res_name, file_location = fname) + fdel(fname) if (CONFIG_GET(flag/cache_assets) && cross_round_cachable) write_to_cache() @@ -229,10 +279,10 @@ GLOBAL_LIST_EMPTY(asset_datums) return ..() /datum/asset/spritesheet/send(client/client) - if (!name) + if(!name) return - if (!should_refresh()) + if(!should_refresh()) return send_from_cache(client) var/all = list("spritesheet_[name].css") @@ -254,33 +304,46 @@ GLOBAL_LIST_EMPTY(asset_datums) /datum/asset/spritesheet/proc/ensure_stripped(sizes_to_strip = sizes) for(var/size_id in sizes_to_strip) var/size = sizes[size_id] - if (size[SPRSZ_STRIPPED]) + if(size[SPRSZ_STRIPPED]) continue // save flattened version - var/png_name = "[name]_[size_id].png" - var/file_directory = "data/spritesheets/[png_name]" - fcopy(size[SPRSZ_ICON], file_directory) - var/error = rustg_dmi_strip_metadata(file_directory) + var/fname = "data/spritesheets/[name]_[size_id].png" + fcopy(size[SPRSZ_ICON], fname) + var/error = rustg_dmi_strip_metadata(fname) if(length(error)) - stack_trace("Failed to strip [png_name]: [error]") - size[SPRSZ_STRIPPED] = icon(file_directory) + stack_trace("Failed to strip [name]_[size_id].png: [error]") + + // Difference from Beestation: Resizing handling + var/icon/stripped = icon(fname) + if(resize != 1) + var/new_width = round(stripped.Width() * resize) + var/new_height = round(stripped.Height() * resize) + // Note: arguments MUST be strings or they don't make it past ffi + var/error_two = rustg_dmi_resize_png(fname, "[new_width]", "[new_height]", "nearest") + if(error_two) + stack_trace("Failed to resize [name]_[size_id].png to [new_width]x[new_height]: [error_two]") + + size[SPRSZ_STRIPPED] = icon(fname) + else + size[SPRSZ_STRIPPED] = stripped // this is useful here for determining if weird sprite issues (like having a white background) are a cause of what we're doing DM-side or not since we can see the full flattened thing at-a-glance. if(CONFIG_GET(flag/save_spritesheets)) - save_to_logs(file_name = png_name, file_location = file_directory) + save_to_logs(file_name = "[name]_[size_id].png", file_location = fname) - fdel(file_directory) + fdel(fname) /datum/asset/spritesheet/proc/generate_css() var/list/out = list() - for (var/size_id in sizes) + for(var/size_id in sizes) var/size = sizes[size_id] var/icon/tiny = size[SPRSZ_ICON] - out += ".[name][size_id]{display:inline-block;width:[tiny.Width()]px;height:[tiny.Height()]px;background:url('[get_background_url("[name]_[size_id].png")]') no-repeat;}" + // Difference from Beestation: * resize on width and height + out += ".[name][size_id]{display:inline-block;width:[round(tiny.Width() * resize)]px;height:[round(tiny.Height() * resize)]px;background:url('[get_background_url("[name]_[size_id].png")]') no-repeat;}" - for (var/sprite_id in sprites) + for(var/sprite_id in sprites) var/sprite = sprites[sprite_id] var/size_id = sprite[SPR_SIZE] var/idx = sprite[SPR_IDX] @@ -288,9 +351,10 @@ GLOBAL_LIST_EMPTY(asset_datums) var/icon/tiny = size[SPRSZ_ICON] var/icon/big = size[SPRSZ_STRIPPED] - var/per_line = big.Width() / tiny.Width() - var/x = (idx % per_line) * tiny.Width() - var/y = round(idx / per_line) * tiny.Height() + // Difference from Beestation: Resizing handling + var/per_line = big.Width() / round(tiny.Width() * resize) + var/x = (idx % per_line) * round(tiny.Width() * resize) + var/y = round(idx / per_line) * round(tiny.Height() * resize) out += ".[name][size_id].[sprite_id]{background-position:-[x]px -[y]px;}" @@ -303,39 +367,31 @@ GLOBAL_LIST_EMPTY(asset_datums) return "[ASSET_CROSS_ROUND_CACHE_DIRECTORY]/spritesheet.[name].json" /datum/asset/spritesheet/proc/read_from_cache() - return read_css_from_cache() && read_data_from_cache() - -/datum/asset/spritesheet/proc/read_css_from_cache() - var/replaced_css = file2text(css_cache_filename()) + var/replaced_css = rustg_file_read("[ASSET_CROSS_ROUND_CACHE_DIRECTORY]/spritesheet.[name].css") var/regex/find_background_urls = regex(@"background:url\('%(.+?)%'\)", "g") while (find_background_urls.Find(replaced_css)) var/asset_id = find_background_urls.group[1] - var/asset_cache_item = SSassets.transport.register_asset(asset_id, "[ASSET_CROSS_ROUND_CACHE_DIRECTORY]/spritesheet.[asset_id]") + var/file_path = "[ASSET_CROSS_ROUND_CACHE_DIRECTORY]/spritesheet.[asset_id]" + // Hashing it here is a *lot* faster. + var/hash = rustg_hash_file("md5", file_path) + var/asset_cache_item = SSassets.transport.register_asset(asset_id, file_path, file_hash=hash) var/asset_url = SSassets.transport.get_asset_url(asset_cache_item = asset_cache_item) replaced_css = replacetext(replaced_css, find_background_urls.match, "background:url('[asset_url]')") LAZYADD(cached_spritesheets_needed, asset_id) - var/finalized_name = "spritesheet_[name].css" - var/replaced_css_filename = "data/spritesheets/[finalized_name]" + var/replaced_css_filename = "data/spritesheets/spritesheet_[name].css" + var/css_hash = rustg_hash_string("md5", replaced_css) rustg_file_write(replaced_css, replaced_css_filename) - SSassets.transport.register_asset(finalized_name, replaced_css_filename) + SSassets.transport.register_asset("spritesheet_[name].css", replaced_css_filename, file_hash=css_hash) if(CONFIG_GET(flag/save_spritesheets)) - save_to_logs(file_name = finalized_name, file_location = replaced_css_filename) + save_to_logs(file_name = "spritesheet_[name].css", file_location = replaced_css_filename) fdel(replaced_css_filename) return TRUE -/datum/asset/spritesheet/proc/read_data_from_cache() - var/json = json_decode(file2text(data_cache_filename())) - - if (islist(json["sprites"])) - sprites = json["sprites"] - - return TRUE - /datum/asset/spritesheet/proc/send_from_cache(client/client) if (isnull(cached_spritesheets_needed)) stack_trace("cached_spritesheets_needed was null when sending assets from [type] from cache") @@ -351,23 +407,15 @@ GLOBAL_LIST_EMPTY(asset_datums) return SSassets.transport.get_asset_url(asset) /datum/asset/spritesheet/proc/write_to_cache() - write_css_to_cache() - write_data_to_cache() - -/datum/asset/spritesheet/proc/write_css_to_cache() for (var/size_id in sizes) - fcopy(SSassets.cache["[name]_[size_id].png"].resource, "[ASSET_CROSS_ROUND_CACHE_DIRECTORY]/spritesheet.[name]_[size_id].png") + var/datum/asset_cache_item/temp = SSassets.cache["[name]_[size_id].png"] + fcopy(temp.resource, "[ASSET_CROSS_ROUND_CACHE_DIRECTORY]/spritesheet.[name]_[size_id].png") generating_cache = TRUE var/mock_css = generate_css() generating_cache = FALSE - rustg_file_write(mock_css, css_cache_filename()) - -/datum/asset/spritesheet/proc/write_data_to_cache() - rustg_file_write(json_encode(list( - "sprites" = sprites, - )), data_cache_filename()) + rustg_file_write(mock_css, "[ASSET_CROSS_ROUND_CACHE_DIRECTORY]/spritesheet.[name].css") /datum/asset/spritesheet/proc/get_cached_url_mappings() var/list/mappings = list() @@ -392,20 +440,17 @@ GLOBAL_LIST_EMPTY(asset_datums) /datum/asset/spritesheet/proc/queuedInsert(sprite_name, icon/I, icon_state="", dir=SOUTH, frame=1, moving=FALSE) I = icon(I, icon_state=icon_state, dir=dir, frame=frame, moving=moving) - if (!I || !length(icon_states(I))) // that direction or state doesn't exist + if(!I || !length(icon_states(I))) // that direction or state doesn't exist return - - //var/start_usage = world.tick_usage - - //any sprite modifications we want to do (aka, coloring a greyscaled asset) - I = ModifyInserted(I) var/size_id = "[I.Width()]x[I.Height()]" var/size = sizes[size_id] - if (sprites[sprite_name]) + if(sprites[sprite_name]) + if(duplicates_allowed) + return // No crash CRASH("duplicate sprite \"[sprite_name]\" in sheet [name] ([type])") - if (size) + if(size) var/position = size[SPRSZ_COUNT]++ // Icons are essentially representations of files + modifications // Because of this, byond keeps them in a cache. It does this in a really dumb way tho @@ -424,26 +469,15 @@ GLOBAL_LIST_EMPTY(asset_datums) sizes[size_id] = size = list(1, I, null) sprites[sprite_name] = list(size_id, 0) - //SSblackbox.record_feedback("tally", "spritesheet_queued_insert_time", TICK_USAGE_TO_MS(start_usage), name) - -/** - * A simple proc handing the Icon for you to modify before it gets turned into an asset. - * - * Arguments: - * * I: icon being turned into an asset - */ -/datum/asset/spritesheet/proc/ModifyInserted(icon/pre_asset) - return pre_asset - /datum/asset/spritesheet/proc/InsertAll(prefix, icon/I, list/directions) - if (length(prefix)) + if(length(prefix)) prefix = "[prefix]-" - if (!directions) + if(!directions) directions = list(SOUTH) - for (var/icon_state_name in icon_states(I)) - for (var/direction in directions) + for(var/icon_state_name in icon_states(I)) + for(var/direction in directions) var/prefix2 = (directions.len > 1) ? "[dir2text(direction)]-" : "" Insert("[prefix][prefix2][icon_state_name]", I, icon_state=icon_state_name, dir=direction) @@ -455,10 +489,10 @@ GLOBAL_LIST_EMPTY(asset_datums) /datum/asset/spritesheet/proc/icon_tag(sprite_name) var/sprite = sprites[sprite_name] - if (!sprite) + if(!sprite) return null var/size_id = sprite[SPR_SIZE] - return {""} + return {""} /datum/asset/spritesheet/proc/icon_class_name(sprite_name) var/sprite = sprites[sprite_name] @@ -486,6 +520,13 @@ GLOBAL_LIST_EMPTY(asset_datums) #undef SPRSZ_ICON #undef SPRSZ_STRIPPED +/datum/asset/spritesheet/simple + _abstract = /datum/asset/spritesheet/simple + var/list/assets + +/datum/asset/spritesheet/simple/create_spritesheets() + for(var/key in assets) + Insert(key, assets[key]) /datum/asset/changelog_item _abstract = /datum/asset/changelog_item @@ -505,14 +546,6 @@ GLOBAL_LIST_EMPTY(asset_datums) return . = list("[item_filename]" = SSassets.transport.get_asset_url(item_filename)) -/datum/asset/spritesheet/simple - _abstract = /datum/asset/spritesheet/simple - var/list/assets - -/datum/asset/spritesheet/simple/create_spritesheets() - for (var/key in assets) - Insert(key, assets[key]) - //Generates assets based on iconstates of a single icon /datum/asset/simple/icon_states _abstract = /datum/asset/simple/icon_states @@ -528,7 +561,7 @@ GLOBAL_LIST_EMPTY(asset_datums) for(var/icon_state_name in icon_states(_icon)) for(var/direction in directions) var/asset = icon(_icon, icon_state_name, direction, frame, movement_states) - if (!asset) + if(!asset) continue asset = fcopy_rsc(asset) //dedupe var/prefix2 = (directions.len > 1) ? "[dir2text(direction)]." : "" @@ -558,36 +591,37 @@ GLOBAL_LIST_EMPTY(asset_datums) var/list/parents = list() /datum/asset/simple/namespaced/register() - if (legacy) + if(legacy) assets |= parents var/list/hashlist = list() - var/list/sorted_assets = sortList(assets) + var/list/created_items = list() - for (var/asset_name in sorted_assets) + var/list/sorted_assets = sortList(assets) + for(var/asset_name in sorted_assets) var/datum/asset_cache_item/ACI = new(asset_name, sorted_assets[asset_name]) - if (!ACI?.hash) + if (!istype(ACI) || !ACI.hash) log_asset("ERROR: Invalid asset: [type]:[asset_name]:[ACI]") continue hashlist += ACI.hash - sorted_assets[asset_name] = ACI + created_items[asset_name] = ACI var/namespace = md5(hashlist.Join()) - for (var/asset_name in parents) + for(var/asset_name in parents) var/datum/asset_cache_item/ACI = new(asset_name, parents[asset_name]) - if (!ACI?.hash) + if (!istype(ACI) || !ACI.hash) log_asset("ERROR: Invalid asset: [type]:[asset_name]:[ACI]") continue ACI.namespace_parent = TRUE - sorted_assets[asset_name] = ACI + created_items[asset_name] = ACI - for (var/asset_name in sorted_assets) - var/datum/asset_cache_item/ACI = sorted_assets[asset_name] - if (!ACI?.hash) + for(var/asset_name in created_items) + var/datum/asset_cache_item/ACI = created_items[asset_name] + if (!istype(ACI) || !ACI.hash) log_asset("ERROR: Invalid asset: [type]:[asset_name]:[ACI]") continue ACI.namespace = namespace - assets = sorted_assets + assets = created_items ..() /// Get a html string that will load a html asset. @@ -612,7 +646,7 @@ GLOBAL_LIST_EMPTY(asset_datums) /datum/asset/json/register() var/filename = "data/[name].json" fdel(filename) - text2file(json_encode(generate()), filename) + rustg_file_write(json_encode(generate()), filename) SSassets.transport.register_asset("[name].json", fcopy_rsc(filename)) fdel(filename) @@ -621,4 +655,5 @@ GLOBAL_LIST_EMPTY(asset_datums) SHOULD_CALL_PARENT(FALSE) CRASH("generate() not implemented for [type]!") -#undef ASSET_CROSS_ROUND_CACHE_DIRECTORY +/datum/asset/json/unregister() + SSassets.transport.unregister_asset("[name].json") diff --git a/code/modules/asset_cache/assets/spritesheets/kitchen_recipes.dm b/code/modules/asset_cache/assets/spritesheets/kitchen_recipes.dm index 2cca2ed670..4fe048a84e 100644 --- a/code/modules/asset_cache/assets/spritesheets/kitchen_recipes.dm +++ b/code/modules/asset_cache/assets/spritesheets/kitchen_recipes.dm @@ -1,26 +1,13 @@ -/datum/asset/spritesheet/kitchen_recipes +/datum/asset/spritesheet_batched/kitchen_recipes name = "kitchen_recipes" -/datum/asset/spritesheet/kitchen_recipes/create_spritesheets() +/datum/asset/spritesheet_batched/kitchen_recipes/create_spritesheets() for(var/datum/recipe/R as anything in subtypesof(/datum/recipe)) - add_atom_icon(R.result, sanitize_css_class_name("[R.type]")) + var/key = sanitize_css_class_name("[R.type]") -/datum/asset/spritesheet/kitchen_recipes/proc/add_atom_icon(typepath, id) - var/icon_file - var/icon_state - var/obj/preview_item = typepath + var/atom/item = R.result + var/datum/universal_icon/icon = get_display_icon_for(item) + if(!icon || !icon.icon_file) + continue - // if(ispath(ingredient_typepath, /datum/reagent)) - // var/datum/reagent/reagent = ingredient_typepath - // preview_item = initial(reagent.default_container) - // var/datum/glass_style/style = GLOB.glass_style_singletons[preview_item]?[reagent] - // if(istype(style)) - // icon_file = style.icon - // icon_state = style.icon_state - - // icon_file ||= initial(preview_item.icon_preview) || initial(preview_item.icon) - // icon_state ||= initial(preview_item.icon_state_preview) || initial(preview_item.icon_state) - icon_file = initial(preview_item.icon) - icon_state = initial(preview_item.icon_state) - - Insert("[id]", icon_file, icon_state) + insert_icon(key, icon) diff --git a/code/modules/asset_cache/iconforge/batched_spritesheet.dm b/code/modules/asset_cache/iconforge/batched_spritesheet.dm new file mode 100644 index 0000000000..c49248f417 --- /dev/null +++ b/code/modules/asset_cache/iconforge/batched_spritesheet.dm @@ -0,0 +1,356 @@ +#define SPR_SIZE "size_id" +#define SPR_IDX "position" +#define CACHE_WAIT "wait" +#define CACHE_INVALID TRUE +#define CACHE_VALID FALSE + +#define WHATTHEFUCKAMIDOING_FILE "data/WHATTHEFUCKFILE.json" + +/datum/asset/spritesheet_batched + _abstract = /datum/asset/spritesheet_batched + var/name + /// list("32x32") + var/list/sizes = list() + /// "foo_bar" -> list("32x32", 5, entry_obj) + var/list/sprites = list() + + // "foo_bar" -> entry_obj + var/list/entries = list() + + /// JSON encoded version of entries. + var/entries_json = null + + /// If this spritesheet exists in a completed state. + var/fully_generated = FALSE + + /// If this asset should be fully loaded on new + /// Defaults to false so we can process this stuff nicely + var/load_immediately = FALSE + /// If we should avoid propogating 'invalid dir' errors from rust-g. Because sometimes, you just don't know what dirs are valid. + var/ignore_dir_errors = FALSE + /// Avoid propogating 'Could not find associated icon state' errors because we know our input data fucking sucks + var/ignore_associated_icon_state_errors = FALSE + + /// Forces use of the smart cache. This is for unit tests, please respect the config <3 + var/force_cache = FALSE + + /// If there is currently an async job, its ID + var/job_id = null + /// If there is currently an async cache job, its ID. + var/cache_job_id = null + + // Fields to store async cache task inputs. + var/cache_data = null + var/cache_sizes_data = null + var/cache_sprites_data = null + var/cache_input_hash = null + var/cache_dmi_hashes = null + var/cache_dmi_hashes_json = null + /// Used to prevent async cache refresh jobs from looping on failure. + var/cache_result = null + +/datum/asset/spritesheet_batched/proc/should_load_immediately() +#ifdef DO_NOT_DEFER_ASSETS + return TRUE +#else + return load_immediately +#endif + +/// Returns true if the cache should be invalidated/doesn't exist. +/datum/asset/spritesheet_batched/should_refresh(yield) + . = CACHE_INVALID // in the case of any errors, we need to regenerate. + if(!fexists("[ASSET_CROSS_ROUND_SMART_CACHE_DIRECTORY]/spritesheet_cache.[name].json")) + return CACHE_INVALID + if(!fexists("data/spritesheets/spritesheet_[name].css")) + return CACHE_INVALID + if(isnull(cache_data) || isnull(cache_dmi_hashes_json)) + cache_data = rustg_file_read("[ASSET_CROSS_ROUND_SMART_CACHE_DIRECTORY]/spritesheet_cache.[name].json") + if(!findtext(cache_data, "{", 1, 2)) // cache isn't valid JSON + log_asset("Cache for spritesheet_[name] was not valid JSON. This is abnormal. Likely tampered with or IO failure.") + return CACHE_INVALID + var/cache_json = json_decode(cache_data) + // Best to invalidate if rustg updates, since the way icons are handled can change. + var/cached_rustg_version = cache_json["rustg_version"] + if(isnull(cached_rustg_version)) + log_asset("Cache for spritesheet_[name] did not contain a rustg_version!") + return CACHE_INVALID + var/rustg_version = rustg_get_version() + if(cached_rustg_version != rustg_version) + log_asset("Invalidated cache for spritesheet_[name] due to rustg updating from [cached_rustg_version] to [rustg_version].") + return CACHE_INVALID + cache_sizes_data = cache_json["sizes"] + cache_sprites_data = cache_json["sprites"] + cache_input_hash = cache_json["input_hash"] + cache_dmi_hashes = cache_json["dmi_hashes"] + if(!length(cache_dmi_hashes) || !length(cache_input_hash) || !length(cache_sizes_data) || !length(cache_sprites_data)) + log_asset("Cache for spritesheet_[name] did not contain the correct data. This is abnormal. Likely tampered with.") + return CACHE_INVALID + cache_dmi_hashes_json = json_encode(cache_dmi_hashes) + var/data_out + + if(yield || !isnull(cache_job_id)) + if(isnull(cache_job_id)) + cache_job_id = rustg_iconforge_cache_valid_async(cache_input_hash, cache_dmi_hashes_json, entries_json) + . = CACHE_WAIT // if we return during this, WAIT!! + UNTIL((data_out = rustg_iconforge_check(cache_job_id)) != RUSTG_JOB_NO_RESULTS_YET) + cache_job_id = null + . = CACHE_INVALID // reset back to normal, invalid on CRASH + else + data_out = rustg_iconforge_cache_valid(cache_input_hash, cache_dmi_hashes_json, entries_json) + if (data_out == RUSTG_JOB_ERROR) + CRASH("Spritesheet [name] cache JOB PANIC") + else if(!findtext(data_out, "{", 1, 2)) + rustg_file_write(cache_data, "[log_path]-spritesheet_cache_debug.[name].json") + rustg_file_write(entries_json, "[log_path]-spritesheet_debug_[name].json") + CRASH("Spritesheet [name] cache check UNKNOWN ERROR: [data_out]") + var/result = json_decode(data_out) + var/fail = result["fail_reason"] + if(length(fail) || result["result"] != "1") + if(findtextEx(fail, "ERROR:")) + CRASH("Spritesheet [name] cache check UNKNOWN [fail]") + log_asset("Invalidated cache for spritesheet_[name]: [fail]") + return CACHE_INVALID + // Populate the sizes and sprites list. + sizes = cache_sizes_data + sprites = cache_sprites_data + log_asset("Validated cache for spritesheet_[name]!") + return CACHE_VALID + +/datum/asset/spritesheet_batched/proc/insert_icon(sprite_name, datum/universal_icon/entry) + if(!istext(sprite_name) || !length(sprite_name)) + CRASH("Invalid sprite_name \"[sprite_name]\" given to insert_icon()! Providing non-strings will break icon generation.") + entries[sprite_name] = entry.to_list() + +/datum/asset/spritesheet_batched/register() + SHOULD_NOT_OVERRIDE(TRUE) + + if (!name) + CRASH("spritesheet [type] cannot register without a name") + + // Create our input data first, so we can compare to the cache. + create_spritesheets() + + if(should_load_immediately()) + realize_spritesheets(yield = FALSE) + else + SSasset_loading.queue_asset(src) + +/datum/asset/spritesheet_batched/unregister() + CRASH("unregister() called on batched spritesheet! Bad!") + +/// Call insert_icon or insert_all_icons here, building a spritesheet! +/datum/asset/spritesheet_batched/proc/create_spritesheets() + SHOULD_CALL_PARENT(FALSE) + CRASH("create_spritesheets() not implemented for [type]!") + +/datum/asset/spritesheet_batched/proc/insert_all_icons(prefix, icon/I, list/directions, prefix_with_dirs = TRUE) + if (length(prefix)) + prefix = "[prefix]-" + + if (!directions) + directions = list(SOUTH) + + for (var/icon_state_name in icon_states(I)) + if(icon_state_name == "") + icon_state_name = "byond-default" + for (var/direction in directions) + var/prefix2 = (directions.len > 1 && prefix_with_dirs) ? "[dir2text(direction)]-" : "" + insert_icon("[prefix][prefix2][icon_state_name]", uni_icon(I, icon_state_name, direction)) + +/datum/asset/spritesheet_batched/proc/realize_spritesheets(yield) + if(fully_generated) + return + if(!length(entries)) + CRASH("Spritesheet [name] ([type]) is empty! What are you doing?") + + if(isnull(entries_json)) + entries_json = json_encode(entries) + + if(isnull(cache_result)) + cache_result = should_refresh(yield) + if(cache_result == CACHE_WAIT) // sleep interrupted by MC. We'll get queried again later. + cache_result = null + return + + // read_from_cache returns false if config is disabled, otherwise it fully loads the spritesheet. + if (cache_result == CACHE_VALID && read_from_cache()) + SSasset_loading.dequeue_asset(src) + fully_generated = TRUE + return + // Remove the cache, since it's invalid if we get to this point. + fdel("[ASSET_CROSS_ROUND_SMART_CACHE_DIRECTORY]/spritesheet_cache.[name].json") + + var/do_cache = CONFIG_GET(flag/smart_cache_assets) || force_cache + var/data_out + if(yield || !isnull(job_id)) + if(isnull(job_id)) + job_id = rustg_iconforge_generate_async("data/spritesheets/", name, entries_json, do_cache) + UNTIL((data_out = rustg_iconforge_check(job_id)) != RUSTG_JOB_NO_RESULTS_YET) + else + //rustg_file_write(entries_json, "fuckoff.json") + data_out = rustg_iconforge_generate("data/spritesheets/", name, entries_json, do_cache) + if (data_out == RUSTG_JOB_ERROR) + CRASH("Spritesheet [name] JOB PANIC") + else if(!findtext(data_out, "{", 1, 2)) + rustg_file_write(entries_json, "[log_path]-spritesheet_debug_[name].json") + CRASH("Spritesheet [name] UNKNOWN ERROR: [data_out]") + var/data = json_decode(data_out) + sizes = data["sizes"] + sprites = data["sprites"] + var/input_hash = data["sprites_hash"] + var/dmi_hashes = data["dmi_hashes"] // this only contains values if do_cache is TRUE. + + for(var/size_id in sizes) + var/file_path = "data/spritesheets/[name]_[size_id].png" + var/file_hash = rustg_hash_file("md5", file_path) + SSassets.transport.register_asset("[name]_[size_id].png", fcopy_rsc(file_path), file_hash) + var/res_name = "spritesheet_[name].css" + var/fname = "data/spritesheets/[res_name]" + + fdel(fname) + var/css = generate_css() + rustg_file_write(css, fname) + var/css_hash = rustg_hash_string("md5", css) + SSassets.transport.register_asset(res_name, fcopy_rsc(fname), file_hash=css_hash) + + if (do_cache) + write_cache_meta(input_hash, dmi_hashes) + fully_generated = TRUE + // If we were ever in there, remove ourselves + SSasset_loading.dequeue_asset(src) + if(data["error"]) + var/err = data["error"] + if(ignore_dir_errors && findtext(err, "is not in the set of valid dirs")) + return + if(ignore_associated_icon_state_errors && findtext(err, "Could not find associated icon state")) + return + CRASH("Error during spritesheet generation for [name]: [err]") + +/datum/asset/spritesheet_batched/queued_generation() + realize_spritesheets(yield = TRUE) + +/datum/asset/spritesheet_batched/ensure_ready() + if(!fully_generated) + realize_spritesheets(yield = FALSE) + return ..() + +/datum/asset/spritesheet_batched/send(client/client) + if (!name) + return + + var/all = list("spritesheet_[name].css") + for(var/size_id in sizes) + all += "[name]_[size_id].png" + . = SSassets.transport.send_assets(client, all) + +/datum/asset/spritesheet_batched/get_url_mappings() + if (!name) + return + + . = list("spritesheet_[name].css" = SSassets.transport.get_asset_url("spritesheet_[name].css")) + for(var/size_id in sizes) + .["[name]_[size_id].png"] = SSassets.transport.get_asset_url("[name]_[size_id].png") + +/datum/asset/spritesheet_batched/proc/generate_css() + var/list/out = list() + + for (var/size_id in sizes) + var/size_split = splittext(size_id, "x") + var/width = text2num(size_split[1]) + var/height = text2num(size_split[2]) + out += ".[name][size_id]{display:inline-block;width:[width]px;height:[height]px;background:url('[get_background_url("[name]_[size_id].png")]') no-repeat;}" + + for (var/sprite_id in sprites) + var/sprite = sprites[sprite_id] + var/size_id = sprite[SPR_SIZE] + var/idx = sprite[SPR_IDX] + + var/size_split = splittext(size_id, "x") + var/width = text2num(size_split[1]) + var/x = idx * width + var/y = 0 + + out += ".[name][size_id].[sprite_id]{background-position:-[x]px -[y]px;}" + + return out.Join("\n") + +/datum/asset/spritesheet_batched/proc/read_from_cache() + if(!CONFIG_GET(flag/smart_cache_assets) && !force_cache) + return FALSE + // this is already guaranteed to exist. + var/css_fname = "data/spritesheets/spritesheet_[name].css" + + // sizes gets filled during should_refresh() + for(var/size_id in sizes) + var/fname = "data/spritesheets/[name]_[size_id].png" + if(!fexists(fname)) + return FALSE + + var/css_hash = rustg_hash_file("md5", css_fname) + SSassets.transport.register_asset("spritesheet_[name].css", fcopy_rsc(css_fname), file_hash=css_hash) + for(var/size_id in sizes) + var/fname = "data/spritesheets/[name]_[size_id].png" + var/hash = rustg_hash_file("md5", fname) + SSassets.transport.register_asset("[name]_[size_id].png", fcopy_rsc(fname), file_hash=hash) + + return TRUE + +/// Returns the URL to put in the background:url of the CSS asset +/datum/asset/spritesheet_batched/proc/get_background_url(asset) + return SSassets.transport.get_asset_url(asset) + +/datum/asset/spritesheet_batched/proc/write_cache_meta(input_hash, dmi_hashes) + var/list/cache_data = list( + "input_hash" = input_hash, + "dmi_hashes" = dmi_hashes, + "sizes" = sizes, + "sprites" = sprites, + "rustg_version" = rustg_get_version() + ) + rustg_file_write(json_encode(cache_data), "[ASSET_CROSS_ROUND_SMART_CACHE_DIRECTORY]/spritesheet_cache.[name].json") + +/** + * Third party helpers + * =================== + */ + +/datum/asset/spritesheet_batched/proc/css_tag() + return {""} + +/datum/asset/spritesheet_batched/proc/css_filename() + return SSassets.transport.get_asset_url("spritesheet_[name].css") + +/datum/asset/spritesheet_batched/proc/icon_tag(sprite_name) + var/sprite = sprites[sprite_name] + if (!sprite) + return null + var/size_id = sprite[SPR_SIZE] + return "" + +/datum/asset/spritesheet_batched/proc/icon_class_name(sprite_name) + var/sprite = sprites[sprite_name] + if (!sprite) + return null + var/size_id = sprite[SPR_SIZE] + return "[name][size_id] [sprite_name]" + +/** + * Returns the size class (ex design32x32) for a given sprite's icon + * + * Arguments: + * * sprite_name - The sprite to get the size of + */ +/datum/asset/spritesheet_batched/proc/icon_size_id(sprite_name) + var/sprite = sprites[sprite_name] + if (!sprite) + return null + var/size_id = sprite[SPR_SIZE] + return "[name][size_id]" + +#undef SPR_SIZE +#undef SPR_IDX +#undef CACHE_WAIT +#undef CACHE_INVALID +#undef CACHE_VALID + +#undef WHATTHEFUCKAMIDOING_FILE diff --git a/code/modules/asset_cache/iconforge/universal_icon.dm b/code/modules/asset_cache/iconforge/universal_icon.dm new file mode 100644 index 0000000000..3446151c9b --- /dev/null +++ b/code/modules/asset_cache/iconforge/universal_icon.dm @@ -0,0 +1,186 @@ +/// This is intended to replace /icon, allowing rustg to generate icons much faster than DM can at scale. +/// Construct these with the uni_icon() proc, in the same manner as BYOND's icon() proc. +/// Additionally supports a number of transform procs (lowercase, rather than BYOND's uppercase) +/// such as Crop, Scale, and Blend (as blend_icon/blend_color). +/datum/universal_icon + var/icon/icon_file + var/icon_state + var/dir + var/frame + var/datum/icon_transformer/transform + +/// Don't instantiate these yourself, use uni_icon. +/datum/universal_icon/New(icon/icon_file, icon_state="", dir=SOUTH, frame=1, datum/icon_transformer/transform=null, color=null) + #ifdef UNIT_TESTS + // This check is kinda slow and shouldn't fail unless a developer makes a mistake. So it'll get caught in unit tests. + if(!isicon(icon_file) || !isfile(icon_file) || "[icon_file]" == "/icon") + // bad! use 'icons/path_to_dmi.dmi' format only + CRASH("FATAL: universal_icon was provided icon_file: [icon_file] - icons provided to batched spritesheets MUST be DMI files, they cannot be /image, /icon, or other runtime generated icons.") + #endif + src.icon_file = icon_file + src.icon_state = icon_state + src.dir = dir + src.frame = frame + if(isnull(transform) && !isnull(color) && uppertext(color) != "#FFFFFF") + var/datum/icon_transformer/T = new() + if(color) + T.blend_color(color, ICON_MULTIPLY) + src.transform = T + else if(!isnull(transform)) + src.transform = transform + else // null = empty list + src.transform = null + +/datum/universal_icon/proc/copy() + RETURN_TYPE(/datum/universal_icon) + var/datum/universal_icon/new_icon = new(icon_file, icon_state, dir, frame) + if(!isnull(src.transform)) + new_icon.transform = src.transform.copy() + return new_icon + +/datum/universal_icon/proc/blend_color(color, blend_mode) + if(!transform) + transform = new + transform.blend_color(color, blend_mode) + return src + +/datum/universal_icon/proc/blend_icon(datum/universal_icon/icon_object, blend_mode) + if(!transform) + transform = new + transform.blend_icon(icon_object, blend_mode) + return src + +/datum/universal_icon/proc/scale(width, height) + if(!transform) + transform = new + transform.scale(width, height) + return src + +/datum/universal_icon/proc/crop(x1, y1, x2, y2) + if(!transform) + transform = new + transform.crop(x1, y1, x2, y2) + return src + +/datum/universal_icon/proc/to_list() + RETURN_TYPE(/list) + return list("icon_file" = "[icon_file]", "icon_state" = icon_state, "dir" = dir, "frame" = frame, "transform" = !isnull(transform) ? transform.to_list() : list()) + +/proc/universal_icon_from_list(list/input_in) + RETURN_TYPE(/datum/universal_icon) + var/list/input = input_in.Copy() // copy, since icon_transformer_from_list will mutate the list. + return uni_icon(input["icon_file"], input["icon_state"], input["dir"], input["frame"], icon_transformer_from_list(input["transform"])) + +/datum/universal_icon/proc/to_json() + return json_encode(to_list()) + +/datum/universal_icon/proc/to_icon() + RETURN_TYPE(/icon) + var/icon/self = icon(src.icon_file, src.icon_state, dir=src.dir, frame=src.frame) + if(istype(src.transform)) + src.transform.apply(self) + return self + +/datum/icon_transformer + var/list/transforms = null + +/datum/icon_transformer/New() + transforms = list() + +/// Applies the contained set of transforms to an icon +/datum/icon_transformer/proc/apply(icon/target) + RETURN_TYPE(/icon) + for(var/transform in src.transforms) + switch(transform["type"]) + if(RUSTG_ICONFORGE_BLEND_COLOR) + target.Blend(target["color"], target["blend_mode"]) + if(RUSTG_ICONFORGE_BLEND_ICON) + var/datum/universal_icon/icon_object = target["icon"] + if(!istype(icon_object)) + stack_trace("Invalid icon found in icon transformer during apply()! [icon_object]") + continue + target.Blend(icon_object.to_icon(), target["blend_mode"]) + if(RUSTG_ICONFORGE_SCALE) + target.Scale(target["width"], target["height"]) + if(RUSTG_ICONFORGE_CROP) + target.Crop(target["x1"], target["y1"], target["x2"], target["y2"]) + return target + +/datum/icon_transformer/proc/copy() + RETURN_TYPE(/datum/icon_transformer) + var/datum/icon_transformer/new_transformer = new() + new_transformer.transforms = list() + for(var/transform in src.transforms) + new_transformer.transforms += list(deep_copy_list_alt(transform)) + return new_transformer + +/datum/icon_transformer/proc/blend_color(color, blend_mode) + transforms += list(list("type" = RUSTG_ICONFORGE_BLEND_COLOR, "color" = color, "blend_mode" = blend_mode)) + +/datum/icon_transformer/proc/blend_icon(datum/universal_icon/icon_object, blend_mode) + transforms += list(list("type" = RUSTG_ICONFORGE_BLEND_ICON, "icon" = icon_object, "blend_mode" = blend_mode)) + +/datum/icon_transformer/proc/scale(width, height) + transforms += list(list("type" = RUSTG_ICONFORGE_SCALE, "width" = width, "height" = height)) + +/datum/icon_transformer/proc/crop(x1, y1, x2, y2) + transforms += list(list("type" = RUSTG_ICONFORGE_CROP, "x1" = x1, "y1" = y1, "x2" = x2, "y2" = y2)) + +/datum/icon_transformer/proc/to_list() + RETURN_TYPE(/list) + var/list/transforms_out = list() + for(var/transform in src.transforms.Copy()) + var/this_transform = transform + if(transform["type"] == RUSTG_ICONFORGE_BLEND_ICON) + var/datum/universal_icon/icon_object = this_transform["icon"] + if(!istype(icon_object)) + stack_trace("Invalid icon found in icon transformer during to_list()! [icon_object]") + continue + this_transform["icon"] = icon_object.to_list() + transforms_out += list(this_transform) + return transforms_out + +/proc/icon_transformer_from_list(list/input) + RETURN_TYPE(/datum/icon_transformer) + var/list/transforms = list() + for(var/transform in input) + var/this_transform = transform + if(transform["type"] == RUSTG_ICONFORGE_BLEND_ICON) + this_transform["icon"] = universal_icon_from_list(transform["icon"]) + transforms += list(this_transform) + var/datum/icon_transformer/transformer = new() + transformer.transforms = transforms + return transformer + +/// Constructs a transformer, with optional color multiply pre-added. +/proc/color_transform(color=null) + RETURN_TYPE(/datum/icon_transformer) + var/datum/icon_transformer/transform = new() + if(color) + transform.blend_color(color, ICON_MULTIPLY) + return transform + +/// Converts a GAGS item to a universal icon by generating blend operations. +// /proc/gags_to_universal_icon(obj/item/path) +// RETURN_TYPE(/datum/universal_icon) +// if(!ispath(path, /obj/item) || !initial(path.greyscale_config) || !initial(path.greyscale_colors)) +// CRASH("gags_to_universal_icon() received an invalid path!") +// var/datum/greyscale_config/config = initial(path.greyscale_config) +// var/colors = initial(path.greyscale_colors) +// var/datum/universal_icon/entry = SSgreyscale.GetColoredIconEntryByType(config, colors, initial(path.icon_state)) +// return entry + +/// Gets the relevant universal icon for an atom, when displayed in TGUI. (see: icon_state_preview) +/// Supports GAGS items and colored items. +/proc/get_display_icon_for(atom/A) + if (!ispath(A, /atom)) + return FALSE + var/icon_file = initial(A.icon) + var/icon_state = initial(A.icon_state) + // if(ispath(A, /obj/item)) + // var/obj/item/I = A + // if(initial(I.icon_state_preview)) + // icon_state = initial(I.icon_state_preview) + // if(initial(I.greyscale_config) && initial(I.greyscale_colors)) + // return gags_to_universal_icon(I) + return uni_icon(icon_file, icon_state, color=initial(A.color)) diff --git a/code/modules/asset_cache/transports/asset_transport.dm b/code/modules/asset_cache/transports/asset_transport.dm index db19277abf..77daee2a60 100644 --- a/code/modules/asset_cache/transports/asset_transport.dm +++ b/code/modules/asset_cache/transports/asset_transport.dm @@ -1,6 +1,3 @@ -/// When sending mutiple assets, how many before we give the client a quaint little sending resources message -#define ASSET_CACHE_TELL_CLIENT_AMOUNT 8 - /// Base browse_rsc asset transport /datum/asset_transport var/name = "Simple browse_rsc asset transport" @@ -59,6 +56,10 @@ SSassets.cache[asset_name] = ACI return ACI +/// Immediately removes an asset from the asset cache. +/datum/asset_transport/proc/unregister_asset(asset_name) + SSassets.cache[asset_name] = null + SSassets.cache.Remove(null) /// Returns a url for a given asset. /// asset_name - Name of the asset. @@ -66,6 +67,8 @@ /datum/asset_transport/proc/get_asset_url(asset_name, datum/asset_cache_item/asset_cache_item) if (!istype(asset_cache_item)) asset_cache_item = SSassets.cache[asset_name] + if (!istype(asset_cache_item)) + return null // To ensure code that breaks on cdns breaks in local testing, we only // use the normal filename on legacy assets and name space assets. var/keep_local_name = dont_mutate_filenames \ @@ -82,6 +85,9 @@ /// asset_list - A list of asset filenames to be sent to the client. Can optionally be assoicated with the asset's asset_cache_item datum. /// Returns TRUE if any assets were sent. /datum/asset_transport/proc/send_assets(client/client, list/asset_list) +#if defined(UNIT_TESTS) + return +#endif if (!istype(client)) if (ismob(client)) var/mob/M = client @@ -143,7 +149,7 @@ /// Precache files without clogging up the browse() queue, used for passively sending files on connection start. -/datum/asset_transport/proc/send_assets_slow(client/client, list/files, filerate = 6) +/datum/asset_transport/proc/send_assets_slow(client/client, list/files, filerate = SLOW_ASSET_SEND_RATE) var/startingfilerate = filerate for (var/file in files) if (!client) diff --git a/code/modules/asset_cache/transports/webroot_transport.dm b/code/modules/asset_cache/transports/webroot_transport.dm index e3cb33b8fa..d67d097b0b 100644 --- a/code/modules/asset_cache/transports/webroot_transport.dm +++ b/code/modules/asset_cache/transports/webroot_transport.dm @@ -16,7 +16,7 @@ /// We also save it to the CDN webroot at this step instead of waiting for send_assets() /// asset_name - the identifier of the asset /// asset - the actual asset file or an asset_cache_item datum. -/datum/asset_transport/webroot/register_asset(asset_name, asset) +/datum/asset_transport/webroot/register_asset(asset_name, asset, file_hash, dmi_file_path) . = ..() var/datum/asset_cache_item/ACI = . diff --git a/code/modules/food/kitchen/microwave.dm b/code/modules/food/kitchen/microwave.dm index 9cd7ccd1bc..9ebc0c15a5 100644 --- a/code/modules/food/kitchen/microwave.dm +++ b/code/modules/food/kitchen/microwave.dm @@ -253,7 +253,7 @@ /obj/machinery/microwave/ui_assets(mob/user) return list( - get_asset_datum(/datum/asset/spritesheet/kitchen_recipes) + get_asset_datum(/datum/asset/spritesheet_batched/kitchen_recipes) ) /obj/machinery/microwave/tgui_static_data(mob/user) diff --git a/code/modules/tgui/tgui_window.dm b/code/modules/tgui/tgui_window.dm index 9e0a5491ab..3bae28e39c 100644 --- a/code/modules/tgui/tgui_window.dm +++ b/code/modules/tgui/tgui_window.dm @@ -322,7 +322,10 @@ if(istype(asset, /datum/asset/spritesheet)) var/datum/asset/spritesheet/spritesheet = asset send_message("asset/stylesheet", spritesheet.css_filename()) - send_message("asset/mappings", asset.get_url_mappings()) + else if(istype(asset, /datum/asset/spritesheet_batched)) + var/datum/asset/spritesheet_batched/spritesheet = asset + send_message("asset/stylesheet", spritesheet.css_filename()) + send_raw_message(asset.get_serialized_url_mappings()) /** * private diff --git a/config/example/resources.txt b/config/example/resources.txt index f9a536e341..338cbcd408 100644 --- a/config/example/resources.txt +++ b/config/example/resources.txt @@ -31,6 +31,13 @@ ## but enabled on production (the default). #CACHE_ASSETS +## Enables 'smart' asset caching, for assets that support it. +## This is a type of asset cache that automatically invalidates itself based on inputs to the asset generation. +## It lowers the generation cost and is safe to enable on development and production. +## This cache is only cleared by the game or manually, but it shouldn't affect the results. +## This setting is independent of `CACHE_ASSETS`, they do not affect each other whatsoever. +SMART_CACHE_ASSETS + ## If this is uncommented, we will save all associated spritesheet PNGs and CSS files to a folder in the round-specific logs folder. ## Useful for developers to debug potential spritesheet issues to determine where the issue is cropping up (either in DM-side sprite generation or in the TGUI-side display of said spritesheet). ## Will only seek to waste disk space if ran on production. diff --git a/librust_g.so b/librust_g.so index 3deb97e87d..c7810fbf80 100755 Binary files a/librust_g.so and b/librust_g.so differ diff --git a/rust_g.dll b/rust_g.dll index 9656c3e4a6..9b400774fa 100644 Binary files a/rust_g.dll and b/rust_g.dll differ diff --git a/tools/ci/install_rust_g.sh b/tools/ci/install_rust_g.sh new file mode 100644 index 0000000000..c237162d8f --- /dev/null +++ b/tools/ci/install_rust_g.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail + +source dependencies.sh + +mkdir -p ~/.byond/bin +wget -nv -O ~/.byond/bin/librust_g.so "https://github.com/tgstation/rust-g/releases/download/$RUST_G_VERSION/librust_g.so" +chmod +x ~/.byond/bin/librust_g.so +ldd ~/.byond/bin/librust_g.so diff --git a/vorestation.dme b/vorestation.dme index 3ec53b3c04..06fbbee290 100644 --- a/vorestation.dme +++ b/vorestation.dme @@ -38,6 +38,7 @@ #include "code\__defines\appearance.dm" #include "code\__defines\assemblies.dm" #include "code\__defines\assert.dm" +#include "code\__defines\assets.dm" #include "code\__defines\atmos.dm" #include "code\__defines\atoms_movable.dm" #include "code\__defines\belly_modes_vr.dm" @@ -1993,6 +1994,8 @@ #include "code\modules\asset_cache\assets\spritesheets\stacks.dm" #include "code\modules\asset_cache\assets\spritesheets\vending.dm" #include "code\modules\asset_cache\assets\spritesheets\vore.dm" +#include "code\modules\asset_cache\iconforge\batched_spritesheet.dm" +#include "code\modules\asset_cache\iconforge\universal_icon.dm" #include "code\modules\asset_cache\transports\asset_transport.dm" #include "code\modules\asset_cache\transports\webroot_transport.dm" #include "code\modules\awaymissions\bluespaceartillery.dm"