[MIRROR] JSON Logging Refactor (#11623)

Co-authored-by: Selis <12716288+ItsSelis@users.noreply.github.com>
Co-authored-by: Kashargul <144968721+Kashargul@users.noreply.github.com>

code/modules/logging/categories/log_category_admin.dm

@@ -0,0 +1,33 @@
+/datum/log_category/admin
+	category = LOG_CATEGORY_ADMIN
+	config_flag = /datum/config_entry/flag/log_admin
+
+/datum/log_category/admin_dsay
+	category = LOG_CATEGORY_ADMIN_DSAY
+	master_category = /datum/log_category/admin
+	config_flag = /datum/config_entry/flag/log_admin
+
+// private categories //
+
+/datum/log_category/admin_private
+	category = LOG_CATEGORY_ADMIN_PRIVATE
+	config_flag = /datum/config_entry/flag/log_admin
+	secret = TRUE
+
+/datum/log_category/admin_asay
+	category = LOG_CATEGORY_ADMIN_PRIVATE_ASAY
+	master_category = /datum/log_category/admin_private
+	config_flag = /datum/config_entry/flag/log_adminchat
+	secret = TRUE
+
+/datum/log_category/admin_msay
+	category = LOG_CATEGORY_ADMIN_PRIVATE_MSAY
+	master_category = /datum/log_category/admin_private
+	config_flag = /datum/config_entry/flag/log_adminchat
+	secret = TRUE
+
+/datum/log_category/admin_esay
+	category = LOG_CATEGORY_ADMIN_PRIVATE_ESAY
+	master_category = /datum/log_category/admin_private
+	config_flag = /datum/config_entry/flag/log_eventchat
+	secret = TRUE

code/modules/logging/categories/log_category_compats.dm

@@ -0,0 +1,7 @@
+/datum/log_category/game_compat
+	category = LOG_CATEGORY_COMPAT_GAME
+	master_category = /datum/log_category/game
+	config_flag = /datum/config_entry/flag/logging_compat_adminprivate
+
+/datum/config_entry/flag/logging_compat_adminprivate
+	default = FALSE

code/modules/logging/categories/log_category_debug.dm

@@ -0,0 +1,23 @@
+/datum/log_category/debug
+	category = LOG_CATEGORY_DEBUG
+
+/datum/log_category/debug_sql
+	category = LOG_CATEGORY_DEBUG_SQL
+	master_category = /datum/log_category/debug
+
+// This is not in the debug master category on purpose, do not add it
+/datum/log_category/debug_runtime
+	category = LOG_CATEGORY_RUNTIME
+
+/datum/log_category/debug_mapping
+	category = LOG_CATEGORY_DEBUG_MAPPING
+	master_category = /datum/log_category/debug
+
+/datum/log_category/debug_mobtag
+	category = LOG_CATEGORY_DEBUG_MOBTAG
+	master_category = /datum/log_category/debug
+
+/datum/log_category/debug_asset
+	category = LOG_CATEGORY_DEBUG_ASSET
+	config_flag = /datum/config_entry/flag/log_asset
+	master_category = /datum/log_category/debug

code/modules/logging/categories/log_category_game.dm

@@ -0,0 +1,48 @@
+/datum/log_category/game
+	category = LOG_CATEGORY_GAME
+	config_flag = /datum/config_entry/flag/log_game
+
+/datum/log_category/game_vote
+	category = LOG_CATEGORY_GAME_VOTE
+	config_flag = /datum/config_entry/flag/log_vote
+	master_category = /datum/log_category/game
+
+/datum/log_category/game_emote
+	category = LOG_CATEGORY_GAME_EMOTE
+	config_flag = /datum/config_entry/flag/log_emote
+	master_category = /datum/log_category/game
+
+/datum/log_category/game_topic
+	category = LOG_CATEGORY_GAME_TOPIC
+	config_flag = /datum/config_entry/flag/log_world_topic
+	master_category = /datum/log_category/game
+
+/datum/log_category/game_say
+	category = LOG_CATEGORY_GAME_SAY
+	config_flag = /datum/config_entry/flag/log_say
+	master_category = /datum/log_category/game
+
+/datum/log_category/game_whisper
+	category = LOG_CATEGORY_GAME_WHISPER
+	config_flag = /datum/config_entry/flag/log_whisper
+	master_category = /datum/log_category/game
+
+/datum/log_category/game_ooc
+	category = LOG_CATEGORY_GAME_OOC
+	config_flag = /datum/config_entry/flag/log_ooc
+	master_category = /datum/log_category/game
+
+/datum/log_category/game_looc
+	category = LOG_CATEGORY_GAME_LOOC
+	config_flag = /datum/config_entry/flag/log_looc
+	master_category = /datum/log_category/game
+
+/datum/log_category/game_prayer
+	category = LOG_CATEGORY_GAME_PRAYER
+	config_flag = /datum/config_entry/flag/log_prayer
+	master_category = /datum/log_category/game
+
+/datum/log_category/game_access
+	category = LOG_CATEGORY_GAME_ACCESS
+	config_flag = /datum/config_entry/flag/log_access
+	master_category = /datum/log_category/game

code/modules/logging/categories/log_category_href.dm

@@ -0,0 +1,6 @@
+/datum/log_category/href
+	category = LOG_CATEGORY_HREF
+
+/datum/log_category/href_tgui
+	category = LOG_CATEGORY_HREF_TGUI
+	master_category = /datum/log_category/href

code/modules/logging/categories/log_category_internal.dm

@@ -0,0 +1,6 @@
+/datum/log_category/internal
+	category = LOG_CATEGORY_INTERNAL_ERROR
+
+/datum/log_category/internal_unknown_category
+	category = LOG_CATEGORY_INTERNAL_CATEGORY_NOT_FOUND
+	master_category = /datum/log_category/internal

code/modules/logging/categories/log_category_misc.dm

@@ -0,0 +1,20 @@
+/datum/log_category/attack
+	category = LOG_CATEGORY_ATTACK
+	config_flag = /datum/config_entry/flag/log_attack
+
+/datum/log_category/supicious_login
+	category = LOG_CATEGORY_SUSPICIOUS_LOGIN
+	config_flag = /datum/config_entry/flag/log_suspicious_login
+
+/datum/log_category/config
+	category = LOG_CATEGORY_CONFIG
+
+// Logs seperately, printed into on server shutdown to store hard deletes and such
+/datum/log_category/qdel
+	category = LOG_CATEGORY_QDEL
+	// We want this human readable so it's easy to see at a glance
+	entry_flags = ENTRY_USE_DATA_W_READABLE
+
+/datum/log_category/vore
+	category = LOG_CATEGORY_VORE
+	config_flag = /datum/config_entry/flag/log_vore

code/modules/logging/categories/log_category_pda.dm

@@ -0,0 +1,3 @@
+/datum/log_category/pda
+	category = LOG_CATEGORY_PDA
+	config_flag = /datum/config_entry/flag/log_pda

code/modules/logging/log_category.dm

@@ -0,0 +1,70 @@
+/// The main datum that contains all log entries for a category
+/datum/log_category
+	/// The category name
+	var/category
+
+	/// The schema version of this log category.
+	/// Expected format of "Major.Minor.Patch"
+	var/schema_version = LOG_CATEGORY_SCHEMA_VERSION_NOT_SET
+
+	/// The master category that contains this category
+	var/datum/log_category/master_category
+
+	/// Flags to apply to our /datum/log_entry's
+	/// See code/__DEFINES/logging/dm
+	var/entry_flags = NONE
+
+	/// If set this config flag is checked to enable this log category
+	var/config_flag
+
+	/// Whether or not this log should not be publically visible
+	var/secret = FALSE
+
+	/// The list of header information for this category. Used for log file re-initialization
+	var/list/category_header
+
+	/// Whether the readable version of the log message is formatted internally instead of by rustg
+	/// IF YOU CHANGE THIS VERIFY LOGS ARE STILL PARSED CORRECTLY
+	var/internal_formatting = FALSE
+
+	/// List of log entries for this category
+	var/list/entries = list()
+
+	/// Total number of entries this round so far
+	var/entry_count = 0
+
+GENERAL_PROTECT_DATUM(/datum/log_category)
+
+/// Add an entry to this category. It is very important that any data you provide doesn't hold references to anything!
+/datum/log_category/proc/create_entry(message, list/data, list/semver_store)
+	var/datum/log_entry/entry = new(
+		// world state contains raw timestamp
+		timestamp = logger.human_readable_timestamp(),
+		category = category,
+		message = message,
+		flags = entry_flags,
+		data = data,
+		semver_store = semver_store,
+	)
+
+	write_entry(entry)
+	entry_count += 1
+	if(entry_count <= CONFIG_MAX_CACHED_LOG_ENTRIES)
+		entries += entry
+
+/// Allows for category specific file splitting. Needs to accept a null entry for the default file.
+/// If master_category it will always return the output of master_category.get_output_file(entry)
+/datum/log_category/proc/get_output_file(list/entry, extension = "log.json")
+	if(master_category)
+		return master_category.get_output_file(entry, extension)
+	if(secret)
+		return "[GLOB.log_directory]/secret/[category].[extension]"
+	return "[GLOB.log_directory]/[category].[extension]"
+
+/// Writes an entry to the output file(s) for the category
+/datum/log_category/proc/write_entry(datum/log_entry/entry)
+	// config isn't loaded? assume we want human readable logs
+	if(isnull(config) || CONFIG_GET(flag/log_as_human_readable))
+		entry.write_readable_entry_to_file(get_output_file(entry, "log"), format_internally = internal_formatting)
+
+	entry.write_entry_to_file(get_output_file(entry))

code/modules/logging/log_entry.dm

@@ -0,0 +1,127 @@
+
+// Schema version must always be the very last element in the array.
+
+// Current Schema: 1.0.0
+// [timestamp, category, message, data, world_state, semver_store, id, schema_version]
+
+/// A datum which contains log information.
+/datum/log_entry
+	/// Next id to assign to a log entry.
+	var/static/next_id = 0
+
+	/// Unique id of the log entry.
+	var/id
+
+	/// Schema version of the log entry.
+	var/schema_version = "1.0.0"
+
+	/// Unix timestamp of the log entry.
+	var/timestamp
+
+	/// Category of the log entry.
+	var/category
+
+	/// Message of the log entry.
+	var/message
+
+	/// Bitfield that describes how exactly to log stuff exactly
+	/// See code/__DEFINES/logging/dm
+	var/flags = NONE
+
+	/// Data of the log entry; optional.
+	var/list/data
+
+	/// Semver store of the log entry, used to store the schema of data entries
+	var/list/semver_store
+
+GENERAL_PROTECT_DATUM(/datum/log_entry)
+
+/datum/log_entry/New(timestamp, category, message, flags, list/data, list/semver_store)
+	..()
+
+	src.id = next_id++
+	src.timestamp = timestamp
+	src.category = category
+	src.flags = flags
+	src.message = message
+	with_data(data)
+	with_semver_store(semver_store)
+
+/datum/log_entry/proc/with_data(list/data)
+	if(!isnull(data))
+		if(!islist(data))
+			src.data = list("data" = data)
+			stack_trace("Log entry data was not a list, it was [data.type].")
+		else
+			src.data = data
+	return src
+
+/datum/log_entry/proc/with_semver_store(list/semver_store)
+	if(isnull(semver_store))
+		return
+	if(!islist(semver_store))
+		stack_trace("Log entry semver store was not a list, it was [semver_store.type]. We cannot reliably convert it to a list.")
+	else
+		src.semver_store = semver_store
+	return src
+
+/// Converts the log entry to a human-readable string.
+/datum/log_entry/proc/to_readable_text(format = TRUE)
+	var/output = ""
+	if(format)
+		output += "\[[timestamp]\] [uppertext(category)]: [message]"
+	else
+		output += "[uppertext(category)]: [message]"
+
+	if(flags & ENTRY_USE_DATA_W_READABLE)
+		output += json_encode(data, JSON_PRETTY_PRINT)
+	return output
+
+#define MANUAL_JSON_ENTRY(list, key, value) list.Add("\"[key]\":[(!isnull(value)) ? json_encode(value) : "null"]")
+
+/// Converts the log entry to a JSON string.
+/datum/log_entry/proc/to_json_text()
+	// I do not trust byond's json encoder, and need to ensure the order doesn't change.
+	var/list/json_entries = list()
+	MANUAL_JSON_ENTRY(json_entries, LOG_ENTRY_KEY_TIMESTAMP, timestamp)
+	MANUAL_JSON_ENTRY(json_entries, LOG_ENTRY_KEY_CATEGORY, category)
+	MANUAL_JSON_ENTRY(json_entries, LOG_ENTRY_KEY_MESSAGE, message)
+	MANUAL_JSON_ENTRY(json_entries, LOG_ENTRY_KEY_DATA, data)
+	MANUAL_JSON_ENTRY(json_entries, LOG_ENTRY_KEY_WORLD_STATE, world.get_world_state_for_logging())
+	MANUAL_JSON_ENTRY(json_entries, LOG_ENTRY_KEY_SEMVER_STORE, semver_store)
+	MANUAL_JSON_ENTRY(json_entries, LOG_ENTRY_KEY_ID, id)
+	MANUAL_JSON_ENTRY(json_entries, LOG_ENTRY_KEY_SCHEMA_VERSION, schema_version)
+	return "{[json_entries.Join(",")]}"
+
+#undef MANUAL_JSON_ENTRY
+
+#define CHECK_AND_TRY_FILE_ERROR_RECOVERY(file) \
+	var/static/in_error_recovery = FALSE; \
+	if(!fexists(##file)) { \
+		if(in_error_recovery) { \
+			in_error_recovery = FALSE; \
+			CRASH("Failed to error recover log file: [file]"); \
+		}; \
+		in_error_recovery = TRUE; \
+		logger.Log(LOG_CATEGORY_INTERNAL_ERROR, "attempting to perform file error recovery: [file]"); \
+		logger.init_category_file(logger.log_categories[category]); \
+		call(src, __PROC__)(arglist(args)); \
+		return; \
+	}; \
+	in_error_recovery = FALSE;
+
+/// Writes the log entry to a file.
+/datum/log_entry/proc/write_entry_to_file(file)
+	CHECK_AND_TRY_FILE_ERROR_RECOVERY(file)
+	WRITE_LOG_NO_FORMAT(file, "[to_json_text()]\n")
+
+/// Writes the log entry to a file as a human-readable string.
+/datum/log_entry/proc/write_readable_entry_to_file(file, format_internally = TRUE)
+	CHECK_AND_TRY_FILE_ERROR_RECOVERY(file)
+	// If it's being formatted internally we need to manually add a newline
+	if(format_internally)
+		WRITE_LOG_NO_FORMAT(file, "[to_readable_text(format = TRUE)]\n")
+	else
+		WRITE_LOG(file, "[to_readable_text(format = FALSE)]")
+
+#undef CHECK_AND_TRY_FILE_ERROR_RECOVERY

code/modules/logging/log_holder.dm

@@ -0,0 +1,356 @@
+GLOBAL_REAL(logger, /datum/log_holder)
+/**
+ * Main datum to manage logging actions
+ */
+/datum/log_holder
+	/// Round ID, if set, that logging is initialized for
+	var/round_id
+	/// When the log_holder first initialized
+	var/logging_start_timestamp
+
+	/// Associative: category -> datum
+	var/list/datum/log_category/log_categories
+	/// typecache list for categories that exist but are disabled
+	var/list/disabled_categories
+	/// category nesting tree for ui purposes
+	var/list/category_group_tree
+
+	/// list of Log args waiting for processing pending log initialization
+	var/list/waiting_log_calls
+
+	/// Whether or not logging as human readable text is enabled
+	var/human_readable_enabled = FALSE
+
+	/// Cached ui_data
+	var/list/data_cache = list()
+
+	/// Last time the ui_data was updated
+	var/last_data_update = 0
+
+	var/initialized = FALSE
+	var/shutdown = FALSE
+
+GENERAL_PROTECT_DATUM(/datum/log_holder)
+
+ADMIN_VERB(log_viewer_new, R_ADMIN|R_DEBUG, "View Round Logs", "View the rounds logs.", ADMIN_CATEGORY_LOGS)
+	logger.tgui_interact(user.mob)
+
+/datum/log_holder/tgui_interact(mob/user, datum/tgui/ui)
+	if(!check_rights_for(user.client, R_ADMIN))
+		return
+
+	ui = SStgui.try_update_ui(user, src, ui)
+	if(isnull(ui))
+		ui = new(user, src, "LogViewer", "Log Viewer")
+		ui.set_autoupdate(FALSE)
+		ui.open()
+
+/datum/log_holder/tgui_state(mob/user)
+	return ADMIN_STATE(R_ADMIN | R_DEBUG)
+
+/datum/log_holder/tgui_static_data(mob/user)
+	var/list/data = list(
+		"round_id" = GLOB.round_id,
+		"logging_start_timestamp" = logging_start_timestamp,
+	)
+
+	var/list/tree = list()
+	data["tree"] = tree
+	var/list/enabled_categories = list()
+	for(var/enabled in log_categories)
+		enabled_categories += enabled
+	tree["enabled"] = enabled_categories
+
+	var/list/disabled_categories = list()
+	for(var/disabled in src.disabled_categories)
+		disabled_categories += disabled
+	tree["disabled"] = disabled_categories
+
+	return data
+
+/datum/log_holder/tgui_data(mob/user)
+	if(!last_data_update || (world.time - last_data_update) > LOG_UPDATE_TIMEOUT)
+		cache_ui_data()
+	return data_cache
+
+/datum/log_holder/proc/cache_ui_data()
+	var/list/category_map = list()
+	for(var/datum/log_category/category as anything in log_categories)
+		category = log_categories[category]
+		var/list/category_data = list()
+
+		var/list/entries = list()
+		for(var/datum/log_entry/entry as anything in category.entries)
+			entries += list(list(
+				"id" = entry.id,
+				"message" = entry.message,
+				"timestamp" = entry.timestamp,
+				"data" = entry.data,
+				"semver" = entry.semver_store,
+			))
+		category_data["entries"] = entries
+		category_data["entry_count"] = category.entry_count
+
+		category_map[category.category] = category_data
+
+	data_cache.Cut()
+	last_data_update = world.time
+
+	data_cache["categories"] = category_map
+	data_cache["last_data_update"] = last_data_update
+
+/datum/log_holder/tgui_act(action, list/params, datum/tgui/ui, datum/tgui_state/state)
+	. = ..()
+	if(.)
+		return
+
+	switch(action)
+		if("refresh")
+			cache_ui_data()
+			SStgui.update_uis(src)
+			return TRUE
+		else
+			stack_trace("unknown ui_act action [action] for [type]")
+
+/// Assembles basic information for logging, creating the log category datums and checking for config flags as required
+/datum/log_holder/proc/init_logging()
+	if(initialized)
+		CRASH("Attempted to call init_logging twice!")
+
+	round_id = GLOB.round_id
+	logging_start_timestamp = rustg_unix_timestamp()
+	log_categories = list()
+	disabled_categories = list()
+
+	human_readable_enabled = CONFIG_GET(flag/log_as_human_readable)
+
+	category_group_tree = assemble_log_category_tree()
+	var/config_flag
+	for(var/datum/log_category/master_category as anything in category_group_tree)
+		var/list/sub_categories = category_group_tree[master_category]
+		sub_categories = sub_categories.Copy()
+		for(var/datum/log_category/sub_category as anything in sub_categories)
+			config_flag = initial(sub_category.config_flag)
+			if(config_flag && !config.Get(config_flag))
+				disabled_categories[initial(sub_category.category)] = TRUE
+				sub_categories -= sub_category
+				continue
+
+		config_flag = initial(master_category.config_flag)
+		if(config_flag && !config.Get(config_flag))
+			disabled_categories[initial(master_category.category)] = TRUE
+			if(!length(sub_categories))
+				continue
+		// enabled, or any of the sub categories are enabled
+		init_log_category(master_category, sub_categories)
+
+	initialized = TRUE
+
+	// process any waiting log calls and then cut the list
+	for(var/list/arg_list as anything in waiting_log_calls)
+		Log(arglist(arg_list))
+	waiting_log_calls?.Cut()
+
+	if(fexists(GLOB.config_error_log))
+		fcopy(GLOB.config_error_log, "[GLOB.log_directory]/config_error.log")
+		fdel(GLOB.config_error_log)
+
+	world._initialize_log_files()
+
+/// Tells the log_holder to not allow any more logging to be done, and dumps all categories to their json file
+/datum/log_holder/proc/shutdown_logging()
+	if(shutdown)
+		CRASH("Attempted to call shutdown_logging twice!")
+	shutdown = TRUE
+
+/// Iterates over all log category types to assemble them into a tree of main category -> (sub category)[] while also checking for loops and sanity errors
+/datum/log_holder/proc/assemble_log_category_tree()
+	var/static/list/category_tree
+	if(category_tree)
+		return category_tree
+
+	category_tree = list()
+	var/list/all_types = subtypesof(/datum/log_category)
+	var/list/known_categories = list()
+	var/list/sub_categories = list()
+
+	// Assemble the master categories
+	for(var/datum/log_category/category_type as anything in all_types)
+		var/category = initial(category_type.category)
+		if(category in known_categories)
+			stack_trace("log category type '[category_type]' has duplicate category '[category]', skipping")
+			continue
+
+		if(!initial(category_type.schema_version))
+			stack_trace("log category type '[category_type]' does not have a valid schema version, skipping")
+			continue
+
+		var/master_category = initial(category_type.master_category)
+		if(master_category)
+			sub_categories[master_category] += list(category_type)
+			continue
+		category_tree[category_type] = list()
+
+	// Sort the sub categories
+	for(var/datum/log_category/master as anything in sub_categories)
+		if(!(master in category_tree))
+			stack_trace("log category [master] is an invalid master category as it's a sub category")
+			continue
+		for(var/datum/log_category/sub_category as anything in sub_categories[master])
+			if(initial(sub_category.secret) != initial(master.secret))
+				stack_trace("log category [sub_category] has a secret status that differs from its master category [master]")
+			category_tree[master] += list(sub_category)
+
+	return category_tree
+
+/// Log entry header used to mark a file is being reset
+#define LOG_CATEGORY_RESET_FILE_MARKER "{\"LOG FILE RESET -- THIS IS AN ERROR\"}"
+#define LOG_CATEGORY_RESET_FILE_MARKER_READABLE "LOG FILE RESET -- THIS IS AN ERROR"
+/// Gets a recovery file for the given path. Caches the last known recovery path for each path.
+/datum/log_holder/proc/get_recovery_file_for(path)
+	var/static/cache
+	if(isnull(cache))
+		cache = list()
+
+	var/count = cache[path] || 0
+	while(fexists("[path].rec[count]"))
+		count++
+	cache[path] = count
+
+	return "[path].rec[count]"
+
+/// Sets up the given category's file and header.
+/datum/log_holder/proc/init_category_file(datum/log_category/category)
+	var/file_path = category.get_output_file(null)
+	if(fexists(file_path)) // already exists? implant a reset marker
+		rustg_file_append(LOG_CATEGORY_RESET_FILE_MARKER, file_path)
+		fcopy(file_path, get_recovery_file_for(file_path))
+	rustg_file_write("[json_encode(category.category_header)]\n", file_path)
+
+	if(!human_readable_enabled)
+		return
+
+	file_path = category.get_output_file(null, "log")
+	if(fexists(file_path))
+		rustg_file_append(LOG_CATEGORY_RESET_FILE_MARKER_READABLE, file_path)
+		fcopy(file_path, get_recovery_file_for(file_path))
+	rustg_file_write("\[[human_readable_timestamp()]\] Starting up round ID [round_id].\n - -------------------------\n", file_path)
+
+#undef LOG_CATEGORY_RESET_FILE_MARKER
+#undef LOG_CATEGORY_RESET_FILE_MARKER_READABLE
+
+/// Initializes the given log category and populates the list of contained categories based on the sub category list
+/datum/log_holder/proc/init_log_category(datum/log_category/category_type, list/datum/log_category/sub_categories)
+	var/datum/log_category/category_instance = new category_type
+
+	var/list/contained_categories = list()
+	for(var/datum/log_category/sub_category as anything in sub_categories)
+		sub_category = new sub_category
+		var/sub_category_actual = sub_category.category
+		sub_category.master_category = category_instance
+		log_categories[sub_category_actual] = sub_category
+
+		if(!semver_to_list(sub_category.schema_version))
+			stack_trace("log category [sub_category_actual] has an invalid schema version '[sub_category.schema_version]'")
+			sub_category.schema_version = LOG_CATEGORY_SCHEMA_VERSION_NOT_SET
+
+		contained_categories += sub_category_actual
+
+	log_categories[category_instance.category] = category_instance
+
+	if(!semver_to_list(category_instance.schema_version))
+		stack_trace("log category [category_instance.category] has an invalid schema version '[category_instance.schema_version]'")
+		category_instance.schema_version = LOG_CATEGORY_SCHEMA_VERSION_NOT_SET
+
+	contained_categories += category_instance.category
+
+	var/list/category_header = list(
+		LOG_HEADER_INIT_TIMESTAMP = logging_start_timestamp,
+		LOG_HEADER_ROUND_ID = GLOB.round_id,
+		LOG_HEADER_SECRET = category_instance.secret,
+		LOG_HEADER_CATEGORY_LIST = contained_categories,
+		LOG_HEADER_CATEGORY = category_instance.category,
+	)
+
+	category_instance.category_header = category_header
+	init_category_file(category_instance, category_header)
+
+/datum/log_holder/proc/human_readable_timestamp()
+	return rustg_formatted_timestamp("%Y-%m-%d %H:%M:%S%.3f")
+
+/// Adds an entry to the given category, if the category is disabled it will not be logged.
+/// If the category does not exist, we will CRASH and log to the error category.
+/// the data list is optional and will be recursively json serialized.
+/datum/log_holder/proc/Log(category, message, list/data)
+	// This is Log because log is a byond internal proc
+
+	// do not include the message because these go into the runtime log and we might be secret!
+	if(!istext(message))
+		message = "[message]"
+		stack_trace("Logging with a non-text message")
+
+	if(!category)
+		category = LOG_CATEGORY_INTERNAL_CATEGORY_NOT_FOUND
+		stack_trace("Logging with a null or empty category")
+
+	if(data && !islist(data))
+		data = list("data" = data)
+		stack_trace("Logging with data this is not a list, it will be converted to a list with a single key 'data'")
+
+	if(!initialized) // we are initialized during /world/proc/SetupLogging which is called in /world/New
+		waiting_log_calls += list(list(category, message, data))
+		return
+
+	if(disabled_categories[category])
+		return
+
+	var/datum/log_category/log_category = log_categories[category]
+	if(!log_category)
+		Log(LOG_CATEGORY_INTERNAL_CATEGORY_NOT_FOUND, message, data)
+		CRASH("Attempted to log to a category that doesn't exist! [category]")
+
+	var/list/semver_store = null
+	if(length(data))
+		semver_store = list()
+		data = recursive_jsonify(data, semver_store)
+	log_category.create_entry(message, data, semver_store)
+
+/// Recursively converts an associative list of datums into their jsonified(list) form
+/datum/log_holder/proc/recursive_jsonify(list/data_list, list/semvers)
+	if(isnull(data_list))
+		return null
+
+	var/list/jsonified_list = list()
+	for(var/key in data_list)
+		var/datum/data = data_list[key]
+
+		if(isnull(data))
+			pass() // nulls are allowed
+
+		else if(islist(data))
+			data = recursive_jsonify(data, semvers)
+
+		else if(isdatum(data))
+			var/list/options_list = list(
+				SCHEMA_VERSION = LOG_CATEGORY_SCHEMA_VERSION_NOT_SET,
+			)
+
+			var/list/serialization_data = data.serialize_list(options_list, semvers)
+			var/current_semver = semvers[data.type]
+			if(!semver_to_list(current_semver))
+				stack_trace("serialization of data had an invalid semver")
+				semvers[data.type] = LOG_CATEGORY_SCHEMA_VERSION_NOT_SET
+
+			if(!length(serialization_data)) // serialize_list wasn't implemented, and errored
+				stack_trace("serialization data was empty")
+				continue
+
+			data = recursive_jsonify(serialization_data, semvers)
+
+		if(islist(data) && !length(data))
+			stack_trace("recursive_jsonify got an empty list after serialization")
+			continue
+
+		jsonified_list[key] = data
+
+	return jsonified_list