diff --git a/aurorastation.dme b/aurorastation.dme index ab54338032d..ce703c65f2f 100644 --- a/aurorastation.dme +++ b/aurorastation.dme @@ -277,6 +277,7 @@ #include "code\__HELPERS\logging\subsystems\mapfinalization.dm" #include "code\__HELPERS\logging\subsystems\odyssey.dm" #include "code\__HELPERS\logging\subsystems\persistence.dm" +#include "code\__HELPERS\logging\subsystems\sentry.dm" #include "code\__HELPERS\logging\subsystems\ZAS.dm" #include "code\__HELPERS\paths\jps.dm" #include "code\__HELPERS\paths\path.dm" @@ -389,6 +390,7 @@ #include "code\controllers\subsystems\records.dm" #include "code\controllers\subsystems\responseteam.dm" #include "code\controllers\subsystems\runechat.dm" +#include "code\controllers\subsystems\sentry.dm" #include "code\controllers\subsystems\skills.dm" #include "code\controllers\subsystems\skybox.dm" #include "code\controllers\subsystems\sound_loops.dm" @@ -3780,6 +3782,7 @@ #include "code\modules\research\xenoarchaeology\tools\tools_pickaxe.dm" #include "code\modules\security levels\keycard authentication.dm" #include "code\modules\security levels\security levels.dm" +#include "code\modules\sentry\sentry_event.dm" #include "code\modules\shareddream\area.dm" #include "code\modules\shareddream\brainghost.dm" #include "code\modules\shareddream\dream_entry.dm" diff --git a/code/__DEFINES/subsystems.dm b/code/__DEFINES/subsystems.dm index 3b55776d7c3..ba0e7057a45 100644 --- a/code/__DEFINES/subsystems.dm +++ b/code/__DEFINES/subsystems.dm @@ -225,6 +225,7 @@ #define INIT_ORDER_DBCORE 95 #define INIT_ORDER_AUTH 94 //Admin permissions should be loaded early on #define INIT_ORDER_SOUNDS 83 +#define INIT_ORDER_SENTRY 79 #define INIT_ORDER_DISCORD 78 #define INIT_ORDER_JOBS 65 // Must init before atoms, to set up properly the dynamic job lists. #define INIT_ORDER_AI_CONTROLLERS 55 //So the controller can get the ref diff --git a/code/__HELPERS/logging/_logging.dm b/code/__HELPERS/logging/_logging.dm index 9a9711dea60..0920ad53d44 100644 --- a/code/__HELPERS/logging/_logging.dm +++ b/code/__HELPERS/logging/_logging.dm @@ -98,6 +98,9 @@ GLOBAL_LIST_INIT(testing_global_profiler, list("_PROFILE_NAME" = "Global")) log_runtime("========= END OF RUNTIME DUMP =========") + if(SSsentry) + SSsentry.capture_exception(e, null, usr) + //pretty print a direction bitflag, can be useful for debugging. /proc/print_dir(var/dir) diff --git a/code/__HELPERS/logging/debug.dm b/code/__HELPERS/logging/debug.dm index 44ce5274dc5..255e0150def 100644 --- a/code/__HELPERS/logging/debug.dm +++ b/code/__HELPERS/logging/debug.dm @@ -63,6 +63,8 @@ /// Logging for DB errors /proc/log_sql(text) WRITE_LOG(GLOB.config.logfiles["sql_error_log"], "SQL: [text]") + if(SSsentry) + SSsentry.capture_message(text, "error", "sql") /// Logging for world/Topic /proc/_log_topic(text) diff --git a/code/__HELPERS/logging/subsystems/dbcore.dm b/code/__HELPERS/logging/subsystems/dbcore.dm index e4e3f7d4f72..94e0067b2e2 100644 --- a/code/__HELPERS/logging/subsystems/dbcore.dm +++ b/code/__HELPERS/logging/subsystems/dbcore.dm @@ -1,3 +1,5 @@ /proc/log_subsystem_dbcore(text) if (GLOB.config?.logsettings["log_subsystems_dbcore"]) WRITE_LOG(GLOB.config.logfiles["world_subsystems_dbcore_log"], "SSdbcore: [text]") + if(SSsentry) + SSsentry.capture_message(text, "error", "sql") diff --git a/code/__HELPERS/logging/subsystems/sentry.dm b/code/__HELPERS/logging/subsystems/sentry.dm new file mode 100644 index 00000000000..d7b3470a992 --- /dev/null +++ b/code/__HELPERS/logging/subsystems/sentry.dm @@ -0,0 +1,4 @@ +/// Logging for the Sentry subsystem itself (not the events it forwards). +/proc/log_subsystem_sentry(text) + if (GLOB.config?.logsettings["log_subsystems_sentry"]) + WRITE_LOG(GLOB.config.logfiles["world_subsystems_sentry_log"], "SSsentry: [text]") diff --git a/code/controllers/configuration.dm b/code/controllers/configuration.dm index 80360837fcc..07baf8b089d 100644 --- a/code/controllers/configuration.dm +++ b/code/controllers/configuration.dm @@ -70,6 +70,7 @@ GLOBAL_LIST_EMPTY(gamemode_cache) "log_subsystems_zas" = FALSE, // ZAS "log_subsystems_zas_debug" = FALSE, // ZAS debug "log_subsystems_http" = TRUE, //HTTP Log + "log_subsystems_sentry" = TRUE, //Sentry subsystem self-diagnostics /*#### MODULES ####*/ @@ -129,6 +130,7 @@ GLOBAL_LIST_EMPTY(gamemode_cache) "world_subsystems_zas" = "subsystems/zas.log", "world_subsystems_zas_debug" = "subsystems/zas.log", "world_subsystems_http" = "subsystems/http.log", + "world_subsystems_sentry_log" = "subsystems/sentry.log", /*#### MODULES ####*/ @@ -1188,6 +1190,41 @@ GENERAL_PROTECT_DATUM(/datum/configuration) SSdiscord.alert_visibility = TRUE else log_config("Unknown setting in discord configuration: '[name]'") + + else if (type == "sentry") + if (!SSsentry) + log_config("Sentry: Attempted to read config/sentry.txt before SSsentry was initialized.") + return + + switch (name) + if ("enabled") + SSsentry._config_enabled_flag = TRUE + if ("dsn") + if (!SSsentry.parse_dsn(value)) + log_config("Sentry: Invalid DSN '[value]' in config/sentry.txt") + if ("environment") + SSsentry.environment = value + if ("server_name") + SSsentry.server_name = value + if ("capture_runtimes") + SSsentry.capture_runtimes = text2num(value) ? TRUE : FALSE + if ("capture_sql") + SSsentry.capture_sql = text2num(value) ? TRUE : FALSE + if ("capture_hard_dels") + SSsentry.capture_hard_dels = text2num(value) ? TRUE : FALSE + if ("capture_profiler") + SSsentry.capture_profiler = text2num(value) ? TRUE : FALSE + if ("profiler_top_n") + SSsentry.profiler_top_n = text2num(value) + if ("dedup_window") + SSsentry.dedup_window = text2num(value) + if ("scrub_ckeys") + SSsentry.scrub_ckeys = text2num(value) ? TRUE : FALSE + if ("max_consecutive_failures") + SSsentry.max_consecutive_failures = text2num(value) + else + log_config("Sentry: Unknown setting '[name]' in config/sentry.txt") + load_logging_config() load_away_sites_config() load_exoplanets_config() diff --git a/code/controllers/subsystems/garbage.dm b/code/controllers/subsystems/garbage.dm index 2fb5dea2b4d..37e9b91fa51 100644 --- a/code/controllers/subsystems/garbage.dm +++ b/code/controllers/subsystems/garbage.dm @@ -312,6 +312,14 @@ SUBSYSTEM_DEF(garbage) log_game("Error: [type]([refID]) took longer than [threshold] seconds to delete (took [round(time/10, 0.1)] seconds to delete)") message_admins("Error: [type]([refID]) took longer than [threshold] seconds to delete (took [round(time/10, 0.1)] seconds to delete).") type_info.qdel_flags |= QDEL_ITEM_ADMINS_WARNED + if(SSsentry) + SSsentry.capture_message( + "Hard delete overrun: [type] ([refID])", + "warning", + "garbage", + tags = list("datum_type" = "[type]"), + extra = list("ref_id" = refID, "time_ms" = tick_usage) + ) type_info.hard_deletes_over_threshold++ var/overrun_limit = 0 // Used to be CONFIG_GET(number/hard_deletes_overrun_limit) if (overrun_limit && type_info.hard_deletes_over_threshold >= overrun_limit) diff --git a/code/controllers/subsystems/profiler.dm b/code/controllers/subsystems/profiler.dm index 781b849b3d3..f7c92e18e2e 100644 --- a/code/controllers/subsystems/profiler.dm +++ b/code/controllers/subsystems/profiler.dm @@ -72,6 +72,9 @@ SUBSYSTEM_DEF(profiler) WRITE_FILE(sendmaps_file, current_sendmaps_data) write_cost = MC_AVERAGE(write_cost, TICK_DELTA_TO_MS(TICK_USAGE_REAL - timer)) + if(SSsentry) + SSsentry.capture_profiler_dump(current_profile_data, fetch_cost, write_cost) + #undef PROFILER_FILENAME #undef SENDMAPS_FILENAME #undef PROFILER_PATH diff --git a/code/controllers/subsystems/sentry.dm b/code/controllers/subsystems/sentry.dm new file mode 100644 index 00000000000..329755aa986 --- /dev/null +++ b/code/controllers/subsystems/sentry.dm @@ -0,0 +1,382 @@ +/** + * # SSsentry — Sentry.io Integration + * + * Captures runtime exceptions, SQL errors, and hard-delete overruns and + * forwards them to sentry.io as structured events via the Sentry Envelope API. + * + * ## Setup + * Copy config/example/sentry.txt to config/sentry.txt, set DSN and uncomment ENABLED. + * The subsystem is off by default; it only activates when both ENABLED and DSN are present. + * + * ## Capture sites (wired elsewhere) + * - world/Error() → runtime exceptions (error_handler.dm) + * - log_exception() → caught exceptions (_logging.dm) + * - log_sql() → DB / ARGPARSE errors (debug.dm) + * - SSgarbage/HardDelete() → over-threshold hard dels (garbage.dm) + * + * ## Kill-switch + * After max_consecutive_failures HTTP errors the subsystem sets active = FALSE + * and stops sending. An admin can reset it in-round via VV (set active = TRUE). + */ + +SUBSYSTEM_DEF(sentry) + name = "Sentry" + init_order = INIT_ORDER_SENTRY + flags = SS_NO_FIRE | SS_NO_DISPLAY + + // ---- config flags ---- + /// Master on/off switch. Set to TRUE when ENABLED + valid DSN are both present. + var/enabled = FALSE + /// Runtime kill-switch: flipped to FALSE after too many consecutive HTTP failures. + var/active = TRUE + /// Capture uncaught and caught runtime exceptions. + var/capture_runtimes = TRUE + /// Capture SQL errors and ARGPARSE failures. + var/capture_sql = TRUE + /// Capture hard-delete overruns. + var/capture_hard_dels = TRUE + /// Capture per-window BYOND profiler dumps as events with the raw JSON attached. Off by default — payloads are large. + var/capture_profiler = FALSE + /// How many of the hottest procs to include in the event extras alongside the attachment. + var/profiler_top_n = 20 + /// Hash ckeys with md5 before sending instead of sending them in the clear. + var/scrub_ckeys = TRUE + + // ---- DSN fields (populated by _parse_dsn) ---- + var/dsn_key = "" + var/dsn_host = "" + var/dsn_project_id = "" + /// Fully constructed envelope POST URL built from the DSN. + var/endpoint_url = "" + + // ---- event context ---- + /// Maps to Sentry's "environment" field (e.g. "production", "staging"). + var/environment = "production" + /// Maps to Sentry's "release" field. Populated from git HEAD SHA at init. + var/release = "" + /// Maps to Sentry's "server_name" field. + var/server_name = "" + + // ---- runtime state ---- + /// Per-round dedup table: erroruid (file:line) → world.time of last send. + var/list/sent_this_round + /// How long (in deciseconds of game time) to suppress duplicate file:line events. Defaults to 600 (60 s). + var/dedup_window = 600 + /// Count of consecutive HTTP failures. Reset on any success. + var/consecutive_failures = 0 + /// Kill-switch threshold. + var/max_consecutive_failures = 10 + + // Internal flag used during config loading — tracks whether ENABLED was seen. + var/_config_enabled_flag = FALSE + +// Hide DSN key from VV to avoid credential exposure. +/datum/controller/subsystem/sentry/can_vv_get(var_name) + if (var_name == NAMEOF(src, dsn_key)) + return FALSE + return ..() + +/datum/controller/subsystem/sentry/vv_edit_var(var_name, var_value) + if (var_name == NAMEOF(src, dsn_key)) + return FALSE + return ..() + +/datum/controller/subsystem/sentry/Initialize(timeofday) + sent_this_round = list() + server_name = GLOB.config?.server_name || "" + + if (fexists("config/sentry.txt")) + GLOB.config.load("config/sentry.txt", "sentry") + else + log_subsystem_sentry("config/sentry.txt not found — disabled.") + return SS_INIT_NO_NEED + + if (!_config_enabled_flag) + log_subsystem_sentry("ENABLED not set in config/sentry.txt — disabled.") + return SS_INIT_NO_NEED + + if (!endpoint_url) + log_subsystem_sentry("Valid DSN not found in config/sentry.txt — disabled.") + return SS_INIT_NO_NEED + + enabled = TRUE + release = rustg_git_revparse("HEAD") + log_subsystem_sentry("Initialized. env=[environment] release=[release] project=[dsn_project_id] host=[dsn_host]") + return SS_INIT_SUCCESS + +/datum/controller/subsystem/sentry/stat_entry(msg) + msg = "enabled=[enabled] active=[active] failures=[consecutive_failures]" + return ..() + +// --------------------------------------------------------------------------- +// DSN parsing (called from configuration.dm type == "sentry" handler) +// --------------------------------------------------------------------------- + +/** + * Parse a Sentry DSN and populate dsn_key, dsn_host, dsn_project_id, endpoint_url. + * + * Expected format: https://KEY@HOST/PROJECT_ID + * Returns TRUE on success, FALSE on malformed input. + */ +/datum/controller/subsystem/sentry/proc/parse_dsn(dsn) + if (!dsn || dsn == "") + return FALSE + + var/rest = dsn + var/protocol = "https" + + if (findtext(rest, "https://") == 1) + rest = copytext(rest, 9) + else if (findtext(rest, "http://") == 1) + rest = copytext(rest, 8) + protocol = "http" + else + return FALSE + + var/at_pos = findtext(rest, "@") + if (!at_pos) + return FALSE + + dsn_key = copytext(rest, 1, at_pos) + rest = copytext(rest, at_pos + 1) + + var/slash_pos = findtext(rest, "/") + if (!slash_pos) + return FALSE + + dsn_host = copytext(rest, 1, slash_pos) + dsn_project_id = copytext(rest, slash_pos + 1) + endpoint_url = "[protocol]://[dsn_host]/api/[dsn_project_id]/envelope/" + return TRUE + +// --------------------------------------------------------------------------- +// Public capture API +// --------------------------------------------------------------------------- + +/** + * Capture an uncaught or caught runtime exception. + * + * Call from world/Error() and log_exception(). + * + * Arguments: + * * e — the /exception datum + * * desclines — optional list of extra context lines from error_handler + * * usr_ref — usr at time of exception (may be null) + */ +/datum/controller/subsystem/sentry/proc/capture_exception(exception/e, list/desclines, usr_ref) + SHOULD_NOT_SLEEP(TRUE) + if (!enabled || !active || !capture_runtimes || isnull(e)) + return + + var/erroruid = "[e.file]:[e.line]" + // Dedup: skip if the same file:line was already sent within dedup_window deciseconds of game time. + if (sent_this_round[erroruid] && (world.time - sent_this_round[erroruid]) < dedup_window) + return + sent_this_round[erroruid] = world.time + + var/datum/sentry_event/ev = new() + ev.level = "error" + ev.logger = "world.Error" + ev.set_exception(e, desclines) + + if (!isnull(usr_ref)) + if (scrub_ckeys) + ev.set_usr_scrubbed(usr_ref) + else + ev.set_usr(usr_ref) + + _send_event(ev) + +/** + * Capture a message-style event (SQL error, hard delete, etc.). + * + * Arguments: + * * message — human-readable description + * * level — Sentry severity: "error", "warning", "info" + * * logger — logical name shown in Sentry (e.g. "sql", "garbage") + * * tags — optional assoc list of short indexed tags + * * extra — optional assoc list of non-indexed context + */ +/datum/controller/subsystem/sentry/proc/capture_message(message, level = "error", logger = "game", list/tags, list/extra) + SHOULD_NOT_SLEEP(TRUE) + if (!enabled || !active) + return + + var/datum/sentry_event/ev = new() + ev.level = level + ev.logger = logger + ev.set_message(message) + + if (tags) + for (var/k in tags) + ev.tags[k] = tags[k] + if (extra) + for (var/k in extra) + ev.extra[k] = extra[k] + + _send_event(ev) + +/** + * Capture a BYOND profiler dump as a Sentry event with the raw JSON attached. + * + * Sentry's Profiling product expects sample-based stack frames; BYOND's profiler + * is aggregated per-proc, so this ships as a regular event (logger="profiler", + * level="info") with the top-N hottest procs in `extra` and the full JSON + * payload as an event attachment for offline inspection. + * + * Arguments: + * * profile_json — raw JSON string from world.Profile(PROFILE_REFRESH, format="json") + * * fetch_ms — SSprofiler fetch_cost at time of dump + * * write_ms — SSprofiler write_cost at time of dump + */ +/datum/controller/subsystem/sentry/proc/capture_profiler_dump(profile_json, fetch_ms, write_ms) + SHOULD_NOT_SLEEP(TRUE) + if (!enabled || !active || !capture_profiler) + return + if (!profile_json || !length(profile_json)) + return + + var/list/parsed = json_decode(profile_json) + if (!islist(parsed)) + return + + var/list/top = _profiler_top_n(parsed, profiler_top_n) + + var/datum/sentry_event/ev = new() + ev.level = "info" + ev.logger = "profiler" + ev.set_message("Profiler dump (procs=[length(parsed)] top=[length(top)])") + ev.tags["fetch_ms"] = "[round(fetch_ms, 0.1)]" + ev.tags["write_ms"] = "[round(write_ms, 0.1)]" + ev.tags["proc_count"] = "[length(parsed)]" + ev.extra["top_procs"] = top + + _send_event(ev, profile_json, "profile.json", "application/json") + +/// Return the top-N profiler entries by self-time, descending. Pass-through fields so format changes don't break. +/datum/controller/subsystem/sentry/proc/_profiler_top_n(list/entries, n) + if (!islist(entries) || !length(entries)) + return list() + var/list/sorted = entries.Copy() + sortTim(sorted, GLOBAL_PROC_REF(cmp_sentry_profiler_self_desc)) + if (length(sorted) > n) + sorted.Cut(n + 1) + return sorted + +/proc/cmp_sentry_profiler_self_desc(list/A, list/B) + var/a_self = islist(A) ? A["self"] : 0 + var/b_self = islist(B) ? B["self"] : 0 + return b_self - a_self + +// --------------------------------------------------------------------------- +// Internal dispatch +// --------------------------------------------------------------------------- + +/datum/controller/subsystem/sentry/proc/_send_event(datum/sentry_event/ev, attachment_body = null, attachment_filename = null, attachment_content_type = null) + SHOULD_NOT_SLEEP(TRUE) + PRIVATE_PROC(TRUE) + + if (!enabled || !active) + return + + // Stamp common context. + ev.tags["round_id"] = "[GLOB.round_id]" + ev.tags["byond_version"] = "[world.version]" + ev.environment = environment + ev.server_name = server_name + ev.release = release + + var/envelope_body = _build_envelope(ev, attachment_body, attachment_filename, attachment_content_type) + if (!envelope_body) + return + + var/list/req_headers = list( + "Content-Type" = "application/x-sentry-envelope", + "X-Sentry-Auth" = "Sentry sentry_version=7,sentry_key=[dsn_key],sentry_client=aurora.sentry/1.0" + ) + + if (!isnull(SShttp)) + var/datum/callback/cb = CALLBACK(src, PROC_REF(_on_response)) + SShttp.create_async_request(RUSTG_HTTP_METHOD_POST, endpoint_url, envelope_body, req_headers, cb) + else + // SShttp not yet available (very early runtime) — fire-and-forget directly via rust_g. + rustg_http_request_async(RUSTG_HTTP_METHOD_POST, endpoint_url, envelope_body, json_encode(req_headers), null) + +/datum/controller/subsystem/sentry/proc/_on_response(datum/http_response/res) + PRIVATE_PROC(TRUE) + + if (!res || res.errored || (res.status_code && res.status_code >= 400)) + consecutive_failures++ + if (consecutive_failures >= max_consecutive_failures) + active = FALSE + var/last_err = res ? (res.errored ? "[res.error]" : "HTTP [res.status_code]") : "null response" + log_subsystem_sentry("Kill-switch triggered after [consecutive_failures] consecutive failures. Last: [last_err]") + admin_notice(SPAN_DANGER("Sentry kill-switch triggered after [consecutive_failures] consecutive HTTP failures ([last_err]). \ + Event reporting is disabled. Reset via VV: SSsentry → active = TRUE."), R_ADMIN|R_MOD|R_DEV) + return + + consecutive_failures = 0 + +// --------------------------------------------------------------------------- +// Envelope serialization +// --------------------------------------------------------------------------- + +/** + * Build a Sentry envelope: header + event item, plus an optional attachment item. + * + * Layout: + * {envelope-header}\n + * {event-item-header}\n + * {event-payload}\n + * [{attachment-item-header}\n{attachment-payload}\n] ← only when attachment_body set + */ +/datum/controller/subsystem/sentry/proc/_build_envelope(datum/sentry_event/ev, attachment_body = null, attachment_filename = null, attachment_content_type = null) + PRIVATE_PROC(TRUE) + + ev.event_id = _generate_event_id() + ev.timestamp = _iso8601_now() + + var/payload_json = json_encode(ev.to_list()) + if (!payload_json) + return null + + var/list/env_header = list( + "event_id" = ev.event_id, + "sent_at" = ev.timestamp, + "sdk" = list("name" = "aurora.sentry", "version" = "1.0") + ) + var/list/item_header = list( + "type" = "event", + "content_type" = "application/json", + "length" = length(payload_json) + ) + + var/envelope = "[json_encode(env_header)]\n[json_encode(item_header)]\n[payload_json]\n" + + if (attachment_body && length(attachment_body)) + var/list/att_header = list( + "type" = "attachment", + "length" = length(attachment_body), + "content_type" = attachment_content_type || "application/octet-stream", + "filename" = attachment_filename || "attachment.bin", + "attachment_type" = "event.attachment" + ) + envelope += "[json_encode(att_header)]\n[attachment_body]\n" + + return envelope + +/// Generate a pseudo-random 32-char hex event ID via md5. +/datum/controller/subsystem/sentry/proc/_generate_event_id() + PRIVATE_PROC(TRUE) + + return md5("[world.time][world.realtime][rand(0, 9999999)]") + +/// Return current time as ISO-8601 UTC string (YYYY-MM-DDTHH:MM:SSZ). +/datum/controller/subsystem/sentry/proc/_iso8601_now() + PRIVATE_PROC(TRUE) + + var/t = time2text(world.realtime, "YYYY-MM-DD hh:mm:ss") + var/space = findtext(t, " ") + if (space) + return "[copytext(t, 1, space)]T[copytext(t, space + 1)]Z" + return t diff --git a/code/defines/procs/dbcore.dm b/code/defines/procs/dbcore.dm index 820f7310c95..6f1e965fac7 100644 --- a/code/defines/procs/dbcore.dm +++ b/code/defines/procs/dbcore.dm @@ -195,8 +195,7 @@ Delayed insert mode was removed in mysql 7 and only works with MyISAM type table var/error = ErrorMsg() if (error) - log_sql("SQL Error: '[error]'") - log_sql(" - during query: [sql_query]") + log_sql("SQL Error: '[error]' during query: [sql_query]") // This is hacky and should probably be changed if (error == "MySQL server has gone away") log_game("MySQL connection drop detected, attempting to reconnect.") @@ -319,11 +318,9 @@ Delayed insert mode was removed in mysql 7 and only works with MyISAM type table parsed += cache[curr_arg] else #ifdef UNIT_TEST - log_world("ERROR: SQL ARGPARSE: Unpopulated argument found in an SQL query.") - log_world("ERROR: SQL ARGPARSE: [curr_arg]. Query: [query_to_parse]") + log_world("ERROR: SQL ARGPARSE: Unpopulated argument found in an SQL query: [curr_arg]. Query: [query_to_parse]") #else - log_sql("SQL ARGPARSE: Unpopulated argument found in an SQL query.") - log_sql("SQL ARGPARSE: [curr_arg]. Query: [query_to_parse]") + log_sql("SQL ARGPARSE: Unpopulated argument found in an SQL query: [curr_arg]. Query: [query_to_parse]") #endif return null diff --git a/code/modules/error_handler/error_handler.dm b/code/modules/error_handler/error_handler.dm index 7041695fdd5..af3fe9f6957 100644 --- a/code/modules/error_handler/error_handler.dm +++ b/code/modules/error_handler/error_handler.dm @@ -137,6 +137,9 @@ GLOBAL_VAR_INIT(total_runtimes_skipped, 0) // This writes the regular format (unwrapping newlines and inserting timestamps as needed). log_runtime("runtime error: [E.name]\n[E.desc]") + + if(SSsentry) + SSsentry.capture_exception(E, desclines, usr) #undef ERROR_USEFUL_LEN diff --git a/code/modules/sentry/sentry_event.dm b/code/modules/sentry/sentry_event.dm new file mode 100644 index 00000000000..3d9be36337e --- /dev/null +++ b/code/modules/sentry/sentry_event.dm @@ -0,0 +1,137 @@ +/** + * # Sentry Event + * + * Payload builder for a single Sentry event. + * Populated by SSsentry capture procs, then serialized via to_list() for + * json_encode() before being sent in the envelope body. + */ +/datum/sentry_event + /// 32-char hex UUID, stamped by SSsentry just before sending. + var/event_id = "" + /// ISO-8601 UTC timestamp, stamped by SSsentry just before sending. + var/timestamp = "" + /// Sentry level: "fatal", "error", "warning", "info", "debug". + var/level = "error" + /// Logical logger name shown in Sentry (e.g. "world.Error", "sql", "garbage"). + var/logger = "" + /// Sentry environment tag, e.g. "production". + var/environment = "" + /// Sentry release string. + var/release = "" + /// Human-readable server identifier. + var/server_name = "" + + // ---- tags & extra ---- + /// Assoc list of short indexed tags visible in Sentry's issue sidebar. + var/list/tags + /// Assoc list of arbitrary extra context (not indexed). + var/list/extra + + // ---- exception fields (mutually exclusive with message) ---- + var/exc_type = "" + var/exc_value = "" + var/exc_file = "" + var/exc_line = 0 + /// Flat list of detail lines from error_handler's desclines (optional). + var/list/exc_desc_lines + + // ---- message fields ---- + var/message_text = "" + + // ---- user context ---- + /// Sentry user id — either raw ckey or md5(ckey) depending on scrub setting. + var/usr_id = "" + +/datum/sentry_event/New() + tags = list() + extra = list() + +// --------------------------------------------------------------------------- +// Payload setters +// --------------------------------------------------------------------------- + +/** + * Populate exception fields from a BYOND /exception datum. + * desclines is the optional list of extra context from world/Error(). + */ +/datum/sentry_event/proc/set_exception(exception/e, list/desclines) + exc_type = e.name || "RuntimeError" + exc_value = e.desc || "" + exc_file = e.file || "" + exc_line = e.line || 0 + if (LAZYLEN(desclines)) + exc_desc_lines = desclines.Copy() + +/// Set message text for SQL-error and hard-delete style events. +/datum/sentry_event/proc/set_message(text) + message_text = text + +/// Set usr context without scrubbing (ckey sent in the clear). +/datum/sentry_event/proc/set_usr(atom/usr_ref) + if (istype(usr_ref, /mob)) + var/mob/m = usr_ref + usr_id = m.ckey || m.name || "unknown" + else if (istype(usr_ref, /client)) + var/client/c = usr_ref + usr_id = c.ckey || "unknown" + +/// Set usr context with ckey hashed (md5) for PII compliance. +/datum/sentry_event/proc/set_usr_scrubbed(atom/usr_ref) + var/raw_ckey = "" + if (istype(usr_ref, /mob)) + var/mob/m = usr_ref + raw_ckey = m.ckey || "" + else if (istype(usr_ref, /client)) + var/client/c = usr_ref + raw_ckey = c.ckey || "" + usr_id = raw_ckey ? md5(raw_ckey) : "anonymous" + +// --------------------------------------------------------------------------- +// Serialization +// --------------------------------------------------------------------------- + +/** + * Return an associative list ready for json_encode(). + * Omits empty/null fields to keep payloads lean. + */ +/datum/sentry_event/proc/to_list() + var/list/out = list( + "event_id" = event_id, + "timestamp" = timestamp, + "platform" = "other", + "level" = level + ) + + if (logger) out["logger"] = logger + if (environment) out["environment"] = environment + if (release) out["release"] = release + if (server_name) out["server_name"] = server_name + + if (length(tags)) out["tags"] = tags + if (length(extra)) out["extra"] = extra + + if (usr_id) + out["user"] = list("id" = usr_id) + + // --- exception payload --- + if (exc_type) + var/list/exc_val = list( + "type" = exc_type, + "value" = exc_value + ) + if (exc_file) + exc_val["stacktrace"] = list( + "frames" = list(list( + "filename" = exc_file, + "lineno" = exc_line + )) + ) + if (LAZYLEN(exc_desc_lines)) + exc_val["extra"] = list("detail" = exc_desc_lines.Join("\n")) + out["exception"] = list("values" = list(exc_val)) + + // --- message payload --- + if (message_text) + out["message"] = list("formatted" = message_text) + + return out diff --git a/config/example/sentry.txt b/config/example/sentry.txt new file mode 100644 index 00000000000..b768bd2fd4b --- /dev/null +++ b/config/example/sentry.txt @@ -0,0 +1,46 @@ +## Sentry.io Integration for Aurora Station +## +## Copy this file to config/sentry.txt and fill in your project DSN. +## Without ENABLED uncommented the subsystem does nothing. +## +## DSN format: https://PUBLIC_KEY@HOST/PROJECT_ID +## Find your DSN at: Settings → Projects → → SDK Setup → DSN + +## Uncomment to activate. +# ENABLED + +## Your project DSN (required). +# DSN https://examplekey@o0.ingest.sentry.io/0 + +## Sentry environment tag. Shown in the Sentry UI. +## Defaults to "production". +# ENVIRONMENT production + +## Server name shown on each event. Defaults to the value of SERVERNAME in config.txt. +# SERVER_NAME my-server + +## Which categories of events to forward (1 = enabled, 0 = disabled). +## CAPTURE_RUNTIMES, CAPTURE_SQL, CAPTURE_HARD_DELS default to 1 (enabled). +## CAPTURE_PROFILER defaults to 0 — each dump attaches the full profiler JSON, +## which can be large. Only enable if you actually want it in Sentry. +# CAPTURE_RUNTIMES 1 +# CAPTURE_SQL 1 +# CAPTURE_HARD_DELS 1 +# CAPTURE_PROFILER 0 + +## How many of the hottest procs (by self-time) to include in event extras +## alongside the attached JSON. Defaults to 20. +# PROFILER_TOP_N 20 + +## How long (in deciseconds of game time) to suppress duplicate file:line exceptions. +## Set to 0 to disable deduplication. Defaults to 600 (60 seconds). +# DEDUP_WINDOW 600 + +## Hash (md5) ckeys before sending instead of forwarding them in the clear. +## Defaults to 1 (enabled). +# SCRUB_CKEYS 1 + +## Disable Sentry after this many consecutive HTTP failures (kill-switch). +## An admin can reset it in-round via VV (SSsentry → active = TRUE). +## Defaults to 10. +# MAX_CONSECUTIVE_FAILURES 10 diff --git a/html/changelogs/arrow768-sentry-sdk.yml b/html/changelogs/arrow768-sentry-sdk.yml new file mode 100644 index 00000000000..b33d0595187 --- /dev/null +++ b/html/changelogs/arrow768-sentry-sdk.yml @@ -0,0 +1,8 @@ +author: arrow768 + +delete-after: True + +changes: + - code_imp: "Added Sentry.io integration (SSsentry) for capturing runtime exceptions, SQL errors, and hard-delete overruns to an external error tracking service." + - code_imp: "SSsentry can now ship per-window BYOND profiler dumps to Sentry as events with the raw JSON attached and the top-N hottest procs in the event extras (opt-in via CAPTURE_PROFILER)." + - server: "Added config/sentry.txt support for configuring the Sentry DSN and capture settings. See config/example/sentry.txt for setup instructions."