mirror of
https://github.com/Aurorastation/Aurora.3.git
synced 2026-08-23 21:12:24 +01:00
Sentry Integration (#22202)
Sends Errors, Runtimes, ... to Sentry.io --------- Signed-off-by: Arrow768 <1331699+Arrow768@users.noreply.github.com> Co-authored-by: Werner <Arrow768@users.noreply.github.com>
This commit is contained in:
Arrow768
@@ -70,6 +70,7 @@ GLOBAL_LIST_EMPTY(gamemode_cache)
|
||||
"log_subsystems_zas" = FALSE, // ZAS
|
||||
"log_subsystems_zas_debug" = FALSE, // ZAS debug
|
||||
"log_subsystems_http" = TRUE, //HTTP Log
|
||||
"log_subsystems_sentry" = TRUE, //Sentry subsystem self-diagnostics
|
||||
|
||||
/*#### MODULES ####*/
|
||||
|
||||
@@ -129,6 +130,7 @@ GLOBAL_LIST_EMPTY(gamemode_cache)
|
||||
"world_subsystems_zas" = "subsystems/zas.log",
|
||||
"world_subsystems_zas_debug" = "subsystems/zas.log",
|
||||
"world_subsystems_http" = "subsystems/http.log",
|
||||
"world_subsystems_sentry_log" = "subsystems/sentry.log",
|
||||
|
||||
/*#### MODULES ####*/
|
||||
|
||||
@@ -1188,6 +1190,41 @@ GENERAL_PROTECT_DATUM(/datum/configuration)
|
||||
SSdiscord.alert_visibility = TRUE
|
||||
else
|
||||
log_config("Unknown setting in discord configuration: '[name]'")
|
||||
|
||||
else if (type == "sentry")
|
||||
if (!SSsentry)
|
||||
log_config("Sentry: Attempted to read config/sentry.txt before SSsentry was initialized.")
|
||||
return
|
||||
|
||||
switch (name)
|
||||
if ("enabled")
|
||||
SSsentry._config_enabled_flag = TRUE
|
||||
if ("dsn")
|
||||
if (!SSsentry.parse_dsn(value))
|
||||
log_config("Sentry: Invalid DSN '[value]' in config/sentry.txt")
|
||||
if ("environment")
|
||||
SSsentry.environment = value
|
||||
if ("server_name")
|
||||
SSsentry.server_name = value
|
||||
if ("capture_runtimes")
|
||||
SSsentry.capture_runtimes = text2num(value) ? TRUE : FALSE
|
||||
if ("capture_sql")
|
||||
SSsentry.capture_sql = text2num(value) ? TRUE : FALSE
|
||||
if ("capture_hard_dels")
|
||||
SSsentry.capture_hard_dels = text2num(value) ? TRUE : FALSE
|
||||
if ("capture_profiler")
|
||||
SSsentry.capture_profiler = text2num(value) ? TRUE : FALSE
|
||||
if ("profiler_top_n")
|
||||
SSsentry.profiler_top_n = text2num(value)
|
||||
if ("dedup_window")
|
||||
SSsentry.dedup_window = text2num(value)
|
||||
if ("scrub_ckeys")
|
||||
SSsentry.scrub_ckeys = text2num(value) ? TRUE : FALSE
|
||||
if ("max_consecutive_failures")
|
||||
SSsentry.max_consecutive_failures = text2num(value)
|
||||
else
|
||||
log_config("Sentry: Unknown setting '[name]' in config/sentry.txt")
|
||||
|
||||
load_logging_config()
|
||||
load_away_sites_config()
|
||||
load_exoplanets_config()
|
||||
|
||||
@@ -312,6 +312,14 @@ SUBSYSTEM_DEF(garbage)
|
||||
log_game("Error: [type]([refID]) took longer than [threshold] seconds to delete (took [round(time/10, 0.1)] seconds to delete)")
|
||||
message_admins("Error: [type]([refID]) took longer than [threshold] seconds to delete (took [round(time/10, 0.1)] seconds to delete).")
|
||||
type_info.qdel_flags |= QDEL_ITEM_ADMINS_WARNED
|
||||
if(SSsentry)
|
||||
SSsentry.capture_message(
|
||||
"Hard delete overrun: [type] ([refID])",
|
||||
"warning",
|
||||
"garbage",
|
||||
tags = list("datum_type" = "[type]"),
|
||||
extra = list("ref_id" = refID, "time_ms" = tick_usage)
|
||||
)
|
||||
type_info.hard_deletes_over_threshold++
|
||||
var/overrun_limit = 0 // Used to be CONFIG_GET(number/hard_deletes_overrun_limit)
|
||||
if (overrun_limit && type_info.hard_deletes_over_threshold >= overrun_limit)
|
||||
|
||||
@@ -72,6 +72,9 @@ SUBSYSTEM_DEF(profiler)
|
||||
WRITE_FILE(sendmaps_file, current_sendmaps_data)
|
||||
write_cost = MC_AVERAGE(write_cost, TICK_DELTA_TO_MS(TICK_USAGE_REAL - timer))
|
||||
|
||||
if(SSsentry)
|
||||
SSsentry.capture_profiler_dump(current_profile_data, fetch_cost, write_cost)
|
||||
|
||||
#undef PROFILER_FILENAME
|
||||
#undef SENDMAPS_FILENAME
|
||||
#undef PROFILER_PATH
|
||||
|
||||
@@ -0,0 +1,382 @@
|
||||
/**
|
||||
* # SSsentry — Sentry.io Integration
|
||||
*
|
||||
* Captures runtime exceptions, SQL errors, and hard-delete overruns and
|
||||
* forwards them to sentry.io as structured events via the Sentry Envelope API.
|
||||
*
|
||||
* ## Setup
|
||||
* Copy config/example/sentry.txt to config/sentry.txt, set DSN and uncomment ENABLED.
|
||||
* The subsystem is off by default; it only activates when both ENABLED and DSN are present.
|
||||
*
|
||||
* ## Capture sites (wired elsewhere)
|
||||
* - world/Error() → runtime exceptions (error_handler.dm)
|
||||
* - log_exception() → caught exceptions (_logging.dm)
|
||||
* - log_sql() → DB / ARGPARSE errors (debug.dm)
|
||||
* - SSgarbage/HardDelete() → over-threshold hard dels (garbage.dm)
|
||||
*
|
||||
* ## Kill-switch
|
||||
* After max_consecutive_failures HTTP errors the subsystem sets active = FALSE
|
||||
* and stops sending. An admin can reset it in-round via VV (set active = TRUE).
|
||||
*/
|
||||
|
||||
SUBSYSTEM_DEF(sentry)
|
||||
name = "Sentry"
|
||||
init_order = INIT_ORDER_SENTRY
|
||||
flags = SS_NO_FIRE | SS_NO_DISPLAY
|
||||
|
||||
// ---- config flags ----
|
||||
/// Master on/off switch. Set to TRUE when ENABLED + valid DSN are both present.
|
||||
var/enabled = FALSE
|
||||
/// Runtime kill-switch: flipped to FALSE after too many consecutive HTTP failures.
|
||||
var/active = TRUE
|
||||
/// Capture uncaught and caught runtime exceptions.
|
||||
var/capture_runtimes = TRUE
|
||||
/// Capture SQL errors and ARGPARSE failures.
|
||||
var/capture_sql = TRUE
|
||||
/// Capture hard-delete overruns.
|
||||
var/capture_hard_dels = TRUE
|
||||
/// Capture per-window BYOND profiler dumps as events with the raw JSON attached. Off by default — payloads are large.
|
||||
var/capture_profiler = FALSE
|
||||
/// How many of the hottest procs to include in the event extras alongside the attachment.
|
||||
var/profiler_top_n = 20
|
||||
/// Hash ckeys with md5 before sending instead of sending them in the clear.
|
||||
var/scrub_ckeys = TRUE
|
||||
|
||||
// ---- DSN fields (populated by _parse_dsn) ----
|
||||
var/dsn_key = ""
|
||||
var/dsn_host = ""
|
||||
var/dsn_project_id = ""
|
||||
/// Fully constructed envelope POST URL built from the DSN.
|
||||
var/endpoint_url = ""
|
||||
|
||||
// ---- event context ----
|
||||
/// Maps to Sentry's "environment" field (e.g. "production", "staging").
|
||||
var/environment = "production"
|
||||
/// Maps to Sentry's "release" field. Populated from git HEAD SHA at init.
|
||||
var/release = ""
|
||||
/// Maps to Sentry's "server_name" field.
|
||||
var/server_name = ""
|
||||
|
||||
// ---- runtime state ----
|
||||
/// Per-round dedup table: erroruid (file:line) → world.time of last send.
|
||||
var/list/sent_this_round
|
||||
/// How long (in deciseconds of game time) to suppress duplicate file:line events. Defaults to 600 (60 s).
|
||||
var/dedup_window = 600
|
||||
/// Count of consecutive HTTP failures. Reset on any success.
|
||||
var/consecutive_failures = 0
|
||||
/// Kill-switch threshold.
|
||||
var/max_consecutive_failures = 10
|
||||
|
||||
// Internal flag used during config loading — tracks whether ENABLED was seen.
|
||||
var/_config_enabled_flag = FALSE
|
||||
|
||||
// Hide DSN key from VV to avoid credential exposure.
|
||||
/datum/controller/subsystem/sentry/can_vv_get(var_name)
|
||||
if (var_name == NAMEOF(src, dsn_key))
|
||||
return FALSE
|
||||
return ..()
|
||||
|
||||
/datum/controller/subsystem/sentry/vv_edit_var(var_name, var_value)
|
||||
if (var_name == NAMEOF(src, dsn_key))
|
||||
return FALSE
|
||||
return ..()
|
||||
|
||||
/datum/controller/subsystem/sentry/Initialize(timeofday)
|
||||
sent_this_round = list()
|
||||
server_name = GLOB.config?.server_name || ""
|
||||
|
||||
if (fexists("config/sentry.txt"))
|
||||
GLOB.config.load("config/sentry.txt", "sentry")
|
||||
else
|
||||
log_subsystem_sentry("config/sentry.txt not found — disabled.")
|
||||
return SS_INIT_NO_NEED
|
||||
|
||||
if (!_config_enabled_flag)
|
||||
log_subsystem_sentry("ENABLED not set in config/sentry.txt — disabled.")
|
||||
return SS_INIT_NO_NEED
|
||||
|
||||
if (!endpoint_url)
|
||||
log_subsystem_sentry("Valid DSN not found in config/sentry.txt — disabled.")
|
||||
return SS_INIT_NO_NEED
|
||||
|
||||
enabled = TRUE
|
||||
release = rustg_git_revparse("HEAD")
|
||||
log_subsystem_sentry("Initialized. env=[environment] release=[release] project=[dsn_project_id] host=[dsn_host]")
|
||||
return SS_INIT_SUCCESS
|
||||
|
||||
/datum/controller/subsystem/sentry/stat_entry(msg)
|
||||
msg = "enabled=[enabled] active=[active] failures=[consecutive_failures]"
|
||||
return ..()
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// DSN parsing (called from configuration.dm type == "sentry" handler)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Parse a Sentry DSN and populate dsn_key, dsn_host, dsn_project_id, endpoint_url.
|
||||
*
|
||||
* Expected format: https://KEY@HOST/PROJECT_ID
|
||||
* Returns TRUE on success, FALSE on malformed input.
|
||||
*/
|
||||
/datum/controller/subsystem/sentry/proc/parse_dsn(dsn)
|
||||
if (!dsn || dsn == "")
|
||||
return FALSE
|
||||
|
||||
var/rest = dsn
|
||||
var/protocol = "https"
|
||||
|
||||
if (findtext(rest, "https://") == 1)
|
||||
rest = copytext(rest, 9)
|
||||
else if (findtext(rest, "http://") == 1)
|
||||
rest = copytext(rest, 8)
|
||||
protocol = "http"
|
||||
else
|
||||
return FALSE
|
||||
|
||||
var/at_pos = findtext(rest, "@")
|
||||
if (!at_pos)
|
||||
return FALSE
|
||||
|
||||
dsn_key = copytext(rest, 1, at_pos)
|
||||
rest = copytext(rest, at_pos + 1)
|
||||
|
||||
var/slash_pos = findtext(rest, "/")
|
||||
if (!slash_pos)
|
||||
return FALSE
|
||||
|
||||
dsn_host = copytext(rest, 1, slash_pos)
|
||||
dsn_project_id = copytext(rest, slash_pos + 1)
|
||||
endpoint_url = "[protocol]://[dsn_host]/api/[dsn_project_id]/envelope/"
|
||||
return TRUE
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Public capture API
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Capture an uncaught or caught runtime exception.
|
||||
*
|
||||
* Call from world/Error() and log_exception().
|
||||
*
|
||||
* Arguments:
|
||||
* * e — the /exception datum
|
||||
* * desclines — optional list of extra context lines from error_handler
|
||||
* * usr_ref — usr at time of exception (may be null)
|
||||
*/
|
||||
/datum/controller/subsystem/sentry/proc/capture_exception(exception/e, list/desclines, usr_ref)
|
||||
SHOULD_NOT_SLEEP(TRUE)
|
||||
if (!enabled || !active || !capture_runtimes || isnull(e))
|
||||
return
|
||||
|
||||
var/erroruid = "[e.file]:[e.line]"
|
||||
// Dedup: skip if the same file:line was already sent within dedup_window deciseconds of game time.
|
||||
if (sent_this_round[erroruid] && (world.time - sent_this_round[erroruid]) < dedup_window)
|
||||
return
|
||||
sent_this_round[erroruid] = world.time
|
||||
|
||||
var/datum/sentry_event/ev = new()
|
||||
ev.level = "error"
|
||||
ev.logger = "world.Error"
|
||||
ev.set_exception(e, desclines)
|
||||
|
||||
if (!isnull(usr_ref))
|
||||
if (scrub_ckeys)
|
||||
ev.set_usr_scrubbed(usr_ref)
|
||||
else
|
||||
ev.set_usr(usr_ref)
|
||||
|
||||
_send_event(ev)
|
||||
|
||||
/**
|
||||
* Capture a message-style event (SQL error, hard delete, etc.).
|
||||
*
|
||||
* Arguments:
|
||||
* * message — human-readable description
|
||||
* * level — Sentry severity: "error", "warning", "info"
|
||||
* * logger — logical name shown in Sentry (e.g. "sql", "garbage")
|
||||
* * tags — optional assoc list of short indexed tags
|
||||
* * extra — optional assoc list of non-indexed context
|
||||
*/
|
||||
/datum/controller/subsystem/sentry/proc/capture_message(message, level = "error", logger = "game", list/tags, list/extra)
|
||||
SHOULD_NOT_SLEEP(TRUE)
|
||||
if (!enabled || !active)
|
||||
return
|
||||
|
||||
var/datum/sentry_event/ev = new()
|
||||
ev.level = level
|
||||
ev.logger = logger
|
||||
ev.set_message(message)
|
||||
|
||||
if (tags)
|
||||
for (var/k in tags)
|
||||
ev.tags[k] = tags[k]
|
||||
if (extra)
|
||||
for (var/k in extra)
|
||||
ev.extra[k] = extra[k]
|
||||
|
||||
_send_event(ev)
|
||||
|
||||
/**
|
||||
* Capture a BYOND profiler dump as a Sentry event with the raw JSON attached.
|
||||
*
|
||||
* Sentry's Profiling product expects sample-based stack frames; BYOND's profiler
|
||||
* is aggregated per-proc, so this ships as a regular event (logger="profiler",
|
||||
* level="info") with the top-N hottest procs in `extra` and the full JSON
|
||||
* payload as an event attachment for offline inspection.
|
||||
*
|
||||
* Arguments:
|
||||
* * profile_json — raw JSON string from world.Profile(PROFILE_REFRESH, format="json")
|
||||
* * fetch_ms — SSprofiler fetch_cost at time of dump
|
||||
* * write_ms — SSprofiler write_cost at time of dump
|
||||
*/
|
||||
/datum/controller/subsystem/sentry/proc/capture_profiler_dump(profile_json, fetch_ms, write_ms)
|
||||
SHOULD_NOT_SLEEP(TRUE)
|
||||
if (!enabled || !active || !capture_profiler)
|
||||
return
|
||||
if (!profile_json || !length(profile_json))
|
||||
return
|
||||
|
||||
var/list/parsed = json_decode(profile_json)
|
||||
if (!islist(parsed))
|
||||
return
|
||||
|
||||
var/list/top = _profiler_top_n(parsed, profiler_top_n)
|
||||
|
||||
var/datum/sentry_event/ev = new()
|
||||
ev.level = "info"
|
||||
ev.logger = "profiler"
|
||||
ev.set_message("Profiler dump (procs=[length(parsed)] top=[length(top)])")
|
||||
ev.tags["fetch_ms"] = "[round(fetch_ms, 0.1)]"
|
||||
ev.tags["write_ms"] = "[round(write_ms, 0.1)]"
|
||||
ev.tags["proc_count"] = "[length(parsed)]"
|
||||
ev.extra["top_procs"] = top
|
||||
|
||||
_send_event(ev, profile_json, "profile.json", "application/json")
|
||||
|
||||
/// Return the top-N profiler entries by self-time, descending. Pass-through fields so format changes don't break.
|
||||
/datum/controller/subsystem/sentry/proc/_profiler_top_n(list/entries, n)
|
||||
if (!islist(entries) || !length(entries))
|
||||
return list()
|
||||
var/list/sorted = entries.Copy()
|
||||
sortTim(sorted, GLOBAL_PROC_REF(cmp_sentry_profiler_self_desc))
|
||||
if (length(sorted) > n)
|
||||
sorted.Cut(n + 1)
|
||||
return sorted
|
||||
|
||||
/proc/cmp_sentry_profiler_self_desc(list/A, list/B)
|
||||
var/a_self = islist(A) ? A["self"] : 0
|
||||
var/b_self = islist(B) ? B["self"] : 0
|
||||
return b_self - a_self
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Internal dispatch
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/datum/controller/subsystem/sentry/proc/_send_event(datum/sentry_event/ev, attachment_body = null, attachment_filename = null, attachment_content_type = null)
|
||||
SHOULD_NOT_SLEEP(TRUE)
|
||||
PRIVATE_PROC(TRUE)
|
||||
|
||||
if (!enabled || !active)
|
||||
return
|
||||
|
||||
// Stamp common context.
|
||||
ev.tags["round_id"] = "[GLOB.round_id]"
|
||||
ev.tags["byond_version"] = "[world.version]"
|
||||
ev.environment = environment
|
||||
ev.server_name = server_name
|
||||
ev.release = release
|
||||
|
||||
var/envelope_body = _build_envelope(ev, attachment_body, attachment_filename, attachment_content_type)
|
||||
if (!envelope_body)
|
||||
return
|
||||
|
||||
var/list/req_headers = list(
|
||||
"Content-Type" = "application/x-sentry-envelope",
|
||||
"X-Sentry-Auth" = "Sentry sentry_version=7,sentry_key=[dsn_key],sentry_client=aurora.sentry/1.0"
|
||||
)
|
||||
|
||||
if (!isnull(SShttp))
|
||||
var/datum/callback/cb = CALLBACK(src, PROC_REF(_on_response))
|
||||
SShttp.create_async_request(RUSTG_HTTP_METHOD_POST, endpoint_url, envelope_body, req_headers, cb)
|
||||
else
|
||||
// SShttp not yet available (very early runtime) — fire-and-forget directly via rust_g.
|
||||
rustg_http_request_async(RUSTG_HTTP_METHOD_POST, endpoint_url, envelope_body, json_encode(req_headers), null)
|
||||
|
||||
/datum/controller/subsystem/sentry/proc/_on_response(datum/http_response/res)
|
||||
PRIVATE_PROC(TRUE)
|
||||
|
||||
if (!res || res.errored || (res.status_code && res.status_code >= 400))
|
||||
consecutive_failures++
|
||||
if (consecutive_failures >= max_consecutive_failures)
|
||||
active = FALSE
|
||||
var/last_err = res ? (res.errored ? "[res.error]" : "HTTP [res.status_code]") : "null response"
|
||||
log_subsystem_sentry("Kill-switch triggered after [consecutive_failures] consecutive failures. Last: [last_err]")
|
||||
admin_notice(SPAN_DANGER("Sentry kill-switch triggered after [consecutive_failures] consecutive HTTP failures ([last_err]). \
|
||||
Event reporting is disabled. Reset via VV: SSsentry → active = TRUE."), R_ADMIN|R_MOD|R_DEV)
|
||||
return
|
||||
|
||||
consecutive_failures = 0
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Envelope serialization
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Build a Sentry envelope: header + event item, plus an optional attachment item.
|
||||
*
|
||||
* Layout:
|
||||
* {envelope-header}\n
|
||||
* {event-item-header}\n
|
||||
* {event-payload}\n
|
||||
* [{attachment-item-header}\n{attachment-payload}\n] ← only when attachment_body set
|
||||
*/
|
||||
/datum/controller/subsystem/sentry/proc/_build_envelope(datum/sentry_event/ev, attachment_body = null, attachment_filename = null, attachment_content_type = null)
|
||||
PRIVATE_PROC(TRUE)
|
||||
|
||||
ev.event_id = _generate_event_id()
|
||||
ev.timestamp = _iso8601_now()
|
||||
|
||||
var/payload_json = json_encode(ev.to_list())
|
||||
if (!payload_json)
|
||||
return null
|
||||
|
||||
var/list/env_header = list(
|
||||
"event_id" = ev.event_id,
|
||||
"sent_at" = ev.timestamp,
|
||||
"sdk" = list("name" = "aurora.sentry", "version" = "1.0")
|
||||
)
|
||||
var/list/item_header = list(
|
||||
"type" = "event",
|
||||
"content_type" = "application/json",
|
||||
"length" = length(payload_json)
|
||||
)
|
||||
|
||||
var/envelope = "[json_encode(env_header)]\n[json_encode(item_header)]\n[payload_json]\n"
|
||||
|
||||
if (attachment_body && length(attachment_body))
|
||||
var/list/att_header = list(
|
||||
"type" = "attachment",
|
||||
"length" = length(attachment_body),
|
||||
"content_type" = attachment_content_type || "application/octet-stream",
|
||||
"filename" = attachment_filename || "attachment.bin",
|
||||
"attachment_type" = "event.attachment"
|
||||
)
|
||||
envelope += "[json_encode(att_header)]\n[attachment_body]\n"
|
||||
|
||||
return envelope
|
||||
|
||||
/// Generate a pseudo-random 32-char hex event ID via md5.
|
||||
/datum/controller/subsystem/sentry/proc/_generate_event_id()
|
||||
PRIVATE_PROC(TRUE)
|
||||
|
||||
return md5("[world.time][world.realtime][rand(0, 9999999)]")
|
||||
|
||||
/// Return current time as ISO-8601 UTC string (YYYY-MM-DDTHH:MM:SSZ).
|
||||
/datum/controller/subsystem/sentry/proc/_iso8601_now()
|
||||
PRIVATE_PROC(TRUE)
|
||||
|
||||
var/t = time2text(world.realtime, "YYYY-MM-DD hh:mm:ss")
|
||||
var/space = findtext(t, " ")
|
||||
if (space)
|
||||
return "[copytext(t, 1, space)]T[copytext(t, space + 1)]Z"
|
||||
return t
|
||||
Reference in New Issue
Block a user