diff --git a/code/__defines/profiler_tracy.dm b/code/__defines/profiler_tracy.dm index ae00286e235..3266ed34131 100644 --- a/code/__defines/profiler_tracy.dm +++ b/code/__defines/profiler_tracy.dm @@ -1,31 +1,62 @@ +// File for the byond-tracy profiler. // Implements https://github.com/mafemergency/byond-tracy -// Client https://github.com/wolfpld/tracy -// As of now, only 0.8.2 is supported as a client, this might change in the future however +// and https://github.com/ParadiseSS13/byond-tracy +// Tracy Client https://github.com/wolfpld/tracy -// In case you need to start the capture as soon as the server boots, uncomment the following lines and recompile: +var/byond_tracy_running = 0 // Whether byond-tracy is currently running or not, no matter what version. +var/byond_tracy_running_v = null // Which version of byond-tracy is currently running, so we call the right binary on destruction. -// /world/New() -// prof_init() -// . = ..() +/world/New() + if(config.enable_byond_tracy) + prof_init("tracy_disk") // This start's Affectedarc07's version of Tracy. Writing a .utracy file to the disk. + . = ..() + +/world/Del() + if(byond_tracy_running) + prof_destroy() + . = ..() /client/proc/profiler_start() set name = "Start Tracy Profiler" set category = "Debug" - set desc = "Starts the tracy profiler, which will await the client connection." - switch(alert("Are you sure? Tracy will remain active until the server restarts.", "Tracy Init", "No", "Yes")) - if("Yes") - prof_init() + set desc = "Starts the tracy profiler, which will await the client connection or save utracy files to the server's disk." + if(!byond_tracy_running) + switch(alert("Are you sure? Tracy will remain active until the server restarts.", "Tracy Init", "No", "Yes")) + if("Yes") + switch(alert("Which version of Tracy would you like to run?", "Tracy Version", "Cancel", "Connection Based", "Disk Based")) + if("Connection Based") + prof_init("tracy") // This start's mafemergency's original version of Tracy. Requiring a direct connection, not writing to the disk. + if("Disk Based") + prof_init("tracy_disk") // This start's Affectedarc07's version of Tracy. Writing a .utracy file to the disk. /** - * Starts Tracy + * Starts Tracy. */ -/proc/prof_init() +/proc/prof_init(var/version) var/lib - switch(world.system_type) - if(MS_WINDOWS) lib = "prof.dll" - if(UNIX) lib = "libprof.so" - else CRASH("Tracy initialization failed: unsupported platform or DLL not found.") + switch(version) + if("tracy") lib = "tracy.dll" + if("tracy_disk") lib = "tracy-disk.dll" var/init = LIBCALL(lib, "init")() if("0" != init) CRASH("[lib] init error: [init]") + + byond_tracy_running = 1 + byond_tracy_running_v = version + +/** + * Stops Tracy. + * Doing this in the middle of the round is not good, don't do it. + */ +/proc/prof_destroy() + var/lib + + switch(byond_tracy_running_v) + if("tracy") lib = "tracy.dll" + if("tracy_disk") lib = "tracy-disk.dll" + + LIBCALL(lib, "destroy")() + + byond_tracy_running = 0 + byond_tracy_running_v = null diff --git a/code/controllers/configuration.dm b/code/controllers/configuration.dm index 085e83c416b..fbba01908c3 100644 --- a/code/controllers/configuration.dm +++ b/code/controllers/configuration.dm @@ -261,6 +261,9 @@ var/list/gamemode_cache = list() var/ipintel_save_bad = 1 var/ipintel_domain = "check.getipintel.net" + // BYOND Tracy + var/enable_byond_tracy = 0 + // Access control/Panic bunker settings. var/access_deny_new_players = 0 var/access_deny_new_accounts = -1 @@ -874,6 +877,9 @@ var/list/gamemode_cache = list() if("ipintel_save_bad") ipintel_save_bad = text2num(value) + if("enable_byond_tracy") + enable_byond_tracy = 1 + if("access_deny_new_accounts") access_deny_new_accounts = text2num(value) >= 0 ? text2num(value) : -1 if("access_deny_vms") diff --git a/config/example/config.txt b/config/example/config.txt index e2d02d2c329..57bbd5bbf07 100644 --- a/config/example/config.txt +++ b/config/example/config.txt @@ -504,6 +504,13 @@ LOG_GELF_ENABLED 0 ## Domain name to query (leave commented out for the default, only needed if you pay getipintel.net for more querys) #IPINTEL_DOMAIN check.getipintel.net +### BYOND Tracy: +### Uncomment this option to enable profiling with byond-tracy. Admins and Developers have the ability to use the appropriate debug verb +### to start the profiler in the middle of the round even if this option is uncommented, otherwise tracy will start on /world/New(). +### Warning: Do not uncomment this configuration option unless you know what you are doing and need it. It will create a gigabyte-large file +### everytime you start the server, which will grow as the server continues to run. +#ENABLE_BYOND_TRACY + ### ACCESS CONTROL: ### This allows you to adjust various parametres by which potentially malicious users can by identified. ### Note that none of these are 100% accurate, and should only be used in cases where a "panic bunker" is warranted. diff --git a/html/changelogs/byond-tracy-io-util.yml b/html/changelogs/byond-tracy-io-util.yml new file mode 100644 index 00000000000..65983df44c4 --- /dev/null +++ b/html/changelogs/byond-tracy-io-util.yml @@ -0,0 +1,7 @@ +author: io-util + +delete-after: True + +changes: + - backend: "Use para-byond-tracy instead of byond-tracy which saves .utracy files to the disk to view later." + - admin: "Modified the Start-Tracy-Profiler adminverb, it is now aware of whether tracy is already running or not." diff --git a/prof.dll b/prof.dll deleted file mode 100644 index d5dce10c043..00000000000 Binary files a/prof.dll and /dev/null differ diff --git a/tools/tracy-client/import-chrome.exe b/tools/tracy-client/import-chrome.exe deleted file mode 100644 index c1a3d5262bb..00000000000 Binary files a/tools/tracy-client/import-chrome.exe and /dev/null differ diff --git a/tools/tracy-client/tracy-0.9.1.pdf b/tools/tracy-client/tracy-0.9.1.pdf deleted file mode 100644 index 860feb749b5..00000000000 Binary files a/tools/tracy-client/tracy-0.9.1.pdf and /dev/null differ diff --git a/tools/tracy-client/update.exe b/tools/tracy-client/update.exe deleted file mode 100644 index d2bfba0dc28..00000000000 Binary files a/tools/tracy-client/update.exe and /dev/null differ diff --git a/tools/tracy-client/Tracy.exe b/tools/tracy/client/Tracy-0.8.2.exe similarity index 68% rename from tools/tracy-client/Tracy.exe rename to tools/tracy/client/Tracy-0.8.2.exe index 28b6436089e..fb537bdb2fb 100644 Binary files a/tools/tracy-client/Tracy.exe and b/tools/tracy/client/Tracy-0.8.2.exe differ diff --git a/tools/tracy-client/capture.exe b/tools/tracy/client/capture.exe similarity index 80% rename from tools/tracy-client/capture.exe rename to tools/tracy/client/capture.exe index 73b2e5ecc45..445e3a13ffa 100644 Binary files a/tools/tracy-client/capture.exe and b/tools/tracy/client/capture.exe differ diff --git a/tools/tracy-client/csvexport.exe b/tools/tracy/client/csvexport.exe similarity index 79% rename from tools/tracy-client/csvexport.exe rename to tools/tracy/client/csvexport.exe index caeb79f2e0c..94e23548171 100644 Binary files a/tools/tracy-client/csvexport.exe and b/tools/tracy/client/csvexport.exe differ diff --git a/tools/tracy/replay/README.MD b/tools/tracy/replay/README.MD new file mode 100644 index 00000000000..28ec1c15126 --- /dev/null +++ b/tools/tracy/replay/README.MD @@ -0,0 +1,7 @@ +# Info + +This directory contains the `replay.py` script. This is used to "replay" custom `.utracy` files generated by the production server. + +`BYOND-Tracy` profiles can use excessive amounts of RAM, upwards of 48GB in a single process if captured normally at runtime. This is not viable for the production server, so they are written as a custom flatfile inside of `data/profiler/`. These need to be read into tracy over the "network" (localhost) via `Tracy.exe` or `capture.exe`. You will need >48GB of RAM for this process. I am not joking. + +The version of `replay.py` in this folder is compatible with the protocol of `Tracy 0.8.2`. Newer versions will not work. It requires the `lz4` python module with the streams extension. This requires manually downloading, building and installing the `python-lz4/python-lz4` repo and building with `PYLZ4_EXPERIMENTAL=TRUE` as an environment variable. diff --git a/tools/tracy/replay/replay.py b/tools/tracy/replay/replay.py new file mode 100644 index 00000000000..a55d0993dd5 --- /dev/null +++ b/tools/tracy/replay/replay.py @@ -0,0 +1,432 @@ +import argparse +import ctypes +import socket +import selectors +import lz4.stream + +# file protocol +FileSignature = 0x6D64796361727475 +FileVersion = 2 +FileEventZoneBegin =15 +FileEventZoneEnd = 17 +FileEventZoneColor = 62 +FileEventFrameMark = 64 + +class FileHeader(ctypes.Structure): + _fields_ = ( + ("signature", ctypes.c_ulonglong), + ("version", ctypes.c_uint), + ("multiplier", ctypes.c_double), + ("init_begin", ctypes.c_longlong), + ("init_end", ctypes.c_longlong), + ("delay", ctypes.c_longlong), + ("resolution", ctypes.c_longlong), + ("epoch", ctypes.c_longlong), + ("exec_time", ctypes.c_longlong), + ("pid", ctypes.c_longlong), + ("sampling_period", ctypes.c_longlong), + ("flags", ctypes.c_byte), + ("cpu_arch", ctypes.c_byte), + ("cpu_manufacturer", ctypes.c_char * 12), + ("cpu_id", ctypes.c_uint), + ("program_name", ctypes.c_char * 64), + ("host_info", ctypes.c_char * 1024) + ) + +class FileZoneBegin(ctypes.Structure): + _fields_ = ( + ("tid", ctypes.c_uint32), + ("srcloc", ctypes.c_uint32), + ("timestamp", ctypes.c_int64) + ) + +class FileZoneEnd(ctypes.Structure): + _fields_ = ( + ("tid", ctypes.c_uint32), + ("timestamp", ctypes.c_int64) + ) + +class FileZoneColor(ctypes.Structure): + _fields_ = ( + ("tid", ctypes.c_uint32), + ("color", ctypes.c_uint32) + ) + +class FileFrameMark(ctypes.Structure): + _fields_ = ( + ("name", ctypes.c_uint32), + ("timestamp", ctypes.c_int64) + ) + +class FileEvent(ctypes.Structure): + class Events(ctypes.Union): + _fields_ = ( + ("zone_begin", FileZoneBegin), + ("zone_end", FileZoneEnd), + ("zone_color", FileZoneColor), + ("frame_mark", FileFrameMark), + ) + + _anonymous_ = ("event",) + _fields_ = ( + ("type", ctypes.c_byte), + ("event", Events) + ) + +# network protocol +NetworkMaxFrameSize = 256 * 1024 +NetworkHandshakeWelcome = b"\x01" +NetworkHandshakeProtocolMismatch = b"\x02" +NetworkEventZoneBegin = 15 +NetworkEventZoneEnd = 17 +NetworkEventTerminate = 55 +NetworkEventThreadContext = 57 +NetworkEventZoneColor = 62 +NetworkEventFrameMark = 64 +NetworkEventSrcloc = 67 +NetworkResponseServerQueryNoop = 87 +NetworkResponseSourceCodeNotAvailable = 88 +NetworkResponseSymbolCodeNotAvailable = 89 +NetworkResponseStringData = 94 +NetworkResponseThreadName = 95 +NetworkQueryTerminate = 0 +NetworkQueryString = 1 +NetworkQueryThreadString = 2 +NetworkQuerySrcloc = 3 +NetworkQueryPlotName = 4 +NetworkQueryFrameName = 5 +NetworkQueryParameter = 6 +NetworkQueryFiberName = 7 +NetworkQueryDisconnect = 8 +NetworkQueryCallstackFrame = 9 +NetworkQueryExternalName = 10 +NetworkQuerySymbol = 11 +NetworkQuerySymbolCode = 12 +NetworkQueryCodeLocation = 13 +NetworkQuerySourceCode = 14 +NetworkQueryDataTransfer = 15 +NetworkQueryDataTransferPart = 16 + + +class NetworkHeader(ctypes.Structure): + _pack_ = 1 + _fields_ = ( + ("multiplier", ctypes.c_double), + ("init_begin", ctypes.c_longlong), + ("init_end", ctypes.c_longlong), + ("delay", ctypes.c_longlong), + ("resolution", ctypes.c_longlong), + ("epoch", ctypes.c_longlong), + ("exec_time", ctypes.c_longlong), + ("pid", ctypes.c_longlong), + ("sampling_period", ctypes.c_longlong), + ("flags", ctypes.c_uint8), + ("cpu_arch", ctypes.c_uint8), + ("cpu_manufacturer", ctypes.c_char * 12), + ("cpu_id", ctypes.c_uint), + ("program_name", ctypes.c_char * 64), + ("host_info", ctypes.c_char * 1024) + ) + +class NetworkThreadContext(ctypes.Structure): + _pack_ = 1 + _fields_ = ( + ("type", ctypes.c_uint8), + ("tid", ctypes.c_uint32) + ) + +class NetworkZoneBegin(ctypes.Structure): + _pack_ = 1 + _fields_ = ( + ("type", ctypes.c_uint8), + ("timestamp", ctypes.c_int64), + ("srcloc", ctypes.c_uint64) + ) + +class NetworkZoneEnd(ctypes.Structure): + _pack_ = 1 + _fields_ = ( + ("type", ctypes.c_uint8), + ("timestamp", ctypes.c_int64) + ) + +class NetworkZoneColor(ctypes.Structure): + _pack_ = 1 + _fields_ = ( + ("type", ctypes.c_uint8), + ("r", ctypes.c_uint8), + ("g", ctypes.c_uint8), + ("b", ctypes.c_uint8) + ) + +class NetworkFrameMark(ctypes.Structure): + _pack_ = 1 + _fields_ = ( + ("type", ctypes.c_uint8), + ("timestamp", ctypes.c_int64), + ("name", ctypes.c_uint64) + ) + +class NetworkSrcloc(ctypes.Structure): + _pack_ = 1 + _fields_ = ( + ("type", ctypes.c_uint8), + ("name", ctypes.c_int64), + ("function", ctypes.c_int64), + ("file", ctypes.c_int64), + ("line", ctypes.c_uint32), + ("r", ctypes.c_uint8), + ("g", ctypes.c_uint8), + ("b", ctypes.c_uint8) + ) + +class NetworkRequest(ctypes.Structure): + _pack_ = 1 + _fields_ = ( + ("type", ctypes.c_uint8), + ("ptr", ctypes.c_int64), + ("extra", ctypes.c_uint32) + ) + +def main(stream): + def file_read_uint(): + ctype = ctypes.c_uint() + stream.readinto(ctype) + return ctype.value + + def file_read_chars(size): + if 0 == size: + return None + ctype = (ctypes.c_char * size)() + stream.readinto(ctype) + return ctype.value + + header = FileHeader() + stream.readinto(header) + if header.signature != FileSignature: + print("incorrect signature") + return + + if header.version != FileVersion: + print("incorrect version") + return + + srclocs_len = file_read_uint() + strings = {} + srclocs = [None] * srclocs_len + + for i in range(srclocs_len): + name_len = file_read_uint() + name = file_read_chars(name_len) + function_len = file_read_uint() + function = file_read_chars(function_len) + file_len = file_read_uint() + file = file_read_chars(file_len) + line = file_read_uint() + color = file_read_uint() + + if name != None: + digest = hash(name) + strings[digest] = name + name = digest + else: + name = 0 + + if function != None: + digest = hash(function) + strings[digest] = function + function = digest + else: + function = 0 + + if file != None: + digest = hash(file) + strings[digest] = file + file = digest + else: + file = 0 + + srclocs[i] = (name, function, file, line, color) + + server = socket.create_server(("127.0.0.1", 8086)) + server.listen(1) + print("listening on 127.0.0.1:8086...") + client, addr = server.accept() + + if client.recv(8) != b"TracyPrf": + print("bad client") + client.close() + return + + protocol = ctypes.c_uint() + client.recv_into(protocol) + + if protocol.value not in (56, 57): + print("bad protocol") + client.sendall(NetworkHandshakeProtocolMismatch) + client.close() + return + + print(f"client accepted from {addr}") + client.sendall(NetworkHandshakeWelcome) + + client.sendall(NetworkHeader( + multiplier = header.multiplier, + init_begin = header.init_begin, + init_end = header.init_end, + delay = header.delay, + resolution = header.resolution, + epoch = header.epoch, + exec_time = header.exec_time, + pid = header.pid, + sampling_period = header.sampling_period, + flags = header.flags, + cpu_arch = header.cpu_arch, + cpu_manufacturer = header.cpu_manufacturer, + cpu_id = header.cpu_id, + program_name = header.program_name, + host_info = header.host_info + )) + + event = FileEvent() + event_sz = ctypes.sizeof(event) + timestamp = 0 + tid = None + buffer = bytearray(NetworkMaxFrameSize // event_sz) + offset = 0 + compressor = lz4.stream.LZ4StreamCompressor( + "double_buffer", + NetworkMaxFrameSize, + store_comp_size = 4 + ) + + def commit(): + nonlocal offset + if offset > 0: + block = compressor.compress(buffer[:offset]) + client.sendall(block) + offset = 0 + + def write_msg(msg): + nonlocal offset, buffer + sz = ctypes.sizeof(msg) + if offset + sz > len(buffer): + commit() + + buffer[offset : offset + sz] = msg + offset += sz + + def thread_context(event): + nonlocal tid, timestamp + if event.zone_begin.tid != tid: + tid = event.zone_begin.tid + timestamp = 0 + write_msg(NetworkThreadContext( + type = NetworkEventThreadContext, + tid = tid + )) + + while event_sz == stream.readinto(event): + if event.type == FileEventZoneBegin: + thread_context(event) + write_msg(NetworkZoneBegin( + type = NetworkEventZoneBegin, + timestamp = event.zone_begin.timestamp - timestamp, + srcloc = event.zone_begin.srcloc + )) + timestamp = event.zone_begin.timestamp + + elif event.type == FileEventZoneEnd: + thread_context(event) + write_msg(NetworkZoneEnd( + type = NetworkEventZoneEnd, + timestamp = event.zone_end.timestamp - timestamp + )) + timestamp = event.zone_end.timestamp + + elif event.type == FileEventZoneColor: + thread_context(event) + write_msg(NetworkZoneColor( + type = NetworkEventZoneColor, + r = (event.zone_color.color >> 0x00) & 0xFF, + g = (event.zone_color.color >> 0x08) & 0xFF, + b = (event.zone_color.color >> 0x10) & 0xFF + )) + + elif event.type == FileEventFrameMark: + write_msg(NetworkFrameMark( + type = NetworkEventFrameMark, + name = 0, + timestamp = event.frame_mark.timestamp + )) + + commit() + + def respond_string(string, ptr, type): + string_sz = len(string) + + class NetworkStringData(ctypes.Structure): + _pack_ = 1 + _fields_ = ( + ("type", ctypes.c_uint8), + ("ptr", ctypes.c_uint64), + ("len", ctypes.c_uint16), + ("str", ctypes.c_char * string_sz) + ) + + write_msg(NetworkStringData( + type = type, + ptr = req.ptr, + len = string_sz, + str = string + )) + + req = NetworkRequest() + client.settimeout(1) + + try: + while ctypes.sizeof(req) == client.recv_into(req): + if req.type == NetworkQuerySrcloc: + srcloc = srclocs[req.ptr] + write_msg(NetworkSrcloc( + type = NetworkEventSrcloc, + name = srcloc[0], + function = srcloc[1], + file = srcloc[2], + line = srcloc[3], + r = (srcloc[4] >> 0x00) & 0xFF, + g = (srcloc[4] >> 0x08) & 0xFF, + b = (srcloc[4] >> 0x10) & 0xFF + )) + + elif req.type == NetworkQueryString: + respond_string(strings[req.ptr], req.ptr, NetworkResponseStringData) + + elif req.type == NetworkQuerySymbolCode: + write_msg(ctypes.c_uint8(NetworkResponseSymbolCodeNotAvailable)) + + elif req.type == NetworkQuerySourceCode: + write_msg(ctypes.c_uint8(NetworkResponseSourceCodeNotAvailable)) + + elif req.type == NetworkQueryDataTransfer: + write_msg(ctypes.c_uint8(NetworkResponseServerQueryNoop)) + + elif req.type == NetworkQueryDataTransferPart: + write_msg(ctypes.c_uint8(NetworkResponseServerQueryNoop)) + + elif req.type == NetworkQueryThreadString: + respond_string(b"main", req.ptr, NetworkResponseThreadName) + + else: + print("unknown req:", req.type) + + commit() + + except socket.timeout: + pass + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("file", type=argparse.FileType("rb")) + args = parser.parse_args() + main(args.file) diff --git a/tracy-disk.dll b/tracy-disk.dll new file mode 100644 index 00000000000..dba348a80b9 Binary files /dev/null and b/tracy-disk.dll differ diff --git a/tracy.dll b/tracy.dll new file mode 100644 index 00000000000..62f486e7426 Binary files /dev/null and b/tracy.dll differ