diff --git a/code/__DEFINES/_init.dm b/code/__DEFINES/_init.dm index e7deaa481f9..5346b087c0d 100644 --- a/code/__DEFINES/_init.dm +++ b/code/__DEFINES/_init.dm @@ -4,6 +4,11 @@ // This exists so that world.Profile() is THE FIRST PROC TO RUN in the init sequence. // This allows us to get the real details of everything lagging at server start. world.Profile(PROFILE_START) + #if defined(ENABLE_BYOND_TRACY) && (DM_BUILD == 1589) + var/tracy_init = CALL_EXT("prof.dll", "init")() // Setup Tracy integration + if(tracy_init != "0") + CRASH("Tracy init error: [tracy_init]") + #endif // After that, the debugger is initialized. // Doing it this early makes it possible to set breakpoints in the New() // of things assigned to global variables or objects included in a compiled map file. diff --git a/code/_compile_options.dm b/code/_compile_options.dm index d418f30c78d..6a14468ac8f 100644 --- a/code/_compile_options.dm +++ b/code/_compile_options.dm @@ -4,6 +4,11 @@ // Uncomment the following line to compile unit tests. // #define UNIT_TESTS +// Uncomment the following line to enable Tracy profiling. +// DO NOT DO THIS UNLESS YOU UNDERSTAND THE IMPLICATIONS +// Your data directory will grow by about a gigabyte every time you launch the server, as well as introducing potential instabilities over multiple BYOND versions. +// #define ENABLE_BYOND_TRACY + #ifdef CIBUILDING #define UNIT_TESTS diff --git a/code/game/world.dm b/code/game/world.dm index 59f53b8c02c..cbeb8144f55 100644 --- a/code/game/world.dm +++ b/code/game/world.dm @@ -299,4 +299,7 @@ GLOBAL_LIST_EMPTY(world_topic_handlers) rustg_close_async_http_client() // Close the HTTP client. If you dont do this, youll get phantom threads which can crash DD from memory access violations disable_auxtools_debugger() // Disables the debugger if running. See above comment rustg_redis_disconnect() // Disconnects the redis connection. See above. + #ifdef ENABLE_BYOND_TRACY + CALL_EXT("prof.dll", "destroy")() // Setup Tracy integration + #endif ..() diff --git a/prof.dll b/prof.dll new file mode 100644 index 00000000000..33a6da22947 Binary files /dev/null and b/prof.dll differ diff --git a/tools/tracy_replay/README.MD b/tools/tracy_replay/README.MD new file mode 100644 index 00000000000..28ec1c15126 --- /dev/null +++ b/tools/tracy_replay/README.MD @@ -0,0 +1,7 @@ +# Info + +This directory contains the `replay.py` script. This is used to "replay" custom `.utracy` files generated by the production server. + +`BYOND-Tracy` profiles can use excessive amounts of RAM, upwards of 48GB in a single process if captured normally at runtime. This is not viable for the production server, so they are written as a custom flatfile inside of `data/profiler/`. These need to be read into tracy over the "network" (localhost) via `Tracy.exe` or `capture.exe`. You will need >48GB of RAM for this process. I am not joking. + +The version of `replay.py` in this folder is compatible with the protocol of `Tracy 0.8.2`. Newer versions will not work. It requires the `lz4` python module with the streams extension. This requires manually downloading, building and installing the `python-lz4/python-lz4` repo and building with `PYLZ4_EXPERIMENTAL=TRUE` as an environment variable. diff --git a/tools/tracy_replay/replay.py b/tools/tracy_replay/replay.py new file mode 100644 index 00000000000..a55d0993dd5 --- /dev/null +++ b/tools/tracy_replay/replay.py @@ -0,0 +1,432 @@ +import argparse +import ctypes +import socket +import selectors +import lz4.stream + +# file protocol +FileSignature = 0x6D64796361727475 +FileVersion = 2 +FileEventZoneBegin =15 +FileEventZoneEnd = 17 +FileEventZoneColor = 62 +FileEventFrameMark = 64 + +class FileHeader(ctypes.Structure): + _fields_ = ( + ("signature", ctypes.c_ulonglong), + ("version", ctypes.c_uint), + ("multiplier", ctypes.c_double), + ("init_begin", ctypes.c_longlong), + ("init_end", ctypes.c_longlong), + ("delay", ctypes.c_longlong), + ("resolution", ctypes.c_longlong), + ("epoch", ctypes.c_longlong), + ("exec_time", ctypes.c_longlong), + ("pid", ctypes.c_longlong), + ("sampling_period", ctypes.c_longlong), + ("flags", ctypes.c_byte), + ("cpu_arch", ctypes.c_byte), + ("cpu_manufacturer", ctypes.c_char * 12), + ("cpu_id", ctypes.c_uint), + ("program_name", ctypes.c_char * 64), + ("host_info", ctypes.c_char * 1024) + ) + +class FileZoneBegin(ctypes.Structure): + _fields_ = ( + ("tid", ctypes.c_uint32), + ("srcloc", ctypes.c_uint32), + ("timestamp", ctypes.c_int64) + ) + +class FileZoneEnd(ctypes.Structure): + _fields_ = ( + ("tid", ctypes.c_uint32), + ("timestamp", ctypes.c_int64) + ) + +class FileZoneColor(ctypes.Structure): + _fields_ = ( + ("tid", ctypes.c_uint32), + ("color", ctypes.c_uint32) + ) + +class FileFrameMark(ctypes.Structure): + _fields_ = ( + ("name", ctypes.c_uint32), + ("timestamp", ctypes.c_int64) + ) + +class FileEvent(ctypes.Structure): + class Events(ctypes.Union): + _fields_ = ( + ("zone_begin", FileZoneBegin), + ("zone_end", FileZoneEnd), + ("zone_color", FileZoneColor), + ("frame_mark", FileFrameMark), + ) + + _anonymous_ = ("event",) + _fields_ = ( + ("type", ctypes.c_byte), + ("event", Events) + ) + +# network protocol +NetworkMaxFrameSize = 256 * 1024 +NetworkHandshakeWelcome = b"\x01" +NetworkHandshakeProtocolMismatch = b"\x02" +NetworkEventZoneBegin = 15 +NetworkEventZoneEnd = 17 +NetworkEventTerminate = 55 +NetworkEventThreadContext = 57 +NetworkEventZoneColor = 62 +NetworkEventFrameMark = 64 +NetworkEventSrcloc = 67 +NetworkResponseServerQueryNoop = 87 +NetworkResponseSourceCodeNotAvailable = 88 +NetworkResponseSymbolCodeNotAvailable = 89 +NetworkResponseStringData = 94 +NetworkResponseThreadName = 95 +NetworkQueryTerminate = 0 +NetworkQueryString = 1 +NetworkQueryThreadString = 2 +NetworkQuerySrcloc = 3 +NetworkQueryPlotName = 4 +NetworkQueryFrameName = 5 +NetworkQueryParameter = 6 +NetworkQueryFiberName = 7 +NetworkQueryDisconnect = 8 +NetworkQueryCallstackFrame = 9 +NetworkQueryExternalName = 10 +NetworkQuerySymbol = 11 +NetworkQuerySymbolCode = 12 +NetworkQueryCodeLocation = 13 +NetworkQuerySourceCode = 14 +NetworkQueryDataTransfer = 15 +NetworkQueryDataTransferPart = 16 + + +class NetworkHeader(ctypes.Structure): + _pack_ = 1 + _fields_ = ( + ("multiplier", ctypes.c_double), + ("init_begin", ctypes.c_longlong), + ("init_end", ctypes.c_longlong), + ("delay", ctypes.c_longlong), + ("resolution", ctypes.c_longlong), + ("epoch", ctypes.c_longlong), + ("exec_time", ctypes.c_longlong), + ("pid", ctypes.c_longlong), + ("sampling_period", ctypes.c_longlong), + ("flags", ctypes.c_uint8), + ("cpu_arch", ctypes.c_uint8), + ("cpu_manufacturer", ctypes.c_char * 12), + ("cpu_id", ctypes.c_uint), + ("program_name", ctypes.c_char * 64), + ("host_info", ctypes.c_char * 1024) + ) + +class NetworkThreadContext(ctypes.Structure): + _pack_ = 1 + _fields_ = ( + ("type", ctypes.c_uint8), + ("tid", ctypes.c_uint32) + ) + +class NetworkZoneBegin(ctypes.Structure): + _pack_ = 1 + _fields_ = ( + ("type", ctypes.c_uint8), + ("timestamp", ctypes.c_int64), + ("srcloc", ctypes.c_uint64) + ) + +class NetworkZoneEnd(ctypes.Structure): + _pack_ = 1 + _fields_ = ( + ("type", ctypes.c_uint8), + ("timestamp", ctypes.c_int64) + ) + +class NetworkZoneColor(ctypes.Structure): + _pack_ = 1 + _fields_ = ( + ("type", ctypes.c_uint8), + ("r", ctypes.c_uint8), + ("g", ctypes.c_uint8), + ("b", ctypes.c_uint8) + ) + +class NetworkFrameMark(ctypes.Structure): + _pack_ = 1 + _fields_ = ( + ("type", ctypes.c_uint8), + ("timestamp", ctypes.c_int64), + ("name", ctypes.c_uint64) + ) + +class NetworkSrcloc(ctypes.Structure): + _pack_ = 1 + _fields_ = ( + ("type", ctypes.c_uint8), + ("name", ctypes.c_int64), + ("function", ctypes.c_int64), + ("file", ctypes.c_int64), + ("line", ctypes.c_uint32), + ("r", ctypes.c_uint8), + ("g", ctypes.c_uint8), + ("b", ctypes.c_uint8) + ) + +class NetworkRequest(ctypes.Structure): + _pack_ = 1 + _fields_ = ( + ("type", ctypes.c_uint8), + ("ptr", ctypes.c_int64), + ("extra", ctypes.c_uint32) + ) + +def main(stream): + def file_read_uint(): + ctype = ctypes.c_uint() + stream.readinto(ctype) + return ctype.value + + def file_read_chars(size): + if 0 == size: + return None + ctype = (ctypes.c_char * size)() + stream.readinto(ctype) + return ctype.value + + header = FileHeader() + stream.readinto(header) + if header.signature != FileSignature: + print("incorrect signature") + return + + if header.version != FileVersion: + print("incorrect version") + return + + srclocs_len = file_read_uint() + strings = {} + srclocs = [None] * srclocs_len + + for i in range(srclocs_len): + name_len = file_read_uint() + name = file_read_chars(name_len) + function_len = file_read_uint() + function = file_read_chars(function_len) + file_len = file_read_uint() + file = file_read_chars(file_len) + line = file_read_uint() + color = file_read_uint() + + if name != None: + digest = hash(name) + strings[digest] = name + name = digest + else: + name = 0 + + if function != None: + digest = hash(function) + strings[digest] = function + function = digest + else: + function = 0 + + if file != None: + digest = hash(file) + strings[digest] = file + file = digest + else: + file = 0 + + srclocs[i] = (name, function, file, line, color) + + server = socket.create_server(("127.0.0.1", 8086)) + server.listen(1) + print("listening on 127.0.0.1:8086...") + client, addr = server.accept() + + if client.recv(8) != b"TracyPrf": + print("bad client") + client.close() + return + + protocol = ctypes.c_uint() + client.recv_into(protocol) + + if protocol.value not in (56, 57): + print("bad protocol") + client.sendall(NetworkHandshakeProtocolMismatch) + client.close() + return + + print(f"client accepted from {addr}") + client.sendall(NetworkHandshakeWelcome) + + client.sendall(NetworkHeader( + multiplier = header.multiplier, + init_begin = header.init_begin, + init_end = header.init_end, + delay = header.delay, + resolution = header.resolution, + epoch = header.epoch, + exec_time = header.exec_time, + pid = header.pid, + sampling_period = header.sampling_period, + flags = header.flags, + cpu_arch = header.cpu_arch, + cpu_manufacturer = header.cpu_manufacturer, + cpu_id = header.cpu_id, + program_name = header.program_name, + host_info = header.host_info + )) + + event = FileEvent() + event_sz = ctypes.sizeof(event) + timestamp = 0 + tid = None + buffer = bytearray(NetworkMaxFrameSize // event_sz) + offset = 0 + compressor = lz4.stream.LZ4StreamCompressor( + "double_buffer", + NetworkMaxFrameSize, + store_comp_size = 4 + ) + + def commit(): + nonlocal offset + if offset > 0: + block = compressor.compress(buffer[:offset]) + client.sendall(block) + offset = 0 + + def write_msg(msg): + nonlocal offset, buffer + sz = ctypes.sizeof(msg) + if offset + sz > len(buffer): + commit() + + buffer[offset : offset + sz] = msg + offset += sz + + def thread_context(event): + nonlocal tid, timestamp + if event.zone_begin.tid != tid: + tid = event.zone_begin.tid + timestamp = 0 + write_msg(NetworkThreadContext( + type = NetworkEventThreadContext, + tid = tid + )) + + while event_sz == stream.readinto(event): + if event.type == FileEventZoneBegin: + thread_context(event) + write_msg(NetworkZoneBegin( + type = NetworkEventZoneBegin, + timestamp = event.zone_begin.timestamp - timestamp, + srcloc = event.zone_begin.srcloc + )) + timestamp = event.zone_begin.timestamp + + elif event.type == FileEventZoneEnd: + thread_context(event) + write_msg(NetworkZoneEnd( + type = NetworkEventZoneEnd, + timestamp = event.zone_end.timestamp - timestamp + )) + timestamp = event.zone_end.timestamp + + elif event.type == FileEventZoneColor: + thread_context(event) + write_msg(NetworkZoneColor( + type = NetworkEventZoneColor, + r = (event.zone_color.color >> 0x00) & 0xFF, + g = (event.zone_color.color >> 0x08) & 0xFF, + b = (event.zone_color.color >> 0x10) & 0xFF + )) + + elif event.type == FileEventFrameMark: + write_msg(NetworkFrameMark( + type = NetworkEventFrameMark, + name = 0, + timestamp = event.frame_mark.timestamp + )) + + commit() + + def respond_string(string, ptr, type): + string_sz = len(string) + + class NetworkStringData(ctypes.Structure): + _pack_ = 1 + _fields_ = ( + ("type", ctypes.c_uint8), + ("ptr", ctypes.c_uint64), + ("len", ctypes.c_uint16), + ("str", ctypes.c_char * string_sz) + ) + + write_msg(NetworkStringData( + type = type, + ptr = req.ptr, + len = string_sz, + str = string + )) + + req = NetworkRequest() + client.settimeout(1) + + try: + while ctypes.sizeof(req) == client.recv_into(req): + if req.type == NetworkQuerySrcloc: + srcloc = srclocs[req.ptr] + write_msg(NetworkSrcloc( + type = NetworkEventSrcloc, + name = srcloc[0], + function = srcloc[1], + file = srcloc[2], + line = srcloc[3], + r = (srcloc[4] >> 0x00) & 0xFF, + g = (srcloc[4] >> 0x08) & 0xFF, + b = (srcloc[4] >> 0x10) & 0xFF + )) + + elif req.type == NetworkQueryString: + respond_string(strings[req.ptr], req.ptr, NetworkResponseStringData) + + elif req.type == NetworkQuerySymbolCode: + write_msg(ctypes.c_uint8(NetworkResponseSymbolCodeNotAvailable)) + + elif req.type == NetworkQuerySourceCode: + write_msg(ctypes.c_uint8(NetworkResponseSourceCodeNotAvailable)) + + elif req.type == NetworkQueryDataTransfer: + write_msg(ctypes.c_uint8(NetworkResponseServerQueryNoop)) + + elif req.type == NetworkQueryDataTransferPart: + write_msg(ctypes.c_uint8(NetworkResponseServerQueryNoop)) + + elif req.type == NetworkQueryThreadString: + respond_string(b"main", req.ptr, NetworkResponseThreadName) + + else: + print("unknown req:", req.type) + + commit() + + except socket.timeout: + pass + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("file", type=argparse.FileType("rb")) + args = parser.parse_args() + main(args.file)