Add paradise's version of byond-tracy for better production profiling (#16485)

* add para-byond-tracy and modify the tracy init

* remove obsolete binaries, add replay.py, add tracy 0.8.2 client, add changelog

* add tracy client files and other version of byond-tracy

code/__defines/profiler_tracy.dm

@@ -1,31 +1,62 @@
+// File for the byond-tracy profiler.
 // Implements https://github.com/mafemergency/byond-tracy
-// Client https://github.com/wolfpld/tracy
-// As of now, only 0.8.2 is supported as a client, this might change in the future however
+// and https://github.com/ParadiseSS13/byond-tracy
+// Tracy Client https://github.com/wolfpld/tracy
 
-// In case you need to start the capture as soon as the server boots, uncomment the following lines and recompile:
+var/byond_tracy_running = 0			// Whether byond-tracy is currently running or not, no matter what version.
+var/byond_tracy_running_v = null	// Which version of byond-tracy is currently running, so we call the right binary on destruction.
 
-// /world/New()
-// 	prof_init()
-// 	. = ..()
+/world/New()
+	if(config.enable_byond_tracy)
+		prof_init("tracy_disk") // This start's Affectedarc07's version of Tracy. Writing a .utracy file to the disk.
+	. = ..()
+
+/world/Del()
+	if(byond_tracy_running)
+		prof_destroy()
+	. = ..()
 
 /client/proc/profiler_start()
 	set name = "Start Tracy Profiler"
 	set category = "Debug"
-	set desc = "Starts the tracy profiler, which will await the client connection."
-	switch(alert("Are you sure? Tracy will remain active until the server restarts.", "Tracy Init", "No", "Yes"))
-		if("Yes")
-			prof_init()
+	set desc = "Starts the tracy profiler, which will await the client connection or save utracy files to the server's disk."
+	if(!byond_tracy_running)
+		switch(alert("Are you sure? Tracy will remain active until the server restarts.", "Tracy Init", "No", "Yes"))
+			if("Yes")
+				switch(alert("Which version of Tracy would you like to run?", "Tracy Version", "Cancel", "Connection Based", "Disk Based"))
+					if("Connection Based")
+						prof_init("tracy") // This start's mafemergency's original version of Tracy. Requiring a direct connection, not writing to the disk.
+					if("Disk Based")
+						prof_init("tracy_disk") // This start's Affectedarc07's version of Tracy. Writing a .utracy file to the disk.
 
 /**
- * Starts Tracy
+ * Starts Tracy.
  */
-/proc/prof_init()
+/proc/prof_init(var/version)
 	var/lib
 
-	switch(world.system_type)
-		if(MS_WINDOWS) lib = "prof.dll"
-		if(UNIX) lib = "libprof.so"
-		else CRASH("Tracy initialization failed: unsupported platform or DLL not found.")
+	switch(version)
+		if("tracy") lib = "tracy.dll"
+		if("tracy_disk") lib = "tracy-disk.dll"
 
 	var/init = LIBCALL(lib, "init")()
 	if("0" != init) CRASH("[lib] init error: [init]")
+
+	byond_tracy_running = 1
+	byond_tracy_running_v = version
+
+/**
+ * Stops Tracy.
+ * Doing this in the middle of the round is not good, don't do it.
+ */
+/proc/prof_destroy()
+	var/lib
+
+	switch(byond_tracy_running_v)
+		if("tracy") lib = "tracy.dll"
+		if("tracy_disk") lib = "tracy-disk.dll"
+
+	LIBCALL(lib, "destroy")()
+
+	byond_tracy_running = 0
+	byond_tracy_running_v = null

code/controllers/configuration.dm

@@ -261,6 +261,9 @@ var/list/gamemode_cache = list()
 	var/ipintel_save_bad = 1
 	var/ipintel_domain = "check.getipintel.net"
 
+	// BYOND Tracy
+	var/enable_byond_tracy = 0
+
 	// Access control/Panic bunker settings.
 	var/access_deny_new_players = 0
 	var/access_deny_new_accounts = -1
@@ -874,6 +877,9 @@ var/list/gamemode_cache = list()
 				if("ipintel_save_bad")
 					ipintel_save_bad = text2num(value)
 
+				if("enable_byond_tracy")
+					enable_byond_tracy = 1
+
 				if("access_deny_new_accounts")
 					access_deny_new_accounts = text2num(value) >= 0 ? text2num(value) : -1
 				if("access_deny_vms")

config/example/config.txt

@@ -504,6 +504,13 @@ LOG_GELF_ENABLED 0
 ## Domain name to query (leave commented out for the default, only needed if you pay getipintel.net for more querys)
 #IPINTEL_DOMAIN check.getipintel.net
 
+### BYOND Tracy:
+### Uncomment this option to enable profiling with byond-tracy. Admins and Developers have the ability to use the appropriate debug verb
+### to start the profiler in the middle of the round even if this option is uncommented, otherwise tracy will start on /world/New().
+### Warning: Do not uncomment this configuration option unless you know what you are doing and need it. It will create a gigabyte-large file
+### everytime you start the server, which will grow as the server continues to run.
+#ENABLE_BYOND_TRACY
+
 ### ACCESS CONTROL:
 ### This allows you to adjust various parametres by which potentially malicious users can by identified.
 ### Note that none of these are 100% accurate, and should only be used in cases where a "panic bunker" is warranted.

html/changelogs/byond-tracy-io-util.yml

@@ -0,0 +1,7 @@
+author: io-util
+
+delete-after: True
+
+changes:
+  - backend: "Use para-byond-tracy instead of byond-tracy which saves .utracy files to the disk to view later."
+  - admin: "Modified the Start-Tracy-Profiler adminverb, it is now aware of whether tracy is already running or not."

tools/tracy/replay/README.MD

@@ -0,0 +1,7 @@
+# Info
+
+This directory contains the `replay.py` script. This is used to "replay" custom `.utracy` files generated by the production server.
+
+`BYOND-Tracy` profiles can use excessive amounts of RAM, upwards of 48GB in a single process if captured normally at runtime. This is not viable for the production server, so they are written as a custom flatfile inside of `data/profiler/`. These need to be read into tracy over the "network" (localhost) via `Tracy.exe` or `capture.exe`. You will need >48GB of RAM for this process. I am not joking.
+
+The version of `replay.py` in this folder is compatible with the protocol of `Tracy 0.8.2`. Newer versions will not work. It requires the `lz4` python module with the streams extension. This requires manually downloading, building and installing the `python-lz4/python-lz4` repo and building with `PYLZ4_EXPERIMENTAL=TRUE` as an environment variable.

tools/tracy/replay/replay.py

@@ -0,0 +1,432 @@
+import argparse
+import ctypes
+import socket
+import selectors
+import lz4.stream
+
+# file protocol
+FileSignature = 0x6D64796361727475
+FileVersion = 2
+FileEventZoneBegin =15
+FileEventZoneEnd = 17
+FileEventZoneColor = 62
+FileEventFrameMark = 64
+
+class FileHeader(ctypes.Structure):
+	_fields_ = (
+		("signature", ctypes.c_ulonglong),
+		("version", ctypes.c_uint),
+		("multiplier", ctypes.c_double),
+		("init_begin", ctypes.c_longlong),
+		("init_end", ctypes.c_longlong),
+		("delay", ctypes.c_longlong),
+		("resolution", ctypes.c_longlong),
+		("epoch", ctypes.c_longlong),
+		("exec_time", ctypes.c_longlong),
+		("pid", ctypes.c_longlong),
+		("sampling_period", ctypes.c_longlong),
+		("flags", ctypes.c_byte),
+		("cpu_arch", ctypes.c_byte),
+		("cpu_manufacturer", ctypes.c_char * 12),
+		("cpu_id", ctypes.c_uint),
+		("program_name", ctypes.c_char * 64),
+		("host_info", ctypes.c_char * 1024)
+	)
+
+class FileZoneBegin(ctypes.Structure):
+	_fields_ = (
+		("tid", ctypes.c_uint32),
+		("srcloc", ctypes.c_uint32),
+		("timestamp", ctypes.c_int64)
+	)
+
+class FileZoneEnd(ctypes.Structure):
+	_fields_ = (
+		("tid", ctypes.c_uint32),
+		("timestamp", ctypes.c_int64)
+	)
+
+class FileZoneColor(ctypes.Structure):
+	_fields_ = (
+		("tid", ctypes.c_uint32),
+		("color", ctypes.c_uint32)
+	)
+
+class FileFrameMark(ctypes.Structure):
+	_fields_ = (
+		("name", ctypes.c_uint32),
+		("timestamp", ctypes.c_int64)
+	)
+
+class FileEvent(ctypes.Structure):
+	class Events(ctypes.Union):
+		_fields_ = (
+			("zone_begin", FileZoneBegin),
+			("zone_end", FileZoneEnd),
+			("zone_color", FileZoneColor),
+			("frame_mark", FileFrameMark),
+		)
+
+	_anonymous_ = ("event",)
+	_fields_ = (
+		("type", ctypes.c_byte),
+		("event", Events)
+	)
+
+# network protocol
+NetworkMaxFrameSize = 256 * 1024
+NetworkHandshakeWelcome = b"\x01"
+NetworkHandshakeProtocolMismatch = b"\x02"
+NetworkEventZoneBegin = 15
+NetworkEventZoneEnd = 17
+NetworkEventTerminate = 55
+NetworkEventThreadContext = 57
+NetworkEventZoneColor = 62
+NetworkEventFrameMark = 64
+NetworkEventSrcloc = 67
+NetworkResponseServerQueryNoop = 87
+NetworkResponseSourceCodeNotAvailable = 88
+NetworkResponseSymbolCodeNotAvailable = 89
+NetworkResponseStringData = 94
+NetworkResponseThreadName = 95
+NetworkQueryTerminate = 0
+NetworkQueryString = 1
+NetworkQueryThreadString = 2
+NetworkQuerySrcloc = 3
+NetworkQueryPlotName = 4
+NetworkQueryFrameName = 5
+NetworkQueryParameter = 6
+NetworkQueryFiberName = 7
+NetworkQueryDisconnect = 8
+NetworkQueryCallstackFrame = 9
+NetworkQueryExternalName = 10
+NetworkQuerySymbol = 11
+NetworkQuerySymbolCode = 12
+NetworkQueryCodeLocation = 13
+NetworkQuerySourceCode = 14
+NetworkQueryDataTransfer = 15
+NetworkQueryDataTransferPart = 16
+
+
+class NetworkHeader(ctypes.Structure):
+	_pack_ = 1
+	_fields_ = (
+		("multiplier", ctypes.c_double),
+		("init_begin", ctypes.c_longlong),
+		("init_end", ctypes.c_longlong),
+		("delay", ctypes.c_longlong),
+		("resolution", ctypes.c_longlong),
+		("epoch", ctypes.c_longlong),
+		("exec_time", ctypes.c_longlong),
+		("pid", ctypes.c_longlong),
+		("sampling_period", ctypes.c_longlong),
+		("flags", ctypes.c_uint8),
+		("cpu_arch", ctypes.c_uint8),
+		("cpu_manufacturer", ctypes.c_char * 12),
+		("cpu_id", ctypes.c_uint),
+		("program_name", ctypes.c_char * 64),
+		("host_info", ctypes.c_char * 1024)
+	)
+
+class NetworkThreadContext(ctypes.Structure):
+	_pack_ = 1
+	_fields_ = (
+		("type", ctypes.c_uint8),
+		("tid", ctypes.c_uint32)
+	)
+
+class NetworkZoneBegin(ctypes.Structure):
+	_pack_ = 1
+	_fields_ = (
+		("type", ctypes.c_uint8),
+		("timestamp", ctypes.c_int64),
+		("srcloc", ctypes.c_uint64)
+	)
+
+class NetworkZoneEnd(ctypes.Structure):
+	_pack_ = 1
+	_fields_ = (
+		("type", ctypes.c_uint8),
+		("timestamp", ctypes.c_int64)
+	)
+
+class NetworkZoneColor(ctypes.Structure):
+	_pack_ = 1
+	_fields_ = (
+		("type", ctypes.c_uint8),
+		("r", ctypes.c_uint8),
+		("g", ctypes.c_uint8),
+		("b", ctypes.c_uint8)
+	)
+
+class NetworkFrameMark(ctypes.Structure):
+	_pack_ = 1
+	_fields_ = (
+		("type", ctypes.c_uint8),
+		("timestamp", ctypes.c_int64),
+		("name", ctypes.c_uint64)
+	)
+
+class NetworkSrcloc(ctypes.Structure):
+	_pack_ = 1
+	_fields_ = (
+		("type", ctypes.c_uint8),
+		("name", ctypes.c_int64),
+		("function", ctypes.c_int64),
+		("file", ctypes.c_int64),
+		("line", ctypes.c_uint32),
+		("r", ctypes.c_uint8),
+		("g", ctypes.c_uint8),
+		("b", ctypes.c_uint8)
+	)
+
+class NetworkRequest(ctypes.Structure):
+	_pack_ = 1
+	_fields_ = (
+		("type", ctypes.c_uint8),
+		("ptr", ctypes.c_int64),
+		("extra", ctypes.c_uint32)
+	)
+
+def main(stream):
+	def file_read_uint():
+		ctype = ctypes.c_uint()
+		stream.readinto(ctype)
+		return ctype.value
+
+	def file_read_chars(size):
+		if 0 == size:
+			return None
+		ctype = (ctypes.c_char * size)()
+		stream.readinto(ctype)
+		return ctype.value
+
+	header = FileHeader()
+	stream.readinto(header)
+	if header.signature != FileSignature:
+		print("incorrect signature")
+		return
+
+	if header.version != FileVersion:
+		print("incorrect version")
+		return
+
+	srclocs_len = file_read_uint()
+	strings = {}
+	srclocs = [None] * srclocs_len
+
+	for i in range(srclocs_len):
+		name_len = file_read_uint()
+		name = file_read_chars(name_len)
+		function_len = file_read_uint()
+		function = file_read_chars(function_len)
+		file_len = file_read_uint()
+		file = file_read_chars(file_len)
+		line = file_read_uint()
+		color = file_read_uint()
+
+		if name != None:
+			digest = hash(name)
+			strings[digest] = name
+			name = digest
+		else:
+			name = 0
+
+		if function != None:
+			digest = hash(function)
+			strings[digest] = function
+			function = digest
+		else:
+			function = 0
+
+		if file != None:
+			digest = hash(file)
+			strings[digest] = file
+			file = digest
+		else:
+			file = 0
+
+		srclocs[i] = (name, function, file, line, color)
+
+	server = socket.create_server(("127.0.0.1", 8086))
+	server.listen(1)
+	print("listening on 127.0.0.1:8086...")
+	client, addr = server.accept()
+
+	if client.recv(8) != b"TracyPrf":
+		print("bad client")
+		client.close()
+		return
+
+	protocol = ctypes.c_uint()
+	client.recv_into(protocol)
+
+	if protocol.value not in (56, 57):
+		print("bad protocol")
+		client.sendall(NetworkHandshakeProtocolMismatch)
+		client.close()
+		return
+
+	print(f"client accepted from {addr}")
+	client.sendall(NetworkHandshakeWelcome)
+
+	client.sendall(NetworkHeader(
+		multiplier = header.multiplier,
+		init_begin = header.init_begin,
+		init_end = header.init_end,
+		delay = header.delay,
+		resolution = header.resolution,
+		epoch = header.epoch,
+		exec_time = header.exec_time,
+		pid = header.pid,
+		sampling_period = header.sampling_period,
+		flags = header.flags,
+		cpu_arch = header.cpu_arch,
+		cpu_manufacturer = header.cpu_manufacturer,
+		cpu_id = header.cpu_id,
+		program_name = header.program_name,
+		host_info = header.host_info
+	))
+
+	event = FileEvent()
+	event_sz = ctypes.sizeof(event)
+	timestamp = 0
+	tid = None
+	buffer = bytearray(NetworkMaxFrameSize // event_sz)
+	offset = 0
+	compressor = lz4.stream.LZ4StreamCompressor(
+		"double_buffer",
+		NetworkMaxFrameSize,
+		store_comp_size = 4
+	)
+
+	def commit():
+		nonlocal offset
+		if offset > 0:
+			block = compressor.compress(buffer[:offset])
+			client.sendall(block)
+			offset = 0
+
+	def write_msg(msg):
+		nonlocal offset, buffer
+		sz = ctypes.sizeof(msg)
+		if offset + sz > len(buffer):
+			commit()
+
+		buffer[offset : offset + sz] = msg
+		offset += sz
+
+	def thread_context(event):
+		nonlocal tid, timestamp
+		if event.zone_begin.tid != tid:
+			tid = event.zone_begin.tid
+			timestamp = 0
+			write_msg(NetworkThreadContext(
+				type = NetworkEventThreadContext,
+				tid = tid
+			))
+
+	while event_sz == stream.readinto(event):
+		if event.type == FileEventZoneBegin:
+			thread_context(event)
+			write_msg(NetworkZoneBegin(
+				type = NetworkEventZoneBegin,
+				timestamp = event.zone_begin.timestamp - timestamp,
+				srcloc = event.zone_begin.srcloc
+			))
+			timestamp = event.zone_begin.timestamp
+
+		elif event.type == FileEventZoneEnd:
+			thread_context(event)
+			write_msg(NetworkZoneEnd(
+				type = NetworkEventZoneEnd,
+				timestamp = event.zone_end.timestamp - timestamp
+			))
+			timestamp = event.zone_end.timestamp
+
+		elif event.type == FileEventZoneColor:
+			thread_context(event)
+			write_msg(NetworkZoneColor(
+				type = NetworkEventZoneColor,
+				r = (event.zone_color.color >> 0x00) & 0xFF,
+				g = (event.zone_color.color >> 0x08) & 0xFF,
+				b = (event.zone_color.color >> 0x10) & 0xFF
+			))
+
+		elif event.type == FileEventFrameMark:
+			write_msg(NetworkFrameMark(
+				type = NetworkEventFrameMark,
+				name = 0,
+				timestamp = event.frame_mark.timestamp
+			))
+
+	commit()
+
+	def respond_string(string, ptr, type):
+		string_sz = len(string)
+
+		class NetworkStringData(ctypes.Structure):
+			_pack_ = 1
+			_fields_ = (
+				("type", ctypes.c_uint8),
+				("ptr", ctypes.c_uint64),
+				("len", ctypes.c_uint16),
+				("str", ctypes.c_char * string_sz)
+			)
+
+		write_msg(NetworkStringData(
+			type = type,
+			ptr = req.ptr,
+			len = string_sz,
+			str = string
+		))
+
+	req = NetworkRequest()
+	client.settimeout(1)
+
+	try:
+		while ctypes.sizeof(req) == client.recv_into(req):
+			if req.type == NetworkQuerySrcloc:
+				srcloc = srclocs[req.ptr]
+				write_msg(NetworkSrcloc(
+					type = NetworkEventSrcloc,
+					name = srcloc[0],
+					function = srcloc[1],
+					file = srcloc[2],
+					line = srcloc[3],
+					r = (srcloc[4] >> 0x00) & 0xFF,
+					g = (srcloc[4] >> 0x08) & 0xFF,
+					b = (srcloc[4] >> 0x10) & 0xFF
+				))
+
+			elif req.type == NetworkQueryString:
+				respond_string(strings[req.ptr], req.ptr, NetworkResponseStringData)
+
+			elif req.type == NetworkQuerySymbolCode:
+				write_msg(ctypes.c_uint8(NetworkResponseSymbolCodeNotAvailable))
+
+			elif req.type == NetworkQuerySourceCode:
+				write_msg(ctypes.c_uint8(NetworkResponseSourceCodeNotAvailable))
+
+			elif req.type == NetworkQueryDataTransfer:
+				write_msg(ctypes.c_uint8(NetworkResponseServerQueryNoop))
+
+			elif req.type == NetworkQueryDataTransferPart:
+				write_msg(ctypes.c_uint8(NetworkResponseServerQueryNoop))
+
+			elif req.type == NetworkQueryThreadString:
+				respond_string(b"main", req.ptr, NetworkResponseThreadName)
+
+			else:
+				print("unknown req:", req.type)
+
+			commit()
+
+	except socket.timeout:
+		pass
+
+if __name__ == "__main__":
+	parser = argparse.ArgumentParser()
+	parser.add_argument("file", type=argparse.FileType("rb"))
+	args = parser.parse_args()
+	main(args.file)