13x Performance for Export chatlog, update rust_g to 0.4.6-vorestation

.travis.yml

@@ -1,5 +1,6 @@
-#pretending we're C because otherwise ruby will initialize, even with "language: dm".
-language: c
+language: generic
+os: linux
+dist: bionic
 
 env:
   global:
@@ -12,10 +13,14 @@ cache:
 addons:
   apt:
     packages:
-      - libc6-i386
+      - libc6:i386
       - libgcc1:i386
       - libstdc++6:i386
-      - libssl-dev:i386
+      - libssl1.1:i386
+      - g++-7
+      - g++-7-multilib
+      - gcc-multilib
+      - zlib1g:i386
 
 before_install:
   - chmod -R +x ./tools/travis
@@ -27,6 +32,7 @@ before_script:
   - shopt -s globstar
 
 script:
+  - ldd librust_g.so
   - ./tools/travis/compile_and_run.sh
 
 # Build-specific settings

_build_dependencies.sh

@@ -4,7 +4,7 @@ export SPACEMANDMM_TAG=suite-1.4
 # For NanoUI + TGUI
 export NODE_VERSION=12
 # For the scripts in tools
-export PHP_VERSION=5.6
+export PHP_VERSION=7.2
 # Byond Major
 export BYOND_MAJOR=513
 # Byond Minor

code/__defines/_compile_options.dm

@@ -7,10 +7,6 @@
 								2 for preloading absolutely everything;
 								*/
 
-#define RUST_G "rust_g"			// If uncommented, we will use the rust-g (https://github.com/tgstation/rust-g) native library for fast
-								// logging. This requires you to have the rust_g.dll or rust_g (renamed from librust_g.so) installed in the root folder or BYOND/bin
-								// The define's value should be the name of library file.
-
 // ZAS Compile Options
 //#define ZASDBG	 	// Uncomment to turn on super detailed ZAS debugging that probably won't even compile.
 #define MULTIZAS		// Uncomment to turn on Multi-Z ZAS Support!

code/__defines/rust_g.dm

@@ -0,0 +1,103 @@
+// rust_g.dm - DM API for rust_g extension library
+//
+// To configure, create a `rust_g.config.dm` and set what you care about from
+// the following options:
+//
+// #define RUST_G "path/to/rust_g"
+// Override the .dll/.so detection logic with a fixed path or with detection
+// logic of your own.
+//
+// #define RUSTG_OVERRIDE_BUILTINS
+// Enable replacement rust-g functions for certain builtins. Off by default.
+
+#ifndef RUST_G
+// Default automatic RUST_G detection.
+// On Windows, looks in the standard places for `rust_g.dll`.
+// On Linux, looks in `.`, `$LD_LIBRARY_PATH`, and `~/.byond/bin` for either of
+// `librust_g.so` (preferred) or `rust_g` (old).
+
+/* This comment bypasses grep checks */ /var/__rust_g
+
+/proc/__detect_rust_g()
+	if (world.system_type == UNIX)
+		if (fexists("./librust_g.so"))
+			// No need for LD_LIBRARY_PATH badness.
+			return __rust_g = "./librust_g.so"
+		else if (fexists("./rust_g"))
+			// Old dumb filename.
+			return __rust_g = "./rust_g"
+		else if (fexists("[world.GetConfig("env", "HOME")]/.byond/bin/rust_g"))
+			// Old dumb filename in `~/.byond/bin`.
+			return __rust_g = "rust_g"
+		else
+			// It's not in the current directory, so try others
+			return __rust_g = "librust_g.so"
+	else
+		return __rust_g = "rust_g"
+
+#define RUST_G (__rust_g || __detect_rust_g())
+#endif
+
+#define RUSTG_JOB_NO_RESULTS_YET "NO RESULTS YET"
+#define RUSTG_JOB_NO_SUCH_JOB "NO SUCH JOB"
+#define RUSTG_JOB_ERROR "JOB PANICKED"
+
+#define rustg_dmi_strip_metadata(fname) call(RUST_G, "dmi_strip_metadata")(fname)
+#define rustg_dmi_create_png(path, width, height, data) call(RUST_G, "dmi_create_png")(path, width, height, data)
+
+#define rustg_noise_get_at_coordinates(seed, x, y) call(RUST_G, "noise_get_at_coordinates")(seed, x, y)
+
+#define rustg_file_read(fname) call(RUST_G, "file_read")(fname)
+#define rustg_file_exists(fname) call(RUST_G, "file_exists")(fname)
+#define rustg_file_write(text, fname) call(RUST_G, "file_write")(text, fname)
+#define rustg_file_append(text, fname) call(RUST_G, "file_append")(text, fname)
+
+#ifdef RUSTG_OVERRIDE_BUILTINS
+#define file2text(fname) rustg_file_read("[fname]")
+#define text2file(text, fname) rustg_file_append(text, "[fname]")
+#endif
+
+#define rustg_git_revparse(rev) call(RUST_G, "rg_git_revparse")(rev)
+#define rustg_git_commit_date(rev) call(RUST_G, "rg_git_commit_date")(rev)
+
+#define rustg_hash_string(algorithm, text) call(RUST_G, "hash_string")(algorithm, text)
+#define rustg_hash_file(algorithm, fname) call(RUST_G, "hash_file")(algorithm, fname)
+
+#define RUSTG_HASH_MD5 "md5"
+#define RUSTG_HASH_SHA1 "sha1"
+#define RUSTG_HASH_SHA256 "sha256"
+#define RUSTG_HASH_SHA512 "sha512"
+
+#ifdef RUSTG_OVERRIDE_BUILTINS
+#define md5(thing) (isfile(thing) ? rustg_hash_file(RUSTG_HASH_MD5, "[thing]") : rustg_hash_string(RUSTG_HASH_MD5, thing))
+#endif
+
+#define rustg_json_is_valid(text) (call(RUST_G, "json_is_valid")(text) == "true")
+
+#define rustg_log_write(fname, text, format) call(RUST_G, "log_write")(fname, text, format)
+/proc/rustg_log_close_all() return call(RUST_G, "log_close_all")()
+
+#define rustg_url_encode(text) call(RUST_G, "url_encode")(text)
+#define rustg_url_decode(text) call(RUST_G, "url_decode")(text)
+
+#ifdef RUSTG_OVERRIDE_BUILTINS
+#define url_encode(text) rustg_url_encode(text)
+#define url_decode(text) rustg_url_decode(text)
+#endif
+
+#define RUSTG_HTTP_METHOD_GET "get"
+#define RUSTG_HTTP_METHOD_PUT "put"
+#define RUSTG_HTTP_METHOD_DELETE "delete"
+#define RUSTG_HTTP_METHOD_PATCH "patch"
+#define RUSTG_HTTP_METHOD_HEAD "head"
+#define RUSTG_HTTP_METHOD_POST "post"
+#define rustg_http_request_blocking(method, url, body, headers) call(RUST_G, "http_request_blocking")(method, url, body, headers)
+#define rustg_http_request_async(method, url, body, headers) call(RUST_G, "http_request_async")(method, url, body, headers)
+#define rustg_http_check_request(req_id) call(RUST_G, "http_check_request")(req_id)
+
+#define rustg_sql_connect_pool(options) call(RUST_G, "sql_connect_pool")(options)
+#define rustg_sql_query_async(handle, query, params) call(RUST_G, "sql_query_async")(handle, query, params)
+#define rustg_sql_query_blocking(handle, query, params) call(RUST_G, "sql_query_blocking")(handle, query, params)
+#define rustg_sql_connected(handle) call(RUST_G, "sql_connected")(handle)
+#define rustg_sql_disconnect_pool(handle) call(RUST_G, "sql_disconnect_pool")(handle)
+#define rustg_sql_check_query(job_id) call(RUST_G, "sql_check_query")("[job_id]")

code/_helpers/logging.dm

@@ -1,25 +1,17 @@
 //print an error message to world.log
 
-// Fall back to using old format if we are not using rust-g
-#ifdef RUST_G
-	#define WRITE_LOG(log, text) call(RUST_G, "log_write")(log, text)
-#else
-	#define WRITE_LOG(log, text) log << "\[[time_stamp()]][text]"
-#endif
+//This is an external call, "true" and "false" are how rust parses out booleans
+#define WRITE_LOG(log, text) rustg_log_write(log, text, "true")
+#define WRITE_LOG_NO_FORMAT(log, text) rustg_log_write(log, text, "false")
 
 /* For logging round startup. */
 /proc/start_log(log)
-	#ifndef RUST_G
-	log = file(log)
-	#endif
 	WRITE_LOG(log, "START: Starting up [log_path].")
 	return log
 
 /* Close open log handles. This should be called as late as possible, and no logging should hapen after. */
 /proc/shutdown_logging()
-	#ifdef RUST_G
-	call(RUST_G, "log_close_all")()
-	#endif
+	rustg_log_close_all()
 
 /proc/error(msg)
 	to_world_log("## ERROR: [msg]")

code/modules/asset_cache/asset_list.dm

@@ -119,18 +119,14 @@ GLOBAL_LIST_EMPTY(asset_datums)
 		if (size[SPRSZ_STRIPPED])
 			continue
 
-		#ifdef RUST_G
 		// save flattened version
 		var/fname = "data/spritesheets/[name]_[size_id].png"
 		fcopy(size[SPRSZ_ICON], fname)
-		var/error = call(RUST_G, "dmi_strip_metadata")(fname)
+		var/error = rustg_dmi_strip_metadata(fname)
 		if(length(error))
 			stack_trace("Failed to strip [name]_[size_id].png: [error]")
 		size[SPRSZ_STRIPPED] = icon(fname)
 		fdel(fname)
-		#else
-		#warn It looks like you don't have RUST_G enabled. Without RUST_G, the RPD icons will not function, so it strongly recommended you reenable it.
-		#endif
 
 /datum/asset/spritesheet/proc/generate_css()
 	var/list/out = list()

code/modules/vchat/vchat_client.dm

@@ -400,22 +400,22 @@ var/to_chat_src
 		to_chat(src, "<span class='warning'>Error: Your chat log is already being prepared. Please wait until it's been downloaded before trying to export it again.</span>")
 		return
 
-	o_file = file(o_file)
-
 	// Write the CSS file to the log
-	o_file << "<html><head><style>"
-	o_file << file2text(file("code/modules/vchat/css/ss13styles.css"))
-	o_file << "</style></head><body>"
+	var/text_blob = "<html><head><style>"
+	text_blob += file2text(file("code/modules/vchat/css/ss13styles.css"))
+	text_blob += "</style></head><body>"
 
 	// Write the messages to the log
 	for(var/list/result in results)
-		o_file << "[result["message"]]<br>"
+		text_blob += "[result["message"]]<br>"
 		CHECK_TICK
 
-	o_file << "</body></html>"
+	text_blob += "</body></html>"
+
+	rustg_file_write(text_blob, o_file)
 
 	// Send the log to the client
-	src << ftp(o_file, "log_[time2text(world.timeofday, "YYYY_MM_DD_(hh_mm)")].html")
+	src << ftp(file(o_file), "log_[time2text(world.timeofday, "YYYY_MM_DD_(hh_mm)")].html")
 
 	// clean up the file on our end
 	spawn(10 SECONDS)

code/modules/vore/eating/vore_vr.dm

@@ -207,9 +207,23 @@ V::::::V           V::::::VO:::::::OOO:::::::ORR:::::R     R:::::REE::::::EEEEEE
 		return FALSE
 
 	//Write it out
+<<<<<<< HEAD
 	if(fexists(path))
 		fdel(path) //Byond only supports APPENDING to files, not replacing.
 	text2file(json_to_file, path)
+||||||| parent of b4fb235ba3... Merge pull request #9246 from ShadowLarkens/vchat_export_optimization
+#ifdef RUST_G
+	call(RUST_G, "file_write")(json_to_file, path)
+#else
+	// Fall back to using old format if we are not using rust-g
+	if(fexists(path))
+		fdel(path) //Byond only supports APPENDING to files, not replacing.
+	text2file(json_to_file, path)
+#endif
+=======
+	rustg_file_write(json_to_file, path)
+
+>>>>>>> b4fb235ba3... Merge pull request #9246 from ShadowLarkens/vchat_export_optimization
 	if(!fexists(path))
 		log_debug("Saving: [path] failed file write")
 		return FALSE

vorestation.dme

@@ -78,6 +78,7 @@
 #include "code\__defines\qdel.dm"
 #include "code\__defines\research.dm"
 #include "code\__defines\roguemining_vr.dm"
+#include "code\__defines\rust_g.dm"
 #include "code\__defines\shields.dm"
 #include "code\__defines\shuttle.dm"
 #include "code\__defines\sound.dm"