diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 65242929651..3dc005acc3a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -114,12 +114,6 @@ jobs: uses: ./.github/actions/restore_or_install_byond with: release: ${{ matrix.byondtype }} - - name: Install RUST_G Deps - run: | - sudo dpkg --add-architecture i386 - sudo apt update || true - sudo apt install zlib1g-dev:i386 - tools/ci/install_rustg.sh - name: Compile & Run Unit Tests run: | tools/ci/install_byond.sh '${{ matrix.byondtype }}' @@ -147,12 +141,6 @@ jobs: sudo systemctl start mysql python3 tools/ci/generate_sql_scripts.py tools/ci/validate_sql.sh - - name: Install RUST_G Deps - run: | - sudo dpkg --add-architecture i386 - sudo apt update || true - sudo apt install zlib1g-dev:i386 - tools/ci/install_rustg.sh - name: Compile & Run Unit Tests run: | tools/ci/install_byond.sh '${{ matrix.byondtype }}' @@ -161,14 +149,3 @@ jobs: mkdir -p data echo '/datum/map/test_tiny' > data/next_map.txt tools/ci/run_server.sh - - windows_dll_tests: - name: Windows RUSTG Validation - runs-on: windows-latest - steps: - - uses: actions/checkout@v5 - - uses: actions/setup-python@v6 - with: - python-version: '3.8.2' # Script was made for 3.8.2 - architecture: 'x86' # This MUST be x86 - - run: python tools/ci/validate_rustg_windows.py diff --git a/code/__DEFINES/_versions.dm b/code/__DEFINES/_versions.dm deleted file mode 100644 index 74eb2e89dac..00000000000 --- a/code/__DEFINES/_versions.dm +++ /dev/null @@ -1,2 +0,0 @@ -/// Version of RUST-G that this codebase wants -#define RUST_G_VERSION "3.4.0-P" diff --git a/code/__DEFINES/rust.dm b/code/__DEFINES/rust.dm index 1ac62650108..9386b982629 100644 --- a/code/__DEFINES/rust.dm +++ b/code/__DEFINES/rust.dm @@ -199,7 +199,7 @@ // MARK: Toast /// (Windows only) Triggers a desktop notification with the specified title and body -/proc/rustlibs_create_toast(title, body) +/proc/rustlibs_create_toast(title, body) return RUSTLIB_CALL(create_toast, title, body) @@ -228,6 +228,25 @@ #define RUSTLIBS_JOB_NO_SUCH_JOB "NO SUCH JOB" #define RUSTLIBS_JOB_ERROR "JOB PANICKED" +// MARK: SQL +/proc/rustlibs_sql_connect_pool(datum/db_connection_request/dbcreq) + return RUSTLIB_CALL(sql_connect_pool, dbcreq) + +/proc/rustlibs_sql_disconnect_pool(handle) + return RUSTLIB_CALL(sql_disconnect_pool, handle) + +/proc/rustlibs_sql_connected(handle) + return RUSTLIB_CALL(sql_connected, handle) + +/proc/rustlibs_sql_query_blocking(datum/db_query/query) + return RUSTLIB_CALL(sql_query_blocking, query) + +/proc/rustlibs_sql_query_async(datum/db_query/query) + return RUSTLIB_CALL(sql_query_async, query) + +/proc/rustlibs_sql_check_query(datum/db_query/query) + return RUSTLIB_CALL(sql_check_query, query) + #undef RUSTLIB_CALL // Indexes for Tiles and InterestingTiles diff --git a/code/__DEFINES/rust_g.dm b/code/__DEFINES/rust_g.dm deleted file mode 100644 index d186ef7d5b1..00000000000 --- a/code/__DEFINES/rust_g.dm +++ /dev/null @@ -1,64 +0,0 @@ -// rust_g.dm - DM API for rust_g extension library -// -// To configure, create a `rust_g.config.dm` and set what you care about from -// the following options: -// -// #define RUST_G "path/to/rust_g" -// Override the .dll/.so detection logic with a fixed path or with detection -// logic of your own. -// -// #define RUSTG_OVERRIDE_BUILTINS -// Enable replacement rust-g functions for certain builtins. Off by default. - -#ifndef RUST_G -// Default automatic RUST_G detection. -// On Windows, looks in the standard places for `rust_g.dll`. -// On Linux, looks in `.`, `$LD_LIBRARY_PATH`, and `~/.byond/bin` for either of -// `librust_g.so` (preferred) or `rust_g` (old). - -/* This comment bypasses grep checks */ /var/__rust_g - -/proc/__detect_rust_g() - if(world.system_type == UNIX) - if(fexists("./librust_g.so")) - // No need for LD_LIBRARY_PATH badness. - return __rust_g = "./librust_g.so" - else if(fexists("./rust_g")) - // Old dumb filename. - return __rust_g = "./rust_g" - else if(fexists("[world.GetConfig("env", "HOME")]/.byond/bin/rust_g")) - // Old dumb filename in `~/.byond/bin`. - return __rust_g = "rust_g" - else - // It's not in the current directory, so try others - return __rust_g = "librust_g.so" - else - return __rust_g = "rust_g.dll" - -#define RUST_G (__rust_g || __detect_rust_g()) -#endif - -// Handle 515 call() -> call_ext() changes -#if DM_VERSION >= 515 -#define RUSTG_CALL call_ext -#else -#define RUSTG_CALL call -#endif - -/// Gets the version of rust_g -/proc/rustg_get_version() return RUSTG_CALL(RUST_G, "get_version")() - -// Jobs Defines // - -#define RUSTG_JOB_NO_RESULTS_YET "NO RESULTS YET" -#define RUSTG_JOB_NO_SUCH_JOB "NO SUCH JOB" -#define RUSTG_JOB_ERROR "JOB PANICKED" - -// SQL Operations // - -#define rustg_sql_connect_pool(options) RUSTG_CALL(RUST_G, "sql_connect_pool")(options) -#define rustg_sql_query_async(handle, query, params) RUSTG_CALL(RUST_G, "sql_query_async")(handle, query, params) -#define rustg_sql_query_blocking(handle, query, params) RUSTG_CALL(RUST_G, "sql_query_blocking")(handle, query, params) -#define rustg_sql_connected(handle) RUSTG_CALL(RUST_G, "sql_connected")(handle) -#define rustg_sql_disconnect_pool(handle) RUSTG_CALL(RUST_G, "sql_disconnect_pool")(handle) -#define rustg_sql_check_query(job_id) RUSTG_CALL(RUST_G, "sql_check_query")("[job_id]") diff --git a/code/__HELPERS/files.dm b/code/__HELPERS/files.dm index ac9129cfe43..0810fc2cb41 100644 --- a/code/__HELPERS/files.dm +++ b/code/__HELPERS/files.dm @@ -89,7 +89,7 @@ /// Used because md5ing files stored in the rsc sometimes gives incorrect md5 results. /proc/md5asfile(file) var/static/notch = 0 - // Its importaint this code can handle md5filepath sleeping instead of hard blocking, if it's converted to use rust_g. + // Its importaint this code can handle md5filepath sleeping instead of hard blocking var/filename = "tmp/md5asfile.[world.realtime].[world.timeofday].[world.time].[world.tick_usage].[notch]" notch = WRAP(notch+1, 0, 2**15) fcopy(file, filename) diff --git a/code/__HELPERS/text.dm b/code/__HELPERS/text.dm index d937c83fffa..1c3c7dccf0d 100644 --- a/code/__HELPERS/text.dm +++ b/code/__HELPERS/text.dm @@ -776,3 +776,9 @@ /proc/wiki_link(article_name, link_text = null) var/url = "[GLOB.configuration.url.wiki_url]/index.php?title=[article_name]" return "[link_text ? link_text : url]" + + +/proc/strip_byond_macros(text) + text = replacetext(text, "\proper", "") + text = replacetext(text, "\improper", "") + return text diff --git a/code/controllers/subsystem/SSblackbox.dm b/code/controllers/subsystem/SSblackbox.dm index 7f24bc8d778..5db1b9fdc25 100644 --- a/code/controllers/subsystem/SSblackbox.dm +++ b/code/controllers/subsystem/SSblackbox.dm @@ -28,7 +28,6 @@ SUBSYSTEM_DEF(blackbox) record_feedback("amount", "byond_version", world.byond_version) record_feedback("amount", "byond_build", world.byond_build) record_feedback("text", "random_seed", 1, num2text(Master.random_seed, 32), 1) // a text string because json_encode turns it into lossy scientific notation - record_feedback("text", "rust_g_filepath", 1, "[RUST_G]", 1) record_feedback("text", "rustlibs_filepath", 1, "[RUSTLIB]", 1) /datum/controller/subsystem/blackbox/fire(resumed = 0) @@ -348,7 +347,7 @@ SUBSYSTEM_DEF(blackbox) "key" = L.key, "job" = L.mind.assigned_role, "special" = L.mind.special_role || "", - "pod" = podname, + "pod" = strip_byond_macros(podname), "laname" = laname, "lakey" = lakey, "gender" = L.gender, diff --git a/code/controllers/subsystem/SSdbcore.dm b/code/controllers/subsystem/SSdbcore.dm index 9974e20ccbb..e4ceaeb0ea9 100644 --- a/code/controllers/subsystem/SSdbcore.dm +++ b/code/controllers/subsystem/SSdbcore.dm @@ -20,7 +20,7 @@ SUBSYSTEM_DEF(dbcore) /// SQL errors that have occured mid round var/total_errors = 0 - /// Connection handle. This is an arbitrary handle returned from rust_g. + /// Connection handle. This is an arbitrary handle returned from rustlibs. var/connection offline_implications = "The server will no longer check for undeleted SQL Queries. No immediate action is needed." @@ -60,7 +60,7 @@ SUBSYSTEM_DEF(dbcore) * Connection Creator * * This proc basically does a few sanity checks before connecting, then attempts to make a connection - * When connecting, RUST_G will initialize a thread pool for queries to use to run asynchronously + * When connecting, rustlibs will initialize a thread pool for queries to use to run asynchronously */ /datum/controller/subsystem/dbcore/proc/Connect() if(IsConnected()) @@ -76,22 +76,26 @@ SUBSYSTEM_DEF(dbcore) failed_connection_timeout = world.time + 50 return FALSE - var/result = json_decode(rustg_sql_connect_pool(json_encode(list( - "host" = GLOB.configuration.database.address, - "port" = GLOB.configuration.database.port, - "user" = GLOB.configuration.database.username, - "pass" = GLOB.configuration.database.password, - "db_name" = GLOB.configuration.database.db, - "read_timeout" = GLOB.configuration.database.async_query_timeout, - "write_timeout" = GLOB.configuration.database.async_query_timeout, - "max_threads" = GLOB.configuration.database.async_thread_limit, - )))) - . = (result["status"] == "ok") + var/datum/db_connection_request/dbcreq = new() + dbcreq.host = GLOB.configuration.database.address + dbcreq.port = GLOB.configuration.database.port + dbcreq.user = GLOB.configuration.database.username + dbcreq.pass = GLOB.configuration.database.password + dbcreq.db_name = GLOB.configuration.database.db + dbcreq.read_timeout = GLOB.configuration.database.async_query_timeout + dbcreq.write_timeout = GLOB.configuration.database.async_query_timeout + dbcreq.min_threads = 1 + dbcreq.max_threads = GLOB.configuration.database.async_thread_limit + + var/datum/db_connection_response/dbcres = rustlibs_sql_connect_pool(dbcreq) + + . = dbcres.ok + if(.) - connection = result["handle"] + connection = dbcres.handle else connection = null - last_error = result["data"] + last_error = dbcres.error_message log_sql("Connect() failed | [last_error]") ++failed_connections @@ -129,8 +133,17 @@ SUBSYSTEM_DEF(dbcore) /datum/controller/subsystem/dbcore/proc/Disconnect() failed_connections = 0 if(connection) - rustg_sql_disconnect_pool(connection) - connection = null + var/dc_result = rustlibs_sql_disconnect_pool(connection) + connection = null + if(isnum(dc_result)) + switch(dc_result) + if(1) + // Do nothing - was a success + return // This return is here just to shut OD up + if(0) + log_sql("Failed to disconnect - pool is already offline") + else + log_sql("Failed to disconnect - [dc_result]") /** * Shutdown Handler @@ -212,7 +225,19 @@ SUBSYSTEM_DEF(dbcore) return FALSE if(!connection) return FALSE - return json_decode(rustg_sql_connected(connection))["status"] == "online" + + var/conn_result = rustlibs_sql_connected(connection) + if(isnum(conn_result)) + switch(conn_result) + if(1) + return TRUE + if(0) + log_sql("Connection is offline") + return FALSE + + else + log_sql("Error checking connection - [conn_result]") + return FALSE /** @@ -301,44 +326,6 @@ SUBSYSTEM_DEF(dbcore) if(log) log_debug("Executed [length(querys)] queries in [stop_watch(start_time)]s") -/** - * # db_query - * - * Datum based handler for all database queries - * - * Holds information regarding inputs, status, and outputs - */ -/datum/db_query - // Inputs - /// The connection being used with this query - var/connection - /// The SQL statement being executed with :parameter placeholders - var/sql - /// An associative list of parameters to be substituted into the statement - var/arguments - - // Status information - /// Is the query currently in progress - var/in_progress - /// What was our last error, if any - var/last_error - /// What was our last activity - var/last_activity - /// When was our last activity - var/last_activity_time - - // Output - /// List of all rows returned - var/list/list/rows - /// Counter of the next row to take - var/next_row_to_take = 1 - /// How many rows were affected by the query - var/affected - /// ID of the last inserted row - var/last_insert_id - /// List of data values populated by NextRow() - var/list/item - // Sets up some vars and throws it into the SS active query list /datum/db_query/New(connection, sql, arguments) SSdbcore.active_queries[src] = TRUE @@ -433,33 +420,36 @@ SUBSYSTEM_DEF(dbcore) * * async - Are we running this query asynchronously */ /datum/db_query/proc/run_query(async) - var/job_result_str - if(async) - var/job_id = rustg_sql_query_async(connection, sql, json_encode(arguments)) - in_progress = TRUE - UNTIL((job_result_str = rustg_sql_check_query(job_id)) != RUSTG_JOB_NO_RESULTS_YET) - in_progress = FALSE + rustlibs_sql_query_async(src) - if(job_result_str == RUSTG_JOB_ERROR) - last_error = job_result_str - return FALSE + if(!in_progress) + CRASH("Query was not set as in progress - this is bad") + + UNTIL(async_query_done()) else - job_result_str = rustg_sql_query_blocking(connection, sql, json_encode(arguments)) + rustlibs_sql_query_blocking(src) - var/result = json_decode(job_result_str) - switch(result["status"]) - if("ok") - rows = result["rows"] - affected = result["affected"] - last_insert_id = result["last_insert_id"] - return TRUE - if("err") - last_error = result["data"] - return FALSE - if("offline") - last_error = "offline" - return FALSE + return isnull(last_error) + +/datum/db_query/proc/async_query_done() + // If we dont have an ID, were blocking, so assume complete + if(isnull(query_id)) + return TRUE + + // If we arent in progress, assume complete + if(!in_progress) + return TRUE + + // We got here, so check the status + rustlibs_sql_check_query(src) + + // If we have no result, were not finished + if(last_error == RUSTLIBS_JOB_NO_RESULTS_YET) + return FALSE + + // If we got here, we have a result to parse + return TRUE // Just tells the admins if a query timed out, and asks if the server hung to help error reporting /datum/db_query/proc/slow_query_check() @@ -520,3 +510,62 @@ SUBSYSTEM_DEF(dbcore) message_admins("Database connection failed: [SSdbcore.ErrorMsg()]") else message_admins("Database connection re-established") + + + +// Dont touch stuff below this line without first checking the rust side +/datum/db_connection_response + var/ok + var/handle + var/error_message + +/datum/db_connection_request + var/host + var/port + var/user + var/pass + var/db_name + var/read_timeout + var/write_timeout + var/min_threads + var/max_threads + +/** + * # db_query + * + * Datum based handler for all database queries + * + * Holds information regarding inputs, status, and outputs + */ +/datum/db_query + // Inputs + /// The connection being used with this query + var/connection + /// The SQL statement being executed with :parameter placeholders + var/sql + /// An associative list of parameters to be substituted into the statement + var/arguments + + // Status information + /// Is the query currently in progress + var/in_progress + /// What is the query ID? + var/query_id + /// What was our last error, if any + var/last_error + /// What was our last activity + var/last_activity + /// When was our last activity + var/last_activity_time + + // Output + /// List of all rows returned + var/list/list/rows + /// Counter of the next row to take + var/next_row_to_take = 1 + /// How many rows were affected by the query + var/affected + /// ID of the last inserted row + var/last_insert_id + /// List of data values populated by NextRow() + var/list/item diff --git a/code/datums/http.dm b/code/datums/http.dm index 4434bd2e7fd..bd9c1c51760 100644 --- a/code/datums/http.dm +++ b/code/datums/http.dm @@ -44,7 +44,7 @@ THE METHODS IN THIS FILE ARE TO BE USED BY THE SUBSYSTEM AS A MANGEMENT HUB * Call this with relevant parameters to form the request you want to make * * Arguments: - * * _method - HTTP Method to use, see code/__DEFINES/rust_g.dm for a full list + * * _method - HTTP Method to use, see code/__DEFINES/rust.dm for a full list * * _url - The URL to send the request to * * _body - The body of the request, if applicable * * _headers - Associative list of HTTP headers to send, if applicab;e diff --git a/code/datums/revision.dm b/code/datums/revision.dm index 7dff785bf27..979bd37f6bd 100644 --- a/code/datums/revision.dm +++ b/code/datums/revision.dm @@ -105,8 +105,6 @@ GLOBAL_PROTECT(revision_info) // Dont mess with this else msg += "Server Commit: Unable to determine" - msg += "RUST-G Build: [rustg_get_version()]" - if(world.TgsAvailable()) var/datum/tgs_version/tgs_ver = world.TgsVersion() var/datum/tgs_version/api_ver = world.TgsApiVersion() diff --git a/code/game/world.dm b/code/game/world.dm index 885e98c0b3a..b46d6b08234 100644 --- a/code/game/world.dm +++ b/code/game/world.dm @@ -12,16 +12,6 @@ GLOBAL_DATUM(test_runner, /datum/test_runner) SSmetrics.world_init_time = REALTIMEOFDAY - // Do sanity checks to ensure RUST actually exists - if((!fexists(RUST_G)) && world.system_type == MS_WINDOWS) - DIRECT_OUTPUT(world.log, "ERROR: RUSTG was not found and is required for the game to function. Server will now exit.") - del(world) - - var/rustg_version = rustg_get_version() - if(rustg_version != RUST_G_VERSION) - DIRECT_OUTPUT(world.log, "ERROR: RUSTG version mismatch. Library is [rustg_version], code wants [RUST_G_VERSION]. Server will now exit.") - del(world) - //temporary file used to record errors with loading config and the database, moved to log directory once logging is set up GLOB.config_error_log = GLOB.world_game_log = GLOB.world_runtime_log = GLOB.sql_log = "data/logs/config_error.log" GLOB.configuration.load_configuration() // Load up the base config.toml diff --git a/code/modules/library/library_catalog.dm b/code/modules/library/library_catalog.dm index d1a6a644626..7994a918bde 100644 --- a/code/modules/library/library_catalog.dm +++ b/code/modules/library/library_catalog.dm @@ -373,7 +373,7 @@ if(!SSdbcore.IsConnected()) return var/num_books = clamp(amount, 1, 50) //you don't need more than 50 random books <3 - var/list/sql_params = list("amount" = num_books ) + var/list/sql_params = list("amount" = num_books) var/sql = "SELECT id, author, title, content, summary, rating, primary_category, secondary_category, tertiary_category, ckey, reports FROM library GROUP BY title ORDER BY rand() LIMIT :amount" var/datum/db_query/query = SSdbcore.NewQuery(sql, sql_params) if(!query.warn_execute()) diff --git a/code/tests/game_tests.dm b/code/tests/game_tests.dm index 79f732f3cfe..c854e7ffdea 100644 --- a/code/tests/game_tests.dm +++ b/code/tests/game_tests.dm @@ -42,7 +42,6 @@ #include "test_origin_tech.dm" #include "test_purchase_reference_test.dm" #include "test_reagent_id_typos.dm" -#include "test_rustg_version.dm" #include "test_spawn_humans.dm" #include "test_spell_targeting_test.dm" #include "test_sql.dm" diff --git a/code/tests/test_rustg_version.dm b/code/tests/test_rustg_version.dm deleted file mode 100644 index de39b377bdc..00000000000 --- a/code/tests/test_rustg_version.dm +++ /dev/null @@ -1,3 +0,0 @@ -/datum/game_test/rustg_version/Run() - var/library_version = rustg_get_version() - TEST_ASSERT_EQUAL(library_version, RUST_G_VERSION, "invalid RUSTG Version") diff --git a/paradise.dme b/paradise.dme index b3d9ad3584f..a29f112cf22 100644 --- a/paradise.dme +++ b/paradise.dme @@ -28,7 +28,6 @@ #include "code\__DEFINES\_readme.dm" #include "code\__DEFINES\_tgs_defines.dm" #include "code\__DEFINES\_tick.dm" -#include "code\__DEFINES\_versions.dm" #include "code\__DEFINES\access_defines.dm" #include "code\__DEFINES\action_button_defines.dm" #include "code\__DEFINES\action_defines.dm" @@ -122,7 +121,6 @@ #include "code\__DEFINES\role_preferences.dm" #include "code\__DEFINES\rolebans.dm" #include "code\__DEFINES\rust.dm" -#include "code\__DEFINES\rust_g.dm" #include "code\__DEFINES\shuttle_defines.dm" #include "code\__DEFINES\sight.dm" #include "code\__DEFINES\silicon_defines.dm" diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 72e458004f6..0594fda67ef 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -266,6 +266,15 @@ version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "block2" version = "0.6.1" @@ -288,6 +297,21 @@ dependencies = [ "piper", ] +[[package]] +name = "btoi" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd6407f73a9b8b6162d8a2ef999fe6afd7cc15902ebf42c5cd296addf17e0ad" +dependencies = [ + "num-traits", +] + +[[package]] +name = "bufstream" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40e38929add23cdf8a366df9b0e088953150724bcbe5fc330b0d8eb3b328eec8" + [[package]] name = "builtins-proc-macro" version = "0.0.0" @@ -425,6 +449,15 @@ dependencies = [ "libloading", ] +[[package]] +name = "cmake" +version = "0.1.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0" +dependencies = [ + "cc", +] + [[package]] name = "collection_literals" version = "1.0.2" @@ -462,6 +495,15 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -490,12 +532,82 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-queue" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.104", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.104", +] + +[[package]] +name = "dashmap" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", + "rayon", + "serde", +] + [[package]] name = "dbpnoise" version = "0.1.2" @@ -540,12 +652,33 @@ dependencies = [ "syn 2.0.104", ] +[[package]] +name = "derive_utils" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccfae181bab5ab6c5478b2ccb69e4c68a02f8c3ec72f6616bfec9dbc599d2ee0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.104", +] + [[package]] name = "diff" version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "dispatch2" version = "0.3.0" @@ -735,6 +868,12 @@ dependencies = [ "spin", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -784,6 +923,16 @@ dependencies = [ "byteorder", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "get-size" version = "0.1.4" @@ -851,12 +1000,24 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + [[package]] name = "hashbrown" version = "0.15.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5" +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + [[package]] name = "hermit-abi" version = "0.5.2" @@ -979,6 +1140,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "1.0.3" @@ -1055,6 +1222,15 @@ dependencies = [ "rustversion", ] +[[package]] +name = "io-enum" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d197db2f7ebf90507296df3aebaf65d69f5dce8559d8dbd82776a6cadab61bbf" +dependencies = [ + "derive_utils", +] + [[package]] name = "itertools" version = "0.10.5" @@ -1099,6 +1275,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + [[package]] name = "lerp" version = "0.4.0" @@ -1197,6 +1379,12 @@ version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" + [[package]] name = "mac-notification-sys" version = "0.6.6" @@ -1250,6 +1438,90 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "mysql" +version = "26.0.0" +source = "git+https://github.com/ZeWaka/rust-mysql-simple.git?tag=v26.0.0#8a47884aeca433942d10796f7eaccb1882e89f46" +dependencies = [ + "bufstream", + "bytes", + "crossbeam-queue", + "flate2", + "io-enum", + "libc", + "lru", + "mysql_common", + "named_pipe", + "pem", + "percent-encoding", + "rustls", + "rustls-pemfile", + "socket2", + "twox-hash", + "url", + "webpki", + "webpki-roots 0.26.11", +] + +[[package]] +name = "mysql-common-derive" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66f62cad7623a9cb6f8f64037f0c4f69c8db8e82914334a83c9788201c2c1bfa" +dependencies = [ + "darling", + "heck", + "num-bigint", + "proc-macro-crate", + "proc-macro-error2", + "proc-macro2", + "quote", + "syn 2.0.104", + "termcolor", + "thiserror 2.0.12", +] + +[[package]] +name = "mysql_common" +version = "0.34.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34a9141e735d5bb02414a7ac03add09522466d4db65bdd827069f76ae0850e58" +dependencies = [ + "base64", + "bitflags 2.9.1", + "btoi", + "byteorder", + "bytes", + "cc", + "cmake", + "crc32fast", + "flate2", + "lazy_static", + "mysql-common-derive", + "num-bigint", + "num-traits", + "rand", + "regex", + "saturating", + "serde", + "serde_json", + "sha1 0.10.6", + "sha2", + "subprocess", + "thiserror 1.0.69", + "uuid", + "zstd", +] + +[[package]] +name = "named_pipe" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad9c443cce91fc3e12f017290db75dde490d685cdaaf508d7159d7cf41f0eb2b" +dependencies = [ + "winapi", +] + [[package]] name = "nanorand" version = "0.7.0" @@ -1309,6 +1581,16 @@ dependencies = [ "zbus", ] +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-complex" version = "0.4.6" @@ -1434,6 +1716,29 @@ version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" +[[package]] +name = "parking_lot_core" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets 0.52.6", +] + +[[package]] +name = "pem" +version = "3.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38af38e8470ac9dee3ce1bae1af9c1671fffc44ddfd8bd1d0a3445bf349a8ef3" +dependencies = [ + "base64", + "serde", +] + [[package]] name = "percent-encoding" version = "2.3.1" @@ -1619,6 +1924,28 @@ dependencies = [ "version_check", ] +[[package]] +name = "proc-macro-error-attr2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +dependencies = [ + "proc-macro-error-attr2", + "proc-macro2", + "quote", + "syn 2.0.104", +] + [[package]] name = "proc-macro-utils" version = "0.8.0" @@ -1771,10 +2098,19 @@ dependencies = [ "itoa", "percent-encoding", "ryu", - "sha1", + "sha1 0.6.1", "url", ] +[[package]] +name = "redox_syscall" +version = "0.5.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77" +dependencies = [ + "bitflags 2.9.1", +] + [[package]] name = "regex" version = "1.11.1" @@ -1854,6 +2190,7 @@ dependencies = [ "bitflags 2.9.1", "byondapi", "chrono", + "dashmap", "dbpnoise", "diff", "dmm-tools", @@ -1863,7 +2200,9 @@ dependencies = [ "fxhash", "git2", "itertools 0.10.5", + "mysql", "notify-rust", + "once_cell", "png", "rand", "redis", @@ -1874,6 +2213,7 @@ dependencies = [ "thread-priority", "toml 0.8.23", "ureq", + "uuid", "walkdir", ] @@ -1892,6 +2232,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "rustls-pki-types" version = "1.12.0" @@ -1933,6 +2282,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "saturating" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ece8e78b2f38ec51c51f5d475df0a7187ba5111b2a28bdc761ee05b075d40a71" + [[package]] name = "scc" version = "2.3.4" @@ -2015,12 +2370,34 @@ dependencies = [ "sha1_smol", ] +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sha1_smol" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "shlex" version = "1.3.0" @@ -2060,6 +2437,16 @@ version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "spin" version = "0.9.8" @@ -2081,6 +2468,22 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subprocess" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c2e86926081dda636c546d8c5e641661049d7562a68f5488be4a1f7f66f6086" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "subtle" version = "2.6.1" @@ -2127,7 +2530,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b1e66e07de489fe43a46678dd0b8df65e0c973909df1b60ba33874e297ba9b9" dependencies = [ "quick-xml", - "thiserror", + "thiserror 2.0.12", "windows", "windows-version", ] @@ -2154,13 +2557,33 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + [[package]] name = "thiserror" version = "2.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" dependencies = [ - "thiserror-impl", + "thiserror-impl 2.0.12", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.104", ] [[package]] @@ -2298,6 +2721,18 @@ dependencies = [ "once_cell", ] +[[package]] +name = "twox-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" + +[[package]] +name = "typenum" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" + [[package]] name = "uds_windows" version = "1.1.0" @@ -2354,6 +2789,17 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "uuid" +version = "1.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" +dependencies = [ + "getrandom 0.3.3", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "vcpkg" version = "0.2.15" @@ -2449,6 +2895,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "webpki" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed63aea5ce73d0ff405984102c42de94fc55a6b75765d621c65262469b3c9b53" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "webpki-roots" version = "0.26.11" @@ -2967,6 +3423,34 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "626bd9fa9734751fc50d6060752170984d7053f5a39061f524cda68023d4db8a" +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "zvariant" version = "5.6.0" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index e5c6b60cdbc..6b1b2ad43ca 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -44,3 +44,11 @@ redis = { version = "0.21.4" } flume = { version = "0.10" } notify-rust = "4.11.7" git2 = { version = "0.20.2", default-features = false } +once_cell = { version = "1.21" } +mysql = { git = "https://github.com/ZeWaka/rust-mysql-simple.git", tag = "v26.0.0", default-features = false } +dashmap = { version = "6.1", features = ["rayon", "serde"] } +uuid = { version = "1.18.1", features = ["v4"] } + +[features] +# Use the native tls stack for the mysql db +default = ["mysql/default-rust", "mysql/rustls-tls-ring"] diff --git a/rust/README.MD b/rust/README.MD index d6a1412b66a..c0d64781ed8 100644 --- a/rust/README.MD +++ b/rust/README.MD @@ -17,11 +17,13 @@ It also imports and tweaks the following [rust-g](https://github.com/tgstation/r - HTTP - JSON - Logging +- MySQL - Noisegen - Redis PubSub - TOML +If you're adding anything to this, **please** use the BYONDAPI stuff instead of `json_encode` and `json_decode` everywhere. + ## Building -BYOND 516.1651 introduced breaking changes to ByondAPI, the interop system to get data other than strings in and out of DLLs. -Because of this, you need to specify the `--no-default-features --features byond-516` to build the 516 compliant lib. Not specifying a feature will build for versions `515.1621` to `516.1650`. Specifying `byond-516` will build for `516.1651` and up. +Just running `cargo build --release` should be enough if you're on Windows. If you are on Linux, you should be familiar with figuring stuff out yourself and consulting 300 pieces of documentation for a single command. diff --git a/rust/src/lib.rs b/rust/src/lib.rs index dc3eb9a132d..4999a3825a2 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -10,5 +10,6 @@ mod rustlibs_json; mod rustlibs_logging; mod rustlibs_noisegen; mod rustlibs_redispubsub; +mod rustlibs_sql; mod rustlibs_toast; mod rustlibs_toml; diff --git a/rust/src/logging/mod.rs b/rust/src/logging/mod.rs index 9ba43002169..222abf2ffae 100644 --- a/rust/src/logging/mod.rs +++ b/rust/src/logging/mod.rs @@ -2,6 +2,7 @@ use byondapi::global_call::call_global; use byondapi::prelude::ByondValue; use byondapi::threadsync::thread_sync; use chrono::prelude::Utc; +use uuid::Uuid; /// Call stack trace dm method with message. pub(crate) fn dm_call_stack_trace(msg: String) -> eyre::Result<()> { @@ -20,10 +21,11 @@ pub(crate) fn setup_panic_handler() { || -> ByondValue { if let Err(error) = dm_call_stack_trace(msg_copy) { let second_msg = format!("BYOND error \n {:#?}", error); + let panic_guid = Uuid::new_v4(); + let ts = Utc::now().format("%Y%m%d_%H%M%S").to_string(); + let file_end = format!("{}_{}", ts, panic_guid); let _ = std::fs::write( - Utc::now() - .format("data/rustlibs_dm_trace_failed_%Y%m%d_%H%M%S.txt") - .to_string(), + format!("data/rustlibs_dm_trace_failed_{}.txt", file_end), second_msg.clone(), ); } @@ -31,11 +33,10 @@ pub(crate) fn setup_panic_handler() { }, true, ); - let _ = std::fs::write( - Utc::now() - .format("data/rustlibs_panic_%Y%m%d_%H%M%S.txt") - .to_string(), - msg.clone(), - ); + // GUID may seem pointless but on the off chance we get 2 panics in the same second its needed + let panic_guid = Uuid::new_v4(); + let ts = Utc::now().format("%Y%m%d_%H%M%S").to_string(); + let file_end = format!("{}_{}", ts, panic_guid); + let _ = std::fs::write(format!("data/rustlibs_panic_{}.txt", file_end), msg.clone()); })) } diff --git a/rust/src/rustlibs_http/mod.rs b/rust/src/rustlibs_http/mod.rs index f4cc874a556..36bc727a69c 100644 --- a/rust/src/rustlibs_http/mod.rs +++ b/rust/src/rustlibs_http/mod.rs @@ -1,5 +1,4 @@ use crate::jobs; -use crate::logging; use byondapi::value::ByondValue; use eyre::Result; use serde::{Deserialize, Serialize}; diff --git a/rust/src/rustlibs_sql/mod.rs b/rust/src/rustlibs_sql/mod.rs new file mode 100644 index 00000000000..e97974d4cf8 --- /dev/null +++ b/rust/src/rustlibs_sql/mod.rs @@ -0,0 +1,538 @@ +use crate::jobs; +use byondapi::{prelude::ValueType, value::ByondValue}; +use dashmap::DashMap; +use eyre::{eyre, Result}; +use mysql::{ + consts::{ColumnFlags, ColumnType::*}, + prelude::Queryable, + OptsBuilder, Params, Pool, PoolConstraints, PoolOpts, Value, +}; +use once_cell::sync::Lazy; +use serde::{Deserialize, Serialize}; +use serde_json::Number; +use std::{collections::HashMap, sync::atomic::AtomicUsize}; +use std::{error::Error, time::Duration}; + +// ---------------------------------------------------------------------------- +// Interface + +const DEFAULT_PORT: u16 = 3306; +// The `mysql` crate defaults to 10 and 100 for these, but that is too large. +const DEFAULT_MIN_THREADS: usize = 1; +const DEFAULT_MAX_THREADS: usize = 10; + +struct ConnectOptions { + host: String, + port: u16, + user: String, + pass: String, + db_name: String, + read_timeout: f32, + write_timeout: f32, + min_threads: usize, + max_threads: usize, +} + +struct LocalQuery { + query: String, + connection: String, + params: Params, +} + +// Needs to support serde shenanigans just so we can use the jobs system +// Gah +#[derive(Serialize, Deserialize)] +struct LocalResponse { + status: String, + // Most of these are optional so we can return the same class back + affected: Option, + last_insert_id: Option, + //columns: Option>, + rows: Option>, +} + +#[byondapi::bind] +fn sql_connect_pool(options_p: ByondValue) -> Result { + let options: ConnectOptions = ConnectOptions { + host: options_p.read_string("host").unwrap(), + port: options_p.read_number("port").unwrap() as u16, + user: options_p.read_string("user").unwrap(), + pass: options_p.read_string("pass").unwrap(), + db_name: options_p.read_string("db_name").unwrap(), + read_timeout: options_p.read_number("read_timeout").unwrap(), + write_timeout: options_p.read_number("write_timeout").unwrap(), + min_threads: options_p.read_number("min_threads").unwrap() as usize, + max_threads: options_p.read_number("max_threads").unwrap() as usize, + }; + + let target_type = ByondValue::new_str("/datum/db_connection_response")?; + let mut response_datum = ByondValue::builtin_new(target_type, &[])?; + + match sql_connect(options) { + Ok(v) => { + response_datum + .write_var("ok", &ByondValue::new_num(1 as f32)) + .unwrap(); + response_datum.write_var("handle", &ByondValue::new_str(v)?)?; + } + Err(e) => { + let error_message = unwrap_box(e); + response_datum + .write_var("ok", &ByondValue::new_num(0 as f32)) + .unwrap(); + response_datum.write_var("error_message", &ByondValue::new_str(error_message)?)?; + } + } + + Ok(response_datum) +} + +#[byondapi::bind] +fn sql_query_blocking(mut options_p: ByondValue) -> Result { + // Sort our params now + let mut local_params: Params = Params::Empty; + + match options_p.read_list("arguments") { + Ok(v) => local_params = byondlist_to_params(v), // Create params from a BYOND list, + Err(_) => {} + } + + let lq: LocalQuery = LocalQuery { + query: options_p.read_string("sql").unwrap(), + connection: options_p.read_string("connection").unwrap(), + params: local_params, + }; + + match do_query(lq) { + Ok(o) => { + let query_response: LocalResponse = serde_json::from_str(&o).unwrap(); + + query_data_to_byond(query_response, options_p) + } + Err(e) => { + // Write error back to BYOND + options_p + .write_var("last_error", &ByondValue::new_str(unwrap_box(e)).unwrap()) + .unwrap() + } + }; + + Ok(ByondValue::null()) +} + +#[byondapi::bind] +fn sql_query_async(mut options_p: ByondValue) -> Result { + let mut local_params: Params = Params::Empty; + + match options_p.read_list("arguments") { + Ok(v) => local_params = byondlist_to_params(v), // Create params from a BYOND list, + Err(_) => {} + } + + let lq: LocalQuery = LocalQuery { + query: options_p.read_string("sql").unwrap(), + connection: options_p.read_string("connection").unwrap(), + params: local_params, + }; + + let job_id = jobs::start(move || match do_query(lq) { + Ok(o) => o, + Err(e) => unwrap_box(e), + }); + + options_p.write_var("query_id", &ByondValue::new_num(job_id as f32))?; + options_p.write_var("in_progress", &ByondValue::new_num(1f32))?; + + Ok(ByondValue::null()) +} + +// hopefully won't panic if queries are running +#[byondapi::bind] +fn sql_disconnect_pool(options_p: ByondValue) -> Result { + let handle_p = options_p.get_string().unwrap(); + let handle = match handle_p.parse::() { + Ok(o) => o, + Err(e) => return Ok(ByondValue::new_str(unwrap_box(e)).unwrap()), + }; + + match POOL.remove(&handle) { + Some(_) => return Ok(ByondValue::new_num(1f32)), + None => return Ok(ByondValue::new_num(0f32)), + } +} + +#[byondapi::bind] +fn sql_connected(options_p: ByondValue) -> Result { + let handle_p = options_p.get_string().unwrap(); + let handle = match handle_p.parse::() { + Ok(o) => o, + Err(e) => return Ok(ByondValue::new_str(unwrap_box(e)).unwrap()), + }; + + match POOL.get(&handle) { + Some(_) => return Ok(ByondValue::new_num(1f32)), + None => return Ok(ByondValue::new_num(0f32)), + } +} + +#[byondapi::bind] +fn sql_check_query(mut query: ByondValue) -> Result { + // logging::setup_panic_handler(); + let id = query.read_var("query_id")?.get_number()? as usize; + match jobs::check(&id) { + // Job id exists, check progress + Some(res) => match res { + Ok(res) => match serde_json::from_str(&res) { + Ok(v) => { + // It decoded fine + let query_response: LocalResponse = v; + query_data_to_byond(query_response, query); + query.write_var("last_error", &ByondValue::null())?; + query.write_var("in_progress", &ByondValue::new_num(0f32))?; + return Ok(ByondValue::null()); + } + Err(e) => { + // It was not fine + if res.len() == 0 { + // But maybe it was + // Send us an dataset back + let localres: LocalResponse = LocalResponse { + status: "ok".to_string(), + affected: None, + last_insert_id: None, + rows: None, + }; + query_data_to_byond(localres, query); + query.write_var("last_error", &ByondValue::null())?; + query.write_var("in_progress", &ByondValue::new_num(0f32))?; + return Ok(ByondValue::null()); + } else { + // But it probably wasnt + query.write_var("last_error", &ByondValue::new_str(jobs::JOB_PANICKED)?)?; + query.write_var("in_progress", &ByondValue::new_num(0f32))?; + return Err(eyre!("{} - {}", e, &res)); + } + } + }, + // Query still executed + Err(flume::TryRecvError::Empty) => { + query.write_var("last_error", &ByondValue::new_str(jobs::NO_RESULTS_YET)?)?; + return Ok(ByondValue::null()); + } + // Something bad happened during the query + Err(flume::TryRecvError::Disconnected) => { + query.write_var("last_error", &ByondValue::new_str(jobs::JOB_PANICKED)?)?; + query.write_var("in_progress", &ByondValue::new_num(0f32))?; + return Ok(ByondValue::null()); + } + }, + // Job id does not exist + None => { + query.write_var( + "last_error", + &ByondValue::new_str(jobs::NO_SUCH_JOB).unwrap(), + )?; + + return Ok(ByondValue::null()); + } + } +} +/* +byond_fn!(fn sql_check_query(id) { + Some(jobs::check(id)) +}); +*/ + +// ---------------------------------------------------------------------------- +// Main connect and query implementation + +static POOL: Lazy> = Lazy::new(DashMap::new); +static NEXT_ID: AtomicUsize = AtomicUsize::new(0); + +fn sql_connect(options: ConnectOptions) -> Result> { + let pool_constraints = + PoolConstraints::new(options.min_threads, options.max_threads).unwrap_or( + PoolConstraints::new_const::(), + ); + + let pool_opts = PoolOpts::with_constraints(PoolOpts::new(), pool_constraints); + + // Library wants optionals passed to this builder + // I cant be arsed with that + // -aa07 + let builder = OptsBuilder::new() + .ip_or_hostname(Some(options.host)) + .tcp_port(options.port) + // Work around addresses like `localhost:3307` defaulting to socket as + // if the port were the default too. + .prefer_socket(options.port == DEFAULT_PORT) + .user(Some(options.user)) + .pass(Some(options.pass)) + .db_name(Some(options.db_name)) + .read_timeout(Some(Duration::from_secs_f32(options.read_timeout))) + .write_timeout(Some(Duration::from_secs_f32(options.write_timeout))) + .pool_opts(pool_opts); + + let pool = Pool::new(builder)?; + + let handle = NEXT_ID.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + POOL.insert(handle, pool); + + Ok(handle.to_string()) +} + +fn do_query(lq: LocalQuery) -> Result> { + let mut conn = { + let pool = match POOL.get(&lq.connection.parse()?) { + Some(s) => s, + None => { + return Ok(serde_json::to_string(&LocalResponse { + status: "offline".to_string(), + affected: None, + //columns: None, + last_insert_id: None, + rows: None, + }) + .unwrap()); + } + }; + pool.get_conn()? + }; + + let query_result = conn.exec_iter(lq.query, lq.params)?; + let affected = query_result.affected_rows(); + let last_insert_id = query_result.last_insert_id(); + let mut columns = Vec::new(); + for col in query_result.columns().as_ref().iter() { + let colstr = col.name_str().to_string(); + columns.push(colstr); + } + + let mut rows: Vec = Vec::new(); + for row in query_result { + let row = row?; + let mut json_row: Vec = Vec::new(); + for (i, col) in row.columns_ref().iter().enumerate() { + let ctype = col.column_type(); + let value = row + .as_ref(i) + .ok_or("length of row was smaller than column count")?; + let converted = match value { + mysql::Value::Bytes(b) => match ctype { + MYSQL_TYPE_VARCHAR | MYSQL_TYPE_STRING | MYSQL_TYPE_VAR_STRING => { + serde_json::Value::String(String::from_utf8_lossy(b).into_owned()) + } + MYSQL_TYPE_BLOB + | MYSQL_TYPE_LONG_BLOB + | MYSQL_TYPE_MEDIUM_BLOB + | MYSQL_TYPE_TINY_BLOB => { + if col.flags().contains(ColumnFlags::BINARY_FLAG) { + serde_json::Value::Array( + b.iter() + .map(|x| serde_json::Value::Number(Number::from(*x))) + .collect(), + ) + } else { + serde_json::Value::String(String::from_utf8_lossy(b).into_owned()) + } + } + _ => serde_json::Value::Null, + }, + mysql::Value::Float(f) => serde_json::Value::Number( + Number::from_f64(f64::from(*f)).unwrap_or_else(|| Number::from(0)), + ), + mysql::Value::Double(f) => serde_json::Value::Number( + Number::from_f64(*f).unwrap_or_else(|| Number::from(0)), + ), + mysql::Value::Int(i) => serde_json::Value::Number(Number::from(*i)), + mysql::Value::UInt(u) => serde_json::Value::Number(Number::from(*u)), + mysql::Value::Date(year, month, day, hour, minute, second, _ms) => { + serde_json::Value::String(format!( + "{year}-{month:02}-{day:02} {hour:02}:{minute:02}:{second:02}" + )) + } + _ => serde_json::Value::Null, + }; + json_row.push(converted); + } + rows.push(serde_json::Value::Array(json_row)); + } + + drop(conn); + + Ok(serde_json::to_string(&LocalResponse { + status: "ok".to_string(), + affected: Some(affected), + last_insert_id: last_insert_id, + //columns: Some(columns), + rows: Some(rows), + }) + .unwrap()) +} + +// ---------------------------------------------------------------------------- +// Helpers + +fn unwrap_box(e: E) -> String { + e.to_string() +} + +fn byondlist_to_params(params: Vec) -> Params { + let mut real_params = Params::Empty; + + if params.len() > 0 { + let mut temp_params_2: HashMap, Value> = HashMap::new(); + + for pair in params.chunks(2) { + if let [k, v] = pair { + let kbytes = k.get_string().unwrap().into_bytes(); + let v_value = byondvalue_to_mysql(v.to_owned()); + + temp_params_2.insert(kbytes, v_value); + } + } + + real_params = Params::Named(temp_params_2); + } + + real_params +} + +fn byondvalue_to_mysql(val: ByondValue) -> mysql::Value { + match ValueType::try_from(val.get_type()) { + Ok(v) => { + match v { + ValueType::Number => { + let num = val.get_number().unwrap(); + // Detect integer values vs. true floats + if num.fract() == 0.0 { + mysql::Value::Int(num as i64) + } else { + mysql::Value::Float(num) + } + } + ValueType::String => mysql::Value::Bytes( + val.get_cstring() + .unwrap() // this is definitely unsafe but OH WELL + .to_string_lossy() + .as_bytes() + .into(), + ), + _ => { + // Default to null if we cry about it + mysql::Value::NULL + } + } + } + Err(_) => { + // Default to null if we cant do anything + mysql::Value::NULL + } + } +} + +fn serde_to_byondvalue(val: serde_json::Value) -> ByondValue { + match val { + serde_json::Value::Bool(b) => ByondValue::new_num(if b { 1f32 } else { 0f32 }), + serde_json::Value::Number(i) => { + let v = i.as_f64().unwrap(); + ByondValue::new_num(v as f32) // Loses precision - but we wont get that in BYOND anyway + } + serde_json::Value::String(s) => ByondValue::new_str(s).unwrap(), + _ => ByondValue::null(), + } +} + +fn query_data_to_byond(raw_data: LocalResponse, mut byond_obj: ByondValue) { + if raw_data.status != "ok" { + byond_obj + .write_var("last_error", &ByondValue::new_str("DB offline").unwrap()) + .unwrap(); + return; + } + + // Lets update where needed + match raw_data.affected { + Some(v) => { + // We will lose precision here but byond doesnt like BIGINT anyway sooooo + byond_obj + .write_var("affected", &ByondValue::new_num(v as f32)) + .unwrap() + } + None => {} + } + + match raw_data.last_insert_id { + Some(v) => { + // We will lose precision here but byond doesnt like BIGINT anyway sooooo + byond_obj + .write_var("last_insert_id", &ByondValue::new_num(v as f32)) + .unwrap() + } + None => {} + } + + // Uncomment this if we ever want to support columns DM side + /* + match raw_data.columns { + Some(v) => { + // Oh god we need to make a BYOND list + let mut column_list = ByondValue::new_list().unwrap(); + column_list + .write_var("len", &ByondValue::new_num(v.len() as f32)) + .unwrap(); + let mut list_idx = 0f32; + for column in v.iter() { + list_idx += 1f32; + // Get it as a proper DM string + let column_bv = ByondValue::new_str(column.to_owned()).unwrap(); + // Write to the list + column_list.write_list_index(list_idx, column_bv).unwrap(); + } + + byond_obj.write_var("columns", &column_list).unwrap() + } + None => {} + } + */ + + // Now for the hard part - mapping the colums and rows out + match raw_data.rows { + Some(v) => { + let mut rows_list = ByondValue::new_list().unwrap(); + // This is very hacky but will be fixed when we have native list sizing stuff in byondapi + // Lummox is aware + rows_list + .write_var("len", &ByondValue::new_num(v.len() as f32)) + .unwrap(); + + let mut row_idx = 0f32; + for row in v.iter() { + let mut this_row = ByondValue::new_list().unwrap(); + row_idx += 1f32; + + // Hacky time~ + this_row + .write_var( + "len", + &ByondValue::new_num(row.as_array().unwrap().len() as f32), + ) + .unwrap(); + + let mut col_idx = 0f32; + for val in row.as_array().unwrap().iter() { + col_idx += 1f32; + this_row + .write_list_index(col_idx, serde_to_byondvalue(val.to_owned())) + .unwrap(); + } + + // Dumb hack until lummox does it + rows_list.write_list_index(row_idx, this_row).unwrap(); + } + + byond_obj.write_var("rows", &rows_list).unwrap() + } + None => {} + } +} diff --git a/rust_g.dll b/rust_g.dll deleted file mode 100644 index 9104d762be1..00000000000 Binary files a/rust_g.dll and /dev/null differ diff --git a/rustlibs.dll b/rustlibs.dll index 0cd18b427ef..3b3eaa6a8c8 100644 Binary files a/rustlibs.dll and b/rustlibs.dll differ diff --git a/rustlibs_prod.dll b/rustlibs_prod.dll index 81ab4e64307..0c762a6a6fa 100644 Binary files a/rustlibs_prod.dll and b/rustlibs_prod.dll differ diff --git a/tools/ci/install_rustg.sh b/tools/ci/install_rustg.sh deleted file mode 100755 index f1b77ef5baa..00000000000 --- a/tools/ci/install_rustg.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -source _build_dependencies.sh - -mkdir -p ~/.byond/bin -wget -O ~/.byond/bin/librust_g.so "https://github.com/ParadiseSS13/rust-g/releases/download/$RUSTG_VERSION/librust_g.so" -chmod +x ~/.byond/bin/librust_g.so -ldd ~/.byond/bin/librust_g.so diff --git a/tools/ci/librustlibs_ci.so b/tools/ci/librustlibs_ci.so index 7f20ecfe8f6..68e70011098 100644 Binary files a/tools/ci/librustlibs_ci.so and b/tools/ci/librustlibs_ci.so differ diff --git a/tools/ci/validate_rustg_windows.py b/tools/ci/validate_rustg_windows.py deleted file mode 100644 index 1c3c883ae5e..00000000000 --- a/tools/ci/validate_rustg_windows.py +++ /dev/null @@ -1,107 +0,0 @@ -# Script to validate RUSTG DLL functions under windows (Running DD under windows in a CI environment is pain) -# Author: AffectedArc07 -# This script is invoked by GitHub actions as part of CI to validate that the windows DLL for RUSTG works and creates proper formats - -# Imports -import os, json -from ctypes import * -from datetime import datetime, timedelta - -# Initial vars -ci_log_file = "ci_log.log" -ci_testing_text = "This is a test message" -ci_toml_file_location = "config/example/config.toml" - -# Helpers -def success(msg): - print("[Y] {}".format(msg)) - -def fail(msg): - print("[X] {}".format(msg)) - exit(1) # Exit with 1 to fail the CI - -# Cleanup -if os.path.exists(ci_log_file): - os.remove(ci_log_file) - -# Check the DLL exists at all -if os.path.exists("rust_g.dll"): - success("RUSTG Dll exists") -else: - fail("RUSTG Dll does NOT exist") - -# Check DM header file exists -if os.path.exists("code/__DEFINES/rust_g.dm"): - success("RUSTG DM header file exists") -else: - fail("RUSTG DM header file does NOT exist") - -# Parse the version from the DM file -f = open("code/__DEFINES/_versions.dm", "r") -lines = f.readlines() -f.close() -dm_version = None -for line in lines: - if line.startswith("#define RUST_G_VERSION"): - dm_version = line.split("#define RUST_G_VERSION")[1].strip().strip("\"") - break - -if not dm_version: - fail("Could not detect RUSTG version inside DM header file") - -# Begin DLL loading -rustg_dll = CDLL("./rust_g.dll") - - -# Set args for version retrieval -rustg_dll.get_version.restype = c_char_p -dll_version = rustg_dll.get_version().decode() - -if dll_version == dm_version: - success("DLL and DM versions match") -else: - fail("DLL and DM version mismatch! Got {} DM version, got {} DLL version".format(dm_version, dll_version)) - -# Now test log writing. This hurt to write. -string_array = c_char_p * 2 -sa = string_array(bytes(ci_log_file, "ascii"), bytes(ci_testing_text, "ascii")) -rustg_dll.log_write.argtypes = [c_int, c_char_p * 2] - -timestamp = datetime.now() - -# Generate valid results for the time now and the next 2 seconds. This is so we can account for spurious CI lag. -valid_results = [] -for count in range(3): - timestamp_text = timestamp.strftime('[%Y-%m-%dT%H:%M:%S]') # 8601 is king - valid_results.append("{} {}".format(timestamp_text, ci_testing_text)) - timestamp = timestamp + timedelta(seconds=1) - -# Invoke this now we have prepared -rustg_dll.log_write(2, sa) - -# Now read the output back -logfile = open(ci_log_file, "r") -logline = logfile.readlines()[0].strip("\n") # Remove newline -logfile.close() - -if logline in valid_results: - success("Log timestamp is valid 8601") -else: - fail("Log timestamp is not valid 8601. Got {}".format(logline)) - -# Make sure we can parse TOML -string_array = c_char_p * 1 -sa = string_array(bytes(ci_toml_file_location, "ascii")) -rustg_dll.toml_file_to_json.argtypes = [c_int, c_char_p * 1] -# Set args for JSON retrieval -rustg_dll.toml_file_to_json.restype = c_char_p - -try: - # Run it - output_json = rustg_dll.toml_file_to_json(1, sa).decode() - json.loads(output_json) - success("toml_file_to_json successful") -except Exception: - fail("Failed to run toml_file_to_json") - -exit(0) # Success diff --git a/tools/tgs_scripts/PreCompile.sh b/tools/tgs_scripts/PreCompile.sh index b23f1fc2a22..aeb6e18a63a 100755 --- a/tools/tgs_scripts/PreCompile.sh +++ b/tools/tgs_scripts/PreCompile.sh @@ -12,39 +12,6 @@ cd "$1" . dependencies.sh cd "$original_dir" -git config --global user.name -if [ $? -eq 1 ] -then - git config --global user.name "paradise_tgs_script" -fi - -git config --global user.email -if [ $? -eq 1 ] -then - git config --global user.email "paradise_tgs_script@invalid.com" -fi - -# update rust-g -if [ ! -d "rust-g" ]; then - echo "Cloning rust-g..." - git clone https://github.com/ParadiseSS13/rust-g - cd rust-g - ~/.cargo/bin/rustup target add i686-unknown-linux-gnu -else - echo "Fetching rust-g..." - cd rust-g - git fetch - ~/.cargo/bin/rustup target add i686-unknown-linux-gnu -fi - -echo "Deploying rust-g..." -git reset --hard "$RUSTG_VERSION" -./apply_patches.sh -cd paradise-rust-g -env PKG_CONFIG_ALLOW_CROSS=1 ~/.cargo/bin/cargo build --release --features all --target=i686-unknown-linux-gnu -mv target/i686-unknown-linux-gnu/release/librust_g.so "$1/librust_g.so" -cd ../../ - echo "Deploying Rustlibs..." cd $1/rust env PKG_CONFIG_ALLOW_CROSS=1 ~/.cargo/bin/cargo build --release --features all --target=i686-unknown-linux-gnu