From 235bbae157c6580c23c6c23e9f6db54138ab1cc8 Mon Sep 17 00:00:00 2001 From: Poojawa Date: Wed, 5 Sep 2018 23:45:53 -0500 Subject: [PATCH] ports BSQL to go along with tools There, now you can shut up about it not being ported you meme --- code/__DEFINES/_protect.dm | 10 + code/__DEFINES/bsql.config.dm | 6 + code/__DEFINES/bsql.dm | 135 ++++++++ code/__DEFINES/tgs.config.dm | 11 +- code/__HELPERS/text.dm | 3 +- .../configuration/entries/dbconfig.dm | 22 +- code/controllers/subsystem/dbcore.dm | 289 ++++++++---------- code/modules/bsql/LICENSE | 7 + code/modules/bsql/core/connection.dm | 68 +++++ code/modules/bsql/core/library.dm | 43 +++ code/modules/bsql/core/operation.dm | 47 +++ code/modules/bsql/core/query.dm | 35 +++ code/modules/bsql/includes.dm | 4 + config/dbconfig.txt | 17 +- tgstation.dme | 4 + 15 files changed, 516 insertions(+), 185 deletions(-) create mode 100644 code/__DEFINES/_protect.dm create mode 100644 code/__DEFINES/bsql.config.dm create mode 100644 code/__DEFINES/bsql.dm create mode 100644 code/modules/bsql/LICENSE create mode 100644 code/modules/bsql/core/connection.dm create mode 100644 code/modules/bsql/core/library.dm create mode 100644 code/modules/bsql/core/operation.dm create mode 100644 code/modules/bsql/core/query.dm create mode 100644 code/modules/bsql/includes.dm diff --git a/code/__DEFINES/_protect.dm b/code/__DEFINES/_protect.dm new file mode 100644 index 0000000000..2bfb984153 --- /dev/null +++ b/code/__DEFINES/_protect.dm @@ -0,0 +1,10 @@ +#define GENERAL_PROTECT_DATUM(Path)\ +##Path/can_vv_get(var_name){\ + return FALSE;\ +}\ +##Path/vv_edit_var(var_name, var_value){\ + return FALSE;\ +}\ +##Path/CanProcCall(procname){\ + return FALSE;\ +} \ No newline at end of file diff --git a/code/__DEFINES/bsql.config.dm b/code/__DEFINES/bsql.config.dm new file mode 100644 index 0000000000..3f2e8c4d70 --- /dev/null +++ b/code/__DEFINES/bsql.config.dm @@ -0,0 +1,6 @@ +#define BSQL_EXTERNAL_CONFIGURATION +#define BSQL_DEL_PROC(path) ##path/Destroy() +#define BSQL_DEL_CALL(obj) qdel(##obj) +#define BSQL_IS_DELETED(obj) (QDELETED(obj)) +#define BSQL_PROTECT_DATUM(path) GENERAL_PROTECT_DATUM(##path) +#define BSQL_ERROR(message) SSdbcore.ReportError(message) diff --git a/code/__DEFINES/bsql.dm b/code/__DEFINES/bsql.dm new file mode 100644 index 0000000000..8f2040449a --- /dev/null +++ b/code/__DEFINES/bsql.dm @@ -0,0 +1,135 @@ +//BSQL - DMAPI +#define BSQL_VERSION "v1.3.0.0" + +//types of connections +#define BSQL_CONNECTION_TYPE_MARIADB "MySql" +#define BSQL_CONNECTION_TYPE_SQLSERVER "SqlServer" + +#define BSQL_DEFAULT_TIMEOUT 5 +#define BSQL_DEFAULT_THREAD_LIMIT 50 + +//Call this before rebooting or shutting down your world to clean up gracefully. This invalidates all active connection and operation datums +/world/proc/BSQL_Shutdown() + return + +/* +Called whenever a library call is made with verbose information, override and do with as you please + message: English debug message +*/ +/world/proc/BSQL_Debug(msg) + return + +/* +Create a new database connection, does not perform the actual connect + connection_type: The BSQL connection_type to use + asyncTimeout: The timeout to use for normal operations, 0 for infinite, defaults to BSQL_DEFAULT_TIMEOUT + blockingTimeout: The timeout to use for blocking operations, must be less than or equal to asyncTimeout, 0 for infinite, defaults to asyncTimeout + threadLimit: The limit of additional threads BSQL will run simultaneously, defaults to BSQL_DEFAULT_THREAD_LIMIT +*/ +/datum/BSQL_Connection/New(connection_type, asyncTimeout, blockingTimeout, threadLimit) + return ..() + +/* +Starts an operation to connect to a database. Should only have 1 successful call + ipaddress: The ip/hostname of the target server + port: The port of the target server + username: The username to login to the target server + password: The password for the target server + database: Optional database to connect to. Must be used when trying to do database operations, `USE x` is not sufficient + Returns: A /datum/BSQL_Operation representing the connection or null if an error occurred +*/ +/datum/BSQL_Connection/proc/BeginConnect(ipaddress, port, username, password, database) + return + +/* +Properly quotes a string for use by the database. The connection must be open for this proc to succeed + str: The string to quote + Returns: The string quoted on success, null on error +*/ +/datum/BSQL_Connection/proc/Quote(str) + return + +/* +Starts an operation for a query + query: The text of the query. Only one query allowed per invocation, no semicolons + Returns: A /datum/BSQL_Operation/Query representing the running query and subsequent result set or null if an error occurred + + Note for MariaDB: The underlying connection is pooled. In order to use connection state based properties (i.e. LAST_INSERT_ID()) you can guarantee multiple queries will use the same connection by running BSQL_DEL_CALL(query) on the finished /datum/BSQL_Operation/Query and then creating the next one with another call to BeginQuery() with no sleeps in between +*/ +/datum/BSQL_Connection/proc/BeginQuery(query) + return + +/* +Checks if the operation is complete. This, in some cases must be called multiple times with false return before a result is present regardless of timespan. For best performance check it once per tick + + Returns: TRUE if the operation is complete, FALSE if it's not, null on error +*/ +/datum/BSQL_Operation/proc/IsComplete() + return + +/* +Blocks the entire game until the given operation completes. IsComplete should not be checked after calling this to avoid potential side effects. + +Returns: TRUE on success, FALSE if the operation wait time exceeded the connection's blockingTimeout setting +*/ +/datum/BSQL_Operation/proc/WaitForCompletion() + return + +/* +Get the error message associated with an operation. Should not be used while IsComplete() returns FALSE + + Returns: The error message, if any. null otherwise +*/ +/datum/BSQL_Operation/proc/GetError() + return + +/* +Get the error code associated with an operation. Should not be used while IsComplete() returns FALSE + + Returns: The error code, if any. null otherwise +*/ +/datum/BSQL_Operation/proc/GetErrorCode() + return + +/* +Gets an associated list of column name -> value representation of the most recent row in the query. Only valid if IsComplete() returns TRUE. If this returns null and no errors are present there are no more results in the query. Important to note that once IsComplete() returns TRUE it must not be called again without checking this or the row values may be lost + + Returns: An associated list of column name -> value for the row. Values will always be either strings or null +*/ +/datum/BSQL_Operation/Query/proc/CurrentRow() + return + + +/* +Code configuration options below + +Define this to avoid modifying this file but the following defines must be declared somewhere else before BSQL/includes.dm is included +*/ +#ifndef BSQL_EXTERNAL_CONFIGURATION + +//Modify this if you disagree with byond's GC schemes. Ensure this is called for all connections and operations when they are deleted or they will leak native resources until /world/proc/BSQL_Shutdown() is called +#define BSQL_DEL_PROC(path) ##path/Del() + +//The equivalent of calling del() in your codebase +#define BSQL_DEL_CALL(obj) del(##obj) + +//Returns TRUE if an object is delete +#define BSQL_IS_DELETED(obj) (obj == null) + +//Modify this to add protections to the connection and query datums +#define BSQL_PROTECT_DATUM(path) + +//Modify this to change up error handling for the library +#define BSQL_ERROR(message) CRASH("BSQL: [##message]") + +#endif + +/* +Copyright 2018 Jordan Brown + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ diff --git a/code/__DEFINES/tgs.config.dm b/code/__DEFINES/tgs.config.dm index 00cbae7629..b05cff7e01 100644 --- a/code/__DEFINES/tgs.config.dm +++ b/code/__DEFINES/tgs.config.dm @@ -7,13 +7,4 @@ #define TGS_ERROR_LOG(message) log_world("TGS: Error: [##message]") #define TGS_NOTIFY_ADMINS(event) message_admins(##event) #define TGS_CLIENT_COUNT GLOB.clients.len -#define TGS_PROTECT_DATUM(Path)\ -##Path/can_vv_get(var_name){\ - return FALSE;\ -}\ -##Path/vv_edit_var(var_name, var_value){\ - return FALSE;\ -}\ -##Path/CanProcCall(procname){\ - return FALSE;\ -} +#define TGS_PROTECT_DATUM(Path) GENERAL_PROTECT_DATUM(##Path) diff --git a/code/__HELPERS/text.dm b/code/__HELPERS/text.dm index 2db7b8bc78..7d6c017b77 100644 --- a/code/__HELPERS/text.dm +++ b/code/__HELPERS/text.dm @@ -15,8 +15,7 @@ // Run all strings to be used in an SQL query through this proc first to properly escape out injection attempts. /proc/sanitizeSQL(t) - var/sqltext = SSdbcore.Quote("[t]"); - return copytext(sqltext, 2, lentext(sqltext));//Quote() adds quotes around input, we already do that + return SSdbcore.Quote("[t]") /proc/format_table_name(table as text) return CONFIG_GET(string/feedback_tableprefix) + table diff --git a/code/controllers/configuration/entries/dbconfig.dm b/code/controllers/configuration/entries/dbconfig.dm index 1eb1186a8b..b034195241 100644 --- a/code/controllers/configuration/entries/dbconfig.dm +++ b/code/controllers/configuration/entries/dbconfig.dm @@ -28,4 +28,24 @@ /datum/config_entry/number/query_debug_log_timeout config_entry_value = 70 min_val = 1 - protection = CONFIG_ENTRY_LOCKED | CONFIG_ENTRY_HIDDEN + protection = CONFIG_ENTRY_LOCKED + deprecated_by = /datum/config_entry/number/blocking_query_timeout + +/datum/config_entry/number/query_debug_log_timeout/DeprecationUpdate(value) + return value + +/datum/config_entry/number/async_query_timeout + config_entry_value = 10 + min_val = 0 + protection = CONFIG_ENTRY_LOCKED + +/datum/config_entry/number/blocking_query_timeout + config_entry_value = 5 + min_val = 0 + protection = CONFIG_ENTRY_LOCKED + +/datum/config_entry/number/bsql_thread_limit + config_entry_value = 50 + min_val = 1 + +/datum/config_entry/flag/bsql_debug diff --git a/code/controllers/subsystem/dbcore.dm b/code/controllers/subsystem/dbcore.dm index 34bf57594d..b9ad66fbce 100644 --- a/code/controllers/subsystem/dbcore.dm +++ b/code/controllers/subsystem/dbcore.dm @@ -5,32 +5,16 @@ SUBSYSTEM_DEF(dbcore) init_order = INIT_ORDER_DBCORE var/const/FAILED_DB_CONNECTION_CUTOFF = 5 - var/const/Default_Cursor = 0 - var/const/Client_Cursor = 1 - var/const/Server_Cursor = 2 - //conversions - var/const/TEXT_CONV = 1 - var/const/RSC_FILE_CONV = 2 - var/const/NUMBER_CONV = 3 - //column flag values: - var/const/IS_NUMERIC = 1 - var/const/IS_BINARY = 2 - var/const/IS_NOT_NULL = 4 - var/const/IS_PRIMARY_KEY = 8 - var/const/IS_UNSIGNED = 16 var/schema_mismatch = 0 var/db_minor = 0 var/db_major = 0 -// TODO: Investigate more recent type additions and see if I can handle them. - Nadrew - - var/_db_con// This variable contains a reference to the actual database connection. var/failed_connections = 0 + var/last_error var/list/active_queries = list() -/datum/controller/subsystem/dbcore/PreInit() - if(!_db_con) - _db_con = _dm_db_new_con() + var/datum/BSQL_Connection/connection + var/datum/BSQL_Operation/connectOperation /datum/controller/subsystem/dbcore/Initialize() //We send warnings to the admins during subsystem init, as the clients will be New'd and messages @@ -54,7 +38,8 @@ SUBSYSTEM_DEF(dbcore) return /datum/controller/subsystem/dbcore/Recover() - _db_con = SSdbcore._db_con + connection = SSdbcore.connection + connectOperation = SSdbcore.connectOperation /datum/controller/subsystem/dbcore/Shutdown() //This is as close as we can get to the true round end before Disconnect() without changing where it's called, defeating the reason this is a subsystem @@ -64,13 +49,14 @@ SUBSYSTEM_DEF(dbcore) qdel(query_round_shutdown) if(IsConnected()) Disconnect() + world.BSQL_Shutdown() //nu /datum/controller/subsystem/dbcore/can_vv_get(var_name) - return var_name != NAMEOF(src, _db_con) && var_name != NAMEOF(src, active_queries) && ..() + return var_name != NAMEOF(src, connection) && var_name != NAMEOF(src, active_queries) && var_name != NAMEOF(src, connectOperation) && ..() /datum/controller/subsystem/dbcore/vv_edit_var(var_name, var_value) - if(var_name == "_db_con") + if(var_name == NAMEOF(src, connection) || var_name == NAMEOF(src, connectOperation)) return FALSE return ..() @@ -90,17 +76,31 @@ SUBSYSTEM_DEF(dbcore) var/address = CONFIG_GET(string/address) var/port = CONFIG_GET(number/port) - _dm_db_connect(_db_con, "dbi:mysql:[db]:[address]:[port]", user, pass, Default_Cursor, null) - . = IsConnected() + connection = new /datum/BSQL_Connection(BSQL_CONNECTION_TYPE_MARIADB, CONFIG_GET(number/async_query_timeout), CONFIG_GET(number/blocking_query_timeout), CONFIG_GET(number/bsql_thread_limit)) + var/error + if(QDELETED(connection)) + connection = null + error = last_error + else + SSdbcore.last_error = null + connectOperation = connection.BeginConnect(address, port, user, pass, db) + if(SSdbcore.last_error) + CRASH(SSdbcore.last_error) + UNTIL(connectOperation.IsComplete()) + error = connectOperation.GetError() + . = !error if (!.) - log_sql("Connect() failed | [ErrorMsg()]") + last_error = error + log_sql("Connect() failed | [error]") ++failed_connections + QDEL_NULL(connection) + QDEL_NULL(connectOperation) /datum/controller/subsystem/dbcore/proc/CheckSchemaVersion() if(CONFIG_GET(flag/sql_enabled)) - if(SSdbcore.Connect()) + if(Connect()) log_world("Database connection established.") - var/datum/DBQuery/query_db_version = SSdbcore.NewQuery("SELECT major, minor FROM [format_table_name("schema_revision")] ORDER BY date DESC LIMIT 1") + var/datum/DBQuery/query_db_version = NewQuery("SELECT major, minor FROM [format_table_name("schema_revision")] ORDER BY date DESC LIMIT 1") query_db_version.Execute() if(query_db_version.NextRow()) db_major = text2num(query_db_version.item[1]) @@ -146,27 +146,36 @@ SUBSYSTEM_DEF(dbcore) /datum/controller/subsystem/dbcore/proc/Disconnect() failed_connections = 0 - return _dm_db_close(_db_con) + QDEL_NULL(connectOperation) + QDEL_NULL(connection) /datum/controller/subsystem/dbcore/proc/IsConnected() if(!CONFIG_GET(flag/sql_enabled)) return FALSE - return _dm_db_is_connected(_db_con) + //block until any connect operations finish + var/datum/BSQL_Connection/_connection = connection + var/datum/BSQL_Operation/op = connectOperation + UNTIL(QDELETED(_connection) || op.IsComplete()) + return !QDELETED(connection) && !op.GetError() /datum/controller/subsystem/dbcore/proc/Quote(str) - return _dm_db_quote(_db_con, str) + if(connection) + return connection.Quote(str) /datum/controller/subsystem/dbcore/proc/ErrorMsg() if(!CONFIG_GET(flag/sql_enabled)) return "Database disabled by configuration" - return _dm_db_error_msg(_db_con) + return last_error -/datum/controller/subsystem/dbcore/proc/NewQuery(sql_query, cursor_handler = Default_Cursor) +/datum/controller/subsystem/dbcore/proc/ReportError(error) + last_error = error + +/datum/controller/subsystem/dbcore/proc/NewQuery(sql_query) if(IsAdminAdvancedProcCall()) log_admin_private("ERROR: Advanced admin proc call led to sql query: [sql_query]. Query has been blocked") message_admins("ERROR: Advanced admin proc call led to sql query. Query has been blocked") return FALSE - return new /datum/DBQuery(sql_query, src, cursor_handler) + return new /datum/DBQuery(sql_query, connection) /* Takes a list of rows (each row being an associated list of column => value) and inserts them via a single mass query. @@ -180,7 +189,7 @@ Delayed insert mode was removed in mysql 7 and only works with MyISAM type table It was included because it is still supported in mariadb. It does not work with duplicate_key and the mysql server ignores it in those cases */ -/datum/controller/subsystem/dbcore/proc/MassInsert(table, list/rows, duplicate_key = FALSE, ignore_errors = FALSE, delayed = FALSE, warn = FALSE) +/datum/controller/subsystem/dbcore/proc/MassInsert(table, list/rows, duplicate_key = FALSE, ignore_errors = FALSE, delayed = FALSE, warn = FALSE, async = TRUE) if (!table || !rows || !istype(rows)) return var/list/columns = list() @@ -230,34 +239,30 @@ Delayed insert mode was removed in mysql 7 and only works with MyISAM type table sqlrowlist = " [sqlrowlist.Join(",\n ")]" var/datum/DBQuery/Query = NewQuery("INSERT[delayed][ignore_errors] INTO [table]\n([columns.Join(", ")])\nVALUES\n[sqlrowlist]\n[duplicate_key]") if (warn) - . = Query.warn_execute() + . = Query.warn_execute(async) else - . = Query.Execute() + . = Query.Execute(async) qdel(Query) - /datum/DBQuery var/sql // The sql query being executed. - var/default_cursor - var/list/columns //list of DB Columns populated by Columns() - var/list/conversions var/list/item //list of data values populated by NextRow() + var/last_activity var/last_activity_time - var/datum/controller/subsystem/dbcore/db_connection - var/_db_query -/datum/DBQuery/New(sql_query, datum/controller/subsystem/dbcore/connection_handler, cursor_handler) + var/last_error + var/skip_next_is_complete + var/in_progress + var/datum/BSQL_Connection/connection + var/datum/BSQL_Operation/Query/query + +/datum/DBQuery/New(sql_query, datum/BSQL_Connection/connection) SSdbcore.active_queries[src] = TRUE Activity("Created") - if(sql_query) - sql = sql_query - if(connection_handler) - db_connection = connection_handler - if(cursor_handler) - default_cursor = cursor_handler item = list() - _db_query = _dm_db_new_query() + src.connection = connection + sql = sql_query /datum/DBQuery/Destroy() Close() @@ -268,147 +273,93 @@ Delayed insert mode was removed in mysql 7 and only works with MyISAM type table //fuck off kevinz return FALSE +/datum/DBQuery/proc/SetQuery(new_sql) + if(in_progress) + CRASH("Attempted to set new sql while waiting on active query") + Close() + sql = new_sql + /datum/DBQuery/proc/Activity(activity) last_activity = activity last_activity_time = world.time -/datum/DBQuery/proc/warn_execute() - . = Execute() +/datum/DBQuery/proc/warn_execute(async = FALSE) + . = Execute(async) if(!.) to_chat(usr, "A SQL error occurred during this operation, check the server logs.") -/datum/DBQuery/proc/SetQuery(new_sql) - Activity("SetQuery") - Close() - sql = new_sql - -/datum/DBQuery/proc/Execute(sql_query = sql, cursor_handler = default_cursor, log_error = TRUE) +/datum/DBQuery/proc/Execute(async = FALSE, log_error = TRUE) Activity("Execute") + if(in_progress) + CRASH("Attempted to start a new query while waiting on the old one") + + if(QDELETED(connection)) + last_error = "No connection!" + return FALSE + var/start_time - var/timeout = CONFIG_GET(number/query_debug_log_timeout) - if(timeout) + var/timed_out + if(!async) start_time = REALTIMEOFDAY Close() - . = _dm_db_execute(_db_query, sql_query, db_connection._db_con, cursor_handler, null) + query = connection.BeginQuery(sql) + if(!async) + timed_out = !query.WaitForCompletion() + else + in_progress = TRUE + UNTIL(query.IsComplete()) + in_progress = FALSE + skip_next_is_complete = TRUE + var/error = QDELETED(query) ? "Query object deleted!" : query.GetError() + last_error = error + . = !error if(!. && log_error) - log_sql("[ErrorMsg()] | Query used: [sql]") - if(timeout) - if((REALTIMEOFDAY - start_time) > timeout) - log_query_debug("Query execution started at [start_time]") - log_query_debug("Query execution ended at [REALTIMEOFDAY]") - log_query_debug("Possible slow query timeout detected.") - log_query_debug("Query used: [sql]") - slow_query_check() + log_sql("[error] | Query used: [sql]") + if(!async && timed_out) + log_query_debug("Query execution started at [start_time]") + log_query_debug("Query execution ended at [REALTIMEOFDAY]") + log_query_debug("Slow query timeout detected.") + log_query_debug("Query used: [sql]") + slow_query_check() /datum/DBQuery/proc/slow_query_check() - message_admins("HEY! A database query may have timed out. Did the server just hang? \[YES\]|\[NO\]") + message_admins("HEY! A database query timed out. Did the server just hang? \[YES\]|\[NO\]") -/datum/DBQuery/proc/NextRow() +/datum/DBQuery/proc/NextRow(async) Activity("NextRow") - return _dm_db_next_row(_db_query,item,conversions) + UNTIL(!in_progress) + if(!skip_next_is_complete) + if(!async) + query.WaitForCompletion() + else + in_progress = TRUE + UNTIL(query.IsComplete()) + in_progress = FALSE + else + skip_next_is_complete = FALSE -/datum/DBQuery/proc/RowsAffected() - return _dm_db_rows_affected(_db_query) + last_error = query.GetError() + var/list/results = query.CurrentRow() + . = results != null -/datum/DBQuery/proc/RowCount() - return _dm_db_row_count(_db_query) + item.Cut() + //populate item array + for(var/I in results) + item += results[I] /datum/DBQuery/proc/ErrorMsg() - return _dm_db_error_msg(_db_query) - -/datum/DBQuery/proc/Columns() - if(!columns) - columns = _dm_db_columns(_db_query, /datum/DBColumn) - return columns - -/datum/DBQuery/proc/GetRowData() - var/list/columns = Columns() - var/list/results - if(columns.len) - results = list() - for(var/C in columns) - results+=C - var/datum/DBColumn/cur_col = columns[C] - results[C] = src.item[(cur_col.position+1)] - return results + return last_error /datum/DBQuery/proc/Close() item.Cut() - columns = null - conversions = null - return _dm_db_close(_db_query) + QDEL_NULL(query) -/datum/DBQuery/proc/Quote(str) - return db_connection.Quote(str) +/world/BSQL_Debug(message) + if(!CONFIG_GET(flag/bsql_debug)) + return -/datum/DBQuery/proc/SetConversion(column,conversion) - if(istext(column)) - column = columns.Find(column) - if(!conversions) - conversions = new /list(column) - else if(conversions.len < column) - conversions.len = column - conversions[column] = conversion - - -/datum/DBColumn - var/name - var/table - var/position //1-based index into item data - var/sql_type - var/flags - var/length - var/max_length - //types - var/const/TINYINT = 1 - var/const/SMALLINT = 2 - var/const/MEDIUMINT = 3 - var/const/INTEGER = 4 - var/const/BIGINT = 5 - var/const/DECIMAL = 6 - var/const/FLOAT = 7 - var/const/DOUBLE = 8 - var/const/DATE = 9 - var/const/DATETIME = 10 - var/const/TIMESTAMP = 11 - var/const/TIME = 12 - var/const/STRING = 13 - var/const/BLOB = 14 - -/datum/DBColumn/New(name_handler, table_handler, position_handler, type_handler, flag_handler, length_handler, max_length_handler) - name = name_handler - table = table_handler - position = position_handler - sql_type = type_handler - flags = flag_handler - length = length_handler - max_length = max_length_handler - -/datum/DBColumn/proc/SqlTypeName(type_handler = sql_type) - switch(type_handler) - if(TINYINT) - return "TINYINT" - if(SMALLINT) - return "SMALLINT" - if(MEDIUMINT) - return "MEDIUMINT" - if(INTEGER) - return "INTEGER" - if(BIGINT) - return "BIGINT" - if(FLOAT) - return "FLOAT" - if(DOUBLE) - return "DOUBLE" - if(DATE) - return "DATE" - if(DATETIME) - return "DATETIME" - if(TIMESTAMP) - return "TIMESTAMP" - if(TIME) - return "TIME" - if(STRING) - return "STRING" - if(BLOB) - return "BLOB" + //strip sensitive stuff + if(findtext(message, ": CreateConnection(")) + message = "CreateConnection CENSORED" + + log_sql("BSQL_DEBUG: [message]") diff --git a/code/modules/bsql/LICENSE b/code/modules/bsql/LICENSE new file mode 100644 index 0000000000..2bee290914 --- /dev/null +++ b/code/modules/bsql/LICENSE @@ -0,0 +1,7 @@ +Copyright 2018 Jordan Brown + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/code/modules/bsql/core/connection.dm b/code/modules/bsql/core/connection.dm new file mode 100644 index 0000000000..0e0b891f65 --- /dev/null +++ b/code/modules/bsql/core/connection.dm @@ -0,0 +1,68 @@ +/datum/BSQL_Connection + var/id + var/connection_type + +BSQL_PROTECT_DATUM(/datum/BSQL_Connection) + +/datum/BSQL_Connection/New(connection_type, asyncTimeout, blockingTimeout, threadLimit) + if(asyncTimeout == null) + asyncTimeout = BSQL_DEFAULT_TIMEOUT + if(blockingTimeout == null) + blockingTimeout = asyncTimeout + if(threadLimit == null) + threadLimit = BSQL_DEFAULT_THREAD_LIMIT + + src.connection_type = connection_type + + world._BSQL_InitCheck(src) + + var/error = world._BSQL_Internal_Call("CreateConnection", connection_type, "[asyncTimeout]", "[blockingTimeout]", "[threadLimit]") + if(error) + BSQL_ERROR(error) + return + + id = world._BSQL_Internal_Call("GetConnection") + if(!id) + BSQL_ERROR("BSQL library failed to provide connect operation for connection id [id]([connection_type])!") + +BSQL_DEL_PROC(/datum/BSQL_Connection) + var/error + if(id) + error = world._BSQL_Internal_Call("ReleaseConnection", id) + . = ..() + if(error) + BSQL_ERROR(error) + +/datum/BSQL_Connection/BeginConnect(ipaddress, port, username, password, database) + var/error = world._BSQL_Internal_Call("OpenConnection", id, ipaddress, "[port]", username, password, database) + if(error) + BSQL_ERROR(error) + return + + var/op_id = world._BSQL_Internal_Call("GetOperation") + if(!op_id) + BSQL_ERROR("Library failed to provide connect operation for connection id [id]([connection_type])!") + return + + return new /datum/BSQL_Operation(src, op_id) + + +/datum/BSQL_Connection/BeginQuery(query) + var/error = world._BSQL_Internal_Call("NewQuery", id, query) + if(error) + BSQL_ERROR(error) + return + + var/op_id = world._BSQL_Internal_Call("GetOperation") + if(!op_id) + BSQL_ERROR("Library failed to provide query operation for connection id [id]([connection_type])!") + return + + return new /datum/BSQL_Operation/Query(src, op_id) + +/datum/BSQL_Connection/Quote(str) + if(!str) + return null; + . = world._BSQL_Internal_Call("QuoteString", id, "[str]") + if(!.) + BSQL_ERROR("Library failed to provide quote for [str]!") \ No newline at end of file diff --git a/code/modules/bsql/core/library.dm b/code/modules/bsql/core/library.dm new file mode 100644 index 0000000000..1b62cf3b6a --- /dev/null +++ b/code/modules/bsql/core/library.dm @@ -0,0 +1,43 @@ +/world/proc/_BSQL_Internal_Call(func, ...) + var/list/call_args = args.Copy(2) + BSQL_Debug("[.....]: [args[1]]([call_args.Join(", ")])") + . = call(_BSQL_Library_Path(), func)(arglist(call_args)) + BSQL_Debug("Result: [. == null ? "NULL" : "\"[.]\""]") + +/world/proc/_BSQL_Library_Path() + return system_type == MS_WINDOWS ? "BSQL.dll" : "libBSQL.so" + +/world/proc/_BSQL_InitCheck(datum/BSQL_Connection/caller) + var/static/library_initialized = FALSE + if(_BSQL_Initialized()) + return + var/libPath = _BSQL_Library_Path() + if(!fexists(libPath)) + BSQL_DEL_CALL(caller) + BSQL_ERROR("Could not find [libPath]!") + return + + var/version = _BSQL_Internal_Call("Version") + if(version != BSQL_VERSION) + BSQL_DEL_CALL(caller) + BSQL_ERROR("BSQL DMAPI version mismatch! Expected [BSQL_VERSION], got [version == null ? "NULL" : version]!") + return + + var/result = _BSQL_Internal_Call("Initialize") + if(result) + BSQL_DEL_CALL(caller) + BSQL_ERROR(result) + return + _BSQL_Initialized(TRUE) + +/world/proc/_BSQL_Initialized(new_val) + var/static/bsql_library_initialized = FALSE + if(new_val != null) + bsql_library_initialized = new_val + return bsql_library_initialized + +/world/BSQL_Shutdown() + if(!_BSQL_Initialized()) + return + _BSQL_Internal_Call("Shutdown") + _BSQL_Initialized(FALSE) diff --git a/code/modules/bsql/core/operation.dm b/code/modules/bsql/core/operation.dm new file mode 100644 index 0000000000..a2cdbbe1ee --- /dev/null +++ b/code/modules/bsql/core/operation.dm @@ -0,0 +1,47 @@ +/datum/BSQL_Operation + var/datum/BSQL_Connection/connection + var/id + +BSQL_PROTECT_DATUM(/datum/BSQL_Operation) + +/datum/BSQL_Operation/New(datum/BSQL_Connection/connection, id) + src.connection = connection + src.id = id + +BSQL_DEL_PROC(/datum/BSQL_Operation) + var/error + if(!BSQL_IS_DELETED(connection)) + error = world._BSQL_Internal_Call("ReleaseOperation", connection.id, id) + . = ..() + if(error) + BSQL_ERROR(error) + +/datum/BSQL_Operation/IsComplete() + if(BSQL_IS_DELETED(connection)) + return TRUE + var/result = world._BSQL_Internal_Call("OpComplete", connection.id, id) + if(!result) + BSQL_ERROR("Error fetching operation [id] for connection [connection.id]!") + return + return result == "DONE" + +/datum/BSQL_Operation/GetError() + if(BSQL_IS_DELETED(connection)) + return "Connection deleted!" + return world._BSQL_Internal_Call("GetError", connection.id, id) + +/datum/BSQL_Operation/GetErrorCode() + if(BSQL_IS_DELETED(connection)) + return -2 + return text2num(world._BSQL_Internal_Call("GetErrorCode", connection.id, id)) + +/datum/BSQL_Operation/WaitForCompletion() + if(BSQL_IS_DELETED(connection)) + return + var/error = world._BSQL_Internal_Call("BlockOnOperation", connection.id, id) + if(error) + if(error == "Operation timed out!") //match this with the implementation + return FALSE + BSQL_ERROR("Error waiting for operation [id] for connection [connection.id]! [error]") + return + return TRUE diff --git a/code/modules/bsql/core/query.dm b/code/modules/bsql/core/query.dm new file mode 100644 index 0000000000..fc09fb06b0 --- /dev/null +++ b/code/modules/bsql/core/query.dm @@ -0,0 +1,35 @@ +/datum/BSQL_Operation/Query + var/last_result_json + var/list/last_result + +BSQL_PROTECT_DATUM(/datum/BSQL_Operation/Query) + +/datum/BSQL_Operation/Query/CurrentRow() + return last_result + +/datum/BSQL_Operation/Query/IsComplete() + //whole different ballgame here + if(BSQL_IS_DELETED(connection)) + return TRUE + var/result = world._BSQL_Internal_Call("ReadyRow", connection.id, id) + switch(result) + if("DONE") + //load the data + LoadQueryResult() + return TRUE + if("NOTDONE") + return FALSE + else + BSQL_ERROR(result) + +/datum/BSQL_Operation/Query/WaitForCompletion() + . = ..() + if(.) + LoadQueryResult() + +/datum/BSQL_Operation/Query/proc/LoadQueryResult() + last_result_json = world._BSQL_Internal_Call("GetRow", connection.id, id) + if(last_result_json) + last_result = json_decode(last_result_json) + else + last_result = null diff --git a/code/modules/bsql/includes.dm b/code/modules/bsql/includes.dm new file mode 100644 index 0000000000..d05dcb6451 --- /dev/null +++ b/code/modules/bsql/includes.dm @@ -0,0 +1,4 @@ +#include "core\connection.dm" +#include "core\library.dm" +#include "core\operation.dm" +#include "core\query.dm" diff --git a/config/dbconfig.txt b/config/dbconfig.txt index ed0ffadede..3a058fe563 100644 --- a/config/dbconfig.txt +++ b/config/dbconfig.txt @@ -29,6 +29,17 @@ FEEDBACK_LOGIN username ## Password used to access the database. FEEDBACK_PASSWORD password -## Time in deciseconds for a query to execute before alerting a for possible slow query timeout. -## While enabled queries and their execution times are logged if they exceed this value. -#QUERY_DEBUG_LOG_TIMEOUT 70 +## Time in seconds for asynchronous queries to timeout +## Set to 0 for infinite +ASYNC_QUERY_TIMEOUT 10 + +## Time in seconds for blocking queries to execute before slow query timeout +## Set to 0 for infinite +## Must be less than or equal to ASYNC_QUERY_TIMEOUT +BLOCKING_QUERY_TIMEOUT 5 + +## The maximum number of additional threads BSQL is allowed to run at once +BSQL_THREAD_LIMIT 50 + +## Uncomment to enable verbose BSQL communication logs +#BSQL_DEBUG diff --git a/tgstation.dme b/tgstation.dme index 5910dfa5e5..7e929528af 100755 --- a/tgstation.dme +++ b/tgstation.dme @@ -17,12 +17,15 @@ #include "code\_compile_options.dm" #include "code\world.dm" #include "code\__DEFINES\_globals.dm" +#include "code\__DEFINES\_protect.dm" #include "code\__DEFINES\_tick.dm" #include "code\__DEFINES\access.dm" #include "code\__DEFINES\admin.dm" #include "code\__DEFINES\antagonists.dm" #include "code\__DEFINES\atmospherics.dm" #include "code\__DEFINES\atom_hud.dm" +#include "code\__DEFINES\bsql.config.dm" +#include "code\__DEFINES\bsql.dm" #include "code\__DEFINES\callbacks.dm" #include "code\__DEFINES\cinematics.dm" #include "code\__DEFINES\citadel_defines.dm" @@ -1366,6 +1369,7 @@ #include "code\modules\awaymissions\mission_code\stationCollision.dm" #include "code\modules\awaymissions\mission_code\undergroundoutpost45.dm" #include "code\modules\awaymissions\mission_code\wildwest.dm" +#include "code\modules\bsql\includes.dm" #include "code\modules\cargo\bounty.dm" #include "code\modules\cargo\bounty_console.dm" #include "code\modules\cargo\console.dm"