Adds BSQL (#38323)

* Add BSQL library v1.1.1.0

* Modify dbcore to use BSQL

* Add missing QDEL_NULL for connectOperation

* Moves BSQL_Shutdown() call to dbcore shutdown

* Fix passing the wrong argument to DBQuery/New()

* Darn it @Jordie0608. Fixes db calls without Connect check

No seriously please make sure I'm not breaking anything

* Queries with a null connection won't runtime

* Fix quoting

* Fix mistake

* Update BSQL to v1.1.2.0

* Update BSQL DMAPI to v1.0.1.0

* Fix connection instatiation

* Does the smart thing in regards to quoting

* Fix braces

* Update BSQL to 1.2.0.0. DMAPI to 1.1.0.0

* Execute/NextRow/MassInsert now have async parameter

* Build BSQL for tests

* Add missing apt source

* Def still need gcc-multilib

* Wut

* Revert "Wut"

This reverts commit d7c98a9a6b27f6db03e9f5cc534650d59d018048.

* Try this then

* Could it really be that simple?

* Literally running out of ideas here

* Update BSQL to v1.2.1.0 DMAPI to v1.1.1.0

* Update BSQL travis version

* Nothing about this makes sense tbqhwyfam

* Whoo boy

* No idea why this isn't working tbh

* Absolute madness

* Ahhhhhhhhhhhhh

* *deep breath*

* "though yet again i was frustrated by failure"

* Add BSQL to Dockerfile

* Pass through MassInsert async param

* BSQL to v1.3.0.0 DMAPI to 1.2.0.0

* Add timeout support

* Wait, something's fucky

* Wtf is this meme?

* Just get good lmao

* Just stop being shit lol

* Stupid verbose logging

* Remove verbosity

* Good god

* BSQL to v1.3.0.1 DMAPI to v1.2.0.1

* BSQL to v1.3.0.2

* Update BSQL travis version

* Update BSQL docker version

* Didn't mean to change that

* Strip connection information from debug logs and make it configgable

* Move this to where CONFIG_GET is defined

.travis.yml

@@ -9,6 +9,7 @@ env:
     - BYOND_MINOR="1427"
     - NODE_VERSION="4"
     - RUST_G_VERSION="0.3.0"
+    - BSQL_VERSION="v1.3.0.2"
 matrix:
   include:
     - env:
@@ -37,15 +38,20 @@ matrix:
       addons:
         mariadb: '10.2'
         apt:
+          sources:
+            - ubuntu-toolchain-r-test
           packages:
             - libstdc++6:i386
             - libssl-dev:i386
             - gcc-multilib
+            - g++-7
+            - g++-7-multilib
+            - libmariadbclient-dev:i386
       cache:
         directories:
           - $HOME/.cargo
           - $HOME/BYOND-${BYOND_MAJOR}.${BYOND_MINOR}
-          - $HOME/libmariadb
+          - $HOME/MariaDB
           - $HOME/.rustup
 
 install:

Dockerfile

@@ -6,12 +6,20 @@ WORKDIR /rust_g
 
 RUN apt-get update && apt-get install -y \
     git \
-    libssl-dev \
     ca-certificates \
+    libc6-dev
+
+FROM build as rust_g
+
+WORKDIR /rust_g
+
+RUN apt-get install -y --no-install-recommends \
+    libssl-dev \
     rustc \
     cargo \
-    pkg-config \
+    pkg-config
-    && git init \
+
+RUN git init \
     && git remote add origin https://github.com/tgstation/rust-g
 
 #TODO: find a way to read these from .travis.yml or a common source eventually
@@ -21,6 +29,38 @@ RUN git fetch --depth 1 origin $RUST_G_VERSION \
     && git checkout FETCH_HEAD \
     && cargo build --release
 
+FROM base as bsql
+
+WORKDIR /bsql
+
+RUN apt-get install -y --no-install-recommends software-properties-common \
+    && add-apt-repository ppa:ubuntu-toolchain-r/test \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+    cmake \
+    make \
+    g++-7 \
+    libstdc++6 \
+    libmariadb-client-lgpl-dev
+
+RUN git init \
+    && git remote add origin https://github.com/tgstation/BSQL 
+
+#TODO: find a way to read these from .travis.yml or a common source eventually
+ENV BSQL_VERSION=v1.3.0.2
+
+RUN git fetch --depth 1 origin $BSQL_VERSION \
+    && git checkout FETCH_HEAD
+
+WORKDIR /bsql/artifacts
+
+ENV CC=gcc-7 CXX=g++-7
+
+RUN ln -s /usr/include/mariadb /usr/include/mysql \
+    && ln -s /usr/lib/i386-linux-gnu /root/MariaDB \
+    && cmake .. \
+    && make
+
 FROM base as dm_base
 
 WORKDIR /tgstation
@@ -64,8 +104,12 @@ RUN apt-get update && apt-get install -y \
     && mkdir -p /root/.byond/bin
 
 COPY --from=rustg /rust_g/target/release/librust_g.so /root/.byond/bin/rust_g
+COPY --from=bsql /bsql/artifacts/src/BSQL/libBSQL.so ./
 COPY --from=build /deploy ./
 
+#bsql fexists memes
+RUN ln -s /tgstation/libBSQL.so /root/.byond/bin/libBSQL.so
+
 VOLUME [ "/tgstation/config", "/tgstation/data" ]
 
 ENTRYPOINT [ "DreamDaemon", "tgstation.dmb", "-port", "1337", "-trusted", "-close", "-verbose" ]

code/__DEFINES/_protect.dm

@@ -0,0 +1,10 @@
+#define GENERAL_PROTECT_DATUM(Path)\
+##Path/can_vv_get(var_name){\
+    return FALSE;\
+}\
+##Path/vv_edit_var(var_name, var_value){\
+    return FALSE;\
+}\
+##Path/CanProcCall(procname){\
+    return FALSE;\
+}

code/__DEFINES/bsql.config.dm

@@ -0,0 +1,6 @@
+#define BSQL_EXTERNAL_CONFIGURATION
+#define BSQL_DEL_PROC(path) ##path/Destroy()
+#define BSQL_DEL_CALL(obj) qdel(##obj)
+#define BSQL_IS_DELETED(obj) (QDELETED(obj))
+#define BSQL_PROTECT_DATUM(path) GENERAL_PROTECT_DATUM(##path)
+#define BSQL_ERROR(message) SSdbcore.ReportError(message)

code/__DEFINES/bsql.dm

@@ -0,0 +1,132 @@
+//BSQL - DMAPI v1.2.0.1
+
+//types of connections
+#define BSQL_CONNECTION_TYPE_MARIADB "MySql"
+#define BSQL_CONNECTION_TYPE_SQLSERVER "SqlServer"
+
+#define BSQL_DEFAULT_TIMEOUT 5
+
+//Call this before rebooting or shutting down your world to clean up gracefully. This invalidates all active connection and operation datums
+/world/proc/BSQL_Shutdown()
+	return
+
+/*
+Called whenever a library call is made with verbose information, override and do with as you please
+  message: English debug message
+*/
+/world/proc/BSQL_Debug(msg)
+	return
+
+/*
+Create a new database connection, does not perform the actual connect
+  connection_type: The BSQL connection_type to use
+  asyncTimeout: The timeout to use for normal operations, 0 for infinite, defaults to BSQL_DEFAULT_TIMEOUT
+  blockingTimeout: The timeout to use for blocking operations, must be less than or equal to asyncTimeout, 0 for infinite, defaults to asyncTimeout
+*/
+/datum/BSQL_Connection/New(connection_type, asyncTimeout, blockingTimeout)
+	return ..()
+
+/*
+Starts an operation to connect to a database. Should only have 1 successful call
+  ipaddress: The ip/hostname of the target server
+  port: The port of the target server
+  username: The username to login to the target server
+  password: The password for the target server
+  database: Optional database to connect to. Must be used when trying to do database operations, `USE x` is not sufficient
+ Returns: A /datum/BSQL_Operation representing the connection or null if an error occurred
+*/
+/datum/BSQL_Connection/proc/BeginConnect(ipaddress, port, username, password, database)
+	return
+
+/*
+Properly quotes a string for use by the database. The connection must be open for this proc to succeed
+  str: The string to quote
+ Returns: The string quoted on success, null on error
+*/
+/datum/BSQL_Connection/proc/Quote(str)
+	return
+
+/*
+Starts an operation for a query
+  query: The text of the query. Only one query allowed per invocation, no semicolons
+ Returns: A /datum/BSQL_Operation/Query representing the running query and subsequent result set or null if an error occurred
+
+ Note for MariaDB: The underlying connection is pooled. In order to use connection state based properties (i.e. LAST_INSERT_ID()) you can guarantee multiple queries will use the same connection by running BSQL_DEL_CALL(query) on the finished /datum/BSQL_Operation/Query and then creating the next one with another call to BeginQuery() with no sleeps in between
+*/
+/datum/BSQL_Connection/proc/BeginQuery(query)
+	return
+
+/*
+Checks if the operation is complete. This, in some cases must be called multiple times with false return before a result is present regardless of timespan. For best performance check it once per tick
+
+ Returns: TRUE if the operation is complete, FALSE if it's not, null on error
+*/
+/datum/BSQL_Operation/proc/IsComplete()
+	return
+
+/*
+Blocks the entire game until the given operation completes. IsComplete should not be checked after calling this to avoid potential side effects.
+
+Returns: TRUE on success, FALSE if the operation wait time exceeded the connection's blockingTimeout setting
+*/
+/datum/BSQL_Operation/proc/WaitForCompletion()
+	return
+
+/*
+Get the error message associated with an operation. Should not be used while IsComplete() returns FALSE
+
+ Returns: The error message, if any. null otherwise
+*/
+/datum/BSQL_Operation/proc/GetError()
+	return
+
+/*
+Get the error code associated with an operation. Should not be used while IsComplete() returns FALSE
+
+ Returns: The error code, if any. null otherwise
+*/
+/datum/BSQL_Operation/proc/GetErrorCode()
+	return
+
+/*
+Gets an associated list of column name -> value representation of the most recent row in the query. Only valid if IsComplete() returns TRUE. If this returns null and no errors are present there are no more results in the query. Important to note that once IsComplete() returns TRUE it must not be called again without checking this or the row values may be lost
+
+ Returns: An associated list of column name -> value for the row. Values will always be either strings or null
+*/
+/datum/BSQL_Operation/Query/proc/CurrentRow()
+	return
+
+
+/*
+Code configuration options below
+
+Define this to avoid modifying this file but the following defines must be declared somewhere else before BSQL/includes.dm is included
+*/
+#ifndef BSQL_EXTERNAL_CONFIGURATION
+
+//Modify this if you disagree with byond's GC schemes. Ensure this is called for all connections and operations when they are deleted or they will leak native resources until /world/proc/BSQL_Shutdown() is called
+#define BSQL_DEL_PROC(path) ##path/Del()
+
+//The equivalent of calling del() in your codebase
+#define BSQL_DEL_CALL(obj) del(##obj)
+
+//Returns TRUE if an object is delete
+#define BSQL_IS_DELETED(obj) (obj == null)
+
+//Modify this to add protections to the connection and query datums
+#define BSQL_PROTECT_DATUM(path)
+
+//Modify this to change up error handling for the library
+#define BSQL_ERROR(message) CRASH("BSQL: [##message]")
+
+#endif
+
+/*
+Copyright 2018 Jordan Brown
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/

code/__DEFINES/tgs.config.dm

@@ -7,13 +7,4 @@
 #define TGS_ERROR_LOG(message) log_world("TGS: Error: [##message]")
 #define TGS_NOTIFY_ADMINS(event) message_admins(##event)
 #define TGS_CLIENT_COUNT GLOB.clients.len
-#define TGS_PROTECT_DATUM(Path)\
+#define TGS_PROTECT_DATUM(Path) GENERAL_PROTECT_DATUM(##Path)
-##Path/can_vv_get(var_name){\
-    return FALSE;\
-}\
-##Path/vv_edit_var(var_name, var_value){\
-    return FALSE;\
-}\
-##Path/CanProcCall(procname){\
-    return FALSE;\
-}

code/__HELPERS/text.dm

@@ -15,8 +15,7 @@
 
 // Run all strings to be used in an SQL query through this proc first to properly escape out injection attempts.
 /proc/sanitizeSQL(t)
-	var/sqltext = SSdbcore.Quote("[t]");
+	return SSdbcore.Quote("[t]")
-	return copytext(sqltext, 2, lentext(sqltext));//Quote() adds quotes around input, we already do that
 
 /proc/format_table_name(table as text)
 	return CONFIG_GET(string/feedback_tableprefix) + table

code/controllers/configuration/entries/dbconfig.dm

@@ -28,4 +28,20 @@
 /datum/config_entry/number/query_debug_log_timeout
 	config_entry_value = 70
 	min_val = 1
-	protection = CONFIG_ENTRY_LOCKED | CONFIG_ENTRY_HIDDEN
+	protection = CONFIG_ENTRY_LOCKED
+	deprecated_by = /datum/config_entry/number/blocking_query_timeout
+
+/datum/config_entry/number/query_debug_log_timeout/DeprecationUpdate(value)
+	return value
+
+/datum/config_entry/number/async_query_timeout
+	config_entry_value = 10
+	min_val = 0
+	protection = CONFIG_ENTRY_LOCKED
+
+/datum/config_entry/number/blocking_query_timeout
+	config_entry_value = 5
+	min_val = 0
+	protection = CONFIG_ENTRY_LOCKED
+
+/datum/config_entry/flag/bsql_debug

code/controllers/subsystem/dbcore.dm

@@ -5,32 +5,16 @@ SUBSYSTEM_DEF(dbcore)
 	init_order = INIT_ORDER_DBCORE
 	var/const/FAILED_DB_CONNECTION_CUTOFF = 5
 
-	var/const/Default_Cursor = 0
-	var/const/Client_Cursor = 1
-	var/const/Server_Cursor = 2
-	//conversions
-	var/const/TEXT_CONV = 1
-	var/const/RSC_FILE_CONV = 2
-	var/const/NUMBER_CONV = 3
-	//column flag values:
-	var/const/IS_NUMERIC = 1
-	var/const/IS_BINARY = 2
-	var/const/IS_NOT_NULL = 4
-	var/const/IS_PRIMARY_KEY = 8
-	var/const/IS_UNSIGNED = 16
 	var/schema_mismatch = 0
 	var/db_minor = 0
 	var/db_major = 0
-// TODO: Investigate more recent type additions and see if I can handle them. - Nadrew
-
-	var/_db_con// This variable contains a reference to the actual database connection.
 	var/failed_connections = 0
 
+	var/last_error
 	var/list/active_queries = list()
 
-/datum/controller/subsystem/dbcore/PreInit()
+	var/datum/BSQL_Connection/connection
-	if(!_db_con)
+	var/datum/BSQL_Operation/connectOperation
-		_db_con = _dm_db_new_con()
 
 /datum/controller/subsystem/dbcore/Initialize()
 	//We send warnings to the admins during subsystem init, as the clients will be New'd and messages
@@ -54,7 +38,8 @@ SUBSYSTEM_DEF(dbcore)
 			return
 
 /datum/controller/subsystem/dbcore/Recover()
-	_db_con = SSdbcore._db_con
+	connection = SSdbcore.connection
+	connectOperation = SSdbcore.connectOperation
 
 /datum/controller/subsystem/dbcore/Shutdown()
 	//This is as close as we can get to the true round end before Disconnect() without changing where it's called, defeating the reason this is a subsystem
@@ -64,13 +49,14 @@ SUBSYSTEM_DEF(dbcore)
 		qdel(query_round_shutdown)
 	if(IsConnected())
 		Disconnect()
+	world.BSQL_Shutdown()
 
 //nu
 /datum/controller/subsystem/dbcore/can_vv_get(var_name)
-	return var_name != NAMEOF(src, _db_con) && var_name != NAMEOF(src, active_queries) && ..()
+	return var_name != NAMEOF(src, connection) && var_name != NAMEOF(src, active_queries) && var_name != NAMEOF(src, connectOperation) && ..()
 
 /datum/controller/subsystem/dbcore/vv_edit_var(var_name, var_value)
-	if(var_name == "_db_con")
+	if(var_name == NAMEOF(src, connection) || var_name == NAMEOF(src, connectOperation))
 		return FALSE
 	return ..()
 
@@ -90,17 +76,30 @@ SUBSYSTEM_DEF(dbcore)
 	var/address = CONFIG_GET(string/address)
 	var/port = CONFIG_GET(number/port)
 
-	_dm_db_connect(_db_con, "dbi:mysql:[db]:[address]:[port]", user, pass, Default_Cursor, null)
+	connection = new /datum/BSQL_Connection(BSQL_CONNECTION_TYPE_MARIADB, CONFIG_GET(number/async_query_timeout), CONFIG_GET(number/blocking_query_timeout))
-	. = IsConnected()
+	var/error
+	if(QDELETED(connection))
+		connection = null
+		error = last_error
+	else
+		SSdbcore.last_error = null
+		connectOperation = connection.BeginConnect(address, port, user, pass, db)
+		if(SSdbcore.last_error)
+			CRASH(SSdbcore.last_error)
+		UNTIL(connectOperation.IsComplete())
+		error = connectOperation.GetError()
+	. = !error
 	if (!.)
-		log_sql("Connect() failed | [ErrorMsg()]")
+		log_sql("Connect() failed | [error]")
 		++failed_connections
+		QDEL_NULL(connection)
+		QDEL_NULL(connectOperation)
 
 /datum/controller/subsystem/dbcore/proc/CheckSchemaVersion()
 	if(CONFIG_GET(flag/sql_enabled))
-		if(SSdbcore.Connect())
+		if(Connect())
 			log_world("Database connection established.")
-			var/datum/DBQuery/query_db_version = SSdbcore.NewQuery("SELECT major, minor FROM [format_table_name("schema_revision")] ORDER BY date DESC LIMIT 1")
+			var/datum/DBQuery/query_db_version = NewQuery("SELECT major, minor FROM [format_table_name("schema_revision")] ORDER BY date DESC LIMIT 1")
 			query_db_version.Execute()
 			if(query_db_version.NextRow())
 				db_major = text2num(query_db_version.item[1])
@@ -146,27 +145,36 @@ SUBSYSTEM_DEF(dbcore)
 
 /datum/controller/subsystem/dbcore/proc/Disconnect()
 	failed_connections = 0
-	return _dm_db_close(_db_con)
+	QDEL_NULL(connectOperation)
+	QDEL_NULL(connection)
 
 /datum/controller/subsystem/dbcore/proc/IsConnected()
 	if(!CONFIG_GET(flag/sql_enabled))
 		return FALSE
-	return _dm_db_is_connected(_db_con)
+	//block until any connect operations finish
+	var/datum/BSQL_Connection/_connection = connection
+	var/datum/BSQL_Operation/op = connectOperation
+	UNTIL(QDELETED(_connection) || op.IsComplete())
+	return !QDELETED(connection) && !op.GetError()
 
 /datum/controller/subsystem/dbcore/proc/Quote(str)
-	return _dm_db_quote(_db_con, str)
+	if(connection)
+		return connection.Quote(str)
 
 /datum/controller/subsystem/dbcore/proc/ErrorMsg()
 	if(!CONFIG_GET(flag/sql_enabled))
 		return "Database disabled by configuration"
-	return _dm_db_error_msg(_db_con)
+	return last_error
 
-/datum/controller/subsystem/dbcore/proc/NewQuery(sql_query, cursor_handler = Default_Cursor)
+/datum/controller/subsystem/dbcore/proc/ReportError(error)
+	last_error = error
+
+/datum/controller/subsystem/dbcore/proc/NewQuery(sql_query)
 	if(IsAdminAdvancedProcCall())
 		log_admin_private("ERROR: Advanced admin proc call led to sql query: [sql_query]. Query has been blocked")
 		message_admins("ERROR: Advanced admin proc call led to sql query. Query has been blocked")
 		return FALSE
-	return new /datum/DBQuery(sql_query, src, cursor_handler)
+	return new /datum/DBQuery(sql_query, connection)
 
 /*
 Takes a list of rows (each row being an associated list of column => value) and inserts them via a single mass query.
@@ -180,7 +188,7 @@ Delayed insert mode was removed in mysql 7 and only works with MyISAM type table
 	It was included because it is still supported in mariadb.
 	It does not work with duplicate_key and the mysql server ignores it in those cases
 */
-/datum/controller/subsystem/dbcore/proc/MassInsert(table, list/rows, duplicate_key = FALSE, ignore_errors = FALSE, delayed = FALSE, warn = FALSE)
+/datum/controller/subsystem/dbcore/proc/MassInsert(table, list/rows, duplicate_key = FALSE, ignore_errors = FALSE, delayed = FALSE, warn = FALSE, async = FALSE)
 	if (!table || !rows || !istype(rows))
 		return
 	var/list/columns = list()
@@ -230,34 +238,30 @@ Delayed insert mode was removed in mysql 7 and only works with MyISAM type table
 	sqlrowlist = "	[sqlrowlist.Join(",\n	")]"
 	var/datum/DBQuery/Query = NewQuery("INSERT[delayed][ignore_errors] INTO [table]\n([columns.Join(", ")])\nVALUES\n[sqlrowlist]\n[duplicate_key]")
 	if (warn)
-		. = Query.warn_execute()
+		. = Query.warn_execute(async)
 	else
-		. = Query.Execute()
+		. = Query.Execute(async)
 	qdel(Query)
 
-
 /datum/DBQuery
 	var/sql // The sql query being executed.
-	var/default_cursor
-	var/list/columns //list of DB Columns populated by Columns()
-	var/list/conversions
 	var/list/item  //list of data values populated by NextRow()
+
 	var/last_activity
 	var/last_activity_time
-	var/datum/controller/subsystem/dbcore/db_connection
-	var/_db_query
 
-/datum/DBQuery/New(sql_query, datum/controller/subsystem/dbcore/connection_handler, cursor_handler)
+	var/last_error
+	var/skip_next_is_complete
+	var/in_progress
+	var/datum/BSQL_Connection/connection
+	var/datum/BSQL_Operation/Query/query
+
+/datum/DBQuery/New(sql_query, datum/BSQL_Connection/connection)
 	SSdbcore.active_queries[src] = TRUE
 	Activity("Created")
-	if(sql_query)
-		sql = sql_query
-	if(connection_handler)
-		db_connection = connection_handler
-	if(cursor_handler)
-		default_cursor = cursor_handler
 	item = list()
-	_db_query = _dm_db_new_query()
+	src.connection = connection
+	sql = sql_query
 
 /datum/DBQuery/Destroy()
 	Close()
@@ -268,147 +272,93 @@ Delayed insert mode was removed in mysql 7 and only works with MyISAM type table
 	//fuck off kevinz
 	return FALSE
 
+/datum/DBQuery/proc/SetQuery(new_sql)
+	if(in_progress)
+		CRASH("Attempted to set new sql while waiting on active query")
+	Close()
+	sql = new_sql
+
 /datum/DBQuery/proc/Activity(activity)
 	last_activity = activity
 	last_activity_time = world.time
 
-/datum/DBQuery/proc/warn_execute()
+/datum/DBQuery/proc/warn_execute(async = FALSE)
-	. = Execute()
+	. = Execute(async)
 	if(!.)
 		to_chat(usr, "<span class='danger'>A SQL error occurred during this operation, check the server logs.</span>")
 
-/datum/DBQuery/proc/SetQuery(new_sql)
+/datum/DBQuery/proc/Execute(async = FALSE, log_error = TRUE)
-	Activity("SetQuery")
-	Close()
-	sql = new_sql
-
-/datum/DBQuery/proc/Execute(sql_query = sql, cursor_handler = default_cursor, log_error = TRUE)
 	Activity("Execute")
+	if(in_progress)
+		CRASH("Attempted to start a new query while waiting on the old one")
+
+	if(QDELETED(connection))
+		last_error = "No connection!"
+		return FALSE
+
 	var/start_time
-	var/timeout = CONFIG_GET(number/query_debug_log_timeout)
+	var/timed_out
-	if(timeout)
+	if(!async)
 		start_time = REALTIMEOFDAY
 	Close()
-	. = _dm_db_execute(_db_query, sql_query, db_connection._db_con, cursor_handler, null)
+	query = connection.BeginQuery(sql)
+	if(!async)
+		timed_out = !query.WaitForCompletion()
+	else
+		in_progress = TRUE
+		UNTIL(query.IsComplete())
+		in_progress = FALSE
+	skip_next_is_complete = TRUE
+	var/error = QDELETED(query) ? "Query object deleted!" : query.GetError()
+	last_error = error
+	. = !error
 	if(!. && log_error)
-		log_sql("[ErrorMsg()] | Query used: [sql]")
+		log_sql("[error] | Query used: [sql]")
-	if(timeout)
+	if(!async && timed_out)
-		if((REALTIMEOFDAY - start_time) > timeout)
+		log_query_debug("Query execution started at [start_time]")
-			log_query_debug("Query execution started at [start_time]")
+		log_query_debug("Query execution ended at [REALTIMEOFDAY]")
-			log_query_debug("Query execution ended at [REALTIMEOFDAY]")
+		log_query_debug("Slow query timeout detected.")
-			log_query_debug("Possible slow query timeout detected.")
+		log_query_debug("Query used: [sql]")
-			log_query_debug("Query used: [sql]")
+		slow_query_check()
-			slow_query_check()
 
 /datum/DBQuery/proc/slow_query_check()
-	message_admins("HEY! A database query may have timed out. Did the server just hang? <a href='?_src_=holder;[HrefToken()];slowquery=yes'>\[YES\]</a>|<a href='?_src_=holder;[HrefToken()];slowquery=no'>\[NO\]</a>")
+	message_admins("HEY! A database query timed out. Did the server just hang? <a href='?_src_=holder;[HrefToken()];slowquery=yes'>\[YES\]</a>|<a href='?_src_=holder;[HrefToken()];slowquery=no'>\[NO\]</a>")
 
-/datum/DBQuery/proc/NextRow()
+/datum/DBQuery/proc/NextRow(async)
 	Activity("NextRow")
-	return _dm_db_next_row(_db_query,item,conversions)
+	UNTIL(!in_progress)
+	if(!skip_next_is_complete)
+		if(!async)
+			query.WaitForCompletion()
+		else
+			in_progress = TRUE
+			UNTIL(query.IsComplete())
+			in_progress = FALSE
+	else
+		skip_next_is_complete = FALSE
 
-/datum/DBQuery/proc/RowsAffected()
+	last_error = query.GetError()
-	return _dm_db_rows_affected(_db_query)
+	var/list/results = query.CurrentRow()
+	. = results != null
 
-/datum/DBQuery/proc/RowCount()
+	item.Cut()
-	return _dm_db_row_count(_db_query)
+	//populate item array
+	for(var/I in results)
+		item += results[I]
 
 /datum/DBQuery/proc/ErrorMsg()
-	return _dm_db_error_msg(_db_query)
+	return last_error
-
-/datum/DBQuery/proc/Columns()
-	if(!columns)
-		columns = _dm_db_columns(_db_query, /datum/DBColumn)
-	return columns
-
-/datum/DBQuery/proc/GetRowData()
-	var/list/columns = Columns()
-	var/list/results
-	if(columns.len)
-		results = list()
-		for(var/C in columns)
-			results+=C
-			var/datum/DBColumn/cur_col = columns[C]
-			results[C] = src.item[(cur_col.position+1)]
-	return results
 
 /datum/DBQuery/proc/Close()
 	item.Cut()
-	columns = null
+	QDEL_NULL(query)
-	conversions = null
-	return _dm_db_close(_db_query)
 
-/datum/DBQuery/proc/Quote(str)
+/world/BSQL_Debug(message)
-	return db_connection.Quote(str)
+	if(!CONFIG_GET(flag/bsql_debug))
+		return
 
-/datum/DBQuery/proc/SetConversion(column,conversion)
+	//strip sensitive stuff
-	if(istext(column))
+	if(findtext(message, ": CreateConnection("))
-		column = columns.Find(column)
+		message = "CreateConnection CENSORED"
-	if(!conversions)
-		conversions = new /list(column)
-	else if(conversions.len < column)
-		conversions.len = column
-	conversions[column] = conversion
 	
-
+	log_sql("BSQL_DEBUG: [message]")
-/datum/DBColumn
-	var/name
-	var/table
-	var/position //1-based index into item data
-	var/sql_type
-	var/flags
-	var/length
-	var/max_length
-	//types
-	var/const/TINYINT = 1
-	var/const/SMALLINT = 2
-	var/const/MEDIUMINT = 3
-	var/const/INTEGER = 4
-	var/const/BIGINT = 5
-	var/const/DECIMAL = 6
-	var/const/FLOAT = 7
-	var/const/DOUBLE = 8
-	var/const/DATE = 9
-	var/const/DATETIME = 10
-	var/const/TIMESTAMP = 11
-	var/const/TIME = 12
-	var/const/STRING = 13
-	var/const/BLOB = 14
-
-/datum/DBColumn/New(name_handler, table_handler, position_handler, type_handler, flag_handler, length_handler, max_length_handler)
-	name = name_handler
-	table = table_handler
-	position = position_handler
-	sql_type = type_handler
-	flags = flag_handler
-	length = length_handler
-	max_length = max_length_handler
-
-/datum/DBColumn/proc/SqlTypeName(type_handler = sql_type)
-	switch(type_handler)
-		if(TINYINT)
-			return "TINYINT"
-		if(SMALLINT)
-			return "SMALLINT"
-		if(MEDIUMINT)
-			return "MEDIUMINT"
-		if(INTEGER)
-			return "INTEGER"
-		if(BIGINT)
-			return "BIGINT"
-		if(FLOAT)
-			return "FLOAT"
-		if(DOUBLE)
-			return "DOUBLE"
-		if(DATE)
-			return "DATE"
-		if(DATETIME)
-			return "DATETIME"
-		if(TIMESTAMP)
-			return "TIMESTAMP"
-		if(TIME)
-			return "TIME"
-		if(STRING)
-			return "STRING"
-		if(BLOB)
-			return "BLOB"

code/modules/bsql/LICENSE

@@ -0,0 +1,7 @@
+Copyright 2018 Jordan Brown
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

code/modules/bsql/core/connection.dm

@@ -0,0 +1,66 @@
+/datum/BSQL_Connection
+	var/id
+	var/connection_type
+
+BSQL_PROTECT_DATUM(/datum/BSQL_Connection)
+
+/datum/BSQL_Connection/New(connection_type, asyncTimeout, blockingTimeout)
+	if(asyncTimeout == null)
+		asyncTimeout = BSQL_DEFAULT_TIMEOUT
+	if(blockingTimeout == null)
+		blockingTimeout = asyncTimeout
+
+	src.connection_type = connection_type
+
+	world._BSQL_InitCheck(src)
+
+	var/error = world._BSQL_Internal_Call("CreateConnection", connection_type, "[asyncTimeout]", "[blockingTimeout]")
+	if(error)
+		BSQL_ERROR(error)
+		return
+
+	id = world._BSQL_Internal_Call("GetConnection")
+	if(!id)
+		BSQL_ERROR("BSQL library failed to provide connect operation for connection id [id]([connection_type])!")
+
+BSQL_DEL_PROC(/datum/BSQL_Connection)
+	var/error
+	if(id)
+		error = world._BSQL_Internal_Call("ReleaseConnection", id)
+	. = ..()
+	if(error)
+		BSQL_ERROR(error)
+
+/datum/BSQL_Connection/BeginConnect(ipaddress, port, username, password, database)
+	var/error = world._BSQL_Internal_Call("OpenConnection", id, ipaddress, "[port]", username, password, database)
+	if(error)
+		BSQL_ERROR(error)
+		return
+
+	var/op_id = world._BSQL_Internal_Call("GetOperation")
+	if(!op_id)
+		BSQL_ERROR("Library failed to provide connect operation for connection id [id]([connection_type])!")
+		return
+
+	return new /datum/BSQL_Operation(src, op_id)
+
+
+/datum/BSQL_Connection/BeginQuery(query)
+	var/error = world._BSQL_Internal_Call("NewQuery", id, query)
+	if(error)
+		BSQL_ERROR(error)
+		return
+
+	var/op_id = world._BSQL_Internal_Call("GetOperation")
+	if(!op_id)
+		BSQL_ERROR("Library failed to provide query operation for connection id [id]([connection_type])!")
+		return
+
+	return new /datum/BSQL_Operation/Query(src, op_id)
+	
+/datum/BSQL_Connection/Quote(str)
+	if(!str)
+		return null;
+	. = world._BSQL_Internal_Call("QuoteString", id, "[str]")
+	if(!.)
+		BSQL_ERROR("Library failed to provide quote for [str]!")

code/modules/bsql/core/library.dm

@@ -0,0 +1,37 @@
+/world/proc/_BSQL_Internal_Call(func, ...)
+	var/list/call_args = args.Copy(2)
+	BSQL_Debug("[.....]: [args[1]]([call_args.Join(", ")])")
+	. = call(_BSQL_Library_Path(), func)(arglist(call_args))
+	BSQL_Debug("Result: [. == null ? "NULL" : "\"[.]\""]")
+
+/world/proc/_BSQL_Library_Path()
+	return system_type == MS_WINDOWS ? "BSQL.dll" : "libBSQL.so"
+
+/world/proc/_BSQL_InitCheck(datum/BSQL_Connection/caller)
+	var/static/library_initialized = FALSE
+	if(_BSQL_Initialized())
+		return
+	var/libPath = _BSQL_Library_Path()
+	if(!fexists(libPath))
+		BSQL_DEL_CALL(caller)
+		BSQL_ERROR("Could not find [libPath]!")
+		return
+
+	var/result = _BSQL_Internal_Call("Initialize")
+	if(result)
+		BSQL_DEL_CALL(caller)
+		BSQL_ERROR(result)
+		return
+	_BSQL_Initialized(TRUE)
+
+/world/proc/_BSQL_Initialized(new_val)
+	var/static/bsql_library_initialized = FALSE
+	if(new_val != null)
+		bsql_library_initialized = new_val
+	return bsql_library_initialized
+
+/world/BSQL_Shutdown()
+	if(!_BSQL_Initialized())
+		return
+	_BSQL_Internal_Call("Shutdown")
+	_BSQL_Initialized(FALSE)

code/modules/bsql/core/operation.dm

@@ -0,0 +1,47 @@
+/datum/BSQL_Operation
+	var/datum/BSQL_Connection/connection
+	var/id
+
+BSQL_PROTECT_DATUM(/datum/BSQL_Operation)
+
+/datum/BSQL_Operation/New(datum/BSQL_Connection/connection, id)
+	src.connection = connection
+	src.id = id
+
+BSQL_DEL_PROC(/datum/BSQL_Operation)
+	var/error
+	if(!BSQL_IS_DELETED(connection))
+		error = world._BSQL_Internal_Call("ReleaseOperation", connection.id, id)
+	. = ..()
+	if(error)
+		BSQL_ERROR(error)
+
+/datum/BSQL_Operation/IsComplete()
+	if(BSQL_IS_DELETED(connection))
+		return TRUE
+	var/result = world._BSQL_Internal_Call("OpComplete", connection.id, id)
+	if(!result)
+		BSQL_ERROR("Error fetching operation [id] for connection [connection.id]!")
+		return
+	return result == "DONE"
+
+/datum/BSQL_Operation/GetError()
+	if(BSQL_IS_DELETED(connection))
+		return "Connection deleted!"
+	return world._BSQL_Internal_Call("GetError", connection.id, id)
+
+/datum/BSQL_Operation/GetErrorCode()
+	if(BSQL_IS_DELETED(connection))
+		return -2
+	return text2num(world._BSQL_Internal_Call("GetErrorCode", connection.id, id))
+
+/datum/BSQL_Operation/WaitForCompletion()
+	if(BSQL_IS_DELETED(connection))
+		return
+	var/error = world._BSQL_Internal_Call("BlockOnOperation", connection.id, id)
+	if(error)
+		if(error == "Operation timed out!")	//match this with the implementation
+			return FALSE
+		BSQL_ERROR("Error waiting for operation [id] for connection [connection.id]! [error]")
+		return
+	return TRUE

code/modules/bsql/core/query.dm

@@ -0,0 +1,35 @@
+/datum/BSQL_Operation/Query
+	var/last_result_json
+	var/list/last_result
+
+BSQL_PROTECT_DATUM(/datum/BSQL_Operation/Query)
+
+/datum/BSQL_Operation/Query/CurrentRow()
+	return last_result
+
+/datum/BSQL_Operation/Query/IsComplete()
+	//whole different ballgame here
+	if(BSQL_IS_DELETED(connection))
+		return TRUE
+	var/result = world._BSQL_Internal_Call("ReadyRow", connection.id, id)
+	switch(result)
+		if("DONE")
+			//load the data
+			LoadQueryResult()
+			return TRUE
+		if("NOTDONE")
+			return FALSE
+		else
+			BSQL_ERROR(result)
+			
+/datum/BSQL_Operation/Query/WaitForCompletion()
+	. = ..()
+	if(.)
+		LoadQueryResult()
+
+/datum/BSQL_Operation/Query/proc/LoadQueryResult()
+	last_result_json = world._BSQL_Internal_Call("GetRow", connection.id, id)
+	if(last_result_json)
+		last_result = json_decode(last_result_json)
+	else
+		last_result = null

code/modules/bsql/includes.dm

@@ -0,0 +1,4 @@
+#include "core\connection.dm"
+#include "core\library.dm"
+#include "core\operation.dm"
+#include "core\query.dm"

config/dbconfig.txt

@@ -29,6 +29,14 @@ FEEDBACK_LOGIN username
 ## Password used to access the database.
 FEEDBACK_PASSWORD password
 
-## Time in deciseconds for a query to execute before alerting a for possible slow query timeout.
+## Time in seconds for asynchronous queries to timeout
-## While enabled queries and their execution times are logged if they exceed this value.
+## Set to 0 for infinite
-#QUERY_DEBUG_LOG_TIMEOUT 70
+ASYNC_QUERY_TIMEOUT 10
+
+## Time in seconds for blocking queries to execute before alerting a for possible slow query timeout 
+## Set to 0 for infinite
+## Must be less than or equal to ASYNC_QUERY_TIMEOUT
+BLOCKING_QUERY_TIMEOUT 5
+
+## Uncomment to enable verbose BSQL communication logs
+#BSQL_DEBUG

tgstation.dme

@@ -17,8 +17,11 @@
 #include "code\_compile_options.dm"
 #include "code\world.dm"
 #include "code\__DEFINES\_globals.dm"
+#include "code\__DEFINES\_protect.dm"
 #include "code\__DEFINES\_tick.dm"
 #include "code\__DEFINES\access.dm"
+#include "code\__DEFINES\bsql.config.dm"
+#include "code\__DEFINES\bsql.dm"
 #include "code\__DEFINES\admin.dm"
 #include "code\__DEFINES\antagonists.dm"
 #include "code\__DEFINES\atmospherics.dm"
@@ -1346,6 +1349,7 @@
 #include "code\modules\awaymissions\mission_code\stationCollision.dm"
 #include "code\modules\awaymissions\mission_code\undergroundoutpost45.dm"
 #include "code\modules\awaymissions\mission_code\wildwest.dm"
+#include "code\modules\bsql\includes.dm"
 #include "code\modules\cargo\bounty.dm"
 #include "code\modules\cargo\bounty_console.dm"
 #include "code\modules\cargo\console.dm"

tools/travis/build_byond.sh

@@ -53,20 +53,6 @@ if [ "$BUILD_TOOLS" = false ]; then
 		#test config
 		cp tools/travis/travis_config.txt config/config.txt
 	
-		# get libmariadb, cache it so limmex doesn't get angery
-		if [ -f $HOME/libmariadb ]; then
-			#travis likes to interpret the cache command as it being a file for some reason
-			rm $HOME/libmariadb
-			mkdir $HOME/libmariadb
-		fi
-		if [ ! -f $HOME/libmariadb/libmariadb.so ]; then
-			wget http://www.byond.com/download/db/mariadb_client-2.0.0-linux.tgz
-			tar -xvf mariadb_client-2.0.0-linux.tgz
-			mv mariadb_client-2.0.0-linux/libmariadb.so $HOME/libmariadb/libmariadb.so
-			rm -rf mariadb_client-2.0.0-linux.tgz mariadb_client-2.0.0-linux
-		fi
-    	ln -s $HOME/libmariadb/libmariadb.so libmariadb.so
-	
 		DreamDaemon yogstation.dmb -close -trusted -verbose -params "test-run&log-directory=travis"
 		cat data/logs/travis/clean_run.lk
 	fi;

tools/travis/build_dependencies.sh

@@ -16,4 +16,39 @@ if [ $BUILD_TOOLS = false ] && [ $BUILD_TESTING = false ]; then
 
     mkdir -p ~/.byond/bin
     ln -s $PWD/target/release/librust_g.so ~/.byond/bin/rust_g
+
+    mkdir -p ../BSQL/artifacts
+    cd ../BSQL
+    git init
+    git remote add origin https://github.com/tgstation/BSQL
+    git fetch --depth 1 origin $BSQL_VERSION
+    git checkout FETCH_HEAD
+
+    if [ -f "$HOME/MariaDB/libmariadb.so.2" ] && [ -f "$HOME/MariaDB/libmariadb.so" ] && [ -d "$HOME/MariaDB/include" ];
+    then
+        echo "Using cached MariaDB library."
+    else
+        echo "Setting up MariaDB."
+        rm -rf "$HOME/MariaDB"
+        mkdir -p "$HOME/MariaDB"
+        wget http://mirrors.kernel.org/ubuntu/pool/universe/m/mariadb-client-lgpl/libmariadb2_2.0.0-1_i386.deb
+        dpkg -x libmariadb2_2.0.0-1_i386.deb /tmp/extract
+        rm libmariadb2_2.0.0-1_i386.deb
+        mv /tmp/extract/usr/lib/i386-linux-gnu/libmariadb.so.2 $HOME/MariaDB/
+        ln -s $HOME/MariaDB/libmariadb.so.2 $HOME/MariaDB/libmariadb.so
+        rm -rf /tmp/extract
+
+        wget http://mirrors.kernel.org/ubuntu/pool/universe/m/mariadb-connector-c/libmariadb-dev_2.3.3-1_i386.deb
+        dpkg -x libmariadb-dev_2.3.3-1_i386.deb /tmp/extract
+        rm libmariadb-dev_2.3.3-1_i386.deb
+        mv /tmp/extract/usr/include $HOME/MariaDB/
+        #fuck what is this even?
+        mv $HOME/MariaDB/include/mariadb $HOME/MariaDB/include/mysql
+    fi
+
+    cd artifacts
+    export CXX=g++-7
+    cmake .. -DMARIA_INCLUDE_DIR=$HOME/MariaDB/include
+    make
+    mv src/BSQL/libBSQL.so ../../
 fi