From e0af0df4350d6cc20217b1edecfe0cf68be1ece3 Mon Sep 17 00:00:00 2001 From: CitadelStationBot Date: Sat, 20 May 2017 23:43:16 -0500 Subject: [PATCH] [MIRROR] Adds Mass Insert proc to sql (#986) * Adds Mass Insert proc to sql * Update blackbox.dm * Update dbcore.dm * Delete dbcore.dm.rej --- code/controllers/subsystem/blackbox.dm | 488 ++++++++++++------------- code/controllers/subsystem/dbcore.dm | 76 +++- 2 files changed, 316 insertions(+), 248 deletions(-) diff --git a/code/controllers/subsystem/blackbox.dm b/code/controllers/subsystem/blackbox.dm index a807202c9e..5b2c67c296 100644 --- a/code/controllers/subsystem/blackbox.dm +++ b/code/controllers/subsystem/blackbox.dm @@ -1,246 +1,242 @@ -SUBSYSTEM_DEF(blackbox) - name = "Blackbox" - wait = 6000 - flags = SS_NO_TICK_CHECK - - var/list/msg_common = list() - var/list/msg_science = list() - var/list/msg_command = list() - var/list/msg_medical = list() - var/list/msg_engineering = list() - var/list/msg_security = list() - var/list/msg_deathsquad = list() - var/list/msg_syndicate = list() - var/list/msg_service = list() - var/list/msg_cargo = list() - var/list/msg_other = list() - - var/list/feedback = list() //list of datum/feedback_variable - -//poll population -/datum/controller/subsystem/blackbox/fire() - if(!SSdbcore.Connect()) - return - var/playercount = 0 - for(var/mob/M in GLOB.player_list) - if(M.client) - playercount += 1 - var/admincount = GLOB.admins.len - var/datum/DBQuery/query_record_playercount = SSdbcore.NewQuery("INSERT INTO [format_table_name("legacy_population")] (playercount, admincount, time, server_ip, server_port) VALUES ([playercount], [admincount], '[SQLtime()]', INET_ATON('[world.internet_address]'), '[world.port]')") - query_record_playercount.Execute() - -/datum/controller/subsystem/blackbox/Recover() - msg_common = SSblackbox.msg_common - msg_science = SSblackbox.msg_science - msg_command = SSblackbox.msg_command - msg_medical = SSblackbox.msg_medical - msg_engineering = SSblackbox.msg_engineering - msg_security = SSblackbox.msg_security - msg_deathsquad = SSblackbox.msg_deathsquad - msg_syndicate = SSblackbox.msg_syndicate - msg_service = SSblackbox.msg_service - msg_cargo = SSblackbox.msg_cargo - msg_other = SSblackbox.msg_other - - - feedback = SSblackbox.feedback - -//no touchie -/datum/controller/subsystem/blackbox/can_vv_get(var_name) - if(var_name == "feedback") - return FALSE - return ..() - -/datum/controller/subsystem/blackbox/vv_edit_var(var_name, var_value) - return FALSE - -/datum/controller/subsystem/blackbox/Shutdown() - var/pda_msg_amt = 0 - var/rc_msg_amt = 0 - - for (var/obj/machinery/message_server/MS in GLOB.message_servers) - if (MS.pda_msgs.len > pda_msg_amt) - pda_msg_amt = MS.pda_msgs.len - if (MS.rc_msgs.len > rc_msg_amt) - rc_msg_amt = MS.rc_msgs.len - - set_details("radio_usage","") - - add_details("radio_usage","COM-[msg_common.len]") - add_details("radio_usage","SCI-[msg_science.len]") - add_details("radio_usage","HEA-[msg_command.len]") - add_details("radio_usage","MED-[msg_medical.len]") - add_details("radio_usage","ENG-[msg_engineering.len]") - add_details("radio_usage","SEC-[msg_security.len]") - add_details("radio_usage","DTH-[msg_deathsquad.len]") - add_details("radio_usage","SYN-[msg_syndicate.len]") - add_details("radio_usage","SRV-[msg_service.len]") - add_details("radio_usage","CAR-[msg_cargo.len]") - add_details("radio_usage","OTH-[msg_other.len]") - add_details("radio_usage","PDA-[pda_msg_amt]") - add_details("radio_usage","RC-[rc_msg_amt]") - - set_details("round_end","[time2text(world.realtime)]") //This one MUST be the last one that gets set. - - if (!SSdbcore.Connect()) - return - - var/sqlrowlist = "" - - for (var/datum/feedback_variable/FV in feedback) - if (sqlrowlist != "") - sqlrowlist += ", " //a comma (,) at the start of the first row to insert will trigger a SQL error - - sqlrowlist += "(null, Now(), [GLOB.round_id], \"[sanitizeSQL(FV.get_variable())]\", [FV.get_value()], \"[sanitizeSQL(FV.get_details())]\")" - - if (sqlrowlist == "") - return - - var/datum/DBQuery/query_feedback_save = SSdbcore.NewQuery("INSERT DELAYED IGNORE INTO [format_table_name("feedback")] VALUES " + sqlrowlist) - query_feedback_save.Execute() - -/datum/controller/subsystem/blackbox/proc/LogBroadcast(blackbox_msg, freq) - switch(freq) - if(1459) - msg_common += blackbox_msg - if(1351) - msg_science += blackbox_msg - if(1353) - msg_command += blackbox_msg - if(1355) - msg_medical += blackbox_msg - if(1357) - msg_engineering += blackbox_msg - if(1359) - msg_security += blackbox_msg - if(1441) - msg_deathsquad += blackbox_msg - if(1213) - msg_syndicate += blackbox_msg - if(1349) - msg_service += blackbox_msg - if(1347) - msg_cargo += blackbox_msg - else - msg_other += blackbox_msg - -/datum/controller/subsystem/blackbox/proc/find_feedback_datum(variable) - for(var/datum/feedback_variable/FV in feedback) - if(FV.get_variable() == variable) - return FV - - var/datum/feedback_variable/FV = new(variable) - feedback += FV - return FV - -/datum/controller/subsystem/blackbox/proc/set_val(variable, value) - var/datum/feedback_variable/FV = find_feedback_datum(variable) - FV.set_value(value) - -/datum/controller/subsystem/blackbox/proc/inc(variable, value) - var/datum/feedback_variable/FV = find_feedback_datum(variable) - FV.inc(value) - -/datum/controller/subsystem/blackbox/proc/dec(variable,value) - var/datum/feedback_variable/FV = find_feedback_datum(variable) - FV.dec(value) - -/datum/controller/subsystem/blackbox/proc/set_details(variable,details) - var/datum/feedback_variable/FV = find_feedback_datum(variable) - FV.set_details(details) - -/datum/controller/subsystem/blackbox/proc/add_details(variable,details) - var/datum/feedback_variable/FV = find_feedback_datum(variable) - FV.add_details(details) - -/datum/controller/subsystem/blackbox/proc/ReportDeath(mob/living/L) - if(!SSdbcore.Connect()) - return - if(!L || !L.key || !L.mind) - return - var/turf/T = get_turf(L) - var/area/placeofdeath = get_area(T.loc) - var/sqlname = sanitizeSQL(L.real_name) - var/sqlkey = sanitizeSQL(L.ckey) - var/sqljob = sanitizeSQL(L.mind.assigned_role) - var/sqlspecial = sanitizeSQL(L.mind.special_role) - var/sqlpod = sanitizeSQL(placeofdeath.name) - var/laname - var/lakey - if(L.lastattacker && ismob(L.lastattacker)) - var/mob/LA = L.lastattacker - laname = sanitizeSQL(LA.real_name) - lakey = sanitizeSQL(LA.key) - var/sqlgender = sanitizeSQL(L.gender) - var/sqlbrute = sanitizeSQL(L.getBruteLoss()) - var/sqlfire = sanitizeSQL(L.getFireLoss()) - var/sqlbrain = sanitizeSQL(L.getBrainLoss()) - var/sqloxy = sanitizeSQL(L.getOxyLoss()) - var/sqltox = sanitizeSQL(L.getStaminaLoss()) - var/sqlclone = sanitizeSQL(L.getStaminaLoss()) - var/sqlstamina = sanitizeSQL(L.getStaminaLoss()) - var/coord = sanitizeSQL("[L.x], [L.y], [L.z]") - var/map = sanitizeSQL(SSmapping.config.map_name) - var/datum/DBQuery/query_report_death = SSdbcore.NewQuery("INSERT INTO [format_table_name("death")] (name, byondkey, job, special, pod, tod, laname, lakey, gender, bruteloss, fireloss, brainloss, oxyloss, toxloss, cloneloss, staminaloss, coord, mapname, server_ip, server_port) VALUES ('[sqlname]', '[sqlkey]', '[sqljob]', '[sqlspecial]', '[sqlpod]', '[SQLtime()]', '[laname]', '[lakey]', '[sqlgender]', [sqlbrute], [sqlfire], [sqlbrain], [sqloxy], [sqltox], [sqlclone], [sqlstamina], '[coord]', '[map]', INET_ATON('[world.internet_address]'), '[world.port]')") - query_report_death.Execute() - - -//feedback variable datum, for storing all kinds of data -/datum/feedback_variable - var/variable - var/value - var/details - -/datum/feedback_variable/New(param_variable, param_value = 0) - variable = param_variable - value = param_value - -/datum/feedback_variable/proc/inc(num = 1) - if (isnum(value)) - value += num - else - value = text2num(value) - if (isnum(value)) - value += num - else - value = num - -/datum/feedback_variable/proc/dec(num = 1) - if (isnum(value)) - value -= num - else - value = text2num(value) - if (isnum(value)) - value -= num - else - value = -num - -/datum/feedback_variable/proc/set_value(num) - if (isnum(num)) - value = num - -/datum/feedback_variable/proc/get_value() - if (!isnum(value)) - return 0 - return value - -/datum/feedback_variable/proc/get_variable() - return variable - -/datum/feedback_variable/proc/set_details(text) - if (istext(text)) - details = text - -/datum/feedback_variable/proc/add_details(text) - if (istext(text)) - text = replacetext(text, " ", "_") - if (!details) - details = text - else - details += " [text]" - -/datum/feedback_variable/proc/get_details() - return details - -/datum/feedback_variable/proc/get_parsed() - return list(variable,value,details) +SUBSYSTEM_DEF(blackbox) + name = "Blackbox" + wait = 6000 + flags = SS_NO_TICK_CHECK + + var/list/msg_common = list() + var/list/msg_science = list() + var/list/msg_command = list() + var/list/msg_medical = list() + var/list/msg_engineering = list() + var/list/msg_security = list() + var/list/msg_deathsquad = list() + var/list/msg_syndicate = list() + var/list/msg_service = list() + var/list/msg_cargo = list() + var/list/msg_other = list() + + var/list/feedback = list() //list of datum/feedback_variable + +//poll population +/datum/controller/subsystem/blackbox/fire() + if(!SSdbcore.Connect()) + return + var/playercount = 0 + for(var/mob/M in GLOB.player_list) + if(M.client) + playercount += 1 + var/admincount = GLOB.admins.len + var/datum/DBQuery/query_record_playercount = SSdbcore.NewQuery("INSERT INTO [format_table_name("legacy_population")] (playercount, admincount, time, server_ip, server_port) VALUES ([playercount], [admincount], '[SQLtime()]', INET_ATON('[world.internet_address]'), '[world.port]')") + query_record_playercount.Execute() + +/datum/controller/subsystem/blackbox/Recover() + msg_common = SSblackbox.msg_common + msg_science = SSblackbox.msg_science + msg_command = SSblackbox.msg_command + msg_medical = SSblackbox.msg_medical + msg_engineering = SSblackbox.msg_engineering + msg_security = SSblackbox.msg_security + msg_deathsquad = SSblackbox.msg_deathsquad + msg_syndicate = SSblackbox.msg_syndicate + msg_service = SSblackbox.msg_service + msg_cargo = SSblackbox.msg_cargo + msg_other = SSblackbox.msg_other + + feedback = SSblackbox.feedback + +//no touchie +/datum/controller/subsystem/blackbox/can_vv_get(var_name) + if(var_name == "feedback") + return FALSE + return ..() + +/datum/controller/subsystem/blackbox/vv_edit_var(var_name, var_value) + return FALSE + +/datum/controller/subsystem/blackbox/Shutdown() + var/pda_msg_amt = 0 + var/rc_msg_amt = 0 + + for (var/obj/machinery/message_server/MS in GLOB.message_servers) + if (MS.pda_msgs.len > pda_msg_amt) + pda_msg_amt = MS.pda_msgs.len + if (MS.rc_msgs.len > rc_msg_amt) + rc_msg_amt = MS.rc_msgs.len + + set_details("radio_usage","") + + add_details("radio_usage","COM-[msg_common.len]") + add_details("radio_usage","SCI-[msg_science.len]") + add_details("radio_usage","HEA-[msg_command.len]") + add_details("radio_usage","MED-[msg_medical.len]") + add_details("radio_usage","ENG-[msg_engineering.len]") + add_details("radio_usage","SEC-[msg_security.len]") + add_details("radio_usage","DTH-[msg_deathsquad.len]") + add_details("radio_usage","SYN-[msg_syndicate.len]") + add_details("radio_usage","SRV-[msg_service.len]") + add_details("radio_usage","CAR-[msg_cargo.len]") + add_details("radio_usage","OTH-[msg_other.len]") + add_details("radio_usage","PDA-[pda_msg_amt]") + add_details("radio_usage","RC-[rc_msg_amt]") + + set_details("round_end","[time2text(world.realtime)]") //This one MUST be the last one that gets set. + + if (!SSdbcore.Connect()) + return + + var/list/sqlrowlist = list() + + for (var/datum/feedback_variable/FV in feedback) + sqlrowlist += list("time" = "Now()", "round_id" = GLOB.round_id, "var_name" = "'[sanitizeSQL(FV.get_variable())]'", "var_value" = FV.get_value(), "details" = "'[sanitizeSQL(FV.get_details())]'") + + if (!length(sqlrowlist)) + return + + SSdbcore.MassInsert(format_table_name("feedback"), sqlrowlist, ignore_errors = TRUE, delayed = TRUE) + + +/datum/controller/subsystem/blackbox/proc/LogBroadcast(blackbox_msg, freq) + switch(freq) + if(1459) + msg_common += blackbox_msg + if(1351) + msg_science += blackbox_msg + if(1353) + msg_command += blackbox_msg + if(1355) + msg_medical += blackbox_msg + if(1357) + msg_engineering += blackbox_msg + if(1359) + msg_security += blackbox_msg + if(1441) + msg_deathsquad += blackbox_msg + if(1213) + msg_syndicate += blackbox_msg + if(1349) + msg_service += blackbox_msg + if(1347) + msg_cargo += blackbox_msg + else + msg_other += blackbox_msg + +/datum/controller/subsystem/blackbox/proc/find_feedback_datum(variable) + for(var/datum/feedback_variable/FV in feedback) + if(FV.get_variable() == variable) + return FV + + var/datum/feedback_variable/FV = new(variable) + feedback += FV + return FV + +/datum/controller/subsystem/blackbox/proc/set_val(variable, value) + var/datum/feedback_variable/FV = find_feedback_datum(variable) + FV.set_value(value) + +/datum/controller/subsystem/blackbox/proc/inc(variable, value) + var/datum/feedback_variable/FV = find_feedback_datum(variable) + FV.inc(value) + +/datum/controller/subsystem/blackbox/proc/dec(variable,value) + var/datum/feedback_variable/FV = find_feedback_datum(variable) + FV.dec(value) + +/datum/controller/subsystem/blackbox/proc/set_details(variable,details) + var/datum/feedback_variable/FV = find_feedback_datum(variable) + FV.set_details(details) + +/datum/controller/subsystem/blackbox/proc/add_details(variable,details) + var/datum/feedback_variable/FV = find_feedback_datum(variable) + FV.add_details(details) + +/datum/controller/subsystem/blackbox/proc/ReportDeath(mob/living/L) + if(!SSdbcore.Connect()) + return + if(!L || !L.key || !L.mind) + return + var/turf/T = get_turf(L) + var/area/placeofdeath = get_area(T.loc) + var/sqlname = sanitizeSQL(L.real_name) + var/sqlkey = sanitizeSQL(L.ckey) + var/sqljob = sanitizeSQL(L.mind.assigned_role) + var/sqlspecial = sanitizeSQL(L.mind.special_role) + var/sqlpod = sanitizeSQL(placeofdeath.name) + var/laname + var/lakey + if(L.lastattacker && ismob(L.lastattacker)) + var/mob/LA = L.lastattacker + laname = sanitizeSQL(LA.real_name) + lakey = sanitizeSQL(LA.key) + var/sqlgender = sanitizeSQL(L.gender) + var/sqlbrute = sanitizeSQL(L.getBruteLoss()) + var/sqlfire = sanitizeSQL(L.getFireLoss()) + var/sqlbrain = sanitizeSQL(L.getBrainLoss()) + var/sqloxy = sanitizeSQL(L.getOxyLoss()) + var/sqltox = sanitizeSQL(L.getStaminaLoss()) + var/sqlclone = sanitizeSQL(L.getStaminaLoss()) + var/sqlstamina = sanitizeSQL(L.getStaminaLoss()) + var/coord = sanitizeSQL("[L.x], [L.y], [L.z]") + var/map = sanitizeSQL(SSmapping.config.map_name) + var/datum/DBQuery/query_report_death = SSdbcore.NewQuery("INSERT INTO [format_table_name("death")] (name, byondkey, job, special, pod, tod, laname, lakey, gender, bruteloss, fireloss, brainloss, oxyloss, toxloss, cloneloss, staminaloss, coord, mapname, server_ip, server_port) VALUES ('[sqlname]', '[sqlkey]', '[sqljob]', '[sqlspecial]', '[sqlpod]', '[SQLtime()]', '[laname]', '[lakey]', '[sqlgender]', [sqlbrute], [sqlfire], [sqlbrain], [sqloxy], [sqltox], [sqlclone], [sqlstamina], '[coord]', '[map]', INET_ATON('[world.internet_address]'), '[world.port]')") + query_report_death.Execute() + + +//feedback variable datum, for storing all kinds of data +/datum/feedback_variable + var/variable + var/value + var/details + +/datum/feedback_variable/New(param_variable, param_value = 0) + variable = param_variable + value = param_value + +/datum/feedback_variable/proc/inc(num = 1) + if (isnum(value)) + value += num + else + value = text2num(value) + if (isnum(value)) + value += num + else + value = num + +/datum/feedback_variable/proc/dec(num = 1) + if (isnum(value)) + value -= num + else + value = text2num(value) + if (isnum(value)) + value -= num + else + value = -num + +/datum/feedback_variable/proc/set_value(num) + if (isnum(num)) + value = num + +/datum/feedback_variable/proc/get_value() + if (!isnum(value)) + return 0 + return value + +/datum/feedback_variable/proc/get_variable() + return variable + +/datum/feedback_variable/proc/set_details(text) + if (istext(text)) + details = text + +/datum/feedback_variable/proc/add_details(text) + if (istext(text)) + text = replacetext(text, " ", "_") + if (!details) + details = text + else + details += " [text]" + +/datum/feedback_variable/proc/get_details() + return details + +/datum/feedback_variable/proc/get_parsed() + return list(variable,value,details) diff --git a/code/controllers/subsystem/dbcore.dm b/code/controllers/subsystem/dbcore.dm index 20c6f993e5..9c812192d8 100644 --- a/code/controllers/subsystem/dbcore.dm +++ b/code/controllers/subsystem/dbcore.dm @@ -77,15 +77,87 @@ SUBSYSTEM_DEF(dbcore) return FALSE return _dm_db_is_connected(_db_con) -/datum/controller/subsystem/dbcore/proc/Quote(str) +/datum/controller/subsystem/dbcore/proc/Quote(str) return _dm_db_quote(_db_con, str) -/datum/controller/subsystem/dbcore/proc/ErrorMsg() +/datum/controller/subsystem/dbcore/proc/ErrorMsg() + if(!config.sql_enabled) + return "Database disabled by configuration" return _dm_db_error_msg(_db_con) /datum/controller/subsystem/dbcore/proc/NewQuery(sql_query, cursor_handler = Default_Cursor) + if(IsAdminAdvancedProcCall()) + log_admin_private("ERROR: Advanced admin proc call led to sql query: [sql_query]. Query has been blocked") + message_admins("ERROR: Advanced admin proc call led to sql query. Query has been blocked") + return FALSE return new /datum/DBQuery(sql_query, src, cursor_handler) +/* +Takes a list of rows (each row being an associated list of column => value) and inserts them via a single mass query. +Rows missing columns present in other rows will resolve to SQL NULL +You are expected to do your own escaping of the data, and expected to provide your own quotes for strings. +The duplicate_key arg can be true to automatically generate this part of the query + or set to a string that is appended to the end of the query +Ignore_errors instructes mysql to continue inserting rows if some of them have errors. + the erroneous row(s) aren't inserted and there isn't really any way to know why or why errored +Delayed insert mode was removed in mysql 7 and only works with MyISAM type tables, + It was included because it is still supported in mariadb. + It does not work with duplicate_key and the mysql server ignores it in those cases +*/ +/datum/controller/subsystem/dbcore/proc/MassInsert(table, list/rows, duplicate_key = FALSE, ignore_errors = FALSE, delayed = FALSE, warn = FALSE) + if (!table || !rows || !istype(rows)) + return + var/list/columns = list() + var/list/sorted_rows = list() + + for (var/list/row in rows) + var/list/sorted_row = list() + sorted_row.len = columns.len + for (var/column in row) + var/idx = columns[column] + if (!idx) + idx = columns.len + 1 + columns[column] = idx + sorted_row.len = columns.len + + sorted_row[idx] = row[column] + sorted_rows[++sorted_rows.len] = sorted_row + + if (duplicate_key == TRUE) + var/list/column_list = list() + for (var/column in columns) + column_list += "[column] = VALUES([column])" + duplicate_key = "ON DUPLICATE KEY UPDATE [column_list.Join(", ")]\n" + else if (duplicate_key == FALSE) + duplicate_key = null + + if (ignore_errors) + ignore_errors = " IGNORE" + else + ignore_errors = null + + if (delayed) + delayed = " DELAYED" + else + delayed = null + + var/list/sqlrowlist = list() + var/len = columns.len + for (var/list/row in sorted_rows) + if (length(row) != len) + row.len = len + for (var/value in row) + if (value == null) + value = "NULL" + sqlrowlist += "([row.Join(", ")])" + + sqlrowlist = " [sqlrowlist.Join(",\n ")]" + var/datum/DBQuery/Query = NewQuery("INSERT[ignore_errors][delayed] INTO [table]\n([columns.Join(", ")])\nVALUES\n[sqlrowlist]\n[duplicate_key]") + if (warn) + return Query.warn_execute() + else + return Query.Execute() + /datum/DBQuery var/sql // The sql query being executed.