From d91555b79e4e04928a2e644fe3262b907028ff83 Mon Sep 17 00:00:00 2001 From: Tom <8881105+tf-4@users.noreply.github.com> Date: Sat, 3 Jun 2023 22:15:41 +0100 Subject: [PATCH] ezdb - A one click script to quickly setting up a development database (#75053) (#21458) * ezdb - A one click script to quickly setting up a development database (#75053) https://user-images.githubusercontent.com/35135081/235344815-8e825ba9-52cf-44e8-b8e2-a2aeb5d47276.mp4 - Downloads a portable MariaDB (doesn't pollute your main system) - Sets up a database with a random password on port 1338 (configurable) - Installs the initial schema - Every time after, will run updates Major versions right now explicitly escape hatch, because those historically come with something like a Python script, and I do not want it to pretend to work. --------- Co-authored-by: san7890 * touchups * oh well --------- Co-authored-by: Mothblocks <35135081+Mothblocks@users.noreply.github.com> Co-authored-by: san7890 Co-authored-by: Useroth <37159550+Useroth@users.noreply.github.com> --- .github/CODEOWNERS | 1 - .github/CONTRIBUTING.md | 3 +- .github/guides/EZDB.md | 12 +++ .gitignore | 4 + ...se_changelog.txt => database_changelog.md} | 95 ++++++++++++------- .../configuration/configuration.dm | 2 + .../configuration/entries/dbconfig.dm | 5 + code/controllers/subsystem/dbcore.dm | 48 +++++++++- tools/ezdb/__main__.py | 15 +++ tools/ezdb/ezdb.bat | 2 + tools/ezdb/ezdb/__init__.py | 0 tools/ezdb/ezdb/changes.py | 31 ++++++ tools/ezdb/ezdb/config.py | 22 +++++ tools/ezdb/ezdb/mysql.py | 68 +++++++++++++ tools/ezdb/ezdb/paths.py | 34 +++++++ tools/ezdb/steps/__init__.py | 11 +++ tools/ezdb/steps/download_mariadb.py | 50 ++++++++++ tools/ezdb/steps/install_database.py | 46 +++++++++ tools/ezdb/steps/install_initial_schema.py | 53 +++++++++++ tools/ezdb/steps/step.py | 10 ++ tools/ezdb/steps/update_schema.py | 38 ++++++++ tools/requirements.txt | 3 + 22 files changed, 517 insertions(+), 36 deletions(-) create mode 100644 .github/guides/EZDB.md rename SQL/{database_changelog.txt => database_changelog.md} (98%) create mode 100644 tools/ezdb/__main__.py create mode 100644 tools/ezdb/ezdb.bat create mode 100644 tools/ezdb/ezdb/__init__.py create mode 100644 tools/ezdb/ezdb/changes.py create mode 100644 tools/ezdb/ezdb/config.py create mode 100644 tools/ezdb/ezdb/mysql.py create mode 100644 tools/ezdb/ezdb/paths.py create mode 100644 tools/ezdb/steps/__init__.py create mode 100644 tools/ezdb/steps/download_mariadb.py create mode 100644 tools/ezdb/steps/install_database.py create mode 100644 tools/ezdb/steps/install_initial_schema.py create mode 100644 tools/ezdb/steps/step.py create mode 100644 tools/ezdb/steps/update_schema.py diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 627a35366d8..df36cf6bfbe 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -52,4 +52,3 @@ # Maptainers /_maps/ @Jolly-66 @KathrinBailey - diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index b2eec459d43..e25f8f54745 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -110,8 +110,9 @@ Things you **CAN'T** do: - [Hard Deletes](./guides/HARDDELETES.md) - [MC Tab Guide](./guides/MC_tab.md) - [Policy Configuration System](./guides/POLICYCONFIG.md) -- [Splitting up pull requests, aka atomization](./guides/ATOMIZATION.md) +- [Quickly setting up a development database with ezdb](./guides/EZDB.md) - [Required Tests (Continuous Integration)](./guides/CI.md) +- [Splitting up pull requests, aka atomization](./guides/ATOMIZATION.md) - [UI Development](../tgui/README.md) - [Visual Effects and Systems](./guides/VISUALS.md) diff --git a/.github/guides/EZDB.md b/.github/guides/EZDB.md new file mode 100644 index 00000000000..428a819ab4d --- /dev/null +++ b/.github/guides/EZDB.md @@ -0,0 +1,12 @@ +# Quickly setting up a development database with ezdb +While you do not need a database to code for tgstation, it is a prerequisite to many important features, especially on the admin side. Thus, if you are working in any code that benefits from it, it can be helpful to have one handy. + +**ezdb** is a tool for quickly setting up an isolated development database. It will manage downloading MariaDB, creating the database, setting it up, and updating it when the code evolves. It is not recommended for use in production servers, but is perfect for quick development. + +To run ezdb, go to `tools/ezdb`, and double-click on ezdb.bat. This will set up the database on port 1338, but you can configure this with `--port`. When it is done, you should be able to launch tgstation as normal and have database access. This runs on the same Python bootstrapper as things like the map merge tool, which can sometimes be flaky. + +If you wish to delete the ezdb database, delete the `db` folder as well as `config/ezdb.txt`. + +To update ezdb, run the script again. This will both look for any updates in the database changelog, as well as update your schema revision. + +Contact Mothblocks if you face any issues in this process. diff --git a/.gitignore b/.gitignore index bcc636cb08e..1c74dfcdf37 100644 --- a/.gitignore +++ b/.gitignore @@ -239,3 +239,7 @@ Tracy.exe # From /tools/define_sanity/check.py - potential output file that we load onto the user's machine that we don't want to have committed. define_sanity_output.txt + +# ezdb +/db/ +/config/ezdb.txt diff --git a/SQL/database_changelog.txt b/SQL/database_changelog.md similarity index 98% rename from SQL/database_changelog.txt rename to SQL/database_changelog.md index aed8e2f4497..7826846980f 100644 --- a/SQL/database_changelog.txt +++ b/SQL/database_changelog.md @@ -2,19 +2,21 @@ Any time you make a change to the schema files, remember to increment the databa Make sure to also update `DB_MAJOR_VERSION` and `DB_MINOR_VERSION`, which can be found in `code/__DEFINES/subsystem.dm`. -The latest database version is 5.25 (5.23 for /tg/); The query to update the schema revision table is: +The latest database version is 5.26 (5.24 for /tg/); The query to update the schema revision table is: -INSERT INTO `schema_revision` (`major`, `minor`) VALUES (5, 25); +```sql +INSERT INTO `schema_revision` (`major`, `minor`) VALUES (5, 26); +``` or -INSERT INTO `SS13_schema_revision` (`major`, `minor`) VALUES (5, 25); + +```sql +INSERT INTO `SS13_schema_revision` (`major`, `minor`) VALUES (5, 26); +``` In any query remember to add a prefix to the table names if you use one. ----------------------------------------------------- -<<<<<<< HEAD:SQL/database_changelog.txt -Version 5.25, 28 December 2022, by Mothblocks -======= -Version 5.24, 17 May 2023, by LemonInTheDark +Version 5.26, 17 May 2023, by LemonInTheDark Modified the library action table to fit ckeys properly, and to properly store ips. ```sql ALTER TABLE `library_action` MODIFY COLUMN `ckey` varchar(32) NOT NULL; @@ -22,11 +24,10 @@ Modified the library action table to fit ckeys properly, and to properly store i ``` ----------------------------------------------------- -Version 5.23, 28 December 2022, by Mothblocks ->>>>>>> 0065f020ed2 (Removes line about adding a new index to the library_action table (#75808)):SQL/database_changelog.md +Version 5.25, 28 December 2022, by Mothblocks Added `tutorial_completions` to mark what ckeys have completed contextual tutorials. -``` +```sql CREATE TABLE `tutorial_completions` ( `id` INT NOT NULL AUTO_INCREMENT, `ckey` VARCHAR(32) NOT NULL, @@ -39,15 +40,15 @@ CREATE TABLE `tutorial_completions` ( Version 5.24, 22 December 2021, by Mothblocks Fixes a bug in `telemetry_connections` that limited the range of IPs. -``` +```sql ALTER TABLE `telemetry_connections` MODIFY COLUMN `address` INT(10) UNSIGNED NOT NULL; ``` - ----------------------------------------------------- + Version 5.23, 15 December 2021, by Mothblocks Adds `telemetry_connections` table for tracking tgui telemetry. -``` +```sql CREATE TABLE `telemetry_connections` ( `id` INT NOT NULL AUTO_INCREMENT, `ckey` VARCHAR(32) NOT NULL, @@ -65,17 +66,16 @@ CREATE TABLE `telemetry_connections` ( Version 5.22, 11 November 2021, by Mothblocks Adds `admin_ckey` field to the `known_alts` table to track who added what. -``` +```sql ALTER TABLE `known_alts` ADD COLUMN `admin_ckey` VARCHAR(32) NOT NULL DEFAULT '*no key*' AFTER `ckey2`; ``` ----------------------------------------------------- - Version 5.21, 10 November 2021, by WalterMeldron Adds an urgent column to tickets for ahelps marked as urgent. -``` +```sql ALTER TABLE `ticket` ADD COLUMN `urgent` TINYINT(1) UNSIGNED NOT NULL DEFAULT '0' AFTER `sender`; ``` @@ -83,7 +83,7 @@ ALTER TABLE `ticket` ADD COLUMN `urgent` TINYINT(1) UNSIGNED NOT NULL DEFAULT '0 Version 5.20, 1 November 2021, by Mothblocks Added `known_alts` table for tracking who not to create suspicious logins for. -``` +```sql CREATE TABLE `known_alts` ( `id` INT NOT NULL AUTO_INCREMENT, `ckey1` VARCHAR(32) NOT NULL, @@ -94,18 +94,10 @@ CREATE TABLE `known_alts` ( ``` ----------------------------------------------------- -Version 5.19, 23 August 2021, by GoldenAlpharex -Added `discord_report` column to the `ban table` - -``` -`discord_reported` TINYINT(1) UNSIGNED NOT NULL DEFAULT '0', /* SKYRAT EDIT - Labelling bans for ease of reporting them over Discord. */ -``` - ------------------------------------------------------ -Version 5.18, 8 October 2021, by MrStonedOne + Mothblocks +Version 5.19, 8 October 2021, by MrStonedOne + Mothblocks Changes any table that requrired a NOT NULL round ID to now accept NULL. In the BSQL past, these were handled as 0, but in the move to rust-g this behavior was lost. -``` +```sql ALTER TABLE `admin_log` CHANGE `round_id` `round_id` INT(11) UNSIGNED NULL; ALTER TABLE `ban` CHANGE `round_id` `round_id` INT(11) UNSIGNED NULL; ALTER TABLE `citation` CHANGE `round_id` `round_id` INT(11) UNSIGNED NULL; @@ -120,11 +112,19 @@ ALTER TABLE `player` CHANGE `lastseen_round_id` `lastseen_round_id` INT(11) UNSI ALTER TABLE `ticket` CHANGE `round_id` `round_id` INT(11) UNSIGNED NULL; ``` +----------------------------------------------------- +Version 5.18, 23 August 2021, by GoldenAlpharex +Added `discord_report` column to the `ban table` + +```sql +`discord_reported` TINYINT(1) UNSIGNED NOT NULL DEFAULT '0', /* SKYRAT EDIT - Labelling bans for ease of reporting them over Discord. */ +``` + ----------------------------------------------------- Version 5.17, 31 July 2021, by Atlanta-Ned Added `library_action` table for tracking reported library books and actions taken on them. -``` +```sql DROP TABLE IF EXISTS `library_action`; CREATE TABLE `library_action` ( `id` int(11) unsigned NOT NULL AUTO_INCREMENT, @@ -143,7 +143,7 @@ CREATE TABLE `library_action` ( Version 5.16, 2 June 2021, by Mothblocks Added verified admin connection log used for 2FA -``` +```sql DROP TABLE IF EXISTS `admin_connections`; CREATE TABLE `admin_connections` ( `id` INT NOT NULL AUTO_INCREMENT, @@ -157,9 +157,10 @@ CREATE TABLE `admin_connections` ( ----------------------------------------------------- -Version 5.15, 24 May 2021, by Anturke +Version 5.15, xx May 2021, by Anturke Added exploration drone adventure table +```sql DROP TABLE IF EXISTS `text_adventures`; CREATE TABLE `text_adventures` ( `id` int(11) NOT NULL AUTO_INCREMENT, @@ -169,12 +170,14 @@ CREATE TABLE `text_adventures` ( `approved` TINYINT(1) NOT NULL DEFAULT FALSE, PRIMARY KEY (`id`) ) ENGINE=InnoDB; +``` ----------------------------------------------------- Version 5.14, 30 April 2021, by Atlanta Ned Added the `citation` table for tracking security citations in the database. +```sql CREATE TABLE `citation` ( `id` INT(10) UNSIGNED NOT NULL AUTO_INCREMENT, `round_id` INT(11) UNSIGNED NOT NULL, @@ -196,6 +199,7 @@ COLLATE='utf8mb4_general_ci' ENGINE=InnoDB AUTO_INCREMENT=1 ; +``` ----------------------------------------------------- @@ -206,6 +210,7 @@ between servers when it comes to bans, statistical data, etc. You might also want to update the server_name column in older records in the tables, however it is not absolutely necessary. +```sql ALTER TABLE `ban` ADD COLUMN `server_name` VARCHAR(32) DEFAULT NULL AFTER `bantime`, ADD COLUMN `global_ban` TINYINT(1) UNSIGNED NOT NULL DEFAULT '1' AFTER `role`; @@ -227,6 +232,7 @@ ALTER TABLE `messages` ALTER TABLE `round` ADD COLUMN `server_name` VARCHAR(32) DEFAULT NULL AFTER `end_datetime`; +``` ----------------------------------------------------- @@ -234,7 +240,9 @@ ALTER TABLE `round` Version 5.12, 29 December 2020, by Missfox Modified table `messages`, adding column `playtime` to show the user's playtime when the note was created. +```sql ALTER TABLE `messages` ADD `playtime` INT(11) NULL DEFAULT(NULL) AFTER `severity` +``` ----------------------------------------------------- @@ -242,11 +250,13 @@ Version 5.11, 7 September 2020, by bobbahbrown, MrStonedOne, and Jordie0608 (Upd Adds indices to support search operations on the adminhelp ticket tables. This is to support improved performance on Atlanta Ned's Statbus. +```sql ALTER TABLE `ticket` ADD INDEX `idx_ticket_act_recip` (`action`, `recipient`), ADD INDEX `idx_ticket_act_send` (`action`, `sender`), ADD INDEX `idx_ticket_tic_rid` (`ticket`, `round_id`), ADD INDEX `idx_ticket_act_time_rid` (`action`, `timestamp`, `round_id`); +``` ----------------------------------------------------- @@ -256,6 +266,7 @@ Changes how the discord verification process works. Adds the discord_links table, and migrates discord id entries from player table to the discord links table in a once off operation and then removes the discord id on the player table +```sql START TRANSACTION; DROP TABLE IF EXISTS `discord_links`; @@ -274,6 +285,7 @@ INSERT INTO `discord_links` (`ckey`, `discord_id`, `one_time_token`, `valid`) SE ALTER TABLE `player` DROP COLUMN `discord_id`; COMMIT; +``` ----------------------------------------------------- @@ -283,6 +295,7 @@ Added the `deleted` column to tables 'poll_option', 'poll_textreply' and 'poll_v Changes table 'poll_question' column `createdby_ckey` to be NOT NULL and index `idx_pquest_time_admin` to be `idx_pquest_time_deleted_id` and 'poll_textreply' column `adminrank` to have no default. Added procedure `set_poll_deleted` that's called when deleting a poll to set deleted to true on each poll table where rows matching a poll_id argument. +```sql ALTER TABLE `poll_option` ADD COLUMN `deleted` TINYINT(1) UNSIGNED NOT NULL DEFAULT '0' AFTER `default_percentage_calc`; @@ -315,19 +328,23 @@ UPDATE `poll_textreply` SET deleted = 1 WHERE pollid = poll_id; END $$ DELIMITER ; +``` ----------------------------------------------------- Version 5.8, 7 April 2020, by Jordie0608 Modified table `messages`, adding column `deleted_ckey` to record who deleted a message. +```sql ALTER TABLE `messages` ADD COLUMN `deleted_ckey` VARCHAR(32) NULL DEFAULT NULL AFTER `deleted`; +``` ----------------------------------------------------- Version 5.7, 10 January 2020 by Atlanta-Ned Added ticket table for tracking ahelp tickets in the database. +```sql DROP TABLE IF EXISTS `ticket`; CREATE TABLE `ticket` ( `id` int(11) unsigned NOT NULL AUTO_INCREMENT, @@ -342,20 +359,23 @@ CREATE TABLE `ticket` ( `sender` varchar(32) DEFAULT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +``` ----------------------------------------------------- Version 5.6, 6 December 2019 by Anturke Added achievement_name and achievement_description columns to achievement_metadata table. - +```sql ALTER TABLE `achievement_metadata` ADD COLUMN (`achievement_name` VARCHAR(64) NULL DEFAULT NULL, `achievement_description` VARCHAR(512) NULL DEFAULT NULL); +``` ----------------------------------------------------- Version 5.5, 26 October 2019 by Anturke Added achievement_metadata table. +```sql DROP TABLE IF EXISTS `achievement_metadata`; CREATE TABLE `achievement_metadata` ( `achievement_key` VARCHAR(32) NOT NULL, @@ -363,7 +383,7 @@ CREATE TABLE `achievement_metadata` ( `achievement_type` enum('achievement','score','award') NULL DEFAULT NULL, PRIMARY KEY (`achievement_key`) ) ENGINE=InnoDB; - +``` ----------------------------------------------------- @@ -371,6 +391,7 @@ Version 5.4, 5 October 2019 by Anturke Added achievements table. See hub migration verb in _achievement_data.dm for details on migrating. +```sql CREATE TABLE `achievements` ( `ckey` VARCHAR(32) NOT NULL, `achievement_key` VARCHAR(32) NOT NULL, @@ -378,26 +399,32 @@ CREATE TABLE `achievements` ( `last_updated` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (`ckey`,`achievement_key`) ) ENGINE=InnoDB; +``` ---------------------------------------------------- Version 5.3, 6 July 2019, by Atlanta-Ned Added a `feedback` column to the admin table, used for linking to individual admin feedback threads. Currently this is only used for statistics tracking tools such as Statbus and isn't used by the game. +```sql ALTER TABLE `admin` ADD `feedback` VARCHAR(255) NULL DEFAULT NULL AFTER `rank`; +``` ---------------------------------------------------- Version 5.2, 30 May 2019, by AffectedArc07 Added a field to the `player` table to track ckey and discord ID relationships +```sql ALTER TABLE `player` ADD COLUMN `discord_id` BIGINT NULL DEFAULT NULL AFTER `flags`; +``` ---------------------------------------------------- Version 5.1, 25 Feb 2018, by MrStonedOne Added four tables to enable storing of stickybans in the database since byond can lose them, and to enable disabling stickybans for a round without depending on a crash free round. Existing stickybans are automagically imported to the tables. +```sql CREATE TABLE `stickyban` ( `ckey` VARCHAR(32) NOT NULL, `reason` VARCHAR(2048) NOT NULL, @@ -430,6 +457,7 @@ CREATE TABLE `stickyban_matched_cid` ( `last_matched` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (`stickyban`, `matched_cid`) ) ENGINE=InnoDB; +``` ---------------------------------------------------- @@ -439,6 +467,8 @@ Modified ban table to remove the need for the `bantype` column, a python script See the file 'ban_conversion_2018-10-28.py' for instructions on how to use the script. A new ban table can be created with the query: + +```sql CREATE TABLE `ban` ( `id` INT(11) UNSIGNED NOT NULL AUTO_INCREMENT, `bantime` DATETIME NOT NULL, @@ -468,6 +498,7 @@ CREATE TABLE `ban` ( KEY `idx_ban_isbanned_details` (`ckey`,`ip`,`computerid`,`role`,`unbanned_datetime`,`expiration_time`), KEY `idx_ban_count` (`bantime`,`a_ckey`,`applies_to_admins`,`unbanned_datetime`,`expiration_time`) ) ENGINE=InnoDB DEFAULT CHARSET=latin1; +``` ---------------------------------------------------- diff --git a/code/controllers/configuration/configuration.dm b/code/controllers/configuration/configuration.dm index 0fc5b2e3fb7..1a4f3cd4011 100644 --- a/code/controllers/configuration/configuration.dm +++ b/code/controllers/configuration/configuration.dm @@ -90,6 +90,8 @@ break if (fexists("[directory]/dev_overrides.txt")) LoadEntries("dev_overrides.txt") + if (fexists("[directory]/ezdb.txt")) + LoadEntries("ezdb.txt") loadmaplist(CONFIG_MAPS_FILE) LoadMOTD() LoadPolicy() diff --git a/code/controllers/configuration/entries/dbconfig.dm b/code/controllers/configuration/entries/dbconfig.dm index 14713c4e1c7..bd40d3a168c 100644 --- a/code/controllers/configuration/entries/dbconfig.dm +++ b/code/controllers/configuration/entries/dbconfig.dm @@ -68,3 +68,8 @@ . = ..() if (.) SSdbcore.max_concurrent_queries = config_entry_value + +/// The exe for mariadbd.exe. +/// Shouldn't really be set on production servers, primarily for EZDB. +/datum/config_entry/string/db_daemon + protection = CONFIG_ENTRY_LOCKED | CONFIG_ENTRY_HIDDEN diff --git a/code/controllers/subsystem/dbcore.dm b/code/controllers/subsystem/dbcore.dm index bbc41b17f65..6cd6ea87561 100644 --- a/code/controllers/subsystem/dbcore.dm +++ b/code/controllers/subsystem/dbcore.dm @@ -37,6 +37,8 @@ SUBSYSTEM_DEF(dbcore) var/connection // Arbitrary handle returned from rust_g. + var/db_daemon_started = FALSE + /datum/controller/subsystem/dbcore/Initialize() //We send warnings to the admins during subsystem init, as the clients will be New'd and messages //will queue properly with goonchat @@ -74,8 +76,6 @@ SUBSYSTEM_DEF(dbcore) CONFIG_SET(number/pooling_max_sql_connections, max(min_sql_connections, max_sql_connections)) log_config("ERROR: POOLING_MAX_SQL_CONNECTIONS ([max_sql_connections]) is set lower than POOLING_MIN_SQL_CONNECTIONS ([min_sql_connections]). Please check your config or the code defaults for sanity") - - /datum/controller/subsystem/dbcore/stat_entry(msg) msg = "P:[length(all_queries)]|Active:[length(queries_active)]|Standby:[length(queries_standby)]" return ..() @@ -188,6 +188,7 @@ SUBSYSTEM_DEF(dbcore) qdel(query_round_shutdown) if(IsConnected()) Disconnect() + stop_db_daemon() //nu /datum/controller/subsystem/dbcore/can_vv_get(var_name) @@ -231,6 +232,8 @@ SUBSYSTEM_DEF(dbcore) if(!CONFIG_GET(flag/sql_enabled)) return FALSE + start_db_daemon() + var/user = CONFIG_GET(string/feedback_login) var/pass = CONFIG_GET(string/feedback_password) var/db = CONFIG_GET(string/feedback_database) @@ -447,6 +450,47 @@ Delayed insert mode was removed in mysql 7 and only works with MyISAM type table . = Query.Execute(async) qdel(Query) +/datum/controller/subsystem/dbcore/proc/start_db_daemon() + set waitfor = FALSE + + if (db_daemon_started) + return + + db_daemon_started = TRUE + + var/daemon = CONFIG_GET(string/db_daemon) + if (!daemon) + return + + ASSERT(fexists(daemon), "Configured db_daemon doesn't exist") + + var/list/result = world.shelleo("echo \"Starting ezdb daemon, do not close this window\" && [daemon]") + var/error_code = result[1] + if (!error_code) + return + + stack_trace("Failed to start DB daemon: [error_code]\n[result[3]]") + +/datum/controller/subsystem/dbcore/proc/stop_db_daemon() + set waitfor = FALSE + + if (!db_daemon_started) + return + + db_daemon_started = FALSE + + var/daemon = CONFIG_GET(string/db_daemon) + if (!daemon) + return + + switch (world.system_type) + if (MS_WINDOWS) + var/list/result = world.shelleo("Get-Process | ? { $_.Path -eq '[daemon]' } | Stop-Process") + ASSERT(result[1], "Failed to stop DB daemon: [result[3]]") + if (UNIX) + var/list/result = world.shelleo("kill $(pgrep -f '[daemon]')") + ASSERT(result[1], "Failed to stop DB daemon: [result[3]]") + /datum/db_query // Inputs var/connection diff --git a/tools/ezdb/__main__.py b/tools/ezdb/__main__.py new file mode 100644 index 00000000000..c817aa4d654 --- /dev/null +++ b/tools/ezdb/__main__.py @@ -0,0 +1,15 @@ +import argparse +from .steps import STEPS + +parser = argparse.ArgumentParser() +parser.add_argument("--port", type = int, default = 1338) + +args = parser.parse_args() + +for step in STEPS: + if not step.should_run(): + continue + + step.run(args) + +print("Done!") diff --git a/tools/ezdb/ezdb.bat b/tools/ezdb/ezdb.bat new file mode 100644 index 00000000000..b4257fe3c36 --- /dev/null +++ b/tools/ezdb/ezdb.bat @@ -0,0 +1,2 @@ +@call "%~dp0\..\bootstrap\python" -m ezdb %* +@pause diff --git a/tools/ezdb/ezdb/__init__.py b/tools/ezdb/ezdb/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tools/ezdb/ezdb/changes.py b/tools/ezdb/ezdb/changes.py new file mode 100644 index 00000000000..607ea1efea3 --- /dev/null +++ b/tools/ezdb/ezdb/changes.py @@ -0,0 +1,31 @@ +import re +from dataclasses import dataclass +from .paths import get_changelog_path + +REGEX_CHANGE = r"-+\s*Version (?P[0-9]+)\.(?P[0-9]+), .+?\`\`\`sql\s*(?P.+?)\s*\`\`\`.*?-{5}" + +@dataclass +class Change: + major_version: int + minor_version: int + sql: str + +def get_changes() -> list[Change]: + with open(get_changelog_path(), "r") as file: + changelog = file.read() + changes = [] + + for change_match in re.finditer(REGEX_CHANGE, changelog, re.MULTILINE | re.DOTALL): + changes.append(Change( + int(change_match.group("major")), + int(change_match.group("minor")), + change_match.group("sql") + )) + + changes.sort(key = lambda change: (change.major_version, change.minor_version), reverse = True) + + return changes + +def get_current_version(): + changes = get_changes() + return (changes[0].major_version, changes[0].minor_version) diff --git a/tools/ezdb/ezdb/config.py b/tools/ezdb/ezdb/config.py new file mode 100644 index 00000000000..87d853cd42b --- /dev/null +++ b/tools/ezdb/ezdb/config.py @@ -0,0 +1,22 @@ +from .paths import get_config_path +from typing import Optional + +def read_config() -> Optional[dict[str, str]]: + config_path = get_config_path() + if not config_path.exists(): + return None + + with config_path.open('r') as file: + lines = file.readlines() + entries = {} + + for line in lines: + if line.startswith("#"): + continue + if " " not in line: + continue + + key, value = line.split(" ", 1) + entries[key.strip()] = value.strip() + + return entries diff --git a/tools/ezdb/ezdb/mysql.py b/tools/ezdb/ezdb/mysql.py new file mode 100644 index 00000000000..249704fe04d --- /dev/null +++ b/tools/ezdb/ezdb/mysql.py @@ -0,0 +1,68 @@ +import atexit +import mysql.connector +import subprocess +from contextlib import closing +from .config import read_config +from .paths import get_mariadb_client_path, get_mariadb_daemon_path + +def open_connection(): + config = read_config() + assert config["FEEDBACK_PASSWORD"] is not None, "No password found in config file" + + connection = mysql.connector.connect( + user = config["FEEDBACK_LOGIN"], + password = config["FEEDBACK_PASSWORD"], + port = int(config["PORT"]), + raise_on_warnings = True, + ) + + connection.autocommit = True + + return closing(connection) + +# We use custom things like delimiters, so we can't use the built-in cursor.execute +def execute_sql(sql: str): + config = read_config() + assert config is not None, "No config file found" + assert config["FEEDBACK_PASSWORD"] is not None, "No password found in config file" + + subprocess.run( + [ + str(get_mariadb_client_path()), + "-u", + "root", + "-p" + config["FEEDBACK_PASSWORD"], + "--port", + config["PORT"], + "--database", + config["FEEDBACK_DATABASE"], + ], + input = sql, + encoding = "utf-8", + check = True, + stderr = subprocess.STDOUT, + ) + +def insert_new_schema_query(major_version: int, minor_version: int): + return f"INSERT INTO `schema_revision` (`major`, `minor`) VALUES ({major_version}, {minor_version})" + +process = None +def start_daemon(): + global process + if process is not None: + return + + print("Starting MariaDB daemon...") + config = read_config() + assert config is not None, "No config file found" + + process = subprocess.Popen( + [ + str(get_mariadb_daemon_path()), + "--port", + config["PORT"], + ], + stderr = subprocess.PIPE, + ) + + atexit.register(process.kill) diff --git a/tools/ezdb/ezdb/paths.py b/tools/ezdb/ezdb/paths.py new file mode 100644 index 00000000000..502e6762b35 --- /dev/null +++ b/tools/ezdb/ezdb/paths.py @@ -0,0 +1,34 @@ +import pathlib + +def get_root_path(): + current_path = pathlib.Path(__file__) + while current_path.name != 'tools': + current_path = current_path.parent + return current_path.parent + +def get_config_path(): + return get_root_path() / 'config' / 'ezdb.txt' + +def get_db_path(): + return get_root_path() / 'db' + +def get_data_path(): + return get_db_path() / 'data' + +def get_mariadb_bin_path(): + return get_db_path() / 'bin' + +def get_mariadb_client_path(): + return get_mariadb_bin_path() / 'mariadb.exe' + +def get_mariadb_daemon_path(): + return get_mariadb_bin_path() / 'mariadbd.exe' + +def get_mariadb_install_db_path(): + return get_mariadb_bin_path() / 'mariadb-install-db.exe' + +def get_initial_schema_path(): + return get_root_path() / 'SQL' / 'tgstation_schema.sql' + +def get_changelog_path(): + return get_root_path() / 'SQL' / 'database_changelog.md' diff --git a/tools/ezdb/steps/__init__.py b/tools/ezdb/steps/__init__.py new file mode 100644 index 00000000000..969a1029d79 --- /dev/null +++ b/tools/ezdb/steps/__init__.py @@ -0,0 +1,11 @@ +from .download_mariadb import DownloadMariaDB +from .install_database import InstallDatabase +from .install_initial_schema import InstallInitialSchema +from .update_schema import UpdateSchema + +STEPS = [ + DownloadMariaDB, + InstallDatabase, + InstallInitialSchema, + UpdateSchema, +] diff --git a/tools/ezdb/steps/download_mariadb.py b/tools/ezdb/steps/download_mariadb.py new file mode 100644 index 00000000000..ecaf01cf6e2 --- /dev/null +++ b/tools/ezdb/steps/download_mariadb.py @@ -0,0 +1,50 @@ +import os +import pathlib +import tempfile +import urllib.request +import zipfile +from ..ezdb.paths import get_config_path, get_data_path, get_db_path, get_mariadb_bin_path, get_mariadb_daemon_path, get_mariadb_install_db_path +from .step import Step + +# Theoretically, this could use the REST API that MariaDB has to find the URL given a version: +# https://downloads.mariadb.org/rest-api/mariadb/10.11 +DOWNLOAD_URL = "http://downloads.mariadb.org/rest-api/mariadb/10.11.2/mariadb-10.11.2-winx64.zip" +FOLDER_NAME = "mariadb-10.11.2-winx64" + +temp_extract_path = get_db_path() / "_temp/" + +class DownloadMariaDB(Step): + @staticmethod + def should_run() -> bool: + return not get_mariadb_bin_path().exists() + + @staticmethod + def run(args): + if temp_extract_path.exists(): + print("Deleting old temporary extract folder") + temp_extract_path.rmdir() + + print("Downloading portable MariaDB...") + + # delete = False so we can write to it + temporary_file = tempfile.NamedTemporaryFile(delete = False) + + try: + urllib.request.urlretrieve(DOWNLOAD_URL, temporary_file.name) + + print("Extracting...") + os.makedirs(temp_extract_path, exist_ok = True) + with zipfile.ZipFile(temporary_file) as zip_file: + for file in zip_file.namelist(): + if file.startswith(f"{FOLDER_NAME}/bin/"): + with zip_file.open(file) as source, open(temp_extract_path / pathlib.Path(file).name, "wb") as target: + target.write(source.read()) + + print("Moving...") + + temp_extract_path.rename(get_mariadb_bin_path()) + finally: + temporary_file.close() + + if temp_extract_path.exists(): + temp_extract_path.rmdir() diff --git a/tools/ezdb/steps/install_database.py b/tools/ezdb/steps/install_database.py new file mode 100644 index 00000000000..698faf781ed --- /dev/null +++ b/tools/ezdb/steps/install_database.py @@ -0,0 +1,46 @@ +import argparse +import secrets +import subprocess +from ..ezdb.paths import get_config_path, get_data_path, get_mariadb_bin_path, get_mariadb_daemon_path, get_mariadb_install_db_path +from .step import Step + +def create_password() -> str: + return secrets.token_urlsafe(40) + +class InstallDatabase(Step): + @staticmethod + def should_run() -> bool: + # If the db folder exists, but the config doesn't, we cancelled + # halfway through and ought to start over by deleting the data folder. + return get_mariadb_bin_path().exists() and (not get_data_path().exists() or not get_config_path().exists()) + + @staticmethod + def run(args: argparse.Namespace): + data_folder = get_data_path() + if data_folder.exists(): + print("Deleting old data folder") + data_folder.rmdir() + + password = create_password() + + print("Installing database...") + + subprocess.run( + [ + str(get_mariadb_install_db_path()), + f"--port={args.port}", + f"--password={password}", + ], + check = True, + stderr = subprocess.STDOUT, + ) + + print("Creating config...") + with open(get_config_path(), "w") as file: + file.write("SQL_ENABLED\n") + file.write(f"PORT {args.port}\n") + file.write(f"FEEDBACK_LOGIN root\n") + file.write(f"FEEDBACK_PASSWORD {password}\n") + file.write("FEEDBACK_DATABASE tgstation\n") + file.write("FEEDBACK_TABLEPREFIX\n") + file.write(f"DB_DAEMON {str(get_mariadb_daemon_path())}") diff --git a/tools/ezdb/steps/install_initial_schema.py b/tools/ezdb/steps/install_initial_schema.py new file mode 100644 index 00000000000..bb7ee519ae7 --- /dev/null +++ b/tools/ezdb/steps/install_initial_schema.py @@ -0,0 +1,53 @@ +from contextlib import closing +from ..ezdb.changes import get_current_version +from ..ezdb.config import read_config +from ..ezdb.mysql import execute_sql, insert_new_schema_query, open_connection, start_daemon +from ..ezdb.paths import get_initial_schema_path +from .step import Step + +class InstallInitialSchema(Step): + @staticmethod + def should_run() -> bool: + start_daemon() + + config = read_config() + assert config is not None, "No config file found" + + database = config["FEEDBACK_DATABASE"] + assert database is not None, "No database found in config file" + + with open_connection() as connection: + with closing(connection.cursor()) as cursor: + cursor.execute(f"SHOW DATABASES LIKE '{database}'") + if cursor.fetchone() is None: + return True + + cursor.execute(f"USE {database}") + cursor.execute("SHOW TABLES LIKE 'schema_revision'") + if cursor.fetchone() is None: + return True + + cursor.execute("SELECT * FROM `schema_revision` LIMIT 1") + if cursor.fetchone() is None: + return True + + return False + + @staticmethod + def run(args): + print("Installing initial schema...") + + config = read_config() + assert config is not None, "No config file found" + + with open_connection() as connection: + with closing(connection.cursor()) as cursor: + database = config["FEEDBACK_DATABASE"] + cursor.execute(f"CREATE DATABASE {database}") + cursor.execute(f"USE {database}") + + (major_version, minor_version) = get_current_version() + + with open(get_initial_schema_path(), 'r') as file: + schema = file.read() + execute_sql(schema + ";" + insert_new_schema_query(major_version, minor_version)) diff --git a/tools/ezdb/steps/step.py b/tools/ezdb/steps/step.py new file mode 100644 index 00000000000..6ae4b492df4 --- /dev/null +++ b/tools/ezdb/steps/step.py @@ -0,0 +1,10 @@ +import argparse + +class Step: + @staticmethod + def should_run() -> bool: + raise NotImplementedError() + + @staticmethod + def run(args: argparse.Namespace): + raise NotImplementedError() diff --git a/tools/ezdb/steps/update_schema.py b/tools/ezdb/steps/update_schema.py new file mode 100644 index 00000000000..e5a1c8d872e --- /dev/null +++ b/tools/ezdb/steps/update_schema.py @@ -0,0 +1,38 @@ +from contextlib import closing +from ..ezdb.changes import get_changes +from ..ezdb.config import read_config +from ..ezdb.mysql import execute_sql, insert_new_schema_query, open_connection +from .step import Step + +class UpdateSchema(Step): + @staticmethod + def should_run() -> bool: + # Last step is always run + return True + + @staticmethod + def run(args): + config = read_config() + assert config is not None, "No config file found" + + database = config["FEEDBACK_DATABASE"] + assert database is not None, "No database found in config file" + + with open_connection() as connection: + with closing(connection.cursor()) as cursor: + cursor.execute(f"USE {database}") + cursor.execute("SELECT major, minor FROM `schema_revision` ORDER BY `major` DESC, `minor` DESC LIMIT 1") + (major_version, minor_version) = cursor.fetchone() + + changes = get_changes() + for change in changes: + if change.major_version != major_version: + print("NOT IMPLEMENTED: Major version change, these historically require extra tooling") + continue + + if change.minor_version > minor_version: + print(f"Running change {change.major_version}.{change.minor_version}") + execute_sql(change.sql + ";" + insert_new_schema_query(change.major_version, change.minor_version)) + else: + print("No updates necessary") + return diff --git a/tools/requirements.txt b/tools/requirements.txt index 76e14dc45e6..032e1f70e90 100644 --- a/tools/requirements.txt +++ b/tools/requirements.txt @@ -5,3 +5,6 @@ Pillow==9.3.0 # changelogs PyYaml==5.4 beautifulsoup4==4.9.3 + +# ezdb +mysql-connector-python==8.0.33