Merge pull request #16560 from AffectedArc07/multi-instance-support

[READY] Multi instance support

.github/CODEOWNERS

@@ -5,6 +5,7 @@
 
 ### AffectedArc07
 # Actual Code
+/code/controllers/subsystem/instancing.dm @AffectedArc07
 /code/controllers/subsystem/mapping.dm @AffectedArc07
 /code/controllers/subsystem/ticker.dm @AffectedArc07
 /tgui/ @AffectedArc07

SQL/paradise_schema.sql

@@ -114,6 +114,7 @@ CREATE TABLE `death` (
   `pod` text NOT NULL COMMENT 'Place of death',
   `coord` text NOT NULL COMMENT 'X, Y, Z POD',
   `tod` datetime NOT NULL COMMENT 'Time of death',
+  `server_id` TEXT NULL DEFAULT NULL,
   `job` text NOT NULL,
   `special` text NOT NULL,
   `name` text NOT NULL,
@@ -197,6 +198,7 @@ CREATE TABLE `ban` (
   `bantime` datetime NOT NULL,
   `ban_round_id` INT(11) NULL DEFAULT NULL,
   `serverip` varchar(32) COLLATE utf8mb4_unicode_ci NOT NULL,
+  `server_id` VARCHAR(50) NULL DEFAULT NULL COLLATE utf8mb4_general_ci,
   `bantype` varchar(32) COLLATE utf8mb4_unicode_ci NOT NULL,
   `reason` mediumtext COLLATE utf8mb4_unicode_ci NOT NULL,
   `job` varchar(32) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
@@ -320,6 +322,7 @@ CREATE TABLE `karma` (
   `receiverspecial` text,
   `isnegative` tinyint(1) DEFAULT NULL,
   `spenderip` text NOT NULL,
+  `server_id` TEXT NULL DEFAULT NULL,
   `time` datetime NOT NULL,
   PRIMARY KEY (`id`)
 ) ENGINE=InnoDB AUTO_INCREMENT=73614 DEFAULT CHARSET=utf8mb4;
@@ -374,6 +377,7 @@ CREATE TABLE `legacy_population` (
   `id` int(11) NOT NULL AUTO_INCREMENT,
   `playercount` int(11) DEFAULT NULL,
   `admincount` int(11) DEFAULT NULL,
+  `server_id` VARCHAR(50) NULL DEFAULT NULL,
   `time` datetime NOT NULL,
   PRIMARY KEY (`id`)
 ) ENGINE=InnoDB AUTO_INCREMENT=2550 DEFAULT CHARSET=utf8mb4;
@@ -532,6 +536,7 @@ CREATE TABLE `connection_log` (
   `ckey` varchar(32) NOT NULL,
   `ip` varchar(32) NOT NULL,
   `computerid` varchar(32) NOT NULL,
+  `server_id` VARCHAR(50) NULL DEFAULT NULL,
   `result` ENUM('ESTABLISHED','DROPPED - IPINTEL','DROPPED - BANNED','DROPPED - INVALID') NOT NULL DEFAULT 'ESTABLISHED' COLLATE 'utf8mb4_general_ci',
   PRIMARY KEY (`id`),
   KEY `ckey` (`ckey`),
@@ -584,6 +589,7 @@ CREATE TABLE `round` (
   `shuttle_name` VARCHAR(64) NULL,
   `map_name` VARCHAR(32) NULL,
   `station_name` VARCHAR(80) NULL,
+  `server_id` VARCHAR(50) NULL,
   PRIMARY KEY (`id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
 
@@ -612,3 +618,14 @@ CREATE TABLE `pai_saves` (
 	PRIMARY KEY (`id`) USING BTREE,
 	UNIQUE INDEX `ckey` (`ckey`) USING BTREE
 ) COLLATE='utf8mb4_general_ci' ENGINE=InnoDB;
+
+--
+-- Table structure for table `instance_data_cache`
+--
+CREATE TABLE `instance_data_cache` (
+	`server_id` VARCHAR(50) NOT NULL COLLATE 'utf8mb4_general_ci',
+	`key_name` VARCHAR(50) NOT NULL COLLATE 'utf8mb4_general_ci',
+	`key_value` VARCHAR(50) NOT NULL COLLATE 'utf8mb4_general_ci',
+	`last_updated` TIMESTAMP NOT NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
+	PRIMARY KEY (`server_id`, `key_name`) USING HASH
+) COLLATE='utf8mb4_general_ci' ENGINE=MEMORY;

SQL/updates/26-27.sql

@@ -0,0 +1,32 @@
+# Updates DB from 26 to 27 -AffectedArc07
+# Adds a new table for instance data caching, and server_id fields on other tables
+
+CREATE TABLE `instance_data_cache` (
+	`server_id` VARCHAR(50) NOT NULL COLLATE 'utf8mb4_general_ci',
+	`key_name` VARCHAR(50) NOT NULL COLLATE 'utf8mb4_general_ci',
+	`key_value` VARCHAR(50) NOT NULL COLLATE 'utf8mb4_general_ci',
+	`last_updated` TIMESTAMP NOT NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
+	PRIMARY KEY (`server_id`, `key_name`) USING HASH
+) COLLATE='utf8mb4_general_ci' ENGINE=MEMORY;
+
+ALTER TABLE `ban`
+	ADD COLUMN `server_id` VARCHAR(50) NULL DEFAULT NULL AFTER `serverip`;
+
+ALTER TABLE `connection_log`
+	ADD COLUMN `server_id` VARCHAR(50) NULL DEFAULT NULL AFTER `computerid`;
+
+ALTER TABLE `death`
+	ADD COLUMN `server_id` TEXT NULL AFTER `tod`;
+
+ALTER TABLE `karma`
+	ADD COLUMN `server_id` TEXT NULL AFTER `spenderip`;
+
+ALTER TABLE `legacy_population`
+	ADD COLUMN `server_id` VARCHAR(50) NULL DEFAULT NULL AFTER `admincount`;
+
+# Notes already has a column for this
+ALTER TABLE `notes`
+	CHANGE COLUMN `server` `server` VARCHAR(50) NULL COLLATE 'utf8mb4_general_ci' AFTER `edits`;
+
+ALTER TABLE `round`
+	ADD COLUMN `server_id` VARCHAR(50) NULL DEFAULT NULL AFTER `station_name`;

code/__DEFINES/misc.dm

@@ -365,7 +365,7 @@
 #define INVESTIGATE_BOMB "bombs"
 
 // The SQL version required by this version of the code
-#define SQL_VERSION 26
+#define SQL_VERSION 27
 
 // Vending machine stuff
 #define CAT_NORMAL 1

code/__HELPERS/unsorted.dm

@@ -2061,11 +2061,12 @@ GLOBAL_DATUM_INIT(dview_mob, /mob/dview, new)
 
 /proc/log_connection(ckey, ip, cid, connection_type)
 	ASSERT(connection_type in list(CONNECTION_TYPE_ESTABLISHED, CONNECTION_TYPE_DROPPED_IPINTEL, CONNECTION_TYPE_DROPPED_BANNED, CONNECTION_TYPE_DROPPED_INVALID))
-	var/datum/db_query/query_accesslog = SSdbcore.NewQuery("INSERT INTO connection_log (`datetime`, `ckey`, `ip`, `computerid`, `result`) VALUES(Now(), :ckey, :ip, :cid, :result)", list(
+	var/datum/db_query/query_accesslog = SSdbcore.NewQuery("INSERT INTO connection_log (`datetime`, `ckey`, `ip`, `computerid`, `result`, `server_id`) VALUES(Now(), :ckey, :ip, :cid, :result, :server_id)", list(
 		"ckey" = ckey,
 		"ip" = "[ip ? ip : ""]", // This is important. NULL is not the same as "", and if you directly open the `.dmb` file, you get a NULL IP.
 		"cid" = cid,
-		"result" = connection_type
+		"result" = connection_type,
+		"server_id" = GLOB.configuration.system.instance_id
 	))
 	query_accesslog.warn_execute()
 	qdel(query_accesslog)

code/controllers/configuration/configuration_core.dm

@@ -46,6 +46,8 @@ GLOBAL_DATUM_INIT(configuration, /datum/server_configuration, new())
 	var/datum/configuration_section/url_configuration/url
 	/// Holder for the voting configuration datum
 	var/datum/configuration_section/vote_configuration/vote
+	/// Raw data. Stored here to avoid passing data between procs constantly
+	var/list/raw_data = list()
 
 
 /datum/server_configuration/Destroy(force)
@@ -53,6 +55,16 @@ GLOBAL_DATUM_INIT(configuration, /datum/server_configuration, new())
 	// This is going to stay existing. I dont care.
 	return QDEL_HINT_LETMELIVE
 
+/datum/server_configuration/vv_get_var(var_name)
+	if(var_name == "raw_data") // NO!
+		return FALSE
+	. = ..()
+
+/datum/server_configuration/vv_edit_var(var_name, var_value)
+	if(var_name == "raw_data") // NO!
+		return FALSE
+	. = ..()
+
 /datum/server_configuration/CanProcCall(procname)
 	return FALSE // No thanks
 
@@ -85,33 +97,66 @@ GLOBAL_DATUM_INIT(configuration, /datum/server_configuration, new())
 	var/config_file = "config/config.toml"
 	if(!fexists(config_file))
 		config_file = "config/example/config.toml" // Fallback to example if user hasnt setup config properly
-	var/raw_config_data = rustg_read_toml_file(config_file)
+	raw_data = rustg_read_toml_file(config_file)
 
 	// Now pass through all our stuff
-	admin.load_data(raw_config_data["admin_configuration"])
-	afk.load_data(raw_config_data["afk_configuration"])
-	custom_sprites.load_data(raw_config_data["custom_sprites_configuration"])
-	database.load_data(raw_config_data["database_configuration"])
-	discord.load_data(raw_config_data["discord_configuration"])
-	event.load_data(raw_config_data["event_configuration"])
-	gamemode.load_data(raw_config_data["gamemode_configuration"])
-	gateway.load_data(raw_config_data["gateway_configuration"])
-	general.load_data(raw_config_data["general_configuration"])
-	ipintel.load_data(raw_config_data["ipintel_configuration"])
-	jobs.load_data(raw_config_data["job_configuration"])
-	logging.load_data(raw_config_data["logging_configuration"])
-	mc.load_data(raw_config_data["mc_configuration"])
-	metrics.load_data(raw_config_data["metrics_configuration"])
-	movement.load_data(raw_config_data["movement_configuration"])
-	overflow.load_data(raw_config_data["overflow_configuration"])
-	ruins.load_data(raw_config_data["ruin_configuration"])
-	system.load_data(raw_config_data["system_configuration"])
-	url.load_data(raw_config_data["url_configuration"])
-	vote.load_data(raw_config_data["voting_configuration"])
+	load_all_sections()
+
+	// Clear our list to save RAM
+	raw_data = list()
 
 	// And report the load
 	DIRECT_OUTPUT(world.log, "Config loaded in [stop_watch(start)]s")
 
+/datum/server_configuration/proc/load_all_sections()
+	safe_load(admin, "admin_configuration")
+	safe_load(afk, "afk_configuration")
+	safe_load(custom_sprites, "custom_sprites_configuration")
+	safe_load(database, "database_configuration")
+	safe_load(discord, "discord_configuration")
+	safe_load(event, "event_configuration")
+	safe_load(gamemode, "gamemode_configuration")
+	safe_load(gateway, "gateway_configuration")
+	safe_load(general, "general_configuration")
+	safe_load(ipintel, "ipintel_configuration")
+	safe_load(jobs, "job_configuration")
+	safe_load(logging, "logging_configuration")
+	safe_load(mc, "mc_configuration")
+	safe_load(metrics, "metrics_configuration")
+	safe_load(movement, "movement_configuration")
+	safe_load(overflow, "overflow_configuration")
+	safe_load(ruins, "ruin_configuration")
+	safe_load(system, "system_configuration")
+	safe_load(url, "url_configuration")
+	safe_load(vote, "voting_configuration")
+
+// Proc to load up instance-specific overrides
+/datum/server_configuration/proc/load_overrides()
+	var/override_file = "config/overrides_[world.port].toml"
+	if(!fexists(override_file))
+		DIRECT_OUTPUT(world.log, "Overrides not found for this instance.")
+		return
+
+	DIRECT_OUTPUT(world.log, "Overrides found for this instance. Loading them.")
+	var/start = start_watch() // Time tracking
+
+	raw_data = rustg_read_toml_file(override_file)
+
+	// Now safely load our overrides.
+	// Due to the nature of config wrappers, only vars that exist in the config file are applied to the config datums.
+	// This means that an override missing a key doesnt null it out from the main server
+	load_all_sections()
+
+	// Clear our list to save RAM
+	raw_data = list()
+
+	// And report the load
+	DIRECT_OUTPUT(world.log, "Config overrides loaded in [stop_watch(start)]s")
+
+// Only loads the data for a config section if that key exists in the JSON
+/datum/server_configuration/proc/safe_load(datum/configuration_section/CS, section)
+	if(!isnull(raw_data[section]))
+		CS.load_data(raw_data[section])
 
 /datum/configuration_section
 	/// See __config_defines.dm

code/controllers/configuration/sections/system_configuration.dm

@@ -18,6 +18,10 @@
 	var/_2fa_auth_host = null
 	/// List of IP addresses which bypass world topic rate limiting
 	var/list/topic_ip_ratelimit_bypass = list()
+	/// Server instance ID
+	var/instance_id = "paradise_main"
+	/// Server internal IP
+	var/internal_ip = "127.0.0.1"
 
 /datum/configuration_section/system_configuration/load_data(list/data)
 	// Use the load wrappers here. That way the default isnt made 'null' if you comment out the config line
@@ -31,3 +35,6 @@
 	CONFIG_LOAD_STR(_2fa_auth_host, data["_2fa_auth_host"])
 
 	CONFIG_LOAD_LIST(topic_ip_ratelimit_bypass, data["topic_ip_ratelimit_bypass"])
+
+	CONFIG_LOAD_STR(instance_id, data["instance_id"])
+	CONFIG_LOAD_STR(internal_ip, data["internal_ip"])

code/controllers/master.dm

@@ -62,7 +62,7 @@ GLOBAL_REAL(Master, /datum/controller/master) = new
 	/// For scheduling different subsystems for different stages of the round
 	var/current_runlevel
 	/// Do we want to sleep until players log in?
-	var/sleep_offline_after_initializations = TRUE
+	var/sleep_offline_after_initializations = FALSE // No we dont
 
 	var/static/restart_clear = 0
 	var/static/restart_timeout = 0

code/controllers/subsystem/blackbox.dm

@@ -287,8 +287,8 @@ SUBSYSTEM_DEF(blackbox)
 		lakey = L.lastattackerckey
 
 	var/datum/db_query/deathquery = SSdbcore.NewQuery({"
-		INSERT INTO death (name, byondkey, job, special, pod, tod, laname, lakey, gender, bruteloss, fireloss, brainloss, oxyloss, coord)
-		VALUES (:name, :key, :job, :special, :pod, NOW(), :laname, :lakey, :gender, :bruteloss, :fireloss, :brainloss, :oxyloss, :coord)"},
+		INSERT INTO death (name, byondkey, job, special, pod, tod, laname, lakey, gender, bruteloss, fireloss, brainloss, oxyloss, coord, server_id)
+		VALUES (:name, :key, :job, :special, :pod, NOW(), :laname, :lakey, :gender, :bruteloss, :fireloss, :brainloss, :oxyloss, :coord, :server_id)"},
 		list(
 			"name" = L.real_name,
 			"key" = L.key,
@@ -302,7 +302,8 @@ SUBSYSTEM_DEF(blackbox)
 			"fireloss" = L.getFireLoss(),
 			"brainloss" = L.getBrainLoss(),
 			"oxyloss" = L.getOxyLoss(),
-			"coord" = "[L.x], [L.y], [L.z]"
+			"coord" = "[L.x], [L.y], [L.z]",
+			"server_id" = GLOB.configuration.system.instance_id
 		)
 	)
 	deathquery.warn_execute()

code/controllers/subsystem/dbcore.dm

@@ -160,8 +160,8 @@ SUBSYSTEM_DEF(dbcore)
 	if(!IsConnected())
 		return
 	var/datum/db_query/query_round_initialize = SSdbcore.NewQuery(
-		"INSERT INTO round (initialize_datetime, server_ip, server_port) VALUES (Now(), INET_ATON(:internet_address), :port)",
-		list("internet_address" = world.internet_address || "0", "port" = "[world.port]")
+		"INSERT INTO round (initialize_datetime, server_ip, server_port, server_id) VALUES (Now(), INET_ATON(:internet_address), :port, :server_id)",
+		list("internet_address" = world.internet_address || "0", "port" = "[world.port]", "server_id" = GLOB.configuration.system.instance_id)
 	)
 	query_round_initialize.Execute(async = FALSE)
 	GLOB.round_id = "[query_round_initialize.last_insert_id]"

code/controllers/subsystem/instancing.dm

@@ -0,0 +1,184 @@
+SUBSYSTEM_DEF(instancing)
+	name = "Instancing"
+	runlevels = RUNLEVEL_INIT | RUNLEVEL_LOBBY | RUNLEVEL_SETUP | RUNLEVEL_GAME | RUNLEVEL_POSTGAME
+	wait = 30 SECONDS
+	flags = SS_KEEP_TIMING
+
+/datum/controller/subsystem/instancing/Initialize(start_timeofday)
+	// Dont even bother if we arent connected
+	if(!SSdbcore.IsConnected())
+		flags |= SS_NO_FIRE
+		return ..()
+	update_heartbeat() // Make sure you do this before announcing to peers, or no one will hear your announcement
+	var/startup_msg = "The server <code>[GLOB.configuration.general.server_name]</code> is now starting up. The map is [SSmapping.map_datum.fluff_name] ([SSmapping.map_datum.technical_name]). You can connect with the <code>Switch Server</code> verb."
+	message_all_peers(startup_msg)
+	return ..()
+
+/datum/controller/subsystem/instancing/fire(resumed)
+	update_heartbeat()
+	update_playercache()
+
+/**
+  * Playercache updater
+  *
+  * Updates the player cache in the DB. Different from heartbeat so we can force invoke it on player operations
+  */
+/datum/controller/subsystem/instancing/proc/update_playercache(optional_ckey)
+	// You may be wondering, why the fuck is an "optional ckey" variable here
+	// Well, this is invoked in client/New(), and needs to read from GLOB.clients
+	// However, this proc sleeps, and if you sleep during client/New() once the client is in GLOB.clients, stuff breaks bad
+	// (See my comment rambling in client/New())
+	// By passing the ckey through, we can sleep in this proc and still get the data
+	if(!SSdbcore.IsConnected())
+		return
+	// First iterate clients to get ckeys
+	var/list/ckeys = list()
+	for(var/client/C in GLOB.clients) // No code review. I am not doing the `as anything` bullshit, because we *do* need the type checks here to avoid null clients which do happen sometimes
+		ckeys += C.ckey
+	// Add our optional
+	if(optional_ckey)
+		ckeys += optional_ckey
+	// Note: We dont have to sort the list here. The only time this is read for is a search,
+	// and order doesnt matter for that.
+	var/ckey_json = json_encode(ckeys)
+
+	// Yes I care about performance savings this much here to mass execute this shit
+	var/list/datum/db_query/queries = list()
+	queries += SSdbcore.NewQuery("UPDATE instance_data_cache SET key_value=:json WHERE key_name='playerlist' AND server_id=:sid", list(
+		"json" = ckey_json,
+		"sid" = GLOB.configuration.system.instance_id
+	))
+	queries += SSdbcore.NewQuery("UPDATE instance_data_cache SET key_value=:count WHERE key_name='playercount' AND server_id=:sid", list(
+		"count" = length(ckeys),
+		"sid" = GLOB.configuration.system.instance_id
+	))
+
+	SSdbcore.MassExecute(queries, TRUE, TRUE, FALSE, FALSE)
+
+/**
+  * Heartbeat updater
+  *
+  * Updates the heartbeat in the DB. Used so other servers can see when this one was alive
+  */
+/datum/controller/subsystem/instancing/proc/update_heartbeat()
+	// this could probably just go in fire() but clean code and profiler ease who cares
+	var/datum/db_query/dbq = SSdbcore.NewQuery("UPDATE instance_data_cache SET key_value=NOW() WHERE key_name='heartbeat' AND server_id=:sid", list(
+		"sid" = GLOB.configuration.system.instance_id
+	))
+	dbq.warn_execute()
+	qdel(dbq)
+
+/**
+  * Seed data
+  *
+  * Seeds all our data into the DB for other servers to discover from.
+  * This is called during world/New() instead of on initialize so it can be done *instantly*
+  */
+/datum/controller/subsystem/instancing/proc/seed_data()
+	// We need to seed a lot of keys, so lets just use a key-value-pair-map to do this easily
+	var/list/kvp_map = list()
+	kvp_map["server_name"] = GLOB.configuration.general.server_name // Name of the server
+	kvp_map["server_port"] = world.port // Server port (used for redirection and topics)
+	kvp_map["topic_key"] = GLOB.configuration.system.topic_key // Server topic key (used for topics)
+	kvp_map["internal_ip"] = GLOB.configuration.system.internal_ip // Server internal IP (used for topics)
+	kvp_map["playercount"] = length(GLOB.clients) // Server client count (used for status info)
+	kvp_map["playerlist"] = json_encode(list()) // Server client list. Used for dupe login checks. This gets filled in later
+	kvp_map["heartbeat"] = SQLtime() // SQL timestamp for heartbeat purposes. Any server without a heartbeat in the last 60 seconds can be considered dead
+	// Also note for above. You may say "But AA you dont need to JSON encode it, just use "\[]"."
+	// Well to that I say, no. This is meant to be JSON regardless, and it should represent that. This proc is ran once during world/New()
+	// An extra nanosecond of load will make zero difference.
+
+	for(var/key in kvp_map)
+		var/datum/db_query/dbq = SSdbcore.NewQuery("INSERT INTO instance_data_cache (server_id, key_name, key_value) VALUES (:sid, :kn, :kv) ON DUPLICATE KEY UPDATE key_value=:kv2", // Is this necessary? Who knows!
+			list(
+				"sid" = GLOB.configuration.system.instance_id,
+				"kn" = key,
+				"kv" = "[kvp_map[key]]", // String encoding IS necessary since these tables use strings, not ints
+				"kv2" = "[kvp_map[key]]", // Dont know if I need the second but better to be safe
+			)
+		)
+		dbq.warn_execute(FALSE) // Do NOT async execute here because world/New() shouldnt sleep. EVER. You get issues if you do.
+		qdel(dbq)
+
+
+/**
+  * Message all peers
+  *
+  * Wrapper for [topic_all_peers] to format the input into a message topic. Will send a server-wide announcement to the other servers
+  *
+  * Arguments:
+  * * message - Message to send to the other servers
+  */
+/datum/controller/subsystem/instancing/proc/message_all_peers(message)
+	if(!SSdbcore.IsConnected())
+		return
+	var/topic_string = "instance_announce&msg=[url_encode(message)]"
+	topic_all_peers(topic_string)
+
+/**
+  * Sends a topic to all peers
+  *
+  * Sends a raw topic to the other servers. WILL APPEND &key=[commskey] ON THE END. PLEASE ACCOUNT FOR THIS.
+  *
+  * Arguments:
+  * * raw_topic - The raw topic to send to the other servers
+  */
+/datum/controller/subsystem/instancing/proc/topic_all_peers(raw_topic)
+	// Someone here is going to say "AA you shouldnt put load on the DB server you can do sorting in BYOND"
+	// Well let me put it this way. The DB server is an entirely different machine to BYOND, with this entire dataset being stored in its RAM, not even on disk
+	// By making the DB server do the work, we can offload from BYOND, which is already strained
+	var/datum/db_query/dbq1 = SSdbcore.NewQuery({"
+		SELECT server_id, key_name, key_value FROM instance_data_cache WHERE server_id IN
+		(SELECT server_id FROM instance_data_cache WHERE server_id !=:sid AND
+		key_name='heartbeat' AND last_updated BETWEEN NOW() - INTERVAL 60 SECOND AND NOW())
+		AND key_name IN ("topic_key", "internal_ip", "server_port")"}, list(
+		"sid" = GLOB.configuration.system.instance_id
+	))
+	if(!dbq1.warn_execute())
+		qdel(dbq1)
+		return
+
+	var/servers_outer = list()
+	while(dbq1.NextRow())
+		if(!servers_outer[dbq1.item[1]])
+			servers_outer[dbq1.item[1]] = list()
+
+		servers_outer[dbq1.item[1]][dbq1.item[2]] = dbq1.item[3] // This should assoc load our data
+
+	qdel(dbq1)
+
+	for(var/server in servers_outer)
+		var/server_data = servers_outer[server]
+		world.Export("byond://[server_data["internal_ip"]]:[server_data["server_port"]]?[raw_topic]&key=[server_data["topic_key"]]")
+
+
+/**
+  * Player checker
+  *
+  * Check all connected peers to see if a player exists in the player cache
+  * This is used to make sure players dont log into 2 servers at once
+  * Arguments:
+  * ckey - The ckey to check if they are logged into another server
+  */
+/datum/controller/subsystem/instancing/proc/check_player(ckey)
+	// Please see above rant on L127
+	var/datum/db_query/dbq1 = SSdbcore.NewQuery({"
+		SELECT server_id, key_value FROM instance_data_cache WHERE server_id IN
+		(SELECT server_id FROM instance_data_cache WHERE server_id != :sid AND
+		key_name='heartbeat' AND last_updated BETWEEN NOW() - INTERVAL 60 SECOND AND NOW())
+		AND key_name IN ("playerlist")"}, list(
+		"sid" = GLOB.configuration.system.instance_id
+	))
+	if(!dbq1.warn_execute())
+		qdel(dbq1)
+		return
+
+	while(dbq1.NextRow())
+		var/list/other_server_cache = json_decode(dbq1.item[2])
+		if(ckey in other_server_cache)
+			var/target_server = dbq1.item[1] // Yes. This var is necessary.
+			qdel(dbq1)
+			return target_server
+
+	qdel(dbq1)
+	return null // If we are here, it means we didnt find our player on another server

code/controllers/subsystem/metrics.dm

@@ -28,6 +28,7 @@ SUBSYSTEM_DEF(metrics)
 	out["elapsed_real"] = (REALTIMEOFDAY - world_init_time)
 	out["client_count"] = length(GLOB.clients)
 	out["round_id"] = text2num(GLOB.round_id) // This is so we can filter the metrics by a single round ID
+	out["server_id"] = GLOB.configuration.system.instance_id // And this is so we can filter by instance ID
 
 	// Funnel in all SS metrics
 	var/list/ss_data = list()

code/controllers/subsystem/statistics.dm

@@ -18,10 +18,11 @@ SUBSYSTEM_DEF(statistics)
 		return
 	else
 		var/datum/db_query/statquery = SSdbcore.NewQuery(
-			"INSERT INTO legacy_population (playercount, admincount, time) VALUES (:playercount, :admincount, NOW())",
+			"INSERT INTO legacy_population (playercount, admincount, time, server_id) VALUES (:playercount, :admincount, NOW(), :server_id)",
 			list(
 				"playercount" = length(GLOB.clients),
-				"admincount" = length(GLOB.admins)
+				"admincount" = length(GLOB.admins),
+				"server_id" = GLOB.configuration.system.instance_id
 			)
 		)
 		statquery.warn_execute()

code/datums/peer_server.dm

@@ -0,0 +1,32 @@
+/**
+  * # Peer server
+  *
+  * This datum is essentially a model class to represent another instance of Paracode running
+  *
+  * It contains communications passes, an internal IP to communicate to, an external IP for players to connect to
+  *
+  */
+/datum/peer_server
+	// The following options will be loaded from configuration
+	/// Peer server internal IP, used for communications
+	var/internal_ip
+	/// Port of the server
+	var/server_port = 0
+	/// Comms key for this server
+	var/commskey
+
+	// The following options will be pulled from the peer at runtime
+	/// Peer server external IP, used for routing players around
+	var/external_ip
+	/// Peer server ID, used internally. Pulled from the peer on startup.
+	var/server_id
+	/// Peer server name, presented to players. Pulled from the peer on startup.
+	var/server_name
+	/// Have we done initial data discovery yet?
+	var/discovered = FALSE
+	/// Is the peer server online?
+	var/online = FALSE
+	/// Playercount of the peer server on last ping
+	var/playercount
+	/// Last world.time that an operation was attempted
+	var/last_operation_time = 0

code/game/verbs/switch_server.dm

@@ -0,0 +1,49 @@
+/client/verb/switch_server()
+	set name = "Switch Server"
+	set desc = "Switch to a different Paradise server"
+	set category = "OOC"
+
+	// First get our peers
+	var/datum/db_query/dbq1 = SSdbcore.NewQuery({"
+		SELECT server_id, key_name, key_value FROM instance_data_cache WHERE server_id IN
+		(SELECT server_id FROM instance_data_cache WHERE
+		key_name='heartbeat' AND last_updated BETWEEN NOW() - INTERVAL 60 SECOND AND NOW())
+		AND key_name IN ("playercount", "server_port", "server_name")"})
+	if(!dbq1.warn_execute())
+		qdel(dbq1)
+		return
+
+	var/servers_outer = list()
+	while(dbq1.NextRow())
+		if(!servers_outer[dbq1.item[1]])
+			servers_outer[dbq1.item[1]] = list()
+
+		servers_outer[dbq1.item[1]][dbq1.item[2]] = dbq1.item[3] // This should assoc load our data
+
+	// Format the server names into an assoc list of K: name V: port
+	var/list/formatted_servers = list()
+	for(var/server in servers_outer)
+		var/server_data = servers_outer[server]
+		formatted_servers["[server_data["server_name"]] - ([server_data["playercount"]] playing)"] = text2num(server_data["server_port"])
+
+	if(length(formatted_servers) == 1)
+		to_chat(usr, "<span class='warning'>You are already connected to the one online instance!</span>")
+		return
+
+	var/selected_server = input(usr, "Select a server", "Server hop") as anything in formatted_servers
+	if(!selected_server)
+		return // Should never happen
+
+	if(formatted_servers[selected_server] == world.port)
+		to_chat(usr, "<span class='warning'>You are already connected to this instance!</span>")
+		return
+
+	// Now we reconnect them
+	to_chat(usr, "<span class='notice'>Now connecting you to: <b>[selected_server]</b></span>")
+
+	if(watchlisted) // I mean why not
+		message_admins("[key_name_admin(usr)] is on the watchlist and just jumped to [selected_server]")
+
+	// Formulate a connection URL
+	var/target = "byond://[world.internet_address]:[formatted_servers[selected_server]]"
+	src << link(target)

code/game/world.dm

@@ -22,11 +22,15 @@ GLOBAL_LIST_INIT(map_transition_config, list(CC_TRANSITION_CONFIG))
 
 	//temporary file used to record errors with loading config and the database, moved to log directory once logging is set up
 	GLOB.config_error_log = GLOB.world_game_log = GLOB.world_runtime_log = GLOB.sql_log = "data/logs/config_error.log"
-	GLOB.configuration.load_configuration()
+	GLOB.configuration.load_configuration() // Load up the base config.toml
+	// Load up overrides for this specific instance, based on port
+	// If this instance is listening on port 6666, the server will look for config/overrides_6666.toml
+	GLOB.configuration.load_overrides()
 
 	// Right off the bat, load up the DB
 	SSdbcore.CheckSchemaVersion() // This doesnt just check the schema version, it also connects to the db! This needs to happen super early! I cannot stress this enough!
 	SSdbcore.SetRoundID() // Set the round ID here
+	SSinstancing.seed_data() // Set us up in the DB
 
 	// Setup all log paths and stamp them with startups, including round IDs
 	SetupLogs()

code/modules/admin/IsBanned.dm

@@ -27,6 +27,12 @@
 		if(A && (A.rights & R_ADMIN))
 			admin = 1
 
+	// Lets see if they are logged in on another paradise server
+	if(SSdbcore.IsConnected())
+		var/other_server_login = SSinstancing.check_player(ckey)
+		if(other_server_login)
+			return list("reason"="duplicate login", "desc"="\nReason: You are already logged in on server '[other_server_login]'. Please contact the server host if you believe this is an error.")
+
 	//Guest Checking
 	if(GLOB.configuration.general.guest_ban && IsGuestKey(key))
 		log_adminwarn("Failed Login: [key] [computer_id] [address] - Guests not allowed")

code/modules/admin/admin_verbs.dm

@@ -45,6 +45,7 @@ GLOBAL_LIST_INIT(admin_verbs_admin, list(
 	/datum/admins/proc/toggleemoji,     /*toggles using emoji in ooc for everyone*/
 	/client/proc/game_panel,			/*game panel, allows to change game-mode etc*/
 	/client/proc/cmd_admin_say,			/*admin-only ooc chat*/
+	/client/proc/gsay,					/*cross-server asay*/
 	/datum/admins/proc/PlayerNotes,
 	/client/proc/cmd_mentor_say,
 	/datum/admins/proc/show_player_notes,
@@ -67,7 +68,8 @@ GLOBAL_LIST_INIT(admin_verbs_admin, list(
 	/client/proc/toggle_mentor_chat,
 	/client/proc/toggle_advanced_interaction, /*toggle admin ability to interact with not only machines, but also atoms such as buttons and doors*/
 	/client/proc/list_ssds_afks,
-	/client/proc/ccbdb_lookup_ckey
+	/client/proc/ccbdb_lookup_ckey,
+	/client/proc/view_instances
 ))
 GLOBAL_LIST_INIT(admin_verbs_ban, list(
 	/client/proc/ban_panel,

code/modules/admin/db_ban/functions.dm

@@ -146,8 +146,8 @@
 		qdel(adm_query)
 
 	var/datum/db_query/query_insert = SSdbcore.NewQuery({"
-		INSERT INTO ban (`id`,`bantime`,`serverip`,`bantype`,`reason`,`job`,`duration`,`rounds`,`expiration_time`,`ckey`,`computerid`,`ip`,`a_ckey`,`a_computerid`,`a_ip`,`who`,`adminwho`,`edits`,`unbanned`,`unbanned_datetime`,`unbanned_ckey`,`unbanned_computerid`,`unbanned_ip`,`ban_round_id`,`unbanned_round_id`)
-		VALUES (null, Now(), :serverip, :bantype_str, :reason, :job, :duration, :rounds, Now() + INTERVAL :duration MINUTE, :ckey, :computerid, :ip, :a_ckey, :a_computerid, :a_ip, :who, :adminwho, '', null, null, null, null, null, :roundid, null)
+		INSERT INTO ban (`id`,`bantime`,`serverip`,`bantype`,`reason`,`job`,`duration`,`rounds`,`expiration_time`,`ckey`,`computerid`,`ip`,`a_ckey`,`a_computerid`,`a_ip`,`who`,`adminwho`,`edits`,`unbanned`,`unbanned_datetime`,`unbanned_ckey`,`unbanned_computerid`,`unbanned_ip`,`ban_round_id`,`unbanned_round_id`, `server_id`)
+		VALUES (null, Now(), :serverip, :bantype_str, :reason, :job, :duration, :rounds, Now() + INTERVAL :duration MINUTE, :ckey, :computerid, :ip, :a_ckey, :a_computerid, :a_ip, :who, :adminwho, '', null, null, null, null, null, :roundid, null, :server_id)
 	"}, list(
 		// Get ready for parameters
 		"serverip" = serverip,
@@ -164,7 +164,8 @@
 		"a_ip" = a_ip,
 		"who" = who,
 		"adminwho" = adminwho,
-		"roundid" = GLOB.round_id
+		"roundid" = GLOB.round_id,
+		"server_id" = GLOB.configuration.system.instance_id
 	))
 	if(!query_insert.warn_execute())
 		qdel(query_insert)

code/modules/admin/sql_notes.dm

@@ -1,5 +1,4 @@
-// Do not attemtp to remove the blank string from the server arg. It will break DB saving.
-/proc/add_note(target_ckey, notetext, timestamp, adminckey, logged = 1, server = "", checkrights = 1, show_after = TRUE, automated = FALSE)
+/proc/add_note(target_ckey, notetext, timestamp, adminckey, logged = 1, checkrights = 1, show_after = TRUE, automated = FALSE)
 	if(checkrights && !check_rights(R_ADMIN|R_MOD))
 		return
 	if(!SSdbcore.IsConnected())
@@ -53,10 +52,6 @@
 	else if(usr && (usr.ckey == ckey(adminckey))) // Don't ckeyize special note sources
 		adminckey = ckey(adminckey)
 
-	if(!server)
-		if(GLOB.configuration.general.server_name)
-			server = GLOB.configuration.general.server_name
-
 	// Force cast this to 1/0 incase someone tries to feed bad data
 	automated = !!automated
 
@@ -67,7 +62,7 @@
 		"targetckey" = target_ckey,
 		"notetext" = notetext,
 		"adminkey" = adminckey,
-		"server" = server,
+		"server" = GLOB.configuration.system.instance_id,
 		"crewnum" = crew_number,
 		"roundid" = GLOB.round_id,
 		"automated" = automated

code/modules/admin/verbs/gsay.dm

@@ -0,0 +1,27 @@
+// GSAY
+// Like asay but global between instances!
+// *Insert changeling hivemind :g joke here*
+
+/client/proc/gsay(msg as text)
+	set name = "gsay"
+	set hidden = TRUE
+	if(!check_rights(R_ADMIN))
+		return
+
+	if(!msg)
+		return
+	msg = sanitize(copytext(msg, 1, MAX_MESSAGE_LEN))
+
+	// To whoever says "Why dont you just topic the full message with formatting"
+	// This lets us use this in other apps, like a discord bot, without HTML parsing
+	// It also removes a way to put whatever HTML we want in the chat window
+	var/built_topic = "gsay&msg=[url_encode(msg)]&usr=[url_encode(usr.ckey)]&src=[url_encode(GLOB.configuration.system.instance_id)]"
+
+	// Send to peers
+	SSinstancing.topic_all_peers(built_topic)
+	// Send to online admins
+	for(var/client/C in GLOB.admins)
+		if(R_ADMIN & C.holder.rights)
+			to_chat(C, "<span class='admin_channel'>GSAY: [usr.ckey]@[GLOB.configuration.system.instance_id]: [msg]</span>")
+
+	SSblackbox.record_feedback("tally", "admin_verb", 1, "gsay") //If you are copy-pasting this, ensure the 2nd parameter is unique to the new proc!

code/modules/admin/verbs/view_instances.dm

@@ -0,0 +1,33 @@
+/client/proc/view_instances()
+	set name = "View Server Instances"
+	set desc = "View the running server instances"
+	set category = "Server"
+
+	if(!check_rights(R_ADMIN))
+		return
+
+	to_chat(usr, "<b>Server instances info</b>")
+	var/datum/db_query/dbq1 = SSdbcore.NewQuery({"
+		SELECT server_id, key_name, key_value FROM instance_data_cache WHERE server_id IN
+		(SELECT server_id FROM instance_data_cache WHERE
+		key_name='heartbeat' AND last_updated BETWEEN NOW() - INTERVAL 60 SECOND AND NOW())
+		AND key_name IN ("playercount")"})
+	if(!dbq1.warn_execute())
+		qdel(dbq1)
+		return
+
+	var/servers_outer = list()
+	while(dbq1.NextRow())
+		if(!servers_outer[dbq1.item[1]])
+			servers_outer[dbq1.item[1]] = list()
+
+		servers_outer[dbq1.item[1]][dbq1.item[2]] = dbq1.item[3] // This should assoc load our data
+
+	qdel(dbq1)
+
+	for(var/server in servers_outer)
+		var/server_data = servers_outer[server]
+		var/players = text2num(server_data["playercount"])
+
+		to_chat(usr, "<code>[server]</code> - [players] player[players == 1 ? "" : "s"] online.")
+	to_chat(usr, "<i>Offline instances are not reported</i>")

code/modules/client/client_procs.dm

@@ -327,12 +327,16 @@
 	if(length(related_accounts_cid))
 		log_admin("[key_name(src)] Alts by CID: [jointext(related_accounts_cid, " ")]")
 
+	// This sleeps so it has to go here. Dont fucking move it.
+	SSinstancing.update_playercache(ckey)
+
 	// This has to go here to avoid issues
 	// If you sleep past this point, you will get SSinput errors as well as goonchat errors
 	// DO NOT STUFF RANDOM SQL QUERIES BELOW THIS POINT WITHOUT USING `INVOKE_ASYNC()` OR SIMILAR
 	// YOU WILL BREAK STUFF. SERIOUSLY. -aa07
 	GLOB.clients += src
 
+
 	spawn() // Goonchat does some non-instant checks in start()
 		chatOutput.start()
 
@@ -448,6 +452,7 @@
 		GLOB.admins -= src
 	GLOB.directory -= ckey
 	GLOB.clients -= src
+	SSinstancing.update_playercache() // Clear us out
 	QDEL_NULL(chatOutput)
 	QDEL_NULL(pai_save)
 	if(movingmob)

code/modules/karma/karma.dm

@@ -16,15 +16,16 @@
 		return
 
 	var/datum/db_query/log_query = SSdbcore.NewQuery({"
-		INSERT INTO karma (spendername, spenderkey, receivername, receiverkey, receiverrole, receiverspecial, spenderip, time)
-		VALUES (:sname, :skey, :rname, :rkey, :rrole, :rspecial, :sip, Now())"}, list(
+		INSERT INTO karma (spendername, spenderkey, receivername, receiverkey, receiverrole, receiverspecial, spenderip, time, server_id)
+		VALUES (:sname, :skey, :rname, :rkey, :rrole, :rspecial, :sip, Now(), :server_id)"}, list(
 			"sname" = spender.name,
 			"skey" = spender.ckey,
 			"rname" = receiver.name,
 			"rkey" = receiver.ckey,
 			"rrole" = receiverrole,
 			"rspecial" = receiverspecial,
-			"sip" = spender.client.address
+			"sip" = spender.client.address,
+			"server_id" = GLOB.configuration.system.instance_id
 		))
 
 	if(!log_query.warn_execute())

code/modules/world_topic/gsay.dm

@@ -0,0 +1,18 @@
+// Just dumps the text in the admin chat box
+/datum/world_topic_handler/gsay
+	topic_key = "gsay"
+	requires_commskey = TRUE
+
+/datum/world_topic_handler/gsay/execute(list/input, key_valid)
+	if(!input["msg"] || !input["usr"] || !input["src"])
+		return json_encode(list("error" = "Malformed request"))
+
+	var/message = input["msg"]
+	var/user = input["usr"]
+	var/source = input["src"]
+
+	// Send to online admins
+	for(var/client/C in GLOB.admins)
+		if(R_ADMIN & C.holder.rights)
+			to_chat(C, "<span class='admin_channel'>GSAY: [user]@[source]: [message]</span>")
+

code/modules/world_topic/instance_announce.dm

@@ -0,0 +1,8 @@
+/datum/world_topic_handler/instance_announce
+	topic_key = "instance_announce"
+	requires_commskey = TRUE
+
+/datum/world_topic_handler/instance_announce/execute(list/input, key_valid)
+	var/msg = input["msg"]
+	to_chat(world, "<center><span class='boldannounce'><big>Attention</big></span></center><hr>[msg]<hr>")
+	SEND_SOUND(world, sound('sound/misc/notice2.ogg')) // Same as captains priority announce

config/example/config.toml

@@ -21,6 +21,7 @@
 # - gamemode_configuration
 # - gateway_configuration
 # - general_configuration
+# - instancing_configuration
 # - ipintel_configuration
 # - job_configuration
 # - logging_configuration
@@ -141,7 +142,7 @@ ipc_screens = [
 # Enable/disable the database on a whole
 sql_enabled = false
 # SQL version. If this is a mismatch, round start will be delayed
-sql_version = 26
+sql_version = 27
 # SQL server address. Can be an IP or DNS name
 sql_address = "127.0.0.1"
 # SQL server port
@@ -724,6 +725,12 @@ shutdown_on_reboot = false
 topic_ip_ratelimit_bypass = ["127.0.0.1"]
 # Turn this to true if you are running a production server
 is_production = false
+# Server instance ID. This is used for tagging the server in the database
+# You do NOT want to change this once you are running in production
+instance_id = "paradise_main"
+# Server internal IP. Used if you are splitting instances over multiple internal IPs.
+# In most cases this is just 127.0.0.1
+internal_ip = "127.0.0.1"
 
 
 ################################################################

paradise.dme

@@ -230,6 +230,7 @@
 #include "code\controllers\subsystem\icon_smooth.dm"
 #include "code\controllers\subsystem\idlenpcpool.dm"
 #include "code\controllers\subsystem\input.dm"
+#include "code\controllers\subsystem\instancing.dm"
 #include "code\controllers\subsystem\ipintel.dm"
 #include "code\controllers\subsystem\jobs.dm"
 #include "code\controllers\subsystem\late_mapping.dm"
@@ -293,6 +294,7 @@
 #include "code\datums\mind.dm"
 #include "code\datums\mixed.dm"
 #include "code\datums\mutable_appearance.dm"
+#include "code\datums\peer_server.dm"
 #include "code\datums\periodic_news.dm"
 #include "code\datums\pipe_datums.dm"
 #include "code\datums\progressbar.dm"
@@ -1162,6 +1164,7 @@
 #include "code\game\verbs\ooc.dm"
 #include "code\game\verbs\randomtip.dm"
 #include "code\game\verbs\suicide.dm"
+#include "code\game\verbs\switch_server.dm"
 #include "code\game\verbs\webmap.dm"
 #include "code\game\verbs\who.dm"
 #include "code\modules\admin\admin.dm"
@@ -1209,6 +1212,7 @@
 #include "code\modules\admin\verbs\freeze.dm"
 #include "code\modules\admin\verbs\getlogs.dm"
 #include "code\modules\admin\verbs\gimmick_team.dm"
+#include "code\modules\admin\verbs\gsay.dm"
 #include "code\modules\admin\verbs\infiltratorteam_syndicate.dm"
 #include "code\modules\admin\verbs\logging_view.dm"
 #include "code\modules\admin\verbs\manage_queue.dm"
@@ -1230,6 +1234,7 @@
 #include "code\modules\admin\verbs\ticklag.dm"
 #include "code\modules\admin\verbs\toggledebugverbs.dm"
 #include "code\modules\admin\verbs\tripAI.dm"
+#include "code\modules\admin\verbs\view_instances.dm"
 #include "code\modules\admin\verbs\vox_raiders.dm"
 #include "code\modules\admin\verbs\SDQL2\SDQL_2.dm"
 #include "code\modules\admin\verbs\SDQL2\SDQL_2_parser.dm"
@@ -2520,6 +2525,8 @@
 #include "code\modules\world_topic\_topic_base.dm"
 #include "code\modules\world_topic\adminmsg.dm"
 #include "code\modules\world_topic\announce.dm"
+#include "code\modules\world_topic\gsay.dm"
+#include "code\modules\world_topic\instance_announce.dm"
 #include "code\modules\world_topic\manifest.dm"
 #include "code\modules\world_topic\ping.dm"
 #include "code\modules\world_topic\players.dm"

tgui/packages/common/target/npmlist.json

@@ -0,0 +1 @@
+{"name":"common","version":"3.0.0"}