Change the word filter configuration to allow providing reasons, fix emotes not working in filters, and implement separate OOC/IC/PDA filters (#61606)

code/__DEFINES/chat_filter.dm

@@ -0,0 +1,9 @@
+/// The index of the word that was filtered in a is_*_filtered proc
+#define CHAT_FILTER_INDEX_WORD 1
+
+/// The index of the reason why a word was filtered in a is_*_filtered proc
+#define CHAT_FILTER_INDEX_REASON 2
+
+/// Given a chat filter result, will send a to_chat to the user telling them about why their message was blocked
+#define REPORT_CHAT_FILTER_TO_USER(user, filter_result) \
+	to_chat(user, span_warning("The word <b>[html_encode(filter_result[CHAT_FILTER_INDEX_WORD])]</b> is prohibited: [html_encode(filter_result[CHAT_FILTER_INDEX_REASON])]"))

code/__DEFINES/rust_g.dm

@@ -102,7 +102,9 @@
 #define rustg_sql_disconnect_pool(handle) call(RUST_G, "sql_disconnect_pool")(handle)
 #define rustg_sql_check_query(job_id) call(RUST_G, "sql_check_query")("[job_id]")
 
-#define rustg_url_encode(text) call(RUST_G, "url_encode")(text)
+#define rustg_read_toml_file(path) json_decode(call(RUST_G, "toml_file_to_json")(path) || "null")
+
+#define rustg_url_encode(text) call(RUST_G, "url_encode")("[text]")
 #define rustg_url_decode(text) call(RUST_G, "url_decode")(text)
 
 #ifdef RUSTG_OVERRIDE_BUILTINS

code/__DEFINES/say.dm

@@ -88,9 +88,6 @@
 #define MAX_BROADCAST_LEN 512
 #define MAX_CHARTER_LEN 80
 
-// Is something in the IC chat filter? This is config dependent.
-#define CHAT_FILTER_CHECK(T) (config.ic_filter_regex && findtext(T, config.ic_filter_regex))
-
 // Audio/Visual Flags. Used to determine what sense are required to notice a message.
 #define MSG_VISUAL (1<<0)
 #define MSG_AUDIBLE (1<<1)

code/__HELPERS/chat_filter.dm

@@ -0,0 +1,38 @@
+// [2] is the group index of the blocked term when it is not using word bounds.
+// This is sanity checked by unit tests.
+#define GET_MATCHED_GROUP(regex) (regex.group[2] || regex.match)
+
+/// Given a text, will return what word is on the IC filter, with the reason.
+/// Returns null if the message is OK.
+/proc/is_ic_filtered(message)
+	if (config.ic_filter_regex?.Find(message))
+		var/matched_group = GET_MATCHED_GROUP(config.ic_filter_regex)
+		return list(
+			matched_group,
+			config.ic_filter_reasons[matched_group] || config.ic_outside_pda_filter_reasons[matched_group] || config.shared_filter_reasons[matched_group],
+		)
+
+	return null
+
+/// Given a text, will return what word is on the IC filter, ignoring words allowed on the PDA, with the reason.
+/// Returns null if the message is OK.
+/proc/is_ic_filtered_for_pdas(message)
+	if (config.ic_outside_pda_filter_regex?.Find(message))
+		var/matched_group = GET_MATCHED_GROUP(config.ic_outside_pda_filter_regex)
+		return list(
+			matched_group,
+			config.ic_filter_reasons[matched_group] || config.shared_filter_reasons[matched_group],
+		)
+
+	return null
+
+/// Given a text, will return what word is on the OOC filter, with the reason.
+/// Returns null if the message is OK.
+/proc/is_ooc_filtered(message)
+	if (config.ooc_filter_regex?.Find(message))
+		var/matched_group = GET_MATCHED_GROUP(config.ooc_filter_regex)
+		return list(matched_group, config.shared_filter_reasons[matched_group])
+
+	return null
+
+#undef GET_MATCHED_GROUP

code/__HELPERS/text.dm

@@ -23,7 +23,7 @@
 
 ///returns nothing with an alert instead of the message if it contains something in the ic filter, and sanitizes normally if the name is fine. It returns nothing so it backs out of the input the same way as if you had entered nothing.
 /proc/sanitize_name(t,allow_numbers=FALSE)
-	if(CHAT_FILTER_CHECK(t))
+	if(is_ic_filtered(t))
 		tgui_alert(usr, "You cannot set a name that contains a word prohibited in IC chat!")
 		return ""
 	var/r = reject_bad_name(t,allow_numbers=allow_numbers,strict=TRUE)
@@ -221,7 +221,7 @@
 			return //(not case sensitive)
 
 	// Protects against names containing IC chat prohibited words.
-	if(CHAT_FILTER_CHECK(t_out))
+	if(is_ic_filtered(t_out))
 		return
 
 	return t_out

code/controllers/configuration/configuration.dm

@@ -24,8 +24,24 @@
 	/// If the configuration is loaded
 	var/loaded = FALSE
 
+	/// A regex that matches words blocked IC
 	var/static/regex/ic_filter_regex
 
+	/// A regex that matches words blocked OOC
+	var/static/regex/ooc_filter_regex
+
+	/// A regex that matches words blocked IC, but not in PDAs
+	var/static/regex/ic_outside_pda_filter_regex
+
+	/// An assoc list of blocked IC words to their reasons
+	var/static/list/ic_filter_reasons
+
+	/// An assoc list of words that are blocked IC, but not in PDAs, to their reasons
+	var/static/list/ic_outside_pda_filter_reasons
+
+	/// An assoc list of words that are blocked both IC and OOC to their reasons
+	var/static/list/shared_filter_reasons
+
 /datum/controller/configuration/proc/admin_reload()
 	if(IsAdminAdvancedProcCall())
 		return
@@ -338,17 +354,84 @@ Example config:
 				log_config("Unknown command in map vote config: '[command]'")
 
 /datum/controller/configuration/proc/LoadChatFilter()
-	var/list/in_character_filter = list()
-	if(!fexists("[directory]/in_character_filter.txt"))
+	if(!fexists("[directory]/word_filter.toml"))
+		load_legacy_chat_filter()
 		return
+
+	log_config("Loading config file word_filter.toml...")
+
+	var/list/word_filter = rustg_read_toml_file("[directory]/word_filter.toml")
+	if (!islist(word_filter))
+		var/message = "The word filter configuration did not output a list, contact someone with configuration access to make sure it's setup properly."
+		log_config(message)
+		DelayedMessageAdmins(message)
+		return
+
+	ic_filter_reasons = try_extract_from_word_filter(word_filter, "ic")
+	ic_outside_pda_filter_reasons = try_extract_from_word_filter(word_filter, "ic_outside_pda")
+	shared_filter_reasons = try_extract_from_word_filter(word_filter, "shared")
+
+	update_chat_filter_regexes()
+
+/datum/controller/configuration/proc/load_legacy_chat_filter()
+	if (!fexists("[directory]/in_character_filter.txt"))
+		return
+
 	log_config("Loading config file in_character_filter.txt...")
-	for(var/line in world.file2list("[directory]/in_character_filter.txt"))
-		if(!line)
+
+	ic_filter_reasons = list()
+	ic_outside_pda_filter_reasons = list()
+	shared_filter_reasons = list()
+
+	for (var/line in world.file2list("[directory]/in_character_filter.txt"))
+		if (!line)
 			continue
-		if(findtextEx(line,"#",1,2))
+		if (findtextEx(line, "#", 1, 2))
 			continue
-		in_character_filter += REGEX_QUOTE(line)
-	ic_filter_regex = in_character_filter.len ? regex("\\b([jointext(in_character_filter, "|")])\\b", "i") : null
+		// The older filter didn't apply to PDA
+		ic_outside_pda_filter_reasons[line] = "No reason available"
+
+	update_chat_filter_regexes()
+
+/// Will update the internal regexes of the chat filter based on the filter reasons
+/datum/controller/configuration/proc/update_chat_filter_regexes()
+	ic_filter_regex = compile_filter_regex(ic_filter_reasons + ic_outside_pda_filter_reasons + shared_filter_reasons)
+	ic_outside_pda_filter_regex = compile_filter_regex(ic_filter_reasons + shared_filter_reasons)
+	ooc_filter_regex = compile_filter_regex(shared_filter_reasons)
+
+/datum/controller/configuration/proc/try_extract_from_word_filter(list/word_filter, key)
+	var/list/banned_words = word_filter[key]
+
+	if (isnull(banned_words))
+		return list()
+	else if (!islist(banned_words))
+		var/message = "The word filter configuration's '[key]' key was invalid, contact someone with configuration access to make sure it's setup properly."
+		log_config(message)
+		DelayedMessageAdmins(message)
+		return list()
+
+	return banned_words
+
+/datum/controller/configuration/proc/compile_filter_regex(list/banned_words)
+	if (isnull(banned_words) || banned_words.len == 0)
+		return null
+
+	var/static/regex/should_join_on_word_bounds = regex(@"^\w+$")
+
+	// Stuff like emoticons needs another split, since there's no way to get ":)" on a word bound.
+	// Furthermore, normal words need to be on word bounds, so "(adminhelp)" gets filtered.
+	var/list/to_join_on_whitespace_splits = list()
+	var/list/to_join_on_word_bounds = list()
+
+	for (var/banned_word in banned_words)
+		if (findtext(banned_word, should_join_on_word_bounds))
+			to_join_on_word_bounds += REGEX_QUOTE(banned_word)
+		else
+			to_join_on_whitespace_splits += REGEX_QUOTE(banned_word)
+
+	var/whitespace_split = @"(?:(?:^|\s+)(" + jointext(to_join_on_whitespace_splits, "|") + @")(?:$|\s+))"
+	var/word_bounds = @"(\b(" + jointext(to_join_on_word_bounds, "|") + "))"
+	return regex("([whitespace_split]|[word_bounds])", "i")
 
 //Message admins when you can.
 /datum/controller/configuration/proc/DelayedMessageAdmins(text)

code/game/atoms.dm

@@ -1319,7 +1319,7 @@
 	if(href_list[VV_HK_AUTO_RENAME] && check_rights(R_VAREDIT))
 		var/newname = input(usr, "What do you want to rename this to?", "Automatic Rename") as null|text
 		// Check the new name against the chat filter. If it triggers the IC chat filter, give an option to confirm.
-		if(newname && !(CHAT_FILTER_CHECK(newname) && tgui_alert(usr, "Your selected name contains words restricted by IC chat filters. Confirm this new name?", "IC Chat Filter Conflict", list("Confirm", "Cancel")) != "Confirm"))
+		if(newname && !(is_ic_filtered(newname) && tgui_alert(usr, "Your selected name contains words restricted by IC chat filters. Confirm this new name?", "IC Chat Filter Conflict", list("Confirm", "Cancel")) != "Confirm"))
 			vv_auto_rename(newname)
 
 	if(href_list[VV_HK_EDIT_FILTERS] && check_rights(R_VAREDIT))

code/game/objects/items/AI_modules.dm

@@ -258,7 +258,7 @@ AI MODULES
 	var/targName = stripped_input(user, "Please enter a new law for the AI.", "Freeform Law Entry", laws[1], CONFIG_GET(number/max_law_len))
 	if(!targName)
 		return
-	if(CHAT_FILTER_CHECK(targName))
+	if(is_ic_filtered(targName))
 		to_chat(user, span_warning("Error: Law contains invalid text.")) // AI LAW 2 SAY U W U WITHOUT THE SPACES
 		return
 	laws[1] = targName
@@ -465,7 +465,7 @@ AI MODULES
 	var/targName = stripped_input(user, "Please enter a new core law for the AI.", "Freeform Law Entry", laws[1], CONFIG_GET(number/max_law_len))
 	if(!targName)
 		return
-	if(CHAT_FILTER_CHECK(targName))
+	if(is_ic_filtered(targName))
 		to_chat(user, span_warning("Error: Law contains invalid text."))
 		return
 	laws[1] = targName
@@ -487,7 +487,7 @@ AI MODULES
 	var/targName = stripped_input(user, "Please enter a new law for the AI.", "Freeform Law Entry", laws[1], CONFIG_GET(number/max_law_len))
 	if(!targName)
 		return
-	if(CHAT_FILTER_CHECK(targName)) // not even the syndicate can uwu
+	if(is_ic_filtered(targName)) // not even the syndicate can uwu
 		to_chat(user, span_warning("Error: Law contains invalid text."))
 		return
 	laws[1] = targName

code/game/objects/items/devices/PDA/PDA.dm

@@ -758,6 +758,12 @@ GLOBAL_LIST_EMPTY(PDAs)
 		return
 	if((last_text && world.time < last_text + 10) || (everyone && last_everyone && world.time < last_everyone + PDA_SPAM_DELAY))
 		return
+
+	var/list/filter_result = is_ic_filtered_for_pdas(message)
+	if (filter_result)
+		REPORT_CHAT_FILTER_TO_USER(user, filter_result)
+		return
+
 	if(prob(1))
 		message += "\nSent from my PDA"
 	// Send the signal

code/modules/admin/view_variables/topic.dm

@@ -32,7 +32,7 @@
 
 			// If the new name is something that would be restricted by IC chat filters,
 			// give the admin a warning but allow them to do it anyway if they want.
-			if(CHAT_FILTER_CHECK(new_name) && tgui_alert(usr, "Your selected name contains words restricted by IC chat filters. Confirm this new name?", "IC Chat Filter Conflict", list("Confirm", "Cancel")) == "Cancel")
+			if(is_ic_filtered(new_name) && tgui_alert(usr, "Your selected name contains words restricted by IC chat filters. Confirm this new name?", "IC Chat Filter Conflict", list("Confirm", "Cancel")) == "Cancel")
 				return
 
 			if( !new_name || !M )

code/modules/antagonists/cult/cult_comms.dm

@@ -20,8 +20,10 @@
 	var/input = stripped_input(usr, "Please choose a message to tell to the other acolytes.", "Voice of Blood", "")
 	if(!input || !IsAvailable())
 		return
-	if(CHAT_FILTER_CHECK(input))
-		to_chat(usr, span_warning("You cannot send a message that contains a word prohibited in IC chat!"))
+
+	var/list/filter_result = is_ic_filtered(input)
+	if(filter_result)
+		REPORT_CHAT_FILTER_TO_USER(usr, filter_result)
 		return
 	cultist_commune(usr, input)
 

code/modules/client/verbs/ooc.dm

@@ -31,6 +31,11 @@ GLOBAL_VAR_INIT(normal_ooc_colour, "#002eb8")
 	msg = copytext_char(sanitize(msg), 1, MAX_MESSAGE_LEN)
 	var/raw_msg = msg
 
+	var/list/filter_result = is_ooc_filtered(msg)
+	if (filter_result)
+		REPORT_CHAT_FILTER_TO_USER(usr, filter_result)
+		return
+
 	if(!msg)
 		return
 

code/modules/mob/living/living_say.dm

@@ -93,19 +93,21 @@ GLOBAL_LIST_INIT(message_modes_stat_limits, list(
 	return new_msg
 
 /mob/living/say(message, bubble_type,list/spans = list(), sanitize = TRUE, datum/language/language = null, ignore_spam = FALSE, forced = null)
-	var/ic_blocked = FALSE
-	if(client && !forced && CHAT_FILTER_CHECK(message))
+	var/list/filter_result
+	if(client && !forced)
 		//The filter doesn't act on the sanitized message, but the raw message.
-		ic_blocked = TRUE
+		filter_result = is_ic_filtered(message)
 
 	if(sanitize)
 		message = trim(copytext_char(sanitize(message), 1, MAX_MESSAGE_LEN))
 	if(!message || message == "")
 		return
 
-	if(ic_blocked)
+	if(filter_result)
 		//The filter warning message shows the sanitized message though.
-		to_chat(src, span_warning("That message contained a word prohibited in IC chat! Consider reviewing the server rules.\n<span replaceRegex='show_filtered_ic_chat'>\"[message]\"</span>"))
+		to_chat(src, span_warning("That message contained a word prohibited in IC chat! Consider reviewing the server rules."))
+		to_chat(src, span_warning("\"[message]\""))
+		REPORT_CHAT_FILTER_TO_USER(src, filter_result)
 		SSblackbox.record_feedback("tally", "ic_blocked_words", 1, lowertext(config.ic_filter_regex.match))
 		return
 	var/list/message_mods = list()

code/modules/unit_tests/_unit_tests.dm

@@ -50,6 +50,7 @@
 #include "breath.dm"
 #include "card_mismatch.dm"
 #include "chain_pull_through_space.dm"
+#include "chat_filter.dm"
 #include "combat.dm"
 #include "component_tests.dm"
 #include "connect_loc.dm"

code/modules/unit_tests/chat_filter.dm

@@ -0,0 +1,81 @@
+#define BLOCKED_IC "This message is not allowed IC, please use a different weird test phrase."
+#define BLOCKED_IC_OUTSIDE_PDA "Kirby dancing is strictly prohibited on this server."
+#define BLOCKED_SHARED "This message is not allowed anywhere, please use a different weird test phrase."
+
+/// Tests the sanity of the chat filter, ensuring it properly blocks words and gives the reason
+/datum/unit_test/chat_filter_sanity
+
+/datum/unit_test/chat_filter_sanity/Run()
+	// Update the chat filters to only have test phrases, just in case the toml is different
+	config.shared_filter_reasons = list("blockedinshared" = BLOCKED_SHARED)
+	config.ic_filter_reasons = list("blockedinic" = BLOCKED_IC)
+	config.ic_outside_pda_filter_reasons = list("<(0_0<)" = BLOCKED_IC_OUTSIDE_PDA)
+	config.update_chat_filter_regexes()
+
+	test_filter(
+		"this message is blockedinic, AND has a comma (which needs word bounds)",
+		"blockedinic",
+		BLOCKED_IC,
+		BLOCKED_IC,
+		null,
+	)
+
+	test_filter(
+		"<(0_0<) <(0_0)> (>0_0)> KIRBY DANCE!!!",
+		"<(0_0<)",
+		BLOCKED_IC_OUTSIDE_PDA,
+		null,
+		null,
+	)
+
+	test_filter(
+		"This message is blockedinshared, meaning it's banned EVERYWHERE",
+		"blockedinshared",
+		BLOCKED_SHARED,
+		BLOCKED_SHARED,
+		BLOCKED_SHARED,
+	)
+
+/datum/unit_test/chat_filter_sanity/proc/test_filter(
+	message,
+	blocked_word,
+	ic_filter_result,
+	pda_filter_result,
+	ooc_filter_result,
+)
+	var/ic_filter = is_ic_filtered(message)
+	var/pda_filter = is_ic_filtered_for_pdas(message)
+	var/ooc_filter = is_ooc_filtered(message)
+
+	test_filter_result("IC", message, ic_filter, ic_filter_result, blocked_word)
+	test_filter_result("PDA", message, pda_filter, pda_filter_result, blocked_word)
+	test_filter_result("OOC", message, ooc_filter, ooc_filter_result, blocked_word)
+
+/datum/unit_test/chat_filter_sanity/proc/test_filter_result(
+	filter_type,
+	message,
+	outcome,
+	expected_reason,
+	expected_blocked_word,
+)
+	if (isnull(outcome) && isnull(expected_reason))
+		return
+
+	if (isnull(outcome))
+		Fail("[message] was not blocked on the [filter_type] filter when it was expected to")
+		return
+
+	if (isnull(expected_reason))
+		Fail("[message] was blocked on the [filter_type] filter when it wasn't expected to: [json_encode(outcome)]")
+		return
+
+	if (outcome[CHAT_FILTER_INDEX_WORD] != expected_blocked_word)
+		Fail("[message] was blocked on the [filter_type] filter, but for a different word: \"[outcome[CHAT_FILTER_INDEX_WORD]]\" (instead of [expected_blocked_word])")
+		return
+
+	if (outcome[CHAT_FILTER_INDEX_REASON] != expected_reason)
+		Fail("[message] was blocked on the [filter_type] filter, but for a different reason: \"[outcome[CHAT_FILTER_INDEX_REASON]]\" (instead of [expected_reason])")
+
+#undef BLOCKED_IC
+#undef BLOCKED_IC_OUTSIDE_PDA
+#undef BLOCKED_SHARED

config/in_character_filter.txt

@@ -1,7 +0,0 @@
-###############################################################################################
-# Words that will block in character chat messages from sending.                              #
-# Case is not important. Commented-out examples are listed below, just remove the "#".        #
-###############################################################################################
-#lol
-#omg
-#wtf

config/word_filter.toml

@@ -0,0 +1,15 @@
+# This configuration file forms the in-game chat filter.
+# Values are stored as "word" = "reason for being banned".
+
+# Anything in here will be blocked from IC interactions, dead chat, and OOC
+[shared]
+"blockedinshared" = "This message is not allowed anywhere, please use a different weird test phrase."
+
+# Anything in here is blocked just from IC interactions, such as chat or names, but is allowed in dead chat and OOC
+[ic]
+"blockedinic" = "This message is not allowed IC, please use a different weird test phrase."
+
+# Anything in here is blocked from all IC interactions *except* PDAs.
+# On /tg/, netspeak is OK in PDAs.
+[ic_outside_pda]
+"<(0_0<)" = "Kirby dancing is strictly prohibited on this server."

dependencies.sh

@@ -8,7 +8,7 @@ export BYOND_MAJOR=514
 export BYOND_MINOR=1560
 
 #rust_g git tag
-export RUST_G_VERSION=0.4.10
+export RUST_G_VERSION=0.5.0
 
 #node version
 export NODE_VERSION=12

tgstation.dme

@@ -43,6 +43,7 @@
 #include "code\__DEFINES\callbacks.dm"
 #include "code\__DEFINES\cargo.dm"
 #include "code\__DEFINES\chat.dm"
+#include "code\__DEFINES\chat_filter.dm"
 #include "code\__DEFINES\cinematics.dm"
 #include "code\__DEFINES\cleaning.dm"
 #include "code\__DEFINES\colors.dm"
@@ -179,6 +180,7 @@
 #include "code\__HELPERS\atmospherics.dm"
 #include "code\__HELPERS\bitflag_lists.dm"
 #include "code\__HELPERS\chat.dm"
+#include "code\__HELPERS\chat_filter.dm"
 #include "code\__HELPERS\cmp.dm"
 #include "code\__HELPERS\colors.dm"
 #include "code\__HELPERS\config.dm"