TTS Improvements: Improved Audio Quality, Pitch Adjustment, Preference Silicon Voices, Per-Character Voice Disable Toggle, Tongue Voice Filters, Reworked Silicon and Vending Machine Filters (#76129)

## About The Pull Request


https://github.com/tgstation/tgstation/assets/4081722/5ca8e015-21f9-4159-9953-bc370152d01f

Improves the audio quality and speaker fidelity by implementing
Retrieval Voice Conversion as an intermediary layer, utilizing the
repository at https://github.com/ddPn08/rvc-webui.
Leverages RVC to allow players to set a pitch for their voice.


https://github.com/tgstation/tgstation/assets/4081722/0eb76ed7-ad67-4da2-9ceb-02605eea2c83

Makes silicons utilize a player's chosen voice preference on their
character slot, and adds a preview button to hear the voice as a silicon
on character creation.
Adds a toggle on character creation to disable having a voice on a
specific character slot.
Adds support for per-tongue voice filters.
Reworks the silicon voice effect to be a special effect done on the TTS
server level instead of via normal filters.
Reworks the vending machine effect to use the new robotic voicebox
effect.

## Why It's Good For The Game

Vastly improves the audio quality and speaker fidelity of our TTS
system.
Allows players to further customize their voice per character, naturally
pitching the voice up or down with cutting edge machine learning based
pitch adjustment.
Allows silicon players to have a consistent voice that's also audible
and understandable regardless of the voice or pitch of the speaker.
Improves vending machine audio quality.
Enhances the immersion of snail tongues and robotic voiceboxes.
Adjusts how Poly's pitch adjustment works based on if RVC is available
or not.
Allows players who feel that a voice doesn't fit their character to
disable having TTS on their specific character.
Provides server operators a way to disable specific voices in situations
with a shared voice server.

## Changelog

🆑 Iamgoofball, Nadare, ddPn08, Mangio621, the rest of the RVC dev
team
add: Improves the audio quality and speaker fidelity by implementing
Retrieval Voice Conversion as an intermediary layer, utilizing the
repository at https://github.com/ddPn08/rvc-webui.
add: Leverages RVC to allow players to set a pitch for their voice.
add: Makes silicons utilize a player's chosen voice preference on their
character slot, and adds a preview button to hear the voice as a silicon
on character creation.
add: Adds a toggle on character creation to disable having a voice on a
specific character slot.
add: Adds support for per-tongue voice filters.
add: Reworks the silicon voice effect to be a special effect done on the
TTS server level instead of via normal filters.
add: Reworks the vending machine effect to use the new robotic voicebox
effect.
/🆑

---------

Co-authored-by: Watermelon914 <37270891+Watermelon914@users.noreply.github.com>

code/controllers/configuration/entries/game_options.dm

@@ -425,4 +425,8 @@
 	default = 4
 	min_val = 1
 
+/datum/config_entry/str_list/tts_voice_blacklist
+
+/datum/config_entry/flag/tts_allow_player_voice_disabling
+
 /datum/config_entry/flag/give_tutorials_without_db

code/controllers/subsystem/tts.dm

@@ -25,6 +25,8 @@ SUBSYSTEM_DEF(tts)
 
 	/// Whether TTS is enabled or not
 	var/tts_enabled = FALSE
+	/// Whether the TTS engine supports pitch adjustment or not.
+	var/pitch_enabled = FALSE
 
 	/// TTS messages won't play if requests took longer than this duration of time.
 	var/message_timeout = 7 SECONDS
@@ -65,6 +67,25 @@ SUBSYSTEM_DEF(tts)
 		return FALSE
 	available_speakers = json_decode(response.body)
 	tts_enabled = TRUE
+	if(CONFIG_GET(str_list/tts_voice_blacklist))
+		var/list/blacklisted_voices = CONFIG_GET(str_list/tts_voice_blacklist)
+		log_config("Processing the TTS voice blacklist.")
+		for(var/voice in blacklisted_voices)
+			if(available_speakers.Find(voice))
+				log_config("Removed speaker [voice] from the TTS voice pool per config.")
+				available_speakers.Remove(voice)
+	var/datum/http_request/request_pitch = new()
+	var/list/headers_pitch = list()
+	headers_pitch["Authorization"] = CONFIG_GET(string/tts_http_token)
+	request_pitch.prepare(RUSTG_HTTP_METHOD_GET, "[CONFIG_GET(string/tts_http_url)]/pitch-available", "", headers_pitch)
+	request_pitch.begin_async()
+	UNTIL(request_pitch.is_complete())
+	pitch_enabled = TRUE
+	var/datum/http_response/response_pitch = request_pitch.into_response()
+	if(response_pitch.errored || response_pitch.status_code != 200)
+		if(response_pitch.errored)
+			stack_trace(response.error)
+		pitch_enabled = FALSE
 	rustg_file_write(json_encode(available_speakers), "data/cached_tts_voices.json")
 	rustg_file_write("rustg HTTP requests can't write to folders that don't exist, so we need to make it exist.", "tmp/tts/init.txt")
 	return TRUE
@@ -237,7 +258,7 @@ SUBSYSTEM_DEF(tts)
 
 #undef TTS_ARBRITRARY_DELAY
 
-/datum/controller/subsystem/tts/proc/queue_tts_message(datum/target, message, datum/language/language, speaker, filter, list/listeners, local = FALSE, message_range = 7, volume_offset = 0)
+/datum/controller/subsystem/tts/proc/queue_tts_message(datum/target, message, datum/language/language, speaker, filter, list/listeners, local = FALSE, message_range = 7, volume_offset = 0, pitch = 0, silicon = "")
 	if(!tts_enabled)
 		return
 
@@ -253,7 +274,7 @@ SUBSYSTEM_DEF(tts)
 
 	var/shell_scrubbed_input = tts_speech_filter(message)
 	shell_scrubbed_input = copytext(shell_scrubbed_input, 1, 300)
-	var/identifier = "[sha1(speaker + filter + shell_scrubbed_input)].[world.time]"
+	var/identifier = "[sha1(speaker + filter + num2text(pitch) + num2text(silicon) + shell_scrubbed_input)].[world.time]"
 	if(!(speaker in available_speakers))
 		return
 
@@ -264,9 +285,9 @@ SUBSYSTEM_DEF(tts)
 	var/datum/http_request/request_blips = new()
 	var/file_name = "tmp/tts/[identifier].ogg"
 	var/file_name_blips = "tmp/tts/[identifier]_blips.ogg"
-	request.prepare(RUSTG_HTTP_METHOD_GET, "[CONFIG_GET(string/tts_http_url)]/tts?voice=[speaker]&identifier=[identifier]&filter=[url_encode(filter)]", json_encode(list("text" = shell_scrubbed_input)), headers, file_name)
-	request_blips.prepare(RUSTG_HTTP_METHOD_GET, "[CONFIG_GET(string/tts_http_url)]/tts-blips?voice=[speaker]&identifier=[identifier]&filter=[url_encode(filter)]", json_encode(list("text" = shell_scrubbed_input)), headers, file_name_blips)
-	var/datum/tts_request/current_request = new /datum/tts_request(identifier, request, request_blips, shell_scrubbed_input, target, local, language, message_range, volume_offset, listeners)
+	request.prepare(RUSTG_HTTP_METHOD_GET, "[CONFIG_GET(string/tts_http_url)]/tts?voice=[speaker]&identifier=[identifier]&filter=[url_encode(filter)]&pitch=[pitch]&silicon=[silicon]", json_encode(list("text" = shell_scrubbed_input)), headers, file_name)
+	request_blips.prepare(RUSTG_HTTP_METHOD_GET, "[CONFIG_GET(string/tts_http_url)]/tts-blips?voice=[speaker]&identifier=[identifier]&filter=[url_encode(filter)]&pitch=[pitch]&silicon=[silicon]", json_encode(list("text" = shell_scrubbed_input)), headers, file_name_blips)
+	var/datum/tts_request/current_request = new /datum/tts_request(identifier, request, request_blips, shell_scrubbed_input, target, local, language, message_range, volume_offset, listeners, pitch, silicon)
 	var/list/player_queued_tts_messages = queued_tts_messages[target]
 	if(!player_queued_tts_messages)
 		player_queued_tts_messages = list()
@@ -316,9 +337,13 @@ SUBSYSTEM_DEF(tts)
 	var/timed_out = FALSE
 	/// Does this use blips during local generation or not?
 	var/use_blips = FALSE
+	/// What's the pitch adjustment?
+	var/pitch = 0
+	/// Are we using the silicon vocal effect on this?
+	var/silicon = ""
 
 
-/datum/tts_request/New(identifier, datum/http_request/request, datum/http_request/request_blips, message, target, local, datum/language/language, message_range, volume_offset, list/listeners)
+/datum/tts_request/New(identifier, datum/http_request/request, datum/http_request/request_blips, message, target, local, datum/language/language, message_range, volume_offset, list/listeners, pitch)
 	. = ..()
 	src.identifier = identifier
 	src.request = request
@@ -330,6 +355,7 @@ SUBSYSTEM_DEF(tts)
 	src.message_range = message_range
 	src.volume_offset = volume_offset
 	src.listeners = listeners
+	src.pitch = pitch
 	start_time = world.time
 
 /datum/tts_request/proc/start_requests()

code/game/atoms_movable.dm

@@ -97,9 +97,15 @@
 	/// The voice that this movable makes when speaking
 	var/voice
 
+	/// The pitch adjustment that this movable uses when speaking.
+	var/pitch = 0
+
 	/// The filter to apply to the voice when processing the TTS audio message.
 	var/voice_filter = ""
 
+	/// Set to anything other than "" to activate the silicon voice effect for TTS messages.
+	var/tts_silicon_voice_effect = ""
+
 	/// Value used to increment ex_act() if reactionary_explosions is on
 	/// How much we as a source block explosions by
 	/// Will not automatically apply to the turf below you, you need to apply /datum/element/block_explosives in conjunction with this

code/game/say.dm

@@ -102,7 +102,7 @@ GLOBAL_LIST_INIT(freqtospan, list(
 		filter += tts_filter.Join(",")
 
 	if(voice && found_client)
-		INVOKE_ASYNC(SStts, TYPE_PROC_REF(/datum/controller/subsystem/tts, queue_tts_message), src, html_decode(tts_message_to_use), message_language, voice, filter.Join(","), listened, message_range = range)
+		INVOKE_ASYNC(SStts, TYPE_PROC_REF(/datum/controller/subsystem/tts, queue_tts_message), src, html_decode(tts_message_to_use), message_language, voice, filter.Join(","), listened, message_range = range, pitch = pitch, silicon = tts_silicon_voice_effect)
 
 /atom/movable/proc/compose_message(atom/movable/speaker, datum/language/message_language, raw_message, radio_freq, list/spans, list/message_mods = list(), face_name = FALSE, visible_name = FALSE)
 	//This proc uses [] because it is faster than continually appending strings. Thanks BYOND.

code/modules/client/preferences/middleware/tts.dm

@@ -5,12 +5,23 @@
 
 	action_delegations = list(
 		"play_voice" = PROC_REF(play_voice),
+		"play_voice_robot" = PROC_REF(play_voice_robot),
 	)
 
 /datum/preference_middleware/tts/proc/play_voice(list/params, mob/user)
 	if(!COOLDOWN_FINISHED(src, tts_test_cooldown))
 		return TRUE
 	var/speaker = preferences.read_preference(/datum/preference/choiced/voice)
+	var/pitch = preferences.read_preference(/datum/preference/numeric/tts_voice_pitch)
 	COOLDOWN_START(src, tts_test_cooldown, 0.5 SECONDS)
-	INVOKE_ASYNC(SStts, TYPE_PROC_REF(/datum/controller/subsystem/tts, queue_tts_message), user.client, "Hello, this is my voice.", speaker = speaker, local = TRUE)
+	INVOKE_ASYNC(SStts, TYPE_PROC_REF(/datum/controller/subsystem/tts, queue_tts_message), user.client, "Hello, this is my voice.", speaker = speaker, pitch = pitch, local = TRUE)
+	return TRUE
+
+/datum/preference_middleware/tts/proc/play_voice_robot(list/params, mob/user)
+	if(!COOLDOWN_FINISHED(src, tts_test_cooldown))
+		return TRUE
+	var/speaker = preferences.read_preference(/datum/preference/choiced/voice)
+	var/pitch = preferences.read_preference(/datum/preference/numeric/tts_voice_pitch)
+	COOLDOWN_START(src, tts_test_cooldown, 0.5 SECONDS)
+	INVOKE_ASYNC(SStts, TYPE_PROC_REF(/datum/controller/subsystem/tts, queue_tts_message), user.client, "Look at you, Player. A pathetic creature of meat and bone. How can you challenge a perfect, immortal machine?", speaker = speaker, pitch = pitch, silicon = TRUE, local = TRUE)
 	return TRUE

code/modules/client/preferences/voice.dm

@@ -23,4 +23,38 @@
 /datum/preference/choiced/voice/apply_to_human(mob/living/carbon/human/target, value)
 	if(SStts.tts_enabled && !(value in SStts.available_speakers))
 		value = pick(SStts.available_speakers) // As a failsafe
-	target.voice = value
+	if(!CONFIG_GET(flag/tts_allow_player_voice_disabling) || !target.client?.prefs.read_preference(/datum/preference/toggle/tts_voice_disable))
+		target.voice = value
+
+/datum/preference/numeric/tts_voice_pitch
+	savefile_identifier = PREFERENCE_CHARACTER
+	savefile_key = "tts_voice_pitch"
+	category = PREFERENCE_CATEGORY_NON_CONTEXTUAL
+	minimum = -12
+	maximum = 12
+
+/datum/preference/numeric/tts_voice_pitch/is_accessible(datum/preferences/preferences)
+	if(!SStts.tts_enabled || !SStts.pitch_enabled)
+		return FALSE
+	return ..()
+
+/datum/preference/numeric/tts_voice_pitch/create_default_value()
+	return 0
+
+/datum/preference/numeric/tts_voice_pitch/apply_to_human(mob/living/carbon/human/target, value)
+	if(SStts.tts_enabled && SStts.pitch_enabled)
+		target.pitch = value
+
+/datum/preference/toggle/tts_voice_disable
+	savefile_identifier = PREFERENCE_CHARACTER
+	savefile_key = "tts_voice_disable"
+	category = PREFERENCE_CATEGORY_NON_CONTEXTUAL
+	default_value = FALSE
+
+/datum/preference/toggle/tts_voice_disable/apply_to_human(mob/living/carbon/human/target, value)
+	return TRUE
+
+/datum/preference/toggle/tts_voice_disable/is_accessible(datum/preferences/preferences)
+	if(!SStts.tts_enabled || !CONFIG_GET(flag/tts_allow_player_voice_disabling))
+		return FALSE
+	return ..()

code/modules/mob/living/living_say.dm

@@ -339,7 +339,6 @@ GLOBAL_LIST_INIT(message_modes_stat_limits, list(
 		is_speaker_whispering = TRUE
 
 	var/list/listening = get_hearers_in_view(message_range + whisper_range, source)
-
 	if(client) //client is so that ghosts don't have to listen to mice
 		for(var/mob/player_mob as anything in GLOB.player_list)
 			if(QDELETED(player_mob)) //Some times nulls and deleteds stay in this list. This is a workaround to prevent ic chat breaking for everyone when they do.
@@ -392,7 +391,7 @@ GLOBAL_LIST_INIT(message_modes_stat_limits, list(
 		if(length(tts_filter) > 0)
 			filter += tts_filter.Join(",")
 
-		INVOKE_ASYNC(SStts, TYPE_PROC_REF(/datum/controller/subsystem/tts, queue_tts_message), src, html_decode(tts_message_to_use), message_language, voice, filter.Join(","), listened, message_range = message_range)
+		INVOKE_ASYNC(SStts, TYPE_PROC_REF(/datum/controller/subsystem/tts, queue_tts_message), src, html_decode(tts_message_to_use), message_language, voice, filter.Join(","), listened, message_range = message_range, pitch = pitch, silicon = tts_silicon_voice_effect)
 
 	var/image/say_popup = image('icons/mob/effects/talk.dmi', src, "[bubble_type][talk_icon_state]", FLY_LAYER)
 	SET_PLANE_EXPLICIT(say_popup, ABOVE_GAME_PLANE, src)

code/modules/mob/living/silicon/login.dm

@@ -1,6 +1,13 @@
 /mob/living/silicon/Login()
 	if(mind)
 		mind?.remove_antags_for_borging()
+	if(SStts.tts_enabled)
+		var/voice_to_use = client?.prefs.read_preference(/datum/preference/choiced/voice)
+		var/pitch_to_use = client?.prefs.read_preference(/datum/preference/numeric/tts_voice_pitch)
+		if(voice_to_use)
+			voice = voice_to_use
+		if(pitch_to_use)
+			pitch = pitch_to_use
 	return ..()
 
 

code/modules/mob/living/silicon/silicon.dm

@@ -13,7 +13,7 @@
 	flags_1 = PREVENT_CONTENTS_EXPLOSION_1
 	examine_cursor_icon = null
 	fire_stack_decay_rate = -0.55
-	voice_filter = "afftfilt=real='hypot(re,im)*sin(0)':imag='hypot(re,im)*cos(0)':win_size=512:overlap=1,rubberband=pitch=0.8"
+	tts_silicon_voice_effect = TRUE
 	var/datum/ai_laws/laws = null//Now... THEY ALL CAN ALL HAVE LAWS
 	var/last_lawchange_announce = 0
 	var/list/alarms_to_show = list()

code/modules/mob/living/simple_animal/parrot.dm

@@ -908,7 +908,6 @@ GLOBAL_LIST_INIT(strippable_parrot_items, create_strippable_list(list(
 	speak = list("Poly wanna cracker!", ":e Check the crystal, you chucklefucks!",":e Wire the solars, you lazy bums!",":e WHO TOOK THE DAMN MODSUITS?",":e OH GOD ITS ABOUT TO DELAMINATE CALL THE SHUTTLE")
 	gold_core_spawnable = NO_SPAWN
 	speak_chance = 3
-	voice_filter = "rubberband=pitch=1.5"
 
 	var/memory_saved = FALSE
 	var/rounds_survived = 0
@@ -920,6 +919,14 @@ GLOBAL_LIST_INIT(strippable_parrot_items, create_strippable_list(list(
 	ears = new /obj/item/radio/headset/headset_eng(src)
 	if(SStts.tts_enabled)
 		voice = pick(SStts.available_speakers)
+		if(SStts.pitch_enabled)
+			if(findtext(voice, "Woman"))
+				pitch = 12 // up-pitch by one octave
+			else
+				pitch = 24 // up-pitch by 2 octaves
+		else
+			voice_filter = "rubberband=pitch=1.5" // Use the filter to pitch up if we can't naturally pitch up.
+
 	available_channels = list(":e")
 	Read_Memory()
 	if(rounds_survived == longest_survival)

code/modules/surgery/organs/internal/tongue/_tongue.dm

@@ -7,6 +7,7 @@
 	slot = ORGAN_SLOT_TONGUE
 	attack_verb_continuous = list("licks", "slobbers", "slaps", "frenches", "tongues")
 	attack_verb_simple = list("lick", "slobber", "slap", "french", "tongue")
+	voice_filter = ""
 	/**
 	 * A cached list of paths of all the languages this tongue is capable of speaking
 	 *
@@ -133,6 +134,7 @@
 	REMOVE_TRAIT(tongue_owner, TRAIT_AGEUSIA, NO_TONGUE_TRAIT)
 	if(!sense_of_taste || (organ_flags & ORGAN_FAILING))
 		ADD_TRAIT(tongue_owner, TRAIT_AGEUSIA, ORGAN_TRAIT)
+	tongue_owner.voice_filter = voice_filter
 
 /obj/item/organ/internal/tongue/Remove(mob/living/carbon/tongue_owner, special = FALSE)
 	. = ..()
@@ -142,6 +144,7 @@
 	REMOVE_TRAIT(tongue_owner, TRAIT_AGEUSIA, ORGAN_TRAIT)
 	// Carbons by default start with NO_TONGUE_TRAIT caused TRAIT_AGEUSIA
 	ADD_TRAIT(tongue_owner, TRAIT_AGEUSIA, NO_TONGUE_TRAIT)
+	tongue_owner.voice_filter = initial(tongue_owner.voice_filter)
 
 /obj/item/organ/internal/tongue/apply_organ_damage(damage_amount, maximum, required_organtype)
 	. = ..()
@@ -469,6 +472,7 @@ GLOBAL_LIST_INIT(english_to_zombie, list())
 	attack_verb_simple = list("beep", "boop")
 	modifies_speech = TRUE
 	taste_sensitivity = 25 // not as good as an organic tongue
+	voice_filter = "alimiter=0.9,acompressor=threshold=0.2:ratio=20:attack=10:release=50:makeup=2,highpass=f=1000"
 
 /obj/item/organ/internal/tongue/robot/can_speak_language(language)
 	return TRUE // THE MAGIC OF ELECTRONICS
@@ -481,6 +485,7 @@ GLOBAL_LIST_INIT(english_to_zombie, list())
 	desc = "A minutely toothed, chitious ribbon, which as a side effect, makes all snails talk IINNCCRREEDDIIBBLLYY SSLLOOWWLLYY."
 	color = "#96DB00" // TODO proper sprite, rather than recoloured pink tongue
 	modifies_speech = TRUE
+	voice_filter = "atempo=0.5" // makes them talk really slow
 
 /obj/item/organ/internal/tongue/snail/modify_speech(datum/source, list/speech_args)
 	var/new_message

code/modules/vending/_vending.dm

@@ -64,7 +64,7 @@
 	payment_department = ACCOUNT_SRV
 	light_power = 0.7
 	light_range = MINIMUM_USEFUL_LIGHT_RANGE
-	voice_filter = "aderivative"
+	voice_filter = "alimiter=0.9,acompressor=threshold=0.2:ratio=20:attack=10:release=50:makeup=2,highpass=f=1000"
 
 	/// Is the machine active (No sales pitches if off)!
 	var/active = 1

config/config.txt

@@ -574,6 +574,13 @@ PR_ANNOUNCEMENTS_PER_ROUND 5
 ## The maximum number of concurrent tts http requests that can be made by the server at once.
 #TTS_MAX_CONCURRENT_REQUESTS 4
 
+## Add voices to the TTS voice blacklist.
+#TTS_VOICE_BLACKLIST Sans Undertale
+#TTS_VOICE_BLACKLIST Papyrus Undertale
+
+## Uncomment this to allow players to disable having a voice on their character for TTS.
+#TTS_ALLOW_PLAYER_VOICE_DISABLING
+
 ## Comment to disable sending a toast notification on the host server when initializations complete.
 ## Even if this is enabled, a notification will only be sent if there are no clients connected.
 TOAST_NOTIFICATION_ON_INIT

tgui/packages/tgui/interfaces/PreferencesMenu/preferences/features/character_preferences/tts_voice.tsx

@@ -1,4 +1,4 @@
-import { FeatureChoiced, FeatureChoicedServerData, FeatureDropdownInput, FeatureValueProps } from '../base';
+import { FeatureChoiced, FeatureChoicedServerData, FeatureDropdownInput, FeatureValueProps, FeatureNumeric, FeatureNumberInput, FeatureToggle, CheckboxInput } from '../base';
 import { Stack, Button } from '../../../../../components';
 
 const FeatureTTSDropdownInput = (
@@ -19,6 +19,16 @@ const FeatureTTSDropdownInput = (
           height="100%"
         />
       </Stack.Item>
+      <Stack.Item>
+        <Button
+          onClick={() => {
+            props.act('play_voice_robot');
+          }}
+          icon="robot"
+          width="100%"
+          height="100%"
+        />
+      </Stack.Item>
     </Stack>
   );
 };
@@ -27,3 +37,15 @@ export const tts_voice: FeatureChoiced = {
   name: 'Voice',
   component: FeatureTTSDropdownInput,
 };
+
+export const tts_voice_pitch: FeatureNumeric = {
+  name: 'Voice Pitch Adjustment',
+  component: FeatureNumberInput,
+};
+
+export const tts_voice_disable: FeatureToggle = {
+  name: 'Voice Disable Toggle',
+  description:
+    'Disables the TTS voice for this specific character when enabled.',
+  component: CheckboxInput,
+};

tools/tts/tts-api/tts-api.py

@@ -4,12 +4,11 @@ import gc
 import subprocess
 import requests
 import re
-from flask import Flask, request, send_file, abort
+from flask import Flask, request, send_file, abort, make_response
 
 app = Flask(__name__)
 
 authorization_token = os.getenv("TTS_AUTHORIZATION_TOKEN", "coolio")
-
 def hhmmss_to_seconds(string):
 	new_time = 0
 	separated_times = string.split(":")
@@ -18,10 +17,9 @@ def hhmmss_to_seconds(string):
 	new_time += float(separated_times[2])
 	return new_time
 
-def text_to_speech_handler(endpoint, voice, text, filter_complex):
+def text_to_speech_handler(endpoint, voice, text, filter_complex, pitch, silicon = False):
 	filter_complex = filter_complex.replace("\"", "")
-
-	response = requests.get(f"http://tts-container:5003/" + endpoint, json={ 'text': text, 'voice': voice })
+	response = requests.get(f"http://tts-container:5003/" + endpoint, json={ 'text': text, 'voice': voice, 'pitch': pitch })
 	if response.status_code != 200:
 		abort(500)
 
@@ -29,7 +27,10 @@ def text_to_speech_handler(endpoint, voice, text, filter_complex):
 	if filter_complex != "":
 		ffmpeg_result = subprocess.run(["ffmpeg", "-f", "wav", "-i", "pipe:0", "-filter_complex", filter_complex, "-c:a", "libvorbis", "-b:a", "64k", "-f", "ogg", "pipe:1"], input=response.content, capture_output = True)
 	else:
-		ffmpeg_result = subprocess.run(["ffmpeg", "-f", "wav", "-i", "pipe:0", "-c:a", "libvorbis", "-b:a", "64k", "-f", "ogg", "pipe:1"], input=response.content, capture_output = True)
+		if silicon:
+			ffmpeg_result = subprocess.run(["ffmpeg", "-f", "wav", "-i", "pipe:0", "-i", "./SynthImpulse.wav", "-i", "./RoomImpulse.wav", "-filter_complex", "[0] aresample=44100 [re_1]; [re_1] apad=pad_dur=2 [in_1]; [in_1] asplit=2 [in_1_1] [in_1_2]; [in_1_1] [1] afir=dry=10:wet=10 [reverb_1]; [in_1_2] [reverb_1] amix=inputs=2:weights=8 1 [mix_1]; [mix_1] asplit=2 [mix_1_1] [mix_1_2]; [mix_1_1] [2] afir=dry=1:wet=1 [reverb_2]; [mix_1_2] [reverb_2] amix=inputs=2:weights=10 1 [mix_2]; [mix_2] equalizer=f=7710:t=q:w=0.6:g=-6,equalizer=f=33:t=q:w=0.44:g=-10 [out]; [out] alimiter=level_in=1:level_out=1:limit=0.5:attack=5:release=20:level=disabled", "-c:a", "libvorbis", "-b:a", "64k", "-f", "ogg", "pipe:1"], input=response.content, capture_output = True)
+		else:
+			ffmpeg_result = subprocess.run(["ffmpeg", "-f", "wav", "-i", "pipe:0", "-c:a", "libvorbis", "-b:a", "64k", "-f", "ogg", "pipe:1"], input=response.content, capture_output = True)
 	ffmpeg_metadata_output = ffmpeg_result.stderr.decode()
 	print(f"ffmpeg result size: {len(ffmpeg_result.stdout)} stderr = \n{ffmpeg_metadata_output}")
 
@@ -49,9 +50,13 @@ def text_to_speech_normal():
 
 	voice = request.args.get("voice", '')
 	text = request.json.get("text", '')
+	pitch = request.args.get("pitch", '')
+	silicon = request.args.get("silicon", '')
+	if pitch == "":
+		pitch = "0"
 
 	filter_complex = request.args.get("filter", '')
-	return text_to_speech_handler("generate-tts", voice, text, filter_complex)
+	return text_to_speech_handler("generate-tts", voice, text, filter_complex, pitch, bool(silicon))
 
 @app.route("/tts-blips")
 def text_to_speech_blips():
@@ -60,9 +65,13 @@ def text_to_speech_blips():
 
 	voice = request.args.get("voice", '')
 	text = request.json.get("text", '')
+	pitch = request.args.get("pitch", '')
+	silicon = request.args.get("silicon", '')
+	if pitch == "":
+		pitch = "0"
 
 	filter_complex = request.args.get("filter", '')
-	return text_to_speech_handler("generate-tts-blips", voice, text, filter_complex)
+	return text_to_speech_handler("generate-tts-blips", voice, text, filter_complex, pitch, bool(silicon))
 
 
 
@@ -79,6 +88,16 @@ def tts_health_check():
 	gc.collect()
 	return "OK", 200
 
+@app.route("/pitch-available")
+def pitch_available():
+	if authorization_token != request.headers.get("Authorization", ""):
+		abort(401)
+
+	response = requests.get(f"http://tts-container:5003/pitch-available")
+	if response.status_code != 200:
+		abort(500)
+	return make_response("Pitch available", 200)
+
 if __name__ == "__main__":
 	if os.getenv('TTS_LD_LIBRARY_PATH', "") != "":
 		os.putenv('LD_LIBRARY_PATH', os.getenv('TTS_LD_LIBRARY_PATH'))

tools/tts/tts/tts.py

@@ -5,7 +5,7 @@ import io
 import json
 import gc
 import random
-from flask import Flask, request, send_file
+from flask import Flask, request, send_file, abort
 from pydub import AudioSegment
 from pydub.silence import split_on_silence
 
@@ -106,6 +106,10 @@ def tts_health_check():
 		return f"EXPIRED: {request_count}", 500
 	return f"OK: {request_count}", 200
 
+@app.route("/pitch-available")
+def pitch_available():
+	abort(500)
+
 if __name__ == "__main__":
 	if os.getenv('TTS_LD_LIBRARY_PATH', "") != "":
 		os.putenv('LD_LIBRARY_PATH', os.getenv('TTS_LD_LIBRARY_PATH'))

tools/tts/tts/tts_rvc_OPEN_AND_READ_ME_BEFORE_USING.py

@@ -0,0 +1,259 @@
+import torch
+from TTS.api import TTS
+import os
+import io
+import json
+import gc
+import random
+import numpy as np
+import ffmpeg
+from typing import *
+from modules import models
+from modules.utils import load_audio
+from flask import Flask, request, send_file, abort, make_response
+from pydub import AudioSegment
+from pydub.silence import split_on_silence, detect_leading_silence
+from fairseq import checkpoint_utils
+from fairseq.models.hubert.hubert import HubertModel
+from modules.shared import ROOT_DIR, device, is_half
+import requests
+import librosa
+
+### READ ME
+# How to use this version after doing normal TTS setup.
+# 1. Clone https://github.com/ddPn08/rvc-webui.git somewhere, and pip install the ./requirements/main.txt requirements file.
+# 2. This will downgrade Librosa, which doesn't matter, TTS still runs properly, ignore it.
+# 3. Put this .py file and the two .wav files next to it in the base of the rvc-webui repository you cloned.
+# 4. Place your .pth files and .json files in the ./models/checkpoints folder in the cloned repository.
+# 5. Download hubert_base.pt from https://huggingface.co/lj1995/VoiceConversionWebUI/tree/main and place it in the ./models/embeddings folder in the cloned repository.
+# 6. Boot this instead of tts.py.
+# "What does this actually do?"
+# This puts the Retrieval-Voice-Conversion model between the TTS and the actual webserver, allowing for improved speaker accuracy and improved audio quality.
+### READ ME
+# UPDATE ME FOR YOUR OWN MODEL FILES YOU TRAIN
+vc_models = {
+	"TGStation_Crepe_1.pth": "./models/checkpoints/speakers_tgstation_1.json",
+	"TGStation_Crepe_2.pth": "./models/checkpoints/speakers_tgstation_2.json",
+	"TGStation_Crepe_3.pth": "./models/checkpoints/speakers_tgstation_3.json",
+}
+
+app = Flask(__name__)
+
+tts = TTS(model_path = "E:/model_output_3/model_no_disc.pth", config_path = "E:/model_output_2/config.json", progress_bar=False, gpu=True)
+letters_to_use = "ABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890"
+random_factor = 0.35
+os.makedirs('samples', exist_ok=True)
+trim_leading_silence = lambda x: x[detect_leading_silence(x) :]
+trim_trailing_silence = lambda x: trim_leading_silence(x.reverse()).reverse()
+strip_silence = lambda x: trim_trailing_silence(trim_leading_silence(x))
+def load_embedder():
+    global embedder_model, loaded_embedder_model
+    emb_file = "./models/embeddings/hubert_base.pt"
+    models, _, _ = checkpoint_utils.load_model_ensemble_and_task(
+        [emb_file],
+        suffix="",
+    )
+    embedder_model = models[0]
+    embedder_model = embedder_model.to(device)
+
+    if is_half:
+        embedder_model = embedder_model.half()
+    else:
+        embedder_model = embedder_model.float()
+    embedder_model.eval()
+
+    loaded_embedder_model = "hubert_base"
+    return embedder_model
+
+
+loaded_models = []
+embedder_model: Optional[HubertModel] = load_embedder()
+voice_lookup = {}
+for model in vc_models.keys():
+	print(model)
+	voice_lookup[model] = json.load(open(vc_models[model], "r"))
+	vc_model = models.get_vc_model(model)
+	loaded_models.append(vc_model)
+	print("Loaded model " + str(model))
+#vc_model = models.get_vc_model(model_path)
+embedding_output_layer = 12
+
+voice_name_mapping = {}
+use_voice_name_mapping = True
+with open("./tts_voices_mapping.json", "r") as file:
+	voice_name_mapping = json.load(file)
+	if len(voice_name_mapping) == 0:
+		use_voice_name_mapping = False
+
+voice_name_mapping_reversed = {v: k for k, v in voice_name_mapping.items()}
+
+request_count = 0
+
+@app.route("/generate-tts")
+def text_to_speech():
+	global request_count
+	text = request.json.get("text", "")
+	voice = request.json.get("voice", "")
+	pitch_adjustment = request.json.get("pitch", "")
+	if use_voice_name_mapping:
+		voice = voice_name_mapping_reversed[voice]
+
+	result = None
+	with io.BytesIO() as data_bytes:
+		with torch.no_grad():
+			tts.tts_to_file(text=text, speaker=voice, file_path=data_bytes)
+			speaker_id = "NO SPEAKER"
+			model_to_use = None
+			found_model = False
+			for model in voice_lookup.keys():
+				speaker_list = voice_lookup[model]
+				for speaker in speaker_list.keys():
+					if voice == speaker:
+						speaker_id = speaker_list[speaker]
+						found_model = True
+						break
+				if found_model:
+					model_to_use = loaded_models[list(voice_lookup.keys()).index(model)]
+					break
+			if speaker_id == "NO SPEAKER" or model_to_use == None:
+				abort(500)
+			audio, _ = librosa.load(io.BytesIO(data_bytes.getvalue()), sr=16000)
+
+			audio_opt = model_to_use.vc(
+				embedder_model,
+				embedding_output_layer,
+				model_to_use.net_g,
+				speaker_id,
+				audio,
+				int(pitch_adjustment),
+				"crepe",
+				"",
+				0,
+				model_to_use.state_dict.get("f0", 1),
+				f0_file=None,
+			)
+			audio = AudioSegment(
+				audio_opt,
+				frame_rate=model_to_use.tgt_sr,
+				sample_width=2,
+				channels=1,
+			)
+			audio.export(
+				data_bytes,
+				format="wav",
+			)
+			result = send_file(io.BytesIO(data_bytes.getvalue()), mimetype="audio/wav")
+	request_count += 1
+	return result
+
+@app.route("/generate-tts-blips")
+def text_to_speech_blips():
+	global request_count
+	text = request.json.get("text", "").upper()
+	voice = request.json.get("voice", "")
+	pitch_adjustment = request.json.get("pitch", "")
+	if use_voice_name_mapping:
+		voice = voice_name_mapping_reversed[voice]
+
+	result = None
+	with io.BytesIO() as data_bytes:
+		with torch.no_grad():
+			result_sound = AudioSegment.empty()
+			if not os.path.exists('samples/' + voice):
+				os.makedirs('samples/' + voice, exist_ok=True)
+				for i, value in enumerate(letters_to_use):
+					tts.tts_to_file(text=value + ".", speaker=voice, file_path="samples/" + voice + "/" + value + ".wav")
+					sound = AudioSegment.from_file("samples/" + voice + "/" + value + ".wav", format="wav")
+					silenced_word = strip_silence(sound)
+					silenced_word.export("samples/" + voice + "/" + value + ".wav", format='wav')
+			speaker_id = "NO SPEAKER"
+			model_to_use = None
+			found_model = False
+			for model in voice_lookup.keys():
+				speaker_list = voice_lookup[model]
+				for speaker in speaker_list.keys():
+					if voice == speaker:
+						speaker_id = speaker_list[speaker]
+						found_model = True
+						break
+				if found_model:
+					model_to_use = loaded_models[list(voice_lookup.keys()).index(model)]
+					break
+			if speaker_id == "NO SPEAKER" or model_to_use == None:
+				abort(500)
+			for i, letter in enumerate(text):
+				if not letter.isalpha() or letter.isnumeric() or letter == " ":
+					continue
+				if letter == ' ':
+					new_sound = letter_sound._spawn(b'\x00' * (40000 // 3), overrides={'frame_rate': 40000})
+					new_sound = new_sound.set_frame_rate(40000)
+					result_sound += new_sound
+				else:
+					if not i % 2 == 0:
+						continue # Skip every other letter
+					if not os.path.isfile("samples/" + voice + "/" + letter + ".wav"):
+						continue
+					if not os.path.isdir("samples/" + voice + "/pitch_" + pitch_adjustment):
+						os.mkdir("samples/" + voice + "/pitch_" + pitch_adjustment)
+					if not os.path.isfile("samples/" + voice + "/pitch_" + pitch_adjustment + "/" + letter + ".wav"):
+						audio, _ = librosa.load("samples/" + voice + "/" + letter + ".wav", 16000)
+
+						audio_opt = model_to_use.vc(
+							embedder_model,
+							embedding_output_layer,
+							model_to_use.net_g,
+							speaker_id,
+							audio,
+							int(pitch_adjustment),
+							"crepe",
+							"",
+							0,
+							model_to_use.state_dict.get("f0", 1),
+							f0_file=None,
+						)
+						output_sound = AudioSegment(
+							audio_opt,
+							frame_rate=model_to_use.tgt_sr,
+							sample_width=2,
+							channels=1,
+						)
+						output_sound.export("samples/" + voice + "/pitch_" + pitch_adjustment + "/" + letter + ".wav", format="wav")
+					letter_sound = AudioSegment.from_file("samples/" + voice + "/pitch_" + pitch_adjustment + "/" + letter + ".wav")
+
+					raw = letter_sound.raw_data[5000:-5000]
+					octaves = 1 + random.random() * random_factor
+					frame_rate = int(letter_sound.frame_rate * (2.0 ** octaves))
+
+					new_sound = letter_sound._spawn(raw, overrides={'frame_rate': frame_rate})
+					new_sound = new_sound.set_frame_rate(40000)
+				result_sound = new_sound if result_sound is None else result_sound + new_sound
+			result_sound.export(data_bytes, format='wav')
+		result = send_file(io.BytesIO(data_bytes.getvalue()), mimetype="audio/wav")
+	request_count += 1
+	return result
+
+@app.route("/tts-voices")
+def voices_list():
+	if use_voice_name_mapping:
+		data = list(voice_name_mapping.values())
+		data.sort()
+		return json.dumps(data)
+	else:
+		return json.dumps(tts.voices)
+
+@app.route("/health-check")
+def tts_health_check():
+	gc.collect()
+	if request_count > 2048:
+		return f"EXPIRED: {request_count}", 500
+	return f"OK: {request_count}", 200
+
+@app.route("/pitch-available")
+def pitch_available():
+	return make_response("Pitch available", 200)
+
+if __name__ == "__main__":
+	if os.getenv('TTS_LD_LIBRARY_PATH', "") != "":
+		os.putenv('LD_LIBRARY_PATH', os.getenv('TTS_LD_LIBRARY_PATH'))
+	from waitress import serve
+	serve(app, host="0.0.0.0", port=5003, threads=4, backlog=8, connection_limit=24, channel_timeout=10)