diff --git a/code/_globalvars/lists/languages.dm b/code/_globalvars/lists/languages.dm index 8aaf38596ca..da4d248959e 100644 --- a/code/_globalvars/lists/languages.dm +++ b/code/_globalvars/lists/languages.dm @@ -4,9 +4,12 @@ GLOBAL_LIST_INIT_TYPED(language_datum_instances, /datum/language, init_language_ GLOBAL_LIST_INIT(all_languages, init_all_languages()) /// /List of language prototypes to reference, assoc "name" = typepath GLOBAL_LIST_INIT(language_types_by_name, init_language_types_by_name()) -/// 1000 element long list containing the 1000 most common words in the English language. +/// ~1400 element long list containing the 1000 most common words in the English language in alphabetical order. /// Indexed by word, value is the rank of the word in the list. So accessing it is fasta. -GLOBAL_LIST_INIT(most_common_words, init_common_words()) +GLOBAL_LIST_INIT(most_common_words_alphabetical, init_common_words_by_alphabetical()) +/// ~1000 element long list containing the 1000 most common words in the English language in frequency order. +/// Indexed by word, value is the rank of the word in the list. So accessing it is fasta. +GLOBAL_LIST_INIT(most_common_words_frequency, init_common_words_by_frequency()) /proc/init_language_prototypes() var/list/lang_list = list() @@ -33,10 +36,22 @@ GLOBAL_LIST_INIT(most_common_words, init_common_words()) lang_list[initial(lang_type.name)] = lang_type return lang_list -/proc/init_common_words() +/proc/init_common_words_by_alphabetical() var/list/word_to_commonness_list = list() var/i = 1 - for(var/word in world.file2list("strings/1000_most_common.txt")) + for(var/word in world.file2list("strings/1400_most_common_alpha.txt")) word_to_commonness_list[word] = i i += 1 return word_to_commonness_list + +/proc/init_common_words_by_frequency() + var/list/word_to_commonness_list = list() + var/i = 1 + for(var/word in world.file2list("strings/1000_most_common_frequency.txt")) + word_to_commonness_list[word] = i + i += 1 + // alphabetical list has a few more entries than the frequency list, so + // if there are any words we're missing, add them with a default "commonness" of 500 + for(var/word in world.file2list("strings/1400_most_common_alpha.txt")) + word_to_commonness_list[word] ||= 500 + return word_to_commonness_list diff --git a/code/controllers/subsystem/discord.dm b/code/controllers/subsystem/discord.dm index 2e7936f17e3..2313a1a0a28 100644 --- a/code/controllers/subsystem/discord.dm +++ b/code/controllers/subsystem/discord.dm @@ -154,7 +154,7 @@ SUBSYSTEM_DEF(discord) // While there's a collision in the token, generate a new one (should rarely happen) while(not_unique) //Column is varchar 100, so we trim just in case someone does us the dirty later - one_time_token = trim("[pick(GLOB.most_common_words)]-[pick(GLOB.most_common_words)]-[pick(GLOB.most_common_words)]-[pick(GLOB.most_common_words)]-[pick(GLOB.most_common_words)]-[pick(GLOB.most_common_words)]", 100) + one_time_token = trim("[pick(GLOB.most_common_words_alphabetical)]-[pick(GLOB.most_common_words_alphabetical)]-[pick(GLOB.most_common_words_alphabetical)]-[pick(GLOB.most_common_words_alphabetical)]-[pick(GLOB.most_common_words_alphabetical)]-[pick(GLOB.most_common_words_alphabetical)]", 100) not_unique = find_discord_link_by_token(one_time_token, timebound = TRUE) diff --git a/code/datums/brain_damage/mild.dm b/code/datums/brain_damage/mild.dm index ffdabc0adf7..e6ba2bac38f 100644 --- a/code/datums/brain_damage/mild.dm +++ b/code/datums/brain_damage/mild.dm @@ -217,7 +217,7 @@ word = copytext(word, 1, suffix_foundon) word = html_decode(word) - if(GLOB.most_common_words[LOWER_TEXT(word)]) + if(GLOB.most_common_words_alphabetical[LOWER_TEXT(word)]) new_message += word + suffix else if(prob(30) && message_split.len > 2) diff --git a/code/modules/language/_language.dm b/code/modules/language/_language.dm index c6ca1d8717d..65b107ee598 100644 --- a/code/modules/language/_language.dm +++ b/code/modules/language/_language.dm @@ -195,7 +195,7 @@ SHOULD_NOT_OVERRIDE(TRUE) var/lowertext_input = LOWER_TEXT(input) // The most common words are always cached - if(GLOB.most_common_words[lowertext_input]) + if(GLOB.most_common_words_frequency[lowertext_input]) most_common_cache[lowertext_input] = scrambled_text return // Add it to cache, cutting old entries if the list is too long @@ -263,7 +263,7 @@ if(translate_prob > 0) // the probability of managing to understand a word is based on how common it is (+10%, -15%) // 1000 words in the list, so words outside the list are just treated as "the 1250th most common word" - var/commonness = GLOB.most_common_words[LOWER_TEXT(base_word)] || 1250 + var/commonness = GLOB.most_common_words_frequency[LOWER_TEXT(base_word)] || 1250 translate_prob += (10 * (1 - (min(commonness, 1250) / 500))) if(prob(translate_prob)) scrambled_words += word diff --git a/strings/1000_most_common_frequency.txt b/strings/1000_most_common_frequency.txt new file mode 100644 index 00000000000..51682289f7f --- /dev/null +++ b/strings/1000_most_common_frequency.txt @@ -0,0 +1,1000 @@ +the +of +to +and +a +in +is +it +you +that +he +was +for +on +are +with +as +I +his +they +be +at +one +have +this +from +or +had +by +not +word +but +what +some +we +can +out +other +were +all +there +when +up +use +your +how +said +an +each +she +which +do +their +time +if +will +way +about +many +then +them +write +would +like +so +these +her +long +make +thing +see +him +two +has +look +more +day +could +go +come +did +number +sound +no +most +people +my +over +know +water +than +call +first +who +may +down +side +been +now +find +any +new +work +part +take +get +place +made +live +where +after +back +little +only +round +man +year +came +show +every +good +me +give +our +under +name +very +through +just +form +sentence +great +think +say +help +low +line +differ +turn +cause +much +mean +before +move +right +boy +old +too +same +tell +does +set +three +want +air +well +also +play +small +end +put +home +read +hand +port +large +spell +add +even +land +here +must +big +high +such +follow +act +why +ask +men +change +went +light +kind +off +need +house +picture +try +us +again +animal +point +mother +world +near +build +self +earth +father +head +stand +own +page +should +country +found +answer +school +grow +study +still +learn +plant +cover +food +sun +four +between +state +keep +eye +never +last +let +thought +city +tree +cross +farm +hard +start +might +story +saw +far +sea +draw +left +late +run +don't +while +press +close +night +real +life +few +north +open +seem +together +next +white +children +begin +got +walk +example +ease +paper +group +always +music +those +both +mark +often +letter +until +mile +river +car +feet +care +second +book +carry +took +science +eat +room +friend +began +idea +fish +mountain +stop +once +base +hear +horse +cut +sure +watch +color +face +wood +main +enough +plain +girl +usual +young +ready +above +ever +red +list +though +feel +talk +bird +soon +body +dog +family +direct +pose +leave +song +measure +door +product +black +short +numeral +class +wind +question +happen +complete +ship +area +half +rock +order +fire +south +problem +piece +told +knew +pass +since +top +whole +king +space +heard +best +hour +better +true +during +hundred +five +remember +step +early +hold +west +ground +interest +reach +fast +verb +sing +listen +six +table +travel +less +morning +ten +simple +several +vowel +toward +war +lay +against +pattern +slow +center +love +person +money +serve +appear +road +map +rain +rule +govern +pull +cold +notice +voice +unit +power +town +fine +certain +fly +fall +lead +cry +dark +machine +note +wait +plan +figure +star +box +noun +field +rest +correct +able +pound +done +beauty +drive +stood +contain +front +teach +week +final +gave +green +oh +quick +develop +ocean +warm +free +minute +strong +special +mind +behind +clear +tail +produce +fact +street +inch +multiply +nothing +course +stay +wheel +full +force +blue +object +decide +surface +deep +moon +island +foot +system +busy +test +record +boat +common +gold +possible +plane +stead +dry +wonder +laugh +thousand +ago +ran +check +game +shape +equate +hot +miss +brought +heat +snow +tire +bring +yes +distant +fill +east +paint +language +among +grand +ball +yet +wave +drop +heart +am +present +heavy +dance +engine +position +arm +wide +sail +material +size +vary +settle +speak +weight +general +ice +matter +circle +pair +include +divide +syllable +felt +perhaps +pick +sudden +count +square +reason +length +represent +art +subject +region +energy +hunt +probable +bed +brother +egg +ride +cell +believe +fraction +forest +sit +race +window +store +summer +train +sleep +prove +lone +leg +exercise +wall +catch +mount +wish +sky +board +joy +winter +sat +written +wild +instrument +kept +glass +grass +cow +job +edge +sign +visit +past +soft +fun +bright +gas +weather +month +million +bear +finish +happy +hope +flower +clothe +strange +gone +jump +baby +eight +village +meet +root +buy +raise +solve +metal +whether +push +seven +paragraph +third +shall +held +hair +describe +cook +floor +either +result +burn +hill +safe +cat +century +consider +type +law +bit +coast +copy +phrase +silent +tall +sand +soil +roll +temperature +finger +industry +value +fight +lie +beat +excite +natural +view +sense +ear +else +quite +broke +case +middle +kill +son +lake +moment +scale +loud +spring +observe +child +straight +consonant +nation +dictionary +milk +speed +method +organ +pay +age +section +dress +cloud +surprise +quiet +stone +tiny +climb +cool +design +poor +lot +experiment +bottom +key +iron +single +stick +flat +twenty +skin +smile +crease +hole +trade +melody +trip +office +receive +row +mouth +exact +symbol +die +least +trouble +shout +except +wrote +seed +tone +join +suggest +clean +break +lady +yard +rise +bad +blow +oil +blood +touch +grew +cent +mix +team +wire +cost +lost +brown +wear +garden +equal +sent +choose +fell +fit +flow +fair +bank +collect +save +control +decimal +gentle +woman +captain +practice +separate +difficult +doctor +please +protect +noon +whose +locate +ring +character +insect +caught +period +indicate +radio +spoke +atom +human +history +effect +electric +expect +crop +modern +element +hit +student +corner +party +supply +bone +rail +imagine +provide +agree +thus +capital +won't +chair +danger +fruit +rich +thick +soldier +process +operate +guess +necessary +sharp +wing +create +neighbor +wash +bat +rather +crowd +corn +compare +poem +string +bell +depend +meat +rub +tube +famous +dollar +stream +fear +sight +thin +triangle +planet +hurry +chief +colony +clock +mine +tie +enter +major +fresh +search +send +yellow +gun +allow +print +dead +spot +desert +suit +current +lift +rose +continue +block +chart +hat +sell +success +company +subtract +event +particular +deal +swim +term +opposite +wife +shoe +shoulder +spread +arrange +camp +invent +cotton +born +determine +quart +nine +truck +noise +level +chance +gather +shop +stretch +throw +shine +property +column +molecule +select +wrong +gray +repeat +require +broad +prepare +salt +nose +plural +anger +claim +continent +oxygen +sugar +death +pretty +skill +women +season +solution +magnet +silver +thank +branch +match +suffix +especially +fig +afraid +huge +sister +steel +discuss +forward +similar +guide +experience +score +apple +bought +led +pitch +coat +mass +card +band +rope +slip +win +dream +evening +condition +feed +tool +total +basic +smell +valley +nor +double +seat +arrive +master +track +parent +shore +division +sheet +substance +favor +connect +post +spend +chord +fat +glad +original +share +station +dad +bread +charge +proper +bar +offer +segment +slave +duck +instant +market +degree +populate +chick +dear +enemy +reply +drink +occur +support +speech +nature +range +steam +motion +path +liquid +log +meant +quotient +teeth +shell +neck diff --git a/strings/1000_most_common.txt b/strings/1400_most_common_alpha.txt similarity index 71% rename from strings/1000_most_common.txt rename to strings/1400_most_common_alpha.txt index adbe0f6fcf0..75032024b47 100644 --- a/strings/1000_most_common.txt +++ b/strings/1400_most_common_alpha.txt @@ -31,6 +31,7 @@ although always am amaze +among an and anger @@ -47,6 +48,7 @@ anyway apartment apparently appear +apple approach are area @@ -54,12 +56,15 @@ aren't arent arm around +arrange arrive +art as ask asleep ass at +atom attack attempt attention @@ -72,12 +77,18 @@ bad bag ball band +bank bar barely +base +basic +bat bathroom be +bear beat beautiful +beauty became because become @@ -96,6 +107,7 @@ best better between big +bird bit bite black @@ -103,25 +115,34 @@ blink block blonde blood +blow blue blush +board +boat body +bone book bore +born both bother bottle bottom +bought box boy boyfriend brain +branch +bread break breakfast breath breathe bright bring +broad broke broken brother @@ -140,9 +161,12 @@ by call calm came +camp can can't cant +capital +captain car card care @@ -154,59 +178,102 @@ catch caught cause cell +cent +center +century +certain chair chance change +character +charge +chart chase check cheek chest +chick +chief child children +choose +chord chuckle +circle city +claim class clean clear climb +clock close +clothe clothes +cloud +coast +coat coffee cold +collect college +colony color +column come comment +common +company +compare complete completely computer concern +condition confuse +connect consider +consonant +contain +continent continue control conversation +cook cool +copy +corn corner +correct +cost +cotton couch could couldn't couldnt +count counter +country couple course cover +cow crack crazy +crease +create +crop cross crowd cry cup +current cut cute dad damn dance +danger dark date daughter @@ -216,27 +283,45 @@ deal dear death decide +decimal deep definitely +degree +depend +describe +desert +design desk +determine +develop +dictionary did didn't didnt die +differ different +difficult dinner +direct direction disappear +discuss +distant +divide +division do doctor does doesn't doesnt dog +dollar don't done dont door +double doubt down drag @@ -248,60 +333,88 @@ drive drop drove dry +duck during each ear early +earth +ease easily +east easy eat edge +effect +egg +eight either +electric +element else empty end +enemy +energy +engine enjoy enough enter entire +equal +equate escape especially even evening +event eventually ever every everyone everything +exact exactly +example except excite exclaim excuse +exercise expect +experience +experiment explain expression eye eyebrow face fact +fair fall family +famous far +farm fast +fat father fault +favor favorite fear +feed feel feet fell felt few field +fig fight figure fill +final finally find fine @@ -309,12 +422,16 @@ finger finish fire first +fish fit five fix flash +flat flip floor +flow +flower fly focus follow @@ -322,25 +439,34 @@ food foot for force +forest forget form forward found four +fraction free +fresh friend from front frown +fruit fuck full fun funny further game +garden +gas gasp +gather gave gaze +general +gentle gently get giggle @@ -353,16 +479,22 @@ glance glare glass go -God +god +gold gone gonna good got gotten +govern grab +grand +grass +gray great green greet +grew grey grin grip @@ -372,6 +504,7 @@ group grow guard guess +guide gun guy had @@ -388,6 +521,7 @@ happen happy hard has +hat hate have haven't @@ -399,6 +533,7 @@ head hear heard heart +heat heavy hed held @@ -413,11 +548,14 @@ hey hi hide high +hill him himself his +history hit hold +hole home hope horse @@ -433,43 +571,54 @@ huh human hundred hung +hunt hurry hurt -I -I'd -I'll -I'm -I've +i +i'd +i'll +i'm +i've ice -Id +id idea if ignore -Ill -Im +ill +im imagine immediately important in +inch +include +indicate +industry +insect inside +instant instead +instrument interest interrupt into +invent +iron is +island isn't isnt it it's its -its -Ive +ive jacket jeans jerk job join joke +joy jump just keep @@ -479,6 +628,7 @@ kick kid kill kind +king kiss kitchen knee @@ -487,11 +637,14 @@ knock know known lady +lake land +language large last late laugh +law lay lead lean @@ -501,9 +654,11 @@ leave led left leg +length less let letter +level lie life lift @@ -511,11 +666,16 @@ light like line lip +liquid +list listen little live +locate lock locker +log +lone long look lose @@ -525,48 +685,73 @@ loud love low lunch +machine mad made +magnet +main +major make man manage many +map mark +market marry +mass +master match +material matter may maybe me mean meant +measure +meat meet +melody memory men mention met +metal +method middle might +mile +milk +million mind mine minute mirror miss +mix +modern +molecule mom moment money month mood +moon more morning most mother +motion +mount +mountain mouth move movie mr mrs much +multiply mum mumble music @@ -575,35 +760,51 @@ mutter my myself name +nation +natural +nature near nearly +necessary neck need +neighbor nervous never new next nice night +nine no nod noise none +noon +nor normal +north nose not note nothing notice +noun now number +numeral +object +observe obviously +occur +ocean of off offer office often oh +oil okay old on @@ -612,72 +813,115 @@ one only onto open +operate +opposite or order +organ +original other our out outside over own +oxygen pack +page pain paint pair pants paper +paragraph +parent parents park part +particular party pass past +path +pattern pause pay people perfect perhaps +period person phone +phrase pick picture piece pink piss +pitch place +plain plan +plane +planet +plant play please +plural pocket +poem point police +poor pop +populate +port +pose position possible +post +pound power practically +practice +prepare present press pretend pretty +print +probable probably problem +process +produce +product promise +proper +property +protect +prove +provide pull punch push put +quart question quick quickly quiet quietly quite +quotient race +radio +rail rain raise ran rang +range rather reach read @@ -686,8 +930,11 @@ real realize really reason +receive recognize +record red +region relationship relax remain @@ -695,63 +942,97 @@ remember remind repeat reply +represent +require respond rest +result return +rich ride right ring +rise +river road rock roll room +root +rope rose round +row rub +rule run rush sad safe said +sail +salt same +sand sat save saw say +scale scare school +science +score scream +sea search +season seat second +section see +seed seem seen +segment +select self +sell send sense sent +sentence +separate serious seriously +serve set settle seven several shadow shake +shall +shape share +sharp she she'd she's shed +sheet +shell shes shift +shine +ship shirt shit shock shoe shook shop +shore short shot should @@ -771,17 +1052,24 @@ sight sign silence silent +silver +similar +simple simply since +sing single sir sister sit situation six +size +skill skin sky slam +slave sleep slightly slip @@ -793,9 +1081,14 @@ smile smirk smoke snap +snow so soft softly +soil +soldier +solution +solve some somehow someone @@ -808,30 +1101,46 @@ soon sorry sort sound +south space speak +special +speech +speed +spell spend spent spoke spot +spread +spring +square stair stand star stare start state +station stay +stead +steam +steel step stick still stomach +stone stood stop store story straight strange +stream street +stretch +string strong struggle stuck @@ -839,29 +1148,48 @@ student study stuff stupid +subject +substance +subtract +success such suck sudden suddenly +suffix +sugar suggest +suit summer sun +supply +support suppose sure +surface surprise surround sweet +swim +syllable +symbol +system table +tail take taken talk tall +teach teacher team tear teeth tell +temperature ten +term +test than thank that @@ -882,6 +1210,7 @@ they're theyd theyre thick +thin thing think third @@ -889,11 +1218,13 @@ this those though thought +thousand three threw throat through throw +thus tie tight time @@ -909,29 +1240,38 @@ tongue tonight too took +tool top +total totally touch toward town track +trade trail train +travel tree +triangle trip trouble -TRUE +truck +true +true trust truth try +tube turn -TV +tv twenty two type uncle under understand +unit until up upon @@ -939,17 +1279,26 @@ us use usual usually +valley +value +vary +verb very +view +village visit voice +vowel wait wake walk wall want +war warm warn was +wash wasn't wasnt watch @@ -961,21 +1310,23 @@ we'll we're we've wear +weather week +weight weird well -well went were -were weren't werent +west wet weve what what's whatever whats +wheel when where whether @@ -985,13 +1336,19 @@ whisper white who whole +whose why wide wife +wild will +win wind window +wing +winter wipe +wire wish with within @@ -1015,10 +1372,14 @@ wouldnt wow wrap write +written wrong +wrote +yard yeah year yell +yellow yes yet you