DYOM Translation - Support for modded in languages

.gitignore

@@ -1,4 +1,6 @@
 .ccls
 .ccls-cache
 bin/
-bin_rel/
+bin_rel/
+/.vs
+/build

config.toml

@@ -338,6 +338,15 @@ RandomizeChaosPoints = false
 # Picks missions only from those designated as English by the DYOM website.
 UseEnglishOnlyFilter = false
 
+AutoTranslateToEnglish = true
+RandomSpawn = true
+
+# Languages in translation chain separated by a semicolon, e.g. "pl;el;zh-CN;en" means Polish->Greek->Simplified Chinese->English, see https://cloud.google.com/translate/docs/languages for supported languages and their codes
+TranslationChain = "en"
+
+#Character replacement pairs. Original unicode symbol must be followed by the byte of it's in-game counterpart (GTA SA uses Windows-1252)
+#CharactersMap = "А;41;Б;80;В;8B;Г;82;Д;83;Е;45;Ё;A8;Ж;84;З;88;И;85;Й;86;К;4B;Л;87;М;96;Н;AD;О;4F;П;8C;Р;50;С;43;Т;8F;У;59;Ф;81;Х;58;Ц;89;Ч;8D;Ш;8E;Щ;8A;Ъ;90;Ы;91;Ь;92;Э;93;Ю;94;Я;95;а;61;б;97;в;A2;г;99;д;9A;е;65;ё;B8;ж;9B;з;9F;и;9C;й;9D;к;6B;л;9E;м;AF;н;AE;о;6F;п;A3;р;70;с;63;т;A6;у;79;ф;98;х;78;ц;A0;ч;A4;ш;A5;щ;A1;ъ;A7;ы;A8;ь;A9;э;AA;ю;AB;я;AC"
+
 #######################################################
 [CheatRandomizer]
 

include/dyom.hh

@@ -37,6 +37,7 @@ public:
         bool        EnableTextToSpeech;
         bool        RandomSpawn;
         std::string TranslationChain;
+        std::string CharactersMap;
         double      OverrideTTSVolume;
     } m_Config;
 

include/util/dyom/Translation.hh

@@ -3,6 +3,7 @@
 #include <sstream>
 #include <string>
 #include <regex>
+#include <map>
 
 #include <windows.h>
 #include <wininet.h>
@@ -24,7 +25,8 @@ class DyomTranslator
     void ProcessDidTranslate (std::string translated);
 
 public:
-    DyomTranslator (const std::string &translationChain = "");
+    DyomTranslator (const std::string &translationChain = "",
+                    const std::string &charactersMap = "");
     ~DyomTranslator () { internet.Close (); }
 
     void FixupGxtTokens (std::string &text);
@@ -44,4 +46,5 @@ public:
     }
 
     std::vector<std::string> mTranslationChain;
+    std::map<std::string, char> mCharacterMap;
 };

src/dyom.cc

@@ -139,6 +139,7 @@ DyomRandomizer::Initialise ()
             std::pair ("UseEnglishOnlyFilter", &m_Config.EnglishOnly),
             std::pair ("RandomSpawn", &m_Config.RandomSpawn),
             std::pair ("TranslationChain", &m_Config.TranslationChain),
+            std::pair ("CharactersMap", &m_Config.CharactersMap),
             std::pair ("EnableTextToSpeech", &m_Config.EnableTextToSpeech),
             std::pair ("OverrideTTSVolume", &m_Config.OverrideTTSVolume),
             std::pair ("AutoTranslateToEnglish",
@@ -254,7 +255,8 @@ void
 DyomRandomizer::SaveMission (const std::vector<uint8_t> &data)
 {
     DYOM::DYOMFileStructure dyomFile;
-    DyomTranslator          translator (m_Config.TranslationChain);
+    DyomTranslator          translator (m_Config.TranslationChain,
+                               m_Config.CharactersMap);
 
     dyomFile.Read (data);
 

src/util/dyom/Translation.cc

@@ -55,7 +55,8 @@ EncodeURL (const std::string &s)
 }
 
 /*******************************************************/
-DyomTranslator::DyomTranslator (const std::string &translationChain)
+DyomTranslator::DyomTranslator (const std::string &translationChain,
+                                const std::string &charactersMap)
 {
     internet.Open ("translate.google.com");
 
@@ -68,6 +69,21 @@ DyomTranslator::DyomTranslator (const std::string &translationChain)
             for (std::string token; std::getline (iss, token, ';');)
                 mTranslationChain.push_back (std::move (token));
         }
+
+    if (!charactersMap.empty ())
+        {
+            std::istringstream iss (charactersMap);
+            std::string        token_v;
+            char               ch = '0';
+            for (std::string token; std::getline (iss, token, ';')
+                                    && std::getline (iss, token_v, ';');)
+                {
+                    ch = (char)std::stoi (token_v, nullptr, 16);
+                    mCharacterMap.insert (std::pair<std::string, char> (
+                        std::move (token),
+                        ch));
+                }
+        }
 }
 
 /*******************************************************/
@@ -206,11 +222,22 @@ DyomTranslator::TranslateText (const std::string &text)
             translation = cm[1];
         }
 
+    for (const auto &[key, value] : mCharacterMap)
+        {
+            std::string::size_type pos = 0;
+            while ((pos = translation.find (key, pos)) != std::string::npos)
+                {
+                    translation.replace (pos, key.length(), 1, value);
+                    pos += 1;
+                }
+        }
+
     // translator tends to break tags with spaces, attempt to fix
     DecodeSpecialChars (translation);
     translation
         = std::regex_replace (translation,
                               std::regex ("~\\s*([a-zA-Z0-9]+)\\s*~"), "~$1~");
+
     return translation;
 }
 
@@ -228,6 +255,7 @@ DyomTranslator::DecodeSpecialChars (std::string &text)
     SPECIAL_CHAR ("&amp;", "&");
     SPECIAL_CHAR ("\xe2\x80\x99", "'"); //UTF-8 quotation mark
 
+    if (mCharacterMap.size () < 1)
     // Remove any other non-ascii characters
     text.erase (std::remove_if (text.begin (), text.end (),
                                 [] (char ch) { return ch < 0; }),