Add support for more languages for mnemonics (#2424)

* Add more languages for mnemonics

* Remove check for english if french

* remove dev artifact

---------

Co-authored-by: Wim van der Ham <wfjvdham@gmail.com>
Co-authored-by: Kim Neunert <kim@swanbitcoin.com>
Co-authored-by: k9ert <k9ert@gmx.de>

requirements.in

@@ -19,7 +19,7 @@ pyopenssl==23.0.0
 flask_wtf==1.2.1
 pgpy==0.6.0
 cbor2==5.4.6
-mnemonic==0.20
+mnemonic==0.21
 cryptography==39.0.1
 Flask-APScheduler==1.12.4
 gunicorn==20.1.0

requirements.txt

@@ -443,9 +443,9 @@ markupsafe==2.1.2 \
     #   jinja2
     #   werkzeug
     #   wtforms
-mnemonic==0.20 \
-    --hash=sha256:7c6fb5639d779388027a77944680aee4870f0fcd09b1e42a5525ee2ce4c625f6 \
-    --hash=sha256:acd2168872d0379e7a10873bb3e12bf6c91b35de758135c4fbd1015ef18fafc5
+mnemonic==0.21 \
+    --hash=sha256:1fe496356820984f45559b1540c80ff10de448368929b9c60a2b55744cc88acf \
+    --hash=sha256:72dc9de16ec5ef47287237b9b6943da11647a03fe7cf1f139fc3d7c4a7439288
     # via
     #   -r requirements.in
     #   hwi

src/cryptoadvance/specter/util/mnemonic.py

@@ -14,10 +14,14 @@ and embit (https://github.com/diybitcoinhardware/embit)"""
 MNEMONIC_LANGUAGES = {
     "en": "english",
     "es": "spanish",
-    # "fr": "french", # Not supported, for details see: https://github.com/trezor/python-mnemonic/issues/98)
+    "fr": "french",
     "it": "italian",
-    # "jp": "japanese",
-    # "ko": korean",
+    "jp": "japanese",
+    "cz": "czech",
+    "pt": "portuguese",
+    # "ko": "korean", requires special characters
+    # "ru": "russian", requires special characters
+    # "tr": "turkish", has a lot of overlapping words with other languages
     # "?": chinese_simplified",
     # "?": chinese_traditional",
 }
@@ -38,7 +42,7 @@ def generate_mnemonic(strength=256, language_code="en") -> str:
     "tomorrow question cook lend burden bone own junior stage square leaf father edge decrease pipe tired useful junior calm silver topple require rug clock"
 
     :param strength: 256 (default) will give you 24 words. 128 will result in 12 words
-    :param language_code: only "en", "es", and "it" are supported. If you use an unsupported one, it'll fallback to English.
+    :param language_code: "en", "es", "fr", "it", "jp", "cz" and "pt" are supported. If you use an unsupported one, it'll fallback to English.
     """
     mnemo = initialize_mnemonic(language_code)
     return mnemo.generate(strength=strength)
@@ -50,14 +54,6 @@ def get_language(mnemonic: str) -> str:
         supported_language_found = False
         language = Mnemonic.detect_language(mnemonic)
         words_as_list = mnemonic.split()
-        # If we get French as language we have to double check whether it is not really an English mnemonic because of overlaps in the wordlists
-        if language == "french":
-            count = 0
-            for word in words_as_list:
-                if word in WORDLIST:
-                    count += 1
-            if count == 12:
-                language = "english"
         for key, value in MNEMONIC_LANGUAGES.items():
             if value == language:
                 supported_language_found = True

tests/test_util_mnemonic.py

@@ -1,5 +1,7 @@
 import pytest
 from cryptoadvance.specter.util.mnemonic import *
+from collections import Counter, defaultdict
+from itertools import combinations
 
 ghost_machine = (
     "ghost ghost ghost ghost ghost ghost ghost ghost ghost ghost ghost machine"
@@ -11,25 +13,35 @@ exulter_ivoire = "exulter exulter exulter exulter exulter exulter exulter exulte
 
 gravoso_mummia = "gravoso gravoso gravoso gravoso gravoso gravoso gravoso gravoso gravoso gravoso gravoso mummia"
 
+french = "cylindre corniche attentif boulon toboggan foudre bambin intrigue sélectif adverbe codifier neurone"
+
+japanese = "ちほう そざい そんみん まもる いりょう ねだん たんとう きくばり つみき たんのう けんい なおす"
+
+czech = "poryv troska mozol poledne louka levobok kuna lump korpus zvesela lyra kauce"
+
+portuguese = "assador borbulha sorteio enguia castelo fanfarra vertente duplo legista aferir vazio apito"
+
 
 def test_initialize_mnemonic(caplog):
-    mnemo_en = initialize_mnemonic("en")
-    assert isinstance(mnemo_en, Mnemonic)
-    assert mnemo_en.language == "english"
-    mnemo_es = initialize_mnemonic("es")
-    assert mnemo_es.language == "spanish"
+
+    for key in MNEMONIC_LANGUAGES:
+        mnemo = initialize_mnemonic(key)
+        assert isinstance(mnemo, Mnemonic)
+        assert mnemo.language == MNEMONIC_LANGUAGES[key]
+
     mnemo_undefined = initialize_mnemonic("gu")
     # Defaults to English if language code is undefined
     assert mnemo_undefined.language == "english"
-    mnemo_fr = initialize_mnemonic("fr")
-    # ... or if language is not supported
-    assert mnemo_fr.language == "english"
 
 
 def test_get_language():
     assert get_language(ghost_machine) == "english"
     assert get_language(ganso_madera) == "spanish"
     assert get_language(gravoso_mummia) == "italian"
+    assert get_language(french) == "french"
+    assert get_language(japanese) == "japanese"
+    assert get_language(czech) == "czech"
+    assert get_language(portuguese) == "portuguese"
     # This mnemonic created a problem on Cirrus, since "client" is part of the English and the French wordlists
     assert (
         get_language(
@@ -37,11 +49,7 @@ def test_get_language():
         )
         == "english"
     )
-    with pytest.raises(
-        SpecterError, match="The language French is not supported"
-    ) as se:
-        get_language(exulter_ivoire)
-    with pytest.raises(SpecterError, match="Language not detected") as se:
+    with pytest.raises(SpecterError, match="Language unrecognized for") as se:
         get_language(
             "muh ghost ghost ghost ghost ghost ghost ghost ghost ghost ghost machine"
         )
@@ -78,26 +86,41 @@ def test_mnemonic_to_root():
     )
 
 
+def count_multi_occurrence_languages(word_lists_dict):
+    # Create a Counter to store occurrences of words across different lists
+    word_counter = Counter()
+
+    # Create a defaultdict to store the lists in which each word occurs
+    word_lists_indices = defaultdict(list)
+
+    # Iterate over each language and its corresponding word list
+    for language, word_list in word_lists_dict.items():
+        # Update the counter with the words from the current list
+        word_counter.update(word_list)
+
+        # Update the word_lists_indices with the lists in which each word occurs
+        for word in set(word_list):
+            word_lists_indices[word].append(language)
+
+    # Filter words that occurred more than once across different lists
+    multi_occurrence_languages = Counter()
+    for word, count in word_counter.items():
+        if count > 1:
+            language_combinations = combinations(word_lists_indices[word], 2)
+            multi_occurrence_languages.update(language_combinations)
+
+    return multi_occurrence_languages
+
+
 # There is an overlap of 100 words between the English and the French wordlists
 # These are sanity checks that we don't have further overlaps
 def test_duplicates_in_wordlists():
-    mnemo_en = initialize_mnemonic("en")
-    mnemo_es = initialize_mnemonic("es")
-    mnemo_it = initialize_mnemonic("it")
-    mnemo_fr = initialize_mnemonic("fr")
 
-    wordlist_en = mnemo_en.wordlist
-    wordlist_es = mnemo_es.wordlist
-    wordlist_it = mnemo_it.wordlist
-    wordlist_fr = mnemo_fr.wordlist
+    word_lists_dict = dict()
+    for key in MNEMONIC_LANGUAGES:
+        mnemo = initialize_mnemonic(key)
+        word_lists_dict[key] = mnemo.wordlist
 
-    # EN-IT
-    assert len([word for word in wordlist_en if word in wordlist_it]) == 0
-    # EN-ES
-    assert len([word for word in wordlist_en if word in wordlist_es]) == 0
-    # ES-IT
-    assert len([word for word in wordlist_es if word in wordlist_it]) == 0
-    # ES-FR
-    assert len([word for word in wordlist_es if word in wordlist_fr]) == 0
-    # IT-FR
-    assert len([word for word in wordlist_it if word in wordlist_fr]) == 0
+    multi_occurrence_languages = count_multi_occurrence_languages(word_lists_dict)
+    # only EN-FR should be here
+    assert len(multi_occurrence_languages) == 1