Test TTSTokenizer

TTS/tts/utils/text/tokenizer.py

@@ -21,6 +21,14 @@ class TTSTokenizer:
         phonemizer (Phonemizer):
             A phonemizer object or a dict that maps language codes to phonemizer objects. Defaults to None.
 
+    Example:
+
+        >>> from TTS.tts.utils.text.tokenizer import TTSTokenizer
+        >>> tokenizer = TTSTokenizer(use_phonemes=False, characters=Graphemes())
+        >>> text = "Hello world!"
+        >>> ids = tokenizer.text_to_ids(text)
+        >>> text_hat = tokenizer.ids_to_text(ids)
+        >>> assert text == text_hat
     """
 
     def __init__(
@@ -89,21 +97,22 @@ class TTSTokenizer:
         return [self.characters.bos] + list(char_sequence) + [self.characters.eos]
 
     def intersperse_blank_char(self, char_sequence: List[str], use_blank_char: bool = False):
-        char_to_use = self.characters.blank_char if use_blank_char else self.characters.pad
+        char_to_use = self.characters.blank if use_blank_char else self.characters.pad
         result = [char_to_use] * (len(char_sequence) * 2 + 1)
         result[1::2] = char_sequence
         return result
 
-    def print_logs(self, level: int = 1):
+    def print_logs(self, level: int = 0):
         indent = "\t" * level
         print(f"{indent}| > add_blank: {self.use_phonemes}")
         print(f"{indent}| > use_eos_bos: {self.use_phonemes}")
         print(f"{indent}| > use_phonemes: {self.use_phonemes}")
-        print(f"{indent}| > phonemizer: {self.phonemizer.print_logs(level + 1)}")
+        if self.use_phonemes:
+            print(f"{indent}| > phonemizer: {self.phonemizer.print_logs(level + 1)}")
 
     @staticmethod
     def init_from_config(config: "Coqpit"):
-        """Init Tokenizer object from the config.
+        """Init Tokenizer object from config
 
         Args:
             config (Coqpit): Coqpit model config.

tests/text_tests/test_tokenizer.py

@@ -0,0 +1,88 @@
+from dataclasses import dataclass
+from os import sep
+import unittest
+
+from TTS.tts.utils.text.tokenizer import TTSTokenizer
+from TTS.tts.utils.text.characters import Graphemes, IPAPhonemes, _phonemes, _punctuations, _eos, _bos, _pad, _blank
+from TTS.tts.utils.text.phonemizers import ESpeak
+
+from coqpit import Coqpit
+
+
+class TestTTSTokenizer(unittest.TestCase):
+    def setUp(self):
+        self.tokenizer = TTSTokenizer(use_phonemes=False, characters=Graphemes())
+
+        self.ph = ESpeak('tr')
+        self.tokenizer_ph = TTSTokenizer(use_phonemes=True, characters=IPAPhonemes(), phonemizer=self.ph)
+
+    def test_encode_decode_graphemes(self):
+        text = "This is, a test."
+        ids = self.tokenizer.encode(text)
+        test_hat = self.tokenizer.decode(ids)
+        self.assertEqual(text, test_hat)
+        self.assertEqual(len(ids), len(text))
+
+    def test_text_to_ids_phonemes(self):
+        # TODO: note sure how to extend to cover all the languages and phonemizer.
+        text = "Bu bir Örnek."
+        text_ph = self.ph.phonemize(text, separator="")
+        ids = self.tokenizer_ph.text_to_ids(text)
+        test_hat = self.tokenizer_ph.ids_to_text(ids)
+        self.assertEqual(text_ph, test_hat)
+
+    def test_text_to_ids_phonemes_with_eos_bos(self):
+        text = "Bu bir Örnek."
+        self.tokenizer_ph.use_eos_bos = True
+        text_ph = IPAPhonemes().bos + self.ph.phonemize(text, separator="") + IPAPhonemes().eos
+        ids = self.tokenizer_ph.text_to_ids(text)
+        test_hat = self.tokenizer_ph.ids_to_text(ids)
+        self.assertEqual(text_ph, test_hat)
+
+    def test_text_to_ids_phonemes_with_eos_bos_and_blank(self):
+        text = "Bu bir Örnek."
+        self.tokenizer_ph.use_eos_bos = True
+        self.tokenizer_ph.add_blank = True
+        text_ph = "<BOS><BLNK>b<BLNK>ʊ<BLNK> <BLNK>b<BLNK>ɪ<BLNK>r<BLNK> <BLNK>œ<BLNK>r<BLNK>n<BLNK>ˈ<BLNK>ɛ<BLNK>c<BLNK>.<BLNK><EOS>"
+        ids = self.tokenizer_ph.text_to_ids(text)
+        text_hat = self.tokenizer_ph.ids_to_text(ids)
+        self.assertEqual(text_ph, text_hat)
+
+    def test_print_logs(self):
+        self.tokenizer.print_logs()
+        self.tokenizer_ph.print_logs()
+
+    def test_init_from_config(self):
+
+        @dataclass
+        class Characters(Coqpit):
+            characters: str = _phonemes
+            punctuations: str = _punctuations
+            pad: str = _pad
+            eos: str = _eos
+            bos: str = _bos
+            blank: str = _blank
+            is_unique: bool = True
+            is_sorted: bool = True
+
+        @dataclass
+        class TokenizerConfig(Coqpit):
+            enable_eos_bos_chars: bool = True
+            use_phonemes: bool = True
+            add_blank: bool = False
+            characters: str = Characters()
+            phonemizer: str = "espeak"
+            phoneme_language: str = "tr"
+            text_cleaner: str = "phoneme_cleaners"
+            characters = Characters()
+
+        tokenizer_ph = TTSTokenizer.init_from_config(TokenizerConfig())
+        text = "Bu bir Örnek."
+        text_ph = "<BOS>" + self.ph.phonemize(text, separator="") + "<EOS>"
+        ids = tokenizer_ph.text_to_ids(text)
+        test_hat =  tokenizer_ph.ids_to_text(ids)
+        self.assertEqual(text_ph, test_hat)
+
+
+
+