diff --git a/TTS/tts/utils/text/tokenizer.py b/TTS/tts/utils/text/tokenizer.py index 8c231c14..e79cf5e5 100644 --- a/TTS/tts/utils/text/tokenizer.py +++ b/TTS/tts/utils/text/tokenizer.py @@ -21,6 +21,14 @@ class TTSTokenizer: phonemizer (Phonemizer): A phonemizer object or a dict that maps language codes to phonemizer objects. Defaults to None. + Example: + + >>> from TTS.tts.utils.text.tokenizer import TTSTokenizer + >>> tokenizer = TTSTokenizer(use_phonemes=False, characters=Graphemes()) + >>> text = "Hello world!" + >>> ids = tokenizer.text_to_ids(text) + >>> text_hat = tokenizer.ids_to_text(ids) + >>> assert text == text_hat """ def __init__( @@ -89,21 +97,22 @@ class TTSTokenizer: return [self.characters.bos] + list(char_sequence) + [self.characters.eos] def intersperse_blank_char(self, char_sequence: List[str], use_blank_char: bool = False): - char_to_use = self.characters.blank_char if use_blank_char else self.characters.pad + char_to_use = self.characters.blank if use_blank_char else self.characters.pad result = [char_to_use] * (len(char_sequence) * 2 + 1) result[1::2] = char_sequence return result - def print_logs(self, level: int = 1): + def print_logs(self, level: int = 0): indent = "\t" * level print(f"{indent}| > add_blank: {self.use_phonemes}") print(f"{indent}| > use_eos_bos: {self.use_phonemes}") print(f"{indent}| > use_phonemes: {self.use_phonemes}") - print(f"{indent}| > phonemizer: {self.phonemizer.print_logs(level + 1)}") + if self.use_phonemes: + print(f"{indent}| > phonemizer: {self.phonemizer.print_logs(level + 1)}") @staticmethod def init_from_config(config: "Coqpit"): - """Init Tokenizer object from the config. + """Init Tokenizer object from config Args: config (Coqpit): Coqpit model config. diff --git a/tests/text_tests/test_tokenizer.py b/tests/text_tests/test_tokenizer.py new file mode 100644 index 00000000..6b7982cd --- /dev/null +++ b/tests/text_tests/test_tokenizer.py @@ -0,0 +1,88 @@ +from dataclasses import dataclass +from os import sep +import unittest + +from TTS.tts.utils.text.tokenizer import TTSTokenizer +from TTS.tts.utils.text.characters import Graphemes, IPAPhonemes, _phonemes, _punctuations, _eos, _bos, _pad, _blank +from TTS.tts.utils.text.phonemizers import ESpeak + +from coqpit import Coqpit + + +class TestTTSTokenizer(unittest.TestCase): + def setUp(self): + self.tokenizer = TTSTokenizer(use_phonemes=False, characters=Graphemes()) + + self.ph = ESpeak('tr') + self.tokenizer_ph = TTSTokenizer(use_phonemes=True, characters=IPAPhonemes(), phonemizer=self.ph) + + def test_encode_decode_graphemes(self): + text = "This is, a test." + ids = self.tokenizer.encode(text) + test_hat = self.tokenizer.decode(ids) + self.assertEqual(text, test_hat) + self.assertEqual(len(ids), len(text)) + + def test_text_to_ids_phonemes(self): + # TODO: note sure how to extend to cover all the languages and phonemizer. + text = "Bu bir Örnek." + text_ph = self.ph.phonemize(text, separator="") + ids = self.tokenizer_ph.text_to_ids(text) + test_hat = self.tokenizer_ph.ids_to_text(ids) + self.assertEqual(text_ph, test_hat) + + def test_text_to_ids_phonemes_with_eos_bos(self): + text = "Bu bir Örnek." + self.tokenizer_ph.use_eos_bos = True + text_ph = IPAPhonemes().bos + self.ph.phonemize(text, separator="") + IPAPhonemes().eos + ids = self.tokenizer_ph.text_to_ids(text) + test_hat = self.tokenizer_ph.ids_to_text(ids) + self.assertEqual(text_ph, test_hat) + + def test_text_to_ids_phonemes_with_eos_bos_and_blank(self): + text = "Bu bir Örnek." + self.tokenizer_ph.use_eos_bos = True + self.tokenizer_ph.add_blank = True + text_ph = "bʊ bɪr œrnˈɛc." + ids = self.tokenizer_ph.text_to_ids(text) + text_hat = self.tokenizer_ph.ids_to_text(ids) + self.assertEqual(text_ph, text_hat) + + def test_print_logs(self): + self.tokenizer.print_logs() + self.tokenizer_ph.print_logs() + + def test_init_from_config(self): + + @dataclass + class Characters(Coqpit): + characters: str = _phonemes + punctuations: str = _punctuations + pad: str = _pad + eos: str = _eos + bos: str = _bos + blank: str = _blank + is_unique: bool = True + is_sorted: bool = True + + @dataclass + class TokenizerConfig(Coqpit): + enable_eos_bos_chars: bool = True + use_phonemes: bool = True + add_blank: bool = False + characters: str = Characters() + phonemizer: str = "espeak" + phoneme_language: str = "tr" + text_cleaner: str = "phoneme_cleaners" + characters = Characters() + + tokenizer_ph = TTSTokenizer.init_from_config(TokenizerConfig()) + text = "Bu bir Örnek." + text_ph = "" + self.ph.phonemize(text, separator="") + "" + ids = tokenizer_ph.text_to_ids(text) + test_hat = tokenizer_ph.ids_to_text(ids) + self.assertEqual(text_ph, test_hat) + + + +