coqui-ai
/
TTS
mirror of https://github.com/coqui-ai/TTS.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add notebooks

pull/10/head
erogol 2020-07-20 14:55:00 +02:00
parent 72a6ac54c8
commit d4aacd5f6d
5 changed files with 2386 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -128,3 +128,4 @@ tests/outputs/*
TODO.txt
.vscode/*
data/*
notebooks/data/*

329
notebooks/DDC_TTS_and_MultiBand_MelGAN_Example.ipynb Normal file
View File

@ -0,0 +1,329 @@
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"name": "DDC-TTS_and_MultiBand-MelGAN_Example.ipynb",
"provenance": [],
"collapsed_sections": [],
"toc_visible": true
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"accelerator": "GPU"
},
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "6LWsNd3_M3MP",
"colab_type": "text"
},
"source": [
"# Mozilla TTS on CPU Real-Time Speech Synthesis "
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "FAqrSIWgLyP0",
"colab_type": "text"
},
"source": [
"We use Tacotron2 and MultiBand-Melgan models and LJSpeech dataset.\n",
"\n",
"Tacotron2 is trained using [Double Decoder Consistency](https://erogol.com/solving-attention-problems-of-tts-models-with-double-decoder-consistency/) (DDC) only for 130K steps (3 days) with a single GPU.\n",
"\n",
"MultiBand-Melgan is trained 1.45M steps with real spectrograms.\n",
"\n",
"Note that both model performances can be improved with more training."
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "Ku-dA4DKoeXk",
"colab_type": "text"
},
"source": [
"### Download Models"
]
},
{
"cell_type": "code",
"metadata": {
"id": "jGIgnWhGsxU1",
"colab_type": "code",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 162
},
"outputId": "88725e41-a8dc-4885-b3bf-cac939f38abe",
"tags": []
},
"source": [
"!gdown --id 1dntzjWFg7ufWaTaFy80nRz-Tu02xWZos -O data/tts_model.pth.tar\n",
"!gdown --id 18CQ6G6tBEOfvCHlPqP8EBI4xWbrr9dBc -O data/config.json"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "4dnpE0-kvTsu",
"colab_type": "code",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 235
},
"outputId": "76377c6d-789c-4995-ba00-a21a6e1c401e",
"tags": []
},
"source": [
"!gdown --id 1Ty5DZdOc0F7OTGj9oJThYbL5iVu_2G0K -O data/vocoder_model.pth.tar\n",
"!gdown --id 1Rd0R_nRCrbjEdpOwq6XwZAktvugiBvmu -O data/config_vocoder.json\n",
"!gdown --id 11oY3Tv0kQtxK_JPgxrfesa99maVXHNxU -O data/scale_stats.npy"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {
"id": "Zlgi8fPdpRF0",
"colab_type": "text"
},
"source": [
"### Define TTS function"
]
},
{
"cell_type": "code",
"metadata": {
"id": "f-Yc42nQZG5A",
"colab_type": "code",
"colab": {}
},
"source": [
"def tts(model, text, CONFIG, use_cuda, ap, use_gl, figures=True):\n",
" t_1 = time.time()\n",
" waveform, alignment, mel_spec, mel_postnet_spec, stop_tokens, inputs = synthesis(model, text, CONFIG, use_cuda, ap, speaker_id, style_wav=None,\n",
" truncated=False, enable_eos_bos_chars=CONFIG.enable_eos_bos_chars)\n",
" # mel_postnet_spec = ap._denormalize(mel_postnet_spec.T)\n",
" if not use_gl:\n",
" waveform = vocoder_model.inference(torch.FloatTensor(mel_postnet_spec.T).unsqueeze(0))\n",
" waveform = waveform.flatten()\n",
" if use_cuda:\n",
" waveform = waveform.cpu()\n",
" waveform = waveform.numpy()\n",
" rtf = (time.time() - t_1) / (len(waveform) / ap.sample_rate)\n",
" tps = (time.time() - t_1) / len(waveform)\n",
" print(waveform.shape)\n",
" print(\" > Run-time: {}\".format(time.time() - t_1))\n",
" print(\" > Real-time factor: {}\".format(rtf))\n",
" print(\" > Time per step: {}\".format(tps))\n",
" IPython.display.display(IPython.display.Audio(waveform, rate=CONFIG.audio['sample_rate'])) \n",
" return alignment, mel_postnet_spec, stop_tokens, waveform"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {
"id": "ZksegYQepkFg",
"colab_type": "text"
},
"source": [
"### Load Models"
]
},
{
"cell_type": "code",
"metadata": {
"id": "oVa0kOamprgj",
"colab_type": "code",
"colab": {}
},
"source": [
"import os\n",
"import torch\n",
"import time\n",
"import IPython\n",
"\n",
"from TTS.tts.utils.generic_utils import setup_model\n",
"from TTS.utils.io import load_config\n",
"from TTS.tts.utils.text.symbols import symbols, phonemes\n",
"from TTS.utils.audio import AudioProcessor\n",
"from TTS.tts.utils.synthesis import synthesis"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "EY-sHVO8IFSH",
"colab_type": "code",
"colab": {}
},
"source": [
"# runtime settings\n",
"use_cuda = False"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "_1aIUp2FpxOQ",
"colab_type": "code",
"colab": {}
},
"source": [
"# model paths\n",
"TTS_MODEL = \"data/tts_model.pth.tar\"\n",
"TTS_CONFIG = \"data/config.json\"\n",
"VOCODER_MODEL = \"data/vocoder_model.pth.tar\"\n",
"VOCODER_CONFIG = \"data/config_vocoder.json\""
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "CpgmdBVQplbv",
"colab_type": "code",
"colab": {}
},
"source": [
"# load configs\n",
"TTS_CONFIG = load_config(TTS_CONFIG)\n",
"VOCODER_CONFIG = load_config(VOCODER_CONFIG)"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "zmrQxiozIUVE",
"colab_type": "code",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 471
},
"outputId": "60c4daa0-4c5b-4a2e-fe0d-be437d003a49",
"tags": []
},
"source": [
"# load the audio processor\n",
"TTS_CONFIG.audio['stats_path'] = 'data/scale_stats.npy'\n",
"ap = AudioProcessor(**TTS_CONFIG.audio) "
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "8fLoI4ipqMeS",
"colab_type": "code",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 35
},
"outputId": "b789066e-e305-42ad-b3ca-eba8d9267382",
"tags": []
},
"source": [
"# LOAD TTS MODEL\n",
"# multi speaker \n",
"speaker_id = None\n",
"speakers = []\n",
"\n",
"# load the model\n",
"num_chars = len(phonemes) if TTS_CONFIG.use_phonemes else len(symbols)\n",
"model = setup_model(num_chars, len(speakers), TTS_CONFIG)\n",
"\n",
"# load model state\n",
"cp = torch.load(TTS_MODEL, map_location=torch.device('cpu'))\n",
"\n",
"# load the model\n",
"model.load_state_dict(cp['model'])\n",
"if use_cuda:\n",
" model.cuda()\n",
"model.eval()\n",
"\n",
"# set model stepsize\n",
"if 'r' in cp:\n",
" model.decoder.set_r(cp['r'])"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "zKoq0GgzqzhQ",
"colab_type": "code",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 1000
},
"outputId": "234efc61-f37a-40bc-95a3-b51896018ccb",
"tags": []
},
"source": [
"from TTS.vocoder.utils.generic_utils import setup_generator\n",
"\n",
"# LOAD VOCODER MODEL\n",
"vocoder_model = setup_generator(VOCODER_CONFIG)\n",
"vocoder_model.load_state_dict(torch.load(VOCODER_MODEL, map_location=\"cpu\")[\"model\"])\n",
"vocoder_model.remove_weight_norm()\n",
"vocoder_model.inference_padding = 0\n",
"\n",
"ap_vocoder = AudioProcessor(**VOCODER_CONFIG['audio']) \n",
"if use_cuda:\n",
" vocoder_model.cuda()\n",
"vocoder_model.eval()"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {
"id": "Ws_YkPKsLgo-",
"colab_type": "text"
},
"source": [
"## Run Inference"
]
},
{
"cell_type": "code",
"metadata": {
"id": "FuWxZ9Ey5Puj",
"colab_type": "code",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 134
},
"outputId": "9c06adad-5451-4393-89a1-a2e7dc39ab91",
"tags": []
},
"source": [
"sentence = \"Bill got in the habit of asking himself “Is that thought true?” and if he wasn’t absolutely certain it was, he just let it go.\"\n",
"align, spec, stop_tokens, wav = tts(model, sentence, TTS_CONFIG, use_cuda, ap, use_gl=False, figures=True)"
],
"execution_count": null,
"outputs": []
}
]
}

1328
notebooks/DDC_TTS_and_MultiBand_MelGAN_TFLite_Example.ipynb Normal file
View File

File diff suppressed because one or more lines are too long

316
notebooks/DDC_TTS_and_MultiBand_MelGAN_TF_Example.ipynb Normal file
View File

@ -0,0 +1,316 @@
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"name": "DDC-TTS_and_MultiBand-MelGAN_TF_Example.ipynb",
"provenance": [],
"collapsed_sections": [],
"toc_visible": true
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
}
},
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "6LWsNd3_M3MP",
"colab_type": "text"
},
"source": [
"# Mozilla TTS on CPU Real-Time Speech Synthesis with Tensorflow"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "FAqrSIWgLyP0",
"colab_type": "text"
},
"source": [
"**These models are converted from released [PyTorch models](https://colab.research.google.com/drive/1u_16ZzHjKYFn1HNVuA4Qf_i2MMFB9olY?usp=sharing) using our TF utilities provided in Mozilla TTS.**\n",
"\n",
"These TF models support TF 2.2 and for different versions you might need to\n",
"regenerate them. \n",
"\n",
"We use Tacotron2 and MultiBand-Melgan models and LJSpeech dataset.\n",
"\n",
"Tacotron2 is trained using [Double Decoder Consistency](https://erogol.com/solving-attention-problems-of-tts-models-with-double-decoder-consistency/) (DDC) only for 130K steps (3 days) with a single GPU.\n",
"\n",
"MultiBand-Melgan is trained 1.45M steps with real spectrograms.\n",
"\n",
"Note that both model performances can be improved with more training.\n"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "Ku-dA4DKoeXk",
"colab_type": "text"
},
"source": [
"### Download Models"
]
},
{
"cell_type": "code",
"metadata": {
"id": "jGIgnWhGsxU1",
"colab_type": "code",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 162
},
"outputId": "08b0dddd-4edf-48c9-e8e5-a419b36a5c3d",
"tags": []
},
"source": [
"!gdown --id 1p7OSEEW_Z7ORxNgfZwhMy7IiLE1s0aH7 -O data/tts_model.pkl\n",
"!gdown --id 18CQ6G6tBEOfvCHlPqP8EBI4xWbrr9dBc -O data/config.json"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "4dnpE0-kvTsu",
"colab_type": "code",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 235
},
"outputId": "2fe836eb-c7e7-4f1e-9352-0142126bb19f",
"tags": []
},
"source": [
"!gdown --id 1rHmj7CqD3Sfa716Y3ub_vpIBrQg_b1yF -O data/vocoder_model.pkl\n",
"!gdown --id 1Rd0R_nRCrbjEdpOwq6XwZAktvugiBvmu -O data/config_vocoder.json\n",
"!gdown --id 11oY3Tv0kQtxK_JPgxrfesa99maVXHNxU -O data/scale_stats.npy"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {
"id": "Zlgi8fPdpRF0",
"colab_type": "text"
},
"source": [
"### Define TTS function"
]
},
{
"cell_type": "code",
"metadata": {
"id": "f-Yc42nQZG5A",
"colab_type": "code",
"colab": {}
},
"source": [
"def tts(model, text, CONFIG, p):\n",
" t_1 = time.time()\n",
" waveform, alignment, mel_spec, mel_postnet_spec, stop_tokens, inputs = synthesis(model, text, CONFIG, use_cuda, ap, speaker_id, style_wav=None,\n",
" truncated=False, enable_eos_bos_chars=CONFIG.enable_eos_bos_chars,\n",
" backend='tf')\n",
" waveform = vocoder_model.inference(torch.FloatTensor(mel_postnet_spec.T).unsqueeze(0))\n",
" waveform = waveform.numpy()[0, 0]\n",
" rtf = (time.time() - t_1) / (len(waveform) / ap.sample_rate)\n",
" tps = (time.time() - t_1) / len(waveform)\n",
" print(waveform.shape)\n",
" print(\" > Run-time: {}\".format(time.time() - t_1))\n",
" print(\" > Real-time factor: {}\".format(rtf))\n",
" print(\" > Time per step: {}\".format(tps))\n",
" IPython.display.display(IPython.display.Audio(waveform, rate=CONFIG.audio['sample_rate'])) \n",
" return alignment, mel_postnet_spec, stop_tokens, waveform"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {
"id": "ZksegYQepkFg",
"colab_type": "text"
},
"source": [
"### Load Models"
]
},
{
"cell_type": "code",
"metadata": {
"id": "oVa0kOamprgj",
"colab_type": "code",
"colab": {}
},
"source": [
"import os\n",
"import torch\n",
"import time\n",
"import IPython\n",
"\n",
"from TTS.tts.tf.utils.generic_utils import setup_model\n",
"from TTS.tts.tf.utils.io import load_checkpoint\n",
"from TTS.utils.io import load_config\n",
"from TTS.tts.utils.text.symbols import symbols, phonemes\n",
"from TTS.utils.audio import AudioProcessor\n",
"from TTS.tts.utils.synthesis import synthesis"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "EY-sHVO8IFSH",
"colab_type": "code",
"colab": {}
},
"source": [
"# runtime settings\n",
"use_cuda = False"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "_1aIUp2FpxOQ",
"colab_type": "code",
"colab": {}
},
"source": [
"# model paths\n",
"TTS_MODEL = \"data/tts_model.pkl\"\n",
"TTS_CONFIG = \"data/config.json\"\n",
"VOCODER_MODEL = \"data/vocoder_model.pkl\"\n",
"VOCODER_CONFIG = \"data/config_vocoder.json\""
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "CpgmdBVQplbv",
"colab_type": "code",
"colab": {}
},
"source": [
"# load configs\n",
"TTS_CONFIG = load_config(TTS_CONFIG)\n",
"VOCODER_CONFIG = load_config(VOCODER_CONFIG)"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "zmrQxiozIUVE",
"colab_type": "code",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 471
},
"outputId": "fa71bd05-401f-4e5b-a6f7-60ae765966db",
"tags": []
},
"source": [
"# load the audio processor\n",
"TTS_CONFIG.audio['stats_path'] = 'data/scale_stats.npy'\n",
"ap = AudioProcessor(**TTS_CONFIG.audio) "
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "8fLoI4ipqMeS",
"colab_type": "code",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 72
},
"outputId": "595d990f-930d-4698-ee14-77796b5eed7d",
"tags": []
},
"source": [
"# LOAD TTS MODEL\n",
"# multi speaker \n",
"speaker_id = None\n",
"speakers = []\n",
"\n",
"# load the model\n",
"num_chars = len(phonemes) if TTS_CONFIG.use_phonemes else len(symbols)\n",
"model = setup_model(num_chars, len(speakers), TTS_CONFIG)\n",
"model.build_inference()\n",
"model = load_checkpoint(model, TTS_MODEL)\n",
"model.decoder.set_max_decoder_steps(1000)"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "zKoq0GgzqzhQ",
"colab_type": "code",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 489
},
"outputId": "2cc3deae-144f-4465-da3b-98628d948506"
},
"source": [
"from TTS.vocoder.tf.utils.generic_utils import setup_generator\n",
"from TTS.vocoder.tf.utils.io import load_checkpoint\n",
"\n",
"# LOAD VOCODER MODEL\n",
"vocoder_model = setup_generator(VOCODER_CONFIG)\n",
"vocoder_model.build_inference()\n",
"vocoder_model = load_checkpoint(vocoder_model, VOCODER_MODEL)\n",
"vocoder_model.inference_padding = 0\n",
"\n",
"ap_vocoder = AudioProcessor(**VOCODER_CONFIG['audio']) "
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {
"id": "Ws_YkPKsLgo-",
"colab_type": "text"
},
"source": [
"## Run Inference"
]
},
{
"cell_type": "code",
"metadata": {
"id": "FuWxZ9Ey5Puj",
"colab_type": "code",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 134
},
"outputId": "07ede6e5-06e6-4612-f687-7984d20e5254"
},
"source": [
"sentence = \"Bill got in the habit of asking himself “Is that thought true?” and if he wasn’t absolutely certain it was, he just let it go.\"\n",
"align, spec, stop_tokens, wav = tts(model, sentence, TTS_CONFIG, ap)"
],
"execution_count": null,
"outputs": []
}
]
}

412
notebooks/Tutorial_Converting_PyTorch_to_TF_to_TFlite.ipynb Normal file
View File

@ -0,0 +1,412 @@
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"name": "Tutorial_Converting_PyTorch_to_TF_to_TFlite.ipynb",
"provenance": [],
"collapsed_sections": []
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
}
},
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "6LWsNd3_M3MP",
"colab_type": "text"
},
"source": [
"# Converting Pytorch models to Tensorflow and TFLite by MozillaTTS"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "FAqrSIWgLyP0",
"colab_type": "text"
},
"source": [
"This is a tutorial demonstrating Mozilla TTS capabilities to convert \n",
"trained PyTorch models to Tensorflow and Tflite.\n"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "MBJjGYnoEo4v",
"colab_type": "text"
},
"source": [
"# Installation"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "Ku-dA4DKoeXk",
"colab_type": "text"
},
"source": [
"### Download TF Models and configs"
]
},
{
"cell_type": "code",
"metadata": {
"id": "jGIgnWhGsxU1",
"colab_type": "code",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 162
},
"outputId": "b461952f-8507-4dd2-af06-4e6b8692765d",
"tags": []
},
"source": [
"!gdown --id 1dntzjWFg7ufWaTaFy80nRz-Tu02xWZos -O data/tts_model.pth.tar\n",
"!gdown --id 18CQ6G6tBEOfvCHlPqP8EBI4xWbrr9dBc -O data/config.json"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "4dnpE0-kvTsu",
"colab_type": "code",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 235
},
"outputId": "f67c3138-bda0-4b3e-ffcc-647f9feec23e",
"tags": []
},
"source": [
"!gdown --id 1Ty5DZdOc0F7OTGj9oJThYbL5iVu_2G0K -O data/vocoder_model.pth.tar\n",
"!gdown --id 1Rd0R_nRCrbjEdpOwq6XwZAktvugiBvmu -O data/config_vocoder.json\n",
"!gdown --id 11oY3Tv0kQtxK_JPgxrfesa99maVXHNxU -O data/scale_stats.npy"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {
"id": "3IGvvCRMEwqn",
"colab_type": "text"
},
"source": [
"# Model Conversion PyTorch -> TF -> TFLite"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "tLhz8SAf8Pgp",
"colab_type": "text"
},
"source": [
"## Converting PyTorch to Tensorflow\n"
]
},
{
"cell_type": "code",
"metadata": {
"id": "Xsrvr_WQ8Ib5",
"colab_type": "code",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 1000
},
"outputId": "dae96616-e5f7-41b6-cdb9-5026cfcd3214",
"tags": []
},
"source": [
"# convert TTS model to Tensorflow\n",
"!python ../TTS/bin/convert_tacotron2_torch_to_tf.py --config_path data/config.json --torch_model_path data/tts_model.pth.tar --output_path data/tts_model_tf.pkl"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "VJ4NA5If9ljv",
"colab_type": "code",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 1000
},
"outputId": "1520dca8-1db8-4e07-bc0c-b1d5941c775e",
"tags": []
},
"source": [
"# convert Vocoder model to Tensorflow\n",
"!python ../TTS/bin/convert_melgan_torch_to_tf.py --config_path data/config_vocoder.json --torch_model_path data/vocoder_model.pth.tar --output_path data/vocoder_model_tf.pkl"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {
"id": "7d5vTkBZ-BYQ",
"colab_type": "text"
},
"source": [
"## Converting Tensorflow to TFLite"
]
},
{
"cell_type": "code",
"metadata": {
"id": "33hTfpuU99cg",
"colab_type": "code",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 927
},
"outputId": "8a0e5be1-23a2-4128-ee37-8232adcb8ff0",
"tags": []
},
"source": [
"# convert TTS model to TFLite\n",
"!python ../TTS/bin/convert_tacotron2_tflite.py --config_path data/config.json --tf_model data/tts_model_tf.pkl --output_path data/tts_model.tflite"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "e00Hm75Y-wZ2",
"colab_type": "code",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 364
},
"outputId": "42381b05-3c9d-44f0-dac7-d81efd95eadf",
"tags": []
},
"source": [
"# convert Vocoder model to TFLite\n",
"!python ../TTS/bin/convert_melgan_tflite.py --config_path data/config_vocoder.json --tf_model data/vocoder_model_tf.pkl --output_path data/vocoder_model.tflite"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {
"id": "Zlgi8fPdpRF0",
"colab_type": "text"
},
"source": [
"# Run Inference with TFLite "
]
},
{
"cell_type": "code",
"metadata": {
"id": "f-Yc42nQZG5A",
"colab_type": "code",
"colab": {}
},
"source": [
"def run_vocoder(mel_spec):\n",
" vocoder_inputs = mel_spec[None, :, :]\n",
" # get input and output details\n",
" input_details = vocoder_model.get_input_details()\n",
" # reshape input tensor for the new input shape\n",
" vocoder_model.resize_tensor_input(input_details[0]['index'], vocoder_inputs.shape)\n",
" vocoder_model.allocate_tensors()\n",
" detail = input_details[0]\n",
" vocoder_model.set_tensor(detail['index'], vocoder_inputs)\n",
" # run the model\n",
" vocoder_model.invoke()\n",
" # collect outputs\n",
" output_details = vocoder_model.get_output_details()\n",
" waveform = vocoder_model.get_tensor(output_details[0]['index'])\n",
" return waveform \n",
"\n",
"\n",
"def tts(model, text, CONFIG, p):\n",
" t_1 = time.time()\n",
" waveform, alignment, mel_spec, mel_postnet_spec, stop_tokens, inputs = synthesis(model, text, CONFIG, use_cuda, ap, speaker_id, style_wav=None,\n",
" truncated=False, enable_eos_bos_chars=CONFIG.enable_eos_bos_chars,\n",
" backend='tflite')\n",
" waveform = run_vocoder(mel_postnet_spec.T)\n",
" waveform = waveform[0, 0]\n",
" rtf = (time.time() - t_1) / (len(waveform) / ap.sample_rate)\n",
" tps = (time.time() - t_1) / len(waveform)\n",
" print(waveform.shape)\n",
" print(\" > Run-time: {}\".format(time.time() - t_1))\n",
" print(\" > Real-time factor: {}\".format(rtf))\n",
" print(\" > Time per step: {}\".format(tps))\n",
" IPython.display.display(IPython.display.Audio(waveform, rate=CONFIG.audio['sample_rate'])) \n",
" return alignment, mel_postnet_spec, stop_tokens, waveform"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {
"id": "ZksegYQepkFg",
"colab_type": "text"
},
"source": [
"### Load TF Models"
]
},
{
"cell_type": "code",
"metadata": {
"id": "oVa0kOamprgj",
"colab_type": "code",
"colab": {}
},
"source": [
"import os\n",
"import torch\n",
"import time\n",
"import IPython\n",
"\n",
"from TTS.tts.tf.utils.tflite import load_tflite_model\n",
"from TTS.tts.tf.utils.io import load_checkpoint\n",
"from TTS.utils.io import load_config\n",
"from TTS.tts.utils.text.symbols import symbols, phonemes\n",
"from TTS.utils.audio import AudioProcessor\n",
"from TTS.tts.utils.synthesis import synthesis"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "EY-sHVO8IFSH",
"colab_type": "code",
"colab": {}
},
"source": [
"# runtime settings\n",
"use_cuda = False"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "_1aIUp2FpxOQ",
"colab_type": "code",
"colab": {}
},
"source": [
"# model paths\n",
"TTS_MODEL = \"data/tts_model.tflite\"\n",
"TTS_CONFIG = \"data/config.json\"\n",
"VOCODER_MODEL = \"data/vocoder_model.tflite\"\n",
"VOCODER_CONFIG = \"data/config_vocoder.json\""
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "CpgmdBVQplbv",
"colab_type": "code",
"colab": {}
},
"source": [
"# load configs\n",
"TTS_CONFIG = load_config(TTS_CONFIG)\n",
"VOCODER_CONFIG = load_config(VOCODER_CONFIG)"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "zmrQxiozIUVE",
"colab_type": "code",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 471
},
"outputId": "21cda136-de87-4d55-fd46-7d5306103d90",
"tags": []
},
"source": [
"# load the audio processor\n",
"TTS_CONFIG.audio['stats_path'] = 'data/scale_stats.npy'\n",
"ap = AudioProcessor(**TTS_CONFIG.audio) "
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "8fLoI4ipqMeS",
"colab_type": "code",
"colab": {}
},
"source": [
"# LOAD TTS MODEL\n",
"# multi speaker \n",
"speaker_id = None\n",
"speakers = []\n",
"\n",
"# load the models\n",
"model = load_tflite_model(TTS_MODEL)\n",
"vocoder_model = load_tflite_model(VOCODER_MODEL)"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {
"id": "Ws_YkPKsLgo-",
"colab_type": "text"
},
"source": [
"## Run Sample Sentence"
]
},
{
"cell_type": "code",
"metadata": {
"id": "FuWxZ9Ey5Puj",
"colab_type": "code",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 134
},
"outputId": "535c2df1-c27c-458b-e14b-41a977635aa1",
"tags": []
},
"source": [
"sentence = \"Bill got in the habit of asking himself “Is that thought true?” and if he wasn’t absolutely certain it was, he just let it go.\"\n",
"align, spec, stop_tokens, wav = tts(model, sentence, TTS_CONFIG, ap)"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
]
}