diff --git a/TTS/bin/train_encoder.py b/TTS/bin/train_encoder.py index 6acaeff1..3222c278 100644 --- a/TTS/bin/train_encoder.py +++ b/TTS/bin/train_encoder.py @@ -42,8 +42,12 @@ def setup_loader(ap, is_val=False, verbose=False): dataset = MyDataset(ap, meta_data_eval if is_val else meta_data_train, voice_len=1.6, - num_utter_per_speaker=10, + num_utter_per_speaker=c.num_utters_per_speaker, + num_speakers_in_batch=c.num_speakers_in_batch, skip_speakers=False, + storage_size=c.storage["storage_size"], + sample_from_storage_p=c.storage["sample_from_storage_p"], + additive_noise=c.storage["additive_noise"], verbose=verbose) # sampler = DistributedSampler(dataset) if num_gpus > 1 else None loader = DataLoader(dataset, @@ -60,6 +64,7 @@ def train(model, criterion, optimizer, scheduler, ap, global_step): epoch_time = 0 best_loss = float('inf') avg_loss = 0 + avg_loader_time = 0 end_time = time.time() for _, data in enumerate(data_loader): start_time = time.time() @@ -93,8 +98,11 @@ def train(model, criterion, optimizer, scheduler, ap, global_step): step_time = time.time() - start_time epoch_time += step_time - avg_loss = 0.01 * loss.item( - ) + 0.99 * avg_loss if avg_loss != 0 else loss.item() + # Averaged Loss and Averaged Loader Time + avg_loss = 0.01 * loss.item() \ + + 0.99 * avg_loss if avg_loss != 0 else loss.item() + avg_loader_time = 1/c.num_loader_workers * loader_time + \ + (c.num_loader_workers-1) / c.num_loader_workers * avg_loader_time if avg_loader_time != 0 else loader_time current_lr = optimizer.param_groups[0]['lr'] if global_step % c.steps_plot_stats == 0: @@ -103,7 +111,8 @@ def train(model, criterion, optimizer, scheduler, ap, global_step): "loss": avg_loss, "lr": current_lr, "grad_norm": grad_norm, - "step_time": step_time + "step_time": step_time, + "avg_loader_time": avg_loader_time } tb_logger.tb_train_epoch_stats(global_step, train_stats) figures = { @@ -116,9 +125,9 @@ def train(model, criterion, optimizer, scheduler, ap, global_step): if global_step % c.print_step == 0: print( " | > Step:{} Loss:{:.5f} AvgLoss:{:.5f} GradNorm:{:.5f} " - "StepTime:{:.2f} LoaderTime:{:.2f} LR:{:.6f}".format( + "StepTime:{:.2f} LoaderTime:{:.2f} AvGLoaderTime:{:.2f} LR:{:.6f}".format( global_step, loss.item(), avg_loss, grad_norm, step_time, - loader_time, current_lr), + loader_time, avg_loader_time, current_lr), flush=True) # save best model diff --git a/TTS/speaker_encoder/config.json b/TTS/speaker_encoder/config.json index 11da0cf6..4fbd84cc 100644 --- a/TTS/speaker_encoder/config.json +++ b/TTS/speaker_encoder/config.json @@ -1,6 +1,6 @@ { - "run_name": "Model compatible to CorentinJ/Real-Time-Voice-Cloning", + "run_name": "mueller91", "run_description": "train speaker encoder with voxceleb1, voxceleb2 and libriSpeech ", "audio":{ // Audio processing parameters @@ -23,11 +23,11 @@ "clip_norm": true, // clip normalized values into the range. "mel_fmin": 0.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! "mel_fmax": 8000.0, // maximum freq level for mel-spec. Tune for dataset!! - "do_trim_silence": false, // enable trimming of slience of audio as you load it. LJspeech (false), TWEB (false), Nancy (true) + "do_trim_silence": true, // enable trimming of slience of audio as you load it. LJspeech (false), TWEB (false), Nancy (true) "trim_db": 60 // threshold for timming silence. Set this according to your dataset. }, "reinit_layers": [], - "loss": "ge2e", // "ge2e" to use Generalized End-to-End loss and "angleproto" to use Angular Prototypical loss (new SOTA) + "loss": "angleproto", // "ge2e" to use Generalized End-to-End loss and "angleproto" to use Angular Prototypical loss (new SOTA) "grad_clip": 3.0, // upper limit for gradients for clipping. "epochs": 1000, // total number of epochs to train. "lr": 0.0001, // Initial learning rate. If Noam decay is active, maximum learning rate. @@ -35,27 +35,69 @@ "warmup_steps": 4000, // Noam decay steps to increase the learning rate from 0 to "lr" "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. "steps_plot_stats": 10, // number of steps to plot embeddings. - "num_speakers_in_batch": 32, // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'. - "num_loader_workers": 4, // number of training data loader processes. Don't set it too big. 4-8 are good values. + "num_speakers_in_batch": 64, // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'. + "num_utters_per_speaker": 10, // + "num_loader_workers": 8, // number of training data loader processes. Don't set it too big. 4-8 are good values. "wd": 0.000001, // Weight decay weight. "checkpoint": true, // If true, it saves checkpoints per "save_step" "save_step": 1000, // Number of training steps expected to save traning stats and checkpoints. - "print_step": 1, // Number of steps to log traning on console. - "output_path": "../../checkpoints/voxceleb_librispeech/speaker_encoder/", // DATASET-RELATED: output path for all training outputs. + "print_step": 20, // Number of steps to log traning on console. + "output_path": "../../MozillaTTSOutput/checkpoints/voxceleb_librispeech/speaker_encoder/", // DATASET-RELATED: output path for all training outputs. "model": { "input_dim": 40, "proj_dim": 256, - "lstm_dim": 256, + "lstm_dim": 768, "num_lstm_layers": 3, - "use_lstm_with_projection": false + "use_lstm_with_projection": true + }, + "storage": { + "sample_from_storage_p": 0.66, // the probability with which we'll sample from the DataSet in-memory storage + "storage_size": 15, // the size of the in-memory storage with respect to a single batch + "additive_noise": 1e-5 // add very small gaussian noise to the data in order to increase robustness }, "datasets": [ { - "name": "vctk", - "path": "../../../datasets/VCTK-Corpus-removed-silence/", + "name": "vctk_slim", + "path": "../../../audio-datasets/en/VCTK-Corpus/", "meta_file_train": null, "meta_file_val": null + }, + { + "name": "libri_tts", + "path": "../../../audio-datasets/en/LibriTTS/train-clean-100", + "meta_file_train": null, + "meta_file_val": null + }, + { + "name": "libri_tts", + "path": "../../../audio-datasets/en/LibriTTS/train-clean-360", + "meta_file_train": null, + "meta_file_val": null + }, + { + "name": "libri_tts", + "path": "../../../audio-datasets/en/LibriTTS/train-other-500", + "meta_file_train": null, + "meta_file_val": null + }, + { + "name": "voxceleb1", + "path": "../../../audio-datasets/en/voxceleb1/", + "meta_file_train": null, + "meta_file_val": null + }, + { + "name": "voxceleb2", + "path": "../../../audio-datasets/en/voxceleb2/", + "meta_file_train": null, + "meta_file_val": null + }, + { + "name": "common_voice", + "path": "../../../audio-datasets/en/MozillaCommonVoice", + "meta_file_train": "train.tsv", + "meta_file_val": "test.tsv" } ] } \ No newline at end of file diff --git a/TTS/speaker_encoder/dataset.py b/TTS/speaker_encoder/dataset.py index e67dd716..33cc4f36 100644 --- a/TTS/speaker_encoder/dataset.py +++ b/TTS/speaker_encoder/dataset.py @@ -1,11 +1,15 @@ +import numpy import numpy as np +import queue import torch import random from torch.utils.data import Dataset +from tqdm import tqdm class MyDataset(Dataset): def __init__(self, ap, meta_data, voice_len=1.6, num_speakers_in_batch=64, + storage_size=1, sample_from_storage_p=0.5, additive_noise=0, num_utter_per_speaker=10, skip_speakers=False, verbose=False): """ Args: @@ -24,8 +28,15 @@ class MyDataset(Dataset): self.ap = ap self.verbose = verbose self.__parse_items() + self.storage = queue.Queue(maxsize=storage_size*num_speakers_in_batch) + self.sample_from_storage_p = float(sample_from_storage_p) + self.additive_noise = float(additive_noise) if self.verbose: print("\n > DataLoader initialization") + print(f" | > Speakers per Batch: {num_speakers_in_batch}") + print(f" | > Storage Size: {self.storage.maxsize} speakers, each with {num_utter_per_speaker} utters") + print(f" | > Sample_from_storage_p : {self.sample_from_storage_p}") + print(f" | > Noise added : {self.additive_noise}") print(f" | > Number of instances : {len(self.items)}") print(f" | > Sequence length: {self.seq_len}") print(f" | > Num speakers: {len(self.speakers)}") @@ -51,21 +62,37 @@ class MyDataset(Dataset): return sample def __parse_items(self): - """ - Find unique speaker ids and create a dict mapping utterances from speaker id - """ - speakers = list({item[-1] for item in self.items}) self.speaker_to_utters = {} - self.speakers = [] - for speaker in speakers: - speaker_utters = [item[1] for item in self.items if item[2] == speaker] - if len(speaker_utters) < self.num_utter_per_speaker and self.skip_speakers: - print( - f" [!] Skipped speaker {speaker}. Not enough utterances {self.num_utter_per_speaker} vs {len(speaker_utters)}." - ) + for i in self.items: + path_ = i[1] + speaker_ = i[2] + if speaker_ in self.speaker_to_utters.keys(): + self.speaker_to_utters[speaker_].append(path_) else: - self.speakers.append(speaker) - self.speaker_to_utters[speaker] = speaker_utters + self.speaker_to_utters[speaker_] = [path_, ] + + if self.skip_speakers: + self.speaker_to_utters = {k: v for (k, v) in self.speaker_to_utters.items() if + len(v) >= self.num_utter_per_speaker} + + self.speakers = [k for (k, v) in self.speaker_to_utters.items()] + + # def __parse_items(self): + # """ + # Find unique speaker ids and create a dict mapping utterances from speaker id + # """ + # speakers = list({item[-1] for item in self.items}) + # self.speaker_to_utters = {} + # self.speakers = [] + # for speaker in speakers: + # speaker_utters = [item[1] for item in self.items if item[2] == speaker] + # if len(speaker_utters) < self.num_utter_per_speaker and self.skip_speakers: + # print( + # f" [!] Skipped speaker {speaker}. Not enough utterances {self.num_utter_per_speaker} vs {len(speaker_utters)}." + # ) + # else: + # self.speakers.append(speaker) + # self.speaker_to_utters[speaker] = speaker_utters def __len__(self): return int(1e10) @@ -86,7 +113,7 @@ class MyDataset(Dataset): """ Sample all M utterances for the given speaker. """ - feats = [] + wavs = [] labels = [] for _ in range(self.num_utter_per_speaker): # TODO:dummy but works @@ -102,11 +129,9 @@ class MyDataset(Dataset): break self.speaker_to_utters[speaker].remove(utter) - offset = random.randint(0, wav.shape[0] - self.seq_len) - mel = self.ap.melspectrogram(wav[offset : offset + self.seq_len]) - feats.append(torch.FloatTensor(mel)) + wavs.append(wav) labels.append(speaker) - return feats, labels + return wavs, labels def __getitem__(self, idx): speaker, _ = self.__sample_speaker() @@ -116,7 +141,28 @@ class MyDataset(Dataset): labels = [] feats = [] for speaker in batch: - feats_, labels_ = self.__sample_speaker_utterances(speaker) + if random.random() < self.sample_from_storage_p and self.storage.full(): + # sample from storage (if full), ignoring the speaker + wavs_, labels_ = random.choice(self.storage.queue) + else: + # don't sample from storage, but from HDD + wavs_, labels_ = self.__sample_speaker_utterances(speaker) + # if storage is full, remove an item + if self.storage.full(): + _ = self.storage.get_nowait() + # put the newly loaded item into storage + self.storage.put_nowait((wavs_, labels_)) + + # add random gaussian noise + if self.additive_noise > 0: + noises_ = [numpy.random.normal(0, self.additive_noise, size=len(w)) for w in wavs_] + wavs_ = [wavs_[i] + noises_[i] for i in range(len(wavs_))] + + # get a random subset of each of the wavs and convert to MFCC. + offsets_ = [random.randint(0, wav.shape[0] - self.seq_len) for wav in wavs_] + mels_ = [self.ap.melspectrogram(wavs_[i][offsets_[i]: offsets_[i] + self.seq_len]) for i in range(len(wavs_))] + feats_ = [torch.FloatTensor(mel) for mel in mels_] + labels.append(labels_) feats.extend(feats_) feats = torch.stack(feats) diff --git a/TTS/speaker_encoder/generic_utils.py b/TTS/speaker_encoder/generic_utils.py index bc72c91c..b3ac00dc 100644 --- a/TTS/speaker_encoder/generic_utils.py +++ b/TTS/speaker_encoder/generic_utils.py @@ -23,7 +23,7 @@ def save_checkpoint(model, optimizer, model_loss, out_path, def save_best_model(model, optimizer, model_loss, best_loss, out_path, current_step): - if model_loss < best_loss: + if model_loss < best_loss and current_step > 1000: new_state_dict = model.state_dict() state = { 'model': new_state_dict, @@ -35,7 +35,7 @@ def save_best_model(model, optimizer, model_loss, best_loss, out_path, best_loss = model_loss bestmodel_path = 'best_model.pth.tar' bestmodel_path = os.path.join(out_path, bestmodel_path) - print("\n > BEST MODEL ({0:.5f}) : {1:}".format( - model_loss, bestmodel_path)) + print("\n > NEW BEST MODEL ({0:.5f}) : {1:}".format( + model_loss, os.path.abspath(bestmodel_path))) torch.save(state, bestmodel_path) return best_loss diff --git a/TTS/tts/datasets/preprocess.py b/TTS/tts/datasets/preprocess.py index 4bfad648..02954c04 100644 --- a/TTS/tts/datasets/preprocess.py +++ b/TTS/tts/datasets/preprocess.py @@ -2,6 +2,10 @@ import os from glob import glob import re import sys +from pathlib import Path + +from tqdm import tqdm + from TTS.tts.utils.generic_utils import split_dataset @@ -14,8 +18,8 @@ def load_meta_data(datasets): meta_file_train = dataset['meta_file_train'] meta_file_val = dataset['meta_file_val'] preprocessor = get_preprocessor_by_name(name) - meta_data_train = preprocessor(root_path, meta_file_train) + print(f" | > Found {len(meta_data_train)} files in {Path(root_path).resolve()}") if meta_file_val is None: meta_data_eval, meta_data_train = split_dataset(meta_data_train) else: @@ -167,8 +171,8 @@ def common_voice(root_path, meta_file): cols = line.split("\t") text = cols[2] speaker_name = cols[0] - wav_file = os.path.join(root_path, "clips", cols[1] + ".wav") - items.append([text, wav_file, speaker_name]) + wav_file = os.path.join(root_path, "clips", cols[1].replace(".mp3", ".wav")) + items.append([text, wav_file, 'MCV_' + speaker_name]) return items @@ -187,7 +191,7 @@ def libri_tts(root_path, meta_files=None): cols = line.split('\t') wav_file = os.path.join(_root_path, cols[0] + '.wav') text = cols[1] - items.append([text, wav_file, speaker_name]) + items.append([text, wav_file, 'LTTS_' + speaker_name]) for item in items: assert os.path.exists( item[1]), f" [!] wav files don't exist - {item[1]}" @@ -235,8 +239,7 @@ def vctk(root_path, meta_files=None, wavs_path='wav48'): """homepages.inf.ed.ac.uk/jyamagis/release/VCTK-Corpus.tar.gz""" test_speakers = meta_files items = [] - meta_files = glob(f"{os.path.join(root_path,'txt')}/**/*.txt", - recursive=True) + meta_files = glob(f"{os.path.join(root_path,'txt')}/**/*.txt", recursive=True) for meta_file in meta_files: _, speaker_id, txt_file = os.path.relpath(meta_file, root_path).split(os.sep) @@ -249,6 +252,70 @@ def vctk(root_path, meta_files=None, wavs_path='wav48'): text = file_text.readlines()[0] wav_file = os.path.join(root_path, wavs_path, speaker_id, file_id + '.wav') - items.append([text, wav_file, speaker_id]) + items.append([text, wav_file, 'VCTK_' + speaker_id]) return items + + +def vctk_slim(root_path, meta_files=None, wavs_path='wav48'): + """homepages.inf.ed.ac.uk/jyamagis/release/VCTK-Corpus.tar.gz""" + items = [] + txt_files = glob(f"{os.path.join(root_path,'txt')}/**/*.txt", recursive=True) + for text_file in txt_files: + _, speaker_id, txt_file = os.path.relpath(text_file, + root_path).split(os.sep) + file_id = txt_file.split('.')[0] + if isinstance(meta_files, list): # if is list ignore this speakers ids + if speaker_id in meta_files: + continue + wav_file = os.path.join(root_path, wavs_path, speaker_id, + file_id + '.wav') + items.append([None, wav_file, 'VCTK_' + speaker_id]) + + return items + +# ======================================== VOX CELEB =========================================== +def voxceleb2(root_path, meta_file=None): + """ + :param meta_file Used only for consistency with load_meta_data api + """ + return _voxcel_x(root_path, meta_file, voxcel_idx="2") + + +def voxceleb1(root_path, meta_file=None): + """ + :param meta_file Used only for consistency with load_meta_data api + """ + return _voxcel_x(root_path, meta_file, voxcel_idx="1") + + +def _voxcel_x(root_path, meta_file, voxcel_idx): + assert voxcel_idx in ["1", "2"] + expected_count = 148_000 if voxcel_idx == "1" else 1_000_000 + voxceleb_path = Path(root_path) + cache_to = voxceleb_path / f"metafile_voxceleb{voxcel_idx}.csv" + cache_to.parent.mkdir(exist_ok=True) + + # if not exists meta file, crawl recursively for 'wav' files + if meta_file is not None: + with open(str(meta_file), 'r') as f: + return [x.strip().split('|') for x in f.readlines()] + + elif not cache_to.exists(): + cnt = 0 + meta_data = "" + wav_files = voxceleb_path.rglob("**/*.wav") + for path in tqdm(wav_files, desc=f"Building VoxCeleb {voxcel_idx} Meta file ... this needs to be done only once.", + total=expected_count): + speaker_id = str(Path(path).parent.parent.stem) + assert speaker_id.startswith('id') + text = None # VoxCel does not provide transciptions, and they are not needed for training the SE + meta_data += f"{text}|{path}|voxcel{voxcel_idx}_{speaker_id}\n" + cnt += 1 + with open(str(cache_to), 'w') as f: + f.write(meta_data) + if cnt < expected_count: + raise ValueError(f"Found too few instances for Voxceleb. Should be around {expected_count}, is: {cnt}") + + with open(str(cache_to), 'r') as f: + return [x.strip().split('|') for x in f.readlines()] diff --git a/TTS/tts/utils/generic_utils.py b/TTS/tts/utils/generic_utils.py index 6eaa2358..6358e5a9 100644 --- a/TTS/tts/utils/generic_utils.py +++ b/TTS/tts/utils/generic_utils.py @@ -7,11 +7,9 @@ from TTS.utils.generic_utils import check_argument def split_dataset(items): - is_multi_speaker = False speakers = [item[-1] for item in items] is_multi_speaker = len(set(speakers)) > 1 - eval_split_size = 500 if len(items) * 0.01 > 500 else int( - len(items) * 0.01) + eval_split_size = min(500, int(len(items) * 0.01)) assert eval_split_size > 0, " [!] You do not have enough samples to train. You need at least 100 samples." np.random.seed(0) np.random.shuffle(items) @@ -142,6 +140,11 @@ def check_config(c): check_argument('do_trim_silence', c['audio'], restricted=True, val_type=bool) check_argument('trim_db', c['audio'], restricted=True, val_type=int) + # storage parameters + check_argument('sample_from_storage_p', c['storage'], restricted=True, val_type=float, min_val=0.0, max_val=1.0) + check_argument('storage_size', c['storage'], restricted=True, val_type=int, min_val=1, max_val=100) + check_argument('additive_noise', c['storage'], restricted=True, val_type=float, min_val=0.0, max_val=1.0) + # training parameters check_argument('batch_size', c, restricted=True, val_type=int, min_val=1) check_argument('eval_batch_size', c, restricted=True, val_type=int, min_val=1) diff --git a/TTS/tts/utils/io.py b/TTS/tts/utils/io.py index bf5e13d8..78e9b8b2 100644 --- a/TTS/tts/utils/io.py +++ b/TTS/tts/utils/io.py @@ -50,7 +50,7 @@ def save_best_model(target_loss, best_loss, model, optimizer, current_step, epoc if target_loss < best_loss: file_name = 'best_model.pth.tar' checkpoint_path = os.path.join(output_folder, file_name) - print(" > BEST MODEL : {}".format(checkpoint_path)) + print(" >> BEST MODEL : {}".format(checkpoint_path)) save_model(model, optimizer, current_step, epoch, r, checkpoint_path, model_loss=target_loss, **kwargs) best_loss = target_loss return best_loss diff --git a/tests/inputs/common_voice.tsv b/tests/inputs/common_voice.tsv index a6ea30dd..39fc4190 100644 --- a/tests/inputs/common_voice.tsv +++ b/tests/inputs/common_voice.tsv @@ -1,10 +1,6 @@ -client_id path sentence up_votes down_votes age gender accent -aa7af576605fee2c78c26b85497c64cb9c9fd97228071f8666d9f49f15bce01899bbb930fa60b76d212091d779d83b92e0b54c73cbb21d2c7e1eedc817e41cb3 21fce545b24d9a5af0403b949e95e8dd3c10c4ff3e371f14e4d5b4ebf588670b7c9e618285fc872d94a89ed7f0217d9019fe5de33f1577b49dcd518eacf63c4b Man sollte den Länderfinanzausgleich durch einen Bundesliga-Soli ersetzen. 2 0 fourties male germany -aa7af576605fee2c78c26b85497c64cb9c9fd97228071f8666d9f49f15bce01899bbb930fa60b76d212091d779d83b92e0b54c73cbb21d2c7e1eedc817e41cb3 42758baa4e91ef6b82b78b11a04bc5117a035a8d3bc42c33c0bb3084909af17043a194cfd8cd9839f0d6ef1ea5413acda5de5d1936abcc8ca073e2da7f9488ea Folgende Lektüre kann ich Ihnen zum Thema Kognitionspsychologie empfehlen. 2 0 fourties male germany -aa7af576605fee2c78c26b85497c64cb9c9fd97228071f8666d9f49f15bce01899bbb930fa60b76d212091d779d83b92e0b54c73cbb21d2c7e1eedc817e41cb3 478f172c2dbda6675247e9674ade79a5b49efeefb7c9e99040dcc69a847a01d69398cf180570859b0cdb6fc887717e04cd8b149c723d48d00b5d18f41314667c Touristen winkten den Leuten am Ufer zu. 2 0 fourties male germany -aa7af576605fee2c78c26b85497c64cb9c9fd97228071f8666d9f49f15bce01899bbb930fa60b76d212091d779d83b92e0b54c73cbb21d2c7e1eedc817e41cb3 4854368d6d21cb44103e432b5332f31e8d14030582a40850501bcf9377d699314a5ff27a8206fa89254ddde7f3f1c65d33836f3dfcfa16bcabec08537f2b5f08 Valentin hat das Handtuch geworfen. 2 0 fourties male germany -aa7af576605fee2c78c26b85497c64cb9c9fd97228071f8666d9f49f15bce01899bbb930fa60b76d212091d779d83b92e0b54c73cbb21d2c7e1eedc817e41cb3 a841a9f3e032495dd47560e65fba99eeacb3618c07de8b1351c20188e5b71e33cc52f73315f721a3a24b65763c65bb52fbf3ae052eb5774e834dcb57f296db5c Ohne Gehörschutz bei der Arbeit wäre Klaus wohl nach zwei Wochen taub. 2 0 fourties male germany -aa7af576605fee2c78c26b85497c64cb9c9fd97228071f8666d9f49f15bce01899bbb930fa60b76d212091d779d83b92e0b54c73cbb21d2c7e1eedc817e41cb3 03ab970a5bf5410bc3260b073cce1c7f49c688ace83dc8836b1c0f79a09fea45a27725c769f4a9d2e6181defd016d22642789d7ac51da252b42958a9192bd4c7 Gerrit erinnerte sich daran, dass er einst einen Eid geschworen hatte. 2 0 fourties male germany -aa7af576605fee2c78c26b85497c64cb9c9fd97228071f8666d9f49f15bce01899bbb930fa60b76d212091d779d83b92e0b54c73cbb21d2c7e1eedc817e41cb3 c4a94df443ad5f2c7241413ef7145d5f0de41ae929759073917fe96166da3c7d3a612c920ed7b0f3d5950a38d6205e9dba24af5bfb27e390a220d004e6e26744 Auf das, was jetzt kommt, habe ich nämlich absolut keinen Bock. 2 0 fourties male germany -aa7af576605fee2c78c26b85497c64cb9c9fd97228071f8666d9f49f15bce01899bbb930fa60b76d212091d779d83b92e0b54c73cbb21d2c7e1eedc817e41cb3 104695983b1112229b4a48696405d044dad9ddef713aa6eb1a6240cc16b7b7a2a96354ae9da99783850dde08a982091e48d3037288a3a58269cac9fe70a6bd7a Von Salzburg ist es doch nicht weit bis zum Chiemsee. 2 0 fourties male germany -d5b5da343bb0f65e3580bc2e1902a4f5d004241488d751503f2020bc1c93f89715e355e35f6e25def2b90cb3eea99fda403eb92ae3afbb84d039a54a4ed2d875 ad2f69e053b0e20e01c82b9821fe5787f1cc8e4b0b97f0e4cab1e9a652c577169c8244fb222281a60ee3081854014113e04c4ca43643100b7c01dab0fac11974 Warum werden da keine strafrechtlichen Konsequenzen gezogen? 2 0 thirties male germany +client_id path sentence up_votes down_votes age gender accent locale segment +95324d489b122a800b840e0b0d068f7363a1a6c2cd2e7365672cc7033e38deaa794bd59edcf8196aa35c9791652b9085ac3839a98bb50ebab4a1e8538a94846b common_voice_en_20005954.mp3 The applicants are invited for coffee and visa is given immediately. 3 0 en +95324d489b122a800b840e0b0d068f7363a1a6c2cd2e7365672cc7033e38deaa794bd59edcf8196aa35c9791652b9085ac3839a98bb50ebab4a1e8538a94846b common_voice_en_20005955.mp3 Developmental robotics is related to, but differs from, evolutionary robotics. 2 0 en +95324d489b122a800b840e0b0d068f7363a1a6c2cd2e7365672cc7033e38deaa794bd59edcf8196aa35c9791652b9085ac3839a98bb50ebab4a1e8538a94846b common_voice_en_20005956.mp3 The musical was originally directed and choreographed by Alan Lund. 2 0 en +954a4181ae9fba89d1b1570f2ae148b3ee18ee2311de978e698f598db859f830d93d35574596d713518e8c96cdae01fce7a08c60c2e0a22bcf01e020924440a6 common_voice_en_19737073.mp3 He graduated from Columbia High School, in Brown County, South Dakota. 2 0 en +954a4181ae9fba89d1b1570f2ae148b3ee18ee2311de978e698f598db859f830d93d35574596d713518e8c96cdae01fce7a08c60c2e0a22bcf01e020924440a6 common_voice_en_19737074.mp3 Competition for limited resources has also resulted in some local conflicts. 2 0 en diff --git a/tests/test_preprocessors.py b/tests/test_preprocessors.py index 1c7ad46f..8c7b16b0 100644 --- a/tests/test_preprocessors.py +++ b/tests/test_preprocessors.py @@ -11,18 +11,8 @@ class TestPreprocessors(unittest.TestCase): root_path = get_tests_input_path() meta_file = "common_voice.tsv" items = common_voice(root_path, meta_file) - assert items[0][0] == "Man sollte den Länderfinanzausgleich durch " \ - "einen Bundesliga-Soli ersetzen." - assert items[0][1] == os.path.join(get_tests_input_path(), "clips", - "21fce545b24d9a5af0403b949e95e8dd3" - "c10c4ff3e371f14e4d5b4ebf588670b7c" - "9e618285fc872d94a89ed7f0217d9019f" - "e5de33f1577b49dcd518eacf63c4b.wav") + assert items[0][0] == 'The applicants are invited for coffee and visa is given immediately.' + assert items[0][1] == os.path.join(get_tests_input_path(), "clips", "common_voice_en_20005954.wav") - assert items[-1][0] == "Warum werden da keine strafrechtlichen " \ - "Konsequenzen gezogen?" - assert items[-1][1] == os.path.join(get_tests_input_path(), "clips", - "ad2f69e053b0e20e01c82b9821fe5787f1" - "cc8e4b0b97f0e4cab1e9a652c577169c82" - "44fb222281a60ee3081854014113e04c4c" - "a43643100b7c01dab0fac11974.wav") + assert items[-1][0] == "Competition for limited resources has also resulted in some local conflicts." + assert items[-1][1] == os.path.join(get_tests_input_path(), "clips", "common_voice_en_19737074.wav")