From 1fa791f83e0c0636e2dd66de43b7242fbb8d2f59 Mon Sep 17 00:00:00 2001 From: Eren Golge Date: Thu, 25 Jan 2018 07:07:46 -0800 Subject: [PATCH] Tensorboard plotting --- config.json | 3 ++- train.py | 12 +++++++++++- utils/.generic_utils.py.swp | Bin 20480 -> 0 bytes 3 files changed, 13 insertions(+), 2 deletions(-) delete mode 100644 utils/.generic_utils.py.swp diff --git a/config.json b/config.json index 89a30a6b..d94f8851 100644 --- a/config.json +++ b/config.json @@ -25,5 +25,6 @@ "text_cleaner": "english_cleaners", "data_path": "/data/shared/KeithIto/LJSpeech-1.0", - "output_path": "result" + "output_path": "result", + "log_dir": "/home/erogol/projects/TTS/logs/" } diff --git a/train.py b/train.py index 04122e48..d325a54f 100644 --- a/train.py +++ b/train.py @@ -13,6 +13,7 @@ import torch.nn as nn from torch import optim from torch.autograd import Variable from torch.utils.data import DataLoader +from tensorboardX import SummaryWriter from utils.generic_utils import (Progbar, remove_experiment_folder, create_experiment_folder, save_checkpoint, @@ -38,6 +39,10 @@ def main(args): tmp_path = os.path.join("/tmp/", file_name+'_tts') pickle.dump(c, open(tmp_path, "wb")) + # setup tensorboard + LOG_DIR = c.log_dir + tb = SummaryWriter(LOG_DIR) + # Ctrl+C handler to remove empty experiment folder def signal_handler(signal, frame): print(" !! Pressed Ctrl+C !!") @@ -78,7 +83,7 @@ def main(args): print("\n > Model restored from step %d\n" % args.restore_step) except: - print("\n > Starting a new training\n") + print("\n > Starting a new training") model = model.train() @@ -97,6 +102,7 @@ def main(args): dataloader = DataLoader(dataset, batch_size=c.batch_size, shuffle=True, collate_fn=dataset.collate_fn, drop_last=True, num_workers=32) + print("\n | > Epoch {}".format(epoch)) progbar = Progbar(len(dataset) / c.batch_size) for i, data in enumerate(dataloader): @@ -160,6 +166,10 @@ def main(args): ('linear_loss', linear_loss.data[0]), ('mel_loss', mel_loss.data[0])]) + tb.add_scalar('Train/TotalLoss', loss.data[0], current_step) + tb.add_scalar('Train/LinearLoss', linear_loss.data[0], current_step) + tb.add_scalar('Train/MelLoss', mel_loss.data[0], current_step) + if current_step % c.save_step == 0: checkpoint_path = 'checkpoint_{}.pth.tar'.format(current_step) checkpoint_path = os.path.join(OUT_PATH, checkpoint_path) diff --git a/utils/.generic_utils.py.swp b/utils/.generic_utils.py.swp deleted file mode 100644 index 5b4764055e3bbf304a229c799eb03ce5de898598..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 20480 zcmeI2Ym6jS6~_y~1?91#m?*Gtcc8bccYA7PSX|6#4_S6uGwK@S&Vo8@Dm7iVyKB3< zs;R1;*^#gqNhC@lps1gC#Fz+)4i44h-d0FIQT;xkzK4Z$^|Sh_EKpgXvOr~l$^w-IDhpH=s4P%f zpt3+^fyx4v1^x#Xa9qQ<w=c zf-S%Rubg2RFN0r!$H14sr@?jLC^!ht2G5*s7;hZBx!@e|B6PU}+zxC&y1fJ) z{{_@@;C~H zg@ROMg8Y>_+s;NIU$x^Pb_VvE=f=H!mAVPGyY4$f+!b-XXf$xv^J)}ju$F3gDz@C< zkjLZDFEvN#I@xN>@m-S|}=t^^ThtrnY0EAsqKfQHr~wr0(CqB^Tltw}Wzx;;*fY?UFCG=7>Uf1_gcfUL`@s#8-_x$4BIoM%_|c--S5 zivu-ISG{PA5r~Ug-H=BSlS9~_T1;b^{8w{A5n9RXwb-Fwvp$rvZ+)AzT zf>aaOhrK9fL5IcT(Ex=^%yN!a(&(}w!4{JxiutIKFR%J&o-O(=KSC08uTkbS?*(jk z*c}fsW22-YFvvkM{UTB@*A`jO}lLmZK%T|g-Em}jH(rm4L4wgkM0Eu|A(Zo;P z8?E~>vn6d znY6bxYt6n#50>o2)CQSn`yq8W9r$Ov~Zx!V+{0EUHU?Dw%V_;TDACkqtZEBD)oIfZ%BMrTFrqgX7k1en1JUGAiY z3G>9*_`e9998D(F&=w)_qmbG<#kR7KjeRgwC`;*IC_s;5txO&; z(ai=9`C&(#0H&9EBV6LJ)XrIq>DytpR}t9MxvDo(Z} zii!wBw?(lDhE5bNnpn*_QL&s!mOOWuhoiu^27%+60gi8aYRQ>96;3sjRb?TxcQ>}U zyrs$FR14M#y3)0#b6XAYdTk)i%YrqIv2H}!A18tPxv8~T~1 zntsOCuOEZRApd_i{KvcDXOjP~-`_tBU;b0z05~7K0H2-w`CCB;>;Vn12|NLx{XU@m z`K$4~21g9;AU_W;9w5C0RQ|C;2Yrc;B%k@J^=QCZQ%Ft z)gK051OZ^+-|*3&0{4R(z+vEl%fR{IJn#x^`7(GGkX`Qr)u*z+>#_i?L-Y{~m0B0& zSvKRX?_j=mOwv`zab0yTO%jFILMx3Bsio%6+}I-0^dw&z8tM$jQ4cGuPSW40YoVe? z6rE_4MTwvV!UYsuDhbiTl1vC}T+Au1;jDHi=FXpsbb0-qlhZkt9Gost1zTWqe0pc< zS8GA~Tbe!>{XpDeh7Fw~CJy-%lIISoZAy#K-&9=-OT0DYj&B2|F*TpCjtD5GsT47zBFS@+%kipY*ti1J_E2b6xS<-8ArISR=*JdwrMSnpKo0(y|5cZ+R z8lE;VDxiJJy(ugB9U~*p-%aq@XVAocAhN101 z-%wfod*l60ZeWOdY#ckY)*sl5_jgHjKI+-nxxU>22y!CMja}j zqEwPux5Ne~)RFHq>CuD9LLf=^fEH&mJLJyMQ?QwQi%22Da^&} z;5MlG?1xluQ3DWe*`Q}ByUIXAp;mL&U>A5N_y^(u&w}rP zBVYlX01WUb;s9R*UjQp$3j7=~fZM<=AOcqc3p|Gyz}LYV;NTqaGx-1afqTJMK@WTo zyaJz}-rwH|?f?N;1e*cH0-gci0pA9~Ua!lcs|r^Zs4P%ffGm*RF0gZDmU;5d^--yC z4xvINXS%woQ@r4m3ku{8m<7iUhOe&bBsYJ`=L<0^bN=yibyWvfyL7}YHBEYi6>$KB zWGR!n=aNiz`5T8d^?>+W=AHt*N23|8KakYs)m0t5Ypky7R9AJ>dk__LsIKZ{@3mp& zsIKZ*Z@8;Ew2v9L6E%rJE~;&V~bxzUm}yl|-ei^TvMw Dp$iy(