refactor codes: moved functions to util

Author: unilight

convert.py

@@ -17,9 +17,8 @@
 
 import sys
 from preprocessing.vcc2018.feature_reader import Whole_feature_reader
-from preprocessing.normalizer import MinMaxScaler, StandardScaler
-from preprocessing.utils import read_hdf5, read_txt
-from util.wrapper import load, get_default_logdir_output
+from util.normalizer import MinMaxScaler, StandardScaler
+from util.misc import read_hdf5, read_txt, load, get_default_logdir_output
 
 def main():
 

mcd_calculate.py

@@ -22,7 +22,7 @@
 
 import sys
 from preprocessing.vcc2018.feature_reader import Whole_feature_reader
-from preprocessing.normalizer import MinMaxScaler
+from util.normalizer import MinMaxScaler
 
 def read_and_synthesize(file_list, arch, MCD, input_feat, output_feat):
 

model/cdvae-cls-gan-mcc.py

@@ -1,10 +1,10 @@
+import numpy as np
 import tensorflow as tf
 from tensorflow.contrib import slim
+from util.misc import ValueWindow
 from util.layers import (GaussianKLD, GaussianLogDensity, GaussianSampleLayer,
                          Layernorm, conv2d_nchw_layernorm, lrelu,
                          kl_loss, log_loss, gradient_penalty_loss)
-import numpy as np
-from util.wrapper import ValueWindow
 
 class CDVAECLSGAN(object):
     def __init__(self, arch, normalizers=None):

model/cdvae.py

@@ -1,10 +1,10 @@
+import numpy as np
 import tensorflow as tf
 from tensorflow.contrib import slim
+from util.misc import ValueWindow
 from util.layers import (GaussianKLD, GaussianLogDensity, GaussianSampleLayer,
                          Layernorm, conv2d_nchw_layernorm, lrelu,
                          kl_loss, log_loss)
-import numpy as np
-from util.wrapper import ValueWindow
 
 class CDVAE(object):
     def __init__(self, arch, normalizers=None):

model/vae.py

@@ -1,10 +1,10 @@
+import numpy as np
 import tensorflow as tf
 from tensorflow.contrib import slim
+from util.misc import ValueWindow
 from util.layers import (GaussianKLD, GaussianLogDensity, GaussianSampleLayer,
                          Layernorm, conv2d_nchw_layernorm, lrelu,
                          kl_loss, log_loss)
-import numpy as np
-from util.wrapper import ValueWindow
 
 class VAE(object):
     def __init__(self, arch, normalizers=None):

preprocessing/vcc2018/calc_stats.py

@@ -14,9 +14,9 @@
 from sklearn.preprocessing import StandardScaler
 
 import sys
-sys.path.append(os.path.dirname(os.path.dirname(__file__)))
-from utils import (find_files, read_txt, write_hdf5)
-from vcc2018.feature_reader import Whole_feature_reader
+sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
+from util.misc import (find_files, read_txt, write_hdf5)
+from preprocess.vcc2018.feature_reader import Whole_feature_reader
 
 def calc_stats(file_list, feat_param, spk_list, args):
 

preprocessing/vcc2018/feature_extract.py

@@ -27,8 +27,8 @@
 from scipy.signal import lfilter
 from sprocket.speech.feature_extractor import FeatureExtractor
 
-sys.path.append(os.path.dirname(os.path.dirname(__file__)))
-from utils import (find_files, read_txt, read_hdf5, write_hdf5)
+sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
+from util.misc import (find_files, read_txt, read_hdf5, write_hdf5)
 
 def energy_norm(feat):
     en = np.sum(feat + 1e-8, axis=1, keepdims=True)

synthesize.py

@@ -23,11 +23,11 @@
 import multiprocessing as mp
 
 import sys
-from preprocessing.synthesizer import world_synthesis
-from preprocessing.utils import read_hdf5
 from preprocessing.vcc2018.feature_reader import Whole_feature_reader
-from preprocessing.postfilter import fast_MLGV
-from preprocessing.f0transformation import log_linear_transformation
+from util.synthesizer import world_synthesis
+from util.misc import read_hdf5
+from util.postfilter import fast_MLGV
+from util.f0transformation import log_linear_transformation
 
 def read_and_synthesize(file_list, arch, stats, input_feat, output_feat):
 

train.py

@@ -19,8 +19,8 @@
 
 import sys
 from preprocessing.vcc2018.feature_reader import  Segment_feature_reader
-from preprocessing.normalizer import MinMaxScaler
-from preprocessing.utils import read_hdf5
+from util.normalizer import MinMaxScaler
+from util.misc import read_hdf5
 
 def main():
 

trainer/base.py

@@ -2,8 +2,7 @@
 import numpy as np
 import logging, os
 
-from util.wrapper import load 
-from util.wrapper import ValueWindow
+from util.misc import load, ValueWindow
 
 class Trainer(object):
     def __init__(self, model, train_data, arch, args, dirs, ckpt):

trainer/cdvae-cls-gan.py

@@ -4,7 +4,7 @@
 import numpy as np
 import tensorflow as tf
 from trainer.base import Trainer
-from util.wrapper import ValueWindow
+from util.misc import ValueWindow
 import time
 
 class CDVAECLSGANTrainer(Trainer):

trainer/vae.py

@@ -4,7 +4,7 @@
 import numpy as np
 import tensorflow as tf
 from trainer.base import Trainer
-from util.wrapper import ValueWindow
+from util.misc import ValueWindow
 import time
 
 class VAETrainer(Trainer):

preprocessing/f0transformation.py util/f0transformation.py

@@ -1,20 +1,18 @@
-# -*- coding: utf-8 -*-
 
 # Based on 2017 Tomoki Hayashi (Nagoya University)
 #  Apache 2.0  (http://www.apache.org/licenses/LICENSE-2.0)
 
-from __future__ import division
-from __future__ import print_function
-
-import fnmatch
+import json
 import os
 import sys
-import threading
+from datetime import datetime
 
+import fnmatch
 import h5py
 import numpy as np
 
-from numpy.matlib import repmat
+import tensorflow as tf
+import logging
 
 def read_hdf5(hdf5_name, hdf5_path):
     """FUNCTION TO READ HDF5 DATASET
@@ -118,80 +116,73 @@ def read_txt(file_list):
         filenames = f.readlines()
     return [filename.replace("\n", "") for filename in filenames]
 
-
-class BackgroundGenerator(threading.Thread):
-    """BACKGROUND GENERATOR
-
-    reference:
-        https://stackoverflow.com/questions/7323664/python-generator-pre-fetch
-
-    Args:
-        generator (object): generator instance
-        max_prefetch (int): max number of prefetch
-    """
-
-    def __init__(self, generator, max_prefetch=1):
-        threading.Thread.__init__(self)
-        if sys.version_info.major == 2:
-            from Queue import Queue
+def save(saver, sess, logdir, step):
+    ''' Save a model to logdir/model.ckpt-[step] '''
+    model_name = 'model.ckpt'
+    checkpoint_path = os.path.join(logdir, model_name)
+    print('Storing checkpoint to {} ...'.format(logdir), end="")
+    sys.stdout.flush()
+
+    if not os.path.exists(logdir):
+        os.makedirs(logdir)
+
+    saver.save(sess, checkpoint_path, global_step=step)
+    print(' Done.')
+
+
+def load(saver, sess, logdir, ckpt=None):
+    '''
+    Try to load model form a dir (search for the newest checkpoint)
+    '''
+    if ckpt:
+        ckpt = os.path.join(logdir, ckpt)
+        global_step = int(ckpt.split('/')[-1].split('-')[-1])
+        logging.info('  Global step: {}'.format(global_step))
+        saver.restore(sess, ckpt)
+        return global_step
+    else:
+        ckpt = tf.train.latest_checkpoint(logdir)
+        if ckpt:
+            logging.info('  Checkpoint found: {}'.format(ckpt))
+            global_step = int(ckpt.split('/')[-1].split('-')[-1])
+            logging.info('  Global step: {}'.format(global_step))
+            saver.restore(sess, ckpt)
+            return global_step
         else:
-            from queue import Queue
-        self.queue = Queue(max_prefetch)
-        self.generator = generator
-        self.daemon = True
-        self.start()
-
-    def run(self):
-        for item in self.generator:
-            self.queue.put(item)
-        self.queue.put(None)
-
-    def next(self):
-        next_item = self.queue.get()
-        if next_item is None:
-            raise StopIteration
-        return next_item
-
-    def __next__(self):
-        return self.next()
-
-    def __iter__(self):
-        return self
-
-
-class background(object):
-    """BACKGROUND GENERATOR DECORATOR"""
-
-    def __init__(self, max_prefetch=1):
-        self.max_prefetch = max_prefetch
-
-    def __call__(self, gen):
-        def bg_generator(*args, **kwargs):
-            return BackgroundGenerator(gen(*args, **kwargs))
-        return bg_generator
-
-
-def extend_time(feats, upsampling_factor):
-    """FUNCTION TO EXTEND TIME RESOLUTION
-
-    Args:
-        feats (ndarray): feature vector with the shape (T x D)
-        upsampling_factor (int): upsampling_factor
-
-    Return:
-        (ndarray): extend feats with the shape (upsampling_factor*T x D)
-    """
-    # get number
-    n_frames = feats.shape[0]
-    n_dims = feats.shape[1]
-
-    # extend time
-    feats_extended = np.zeros((n_frames * upsampling_factor, n_dims))
-    for j in range(n_frames):
-        start_idx = j * upsampling_factor
-        end_idx = (j + 1) * upsampling_factor
-        feats_extended[start_idx: end_idx] = repmat(feats[j, :], upsampling_factor, 1)
-
-    return feats_extended
-
-
+            print('No checkpoint found')
+            return None
+
+def get_default_logdir_train(note, logdir_root='logdir'):
+    STARTED_DATESTRING = datetime.now().strftime('%0m%0d-%0H%0M-%0S-%Y')
+    logdir = os.path.join(logdir_root, '{}-{}'.format(STARTED_DATESTRING, note))
+    print('Using default logdir: {}'.format(logdir))        
+    return logdir
+
+def get_default_logdir_output(args):
+    STARTED_DATESTRING = datetime.now().strftime('%0m%0d-%0H%0M-%0S-%Y')
+    logdir = os.path.join(args.logdir, STARTED_DATESTRING+'-{}-{}'.format(args.src, args.trg))
+    print('Logdir: {}'.format(logdir))        
+    return logdir
+
+class ValueWindow():
+    def __init__(self, window_size=100):
+        self._window_size = window_size
+        self._values = []
+
+    def append(self, x):
+        self._values = self._values[-(self._window_size - 1):] + [x]
+
+    @property
+    def sum(self):
+        return sum(self._values)
+
+    @property
+    def count(self):
+        return len(self._values)
+
+    @property
+    def average(self):
+        return self.sum / max(1, self.count)
+
+    def reset(self):
+        self._values = []