From e9db1e9b0eeea0dd31b6d9ff8cbbaff4ecc06e9b Mon Sep 17 00:00:00 2001 From: dvgodoy Date: Wed, 30 May 2018 23:57:36 +0200 Subject: [PATCH 01/13] adding violin plot for weights --- deepreplay/plot.py | 38 ++++++++++++++++++++++- deepreplay/replay.py | 73 ++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 104 insertions(+), 7 deletions(-) diff --git a/deepreplay/plot.py b/deepreplay/plot.py index a6af21e..f1f2c04 100644 --- a/deepreplay/plot.py +++ b/deepreplay/plot.py @@ -3,6 +3,7 @@ import matplotlib import matplotlib.pyplot as plt import matplotlib.ticker as ticker +import pandas as pd import seaborn as sns from collections import namedtuple from matplotlib import animation @@ -14,6 +15,7 @@ LossAndMetricData = namedtuple('LossAndMetricData', ['loss', 'metric', 'metric_name']) ProbHistogramData = namedtuple('ProbHistogramData', ['prob', 'target']) LossHistogramData = namedtuple('LossHistogramData', ['loss']) +WeightsViolinsData = namedtuple('WeightsViolinsData', ['names', 'weights']) def build_2d_grid(xlim, ylim, n_lines=11, n_points=1000): """Returns a 2D grid of boundaries given by `xlim` and `ylim`, @@ -588,4 +590,38 @@ def _update(i, lh, epoch_start=0): lh.ax.xaxis.set_major_formatter(ticker.FormatStrFormatter('%0.1f')) lh.ax.locator_params(tight=True, nbins=4) - return lh.line \ No newline at end of file + return lh.line + +class WeightsViolins(Basic): + def __init__(self, ax): + super(WeightsViolins, self).__init__(ax) + self.weights = None + self.names = None + self._title = 'Weights' + + def load_data(self, weights_violins_data): + self.weights = weights_violins_data.weights + self.names = weights_violins_data.names + self.n_epochs = len(self.weights) + self._prepare_plot() + return self + + def _prepare_plot(self): + self.line = self.ax.plot([], []) + + @staticmethod + def _update(i, wv, epoch_start=0): + epoch = i + epoch_start + + df = pd.concat([pd.DataFrame(layer_weights[0].ravel(), + columns=[layer_name]).melt(var_name='layers', value_name='weights') + for layer_name, layer_weights in zip(wv.names, wv.weights[i])]) + + sns.violinplot(data=df, x='layers', y='weights', ax=wv.ax) + wv.ax.set_xticklabels(wv.names) + wv.ax.set_xlabel('Layers') + wv.ax.set_ylabel('Weights') + wv.ax.set_title('{} - Epoch: {}'.format(wv.title[0], epoch)) + + return wv.line + diff --git a/deepreplay/replay.py b/deepreplay/replay.py index b4335e3..d3a63ad 100644 --- a/deepreplay/replay.py +++ b/deepreplay/replay.py @@ -3,11 +3,17 @@ import h5py import keras.backend as K from keras.models import load_model -from .plot import build_2d_grid, FeatureSpace, ProbabilityHistogram, LossHistogram, LossAndMetric -from .plot import FeatureSpaceData, FeatureSpaceLines, ProbHistogramData, LossHistogramData, LossAndMetricData +from .plot import ( + build_2d_grid, FeatureSpace, ProbabilityHistogram, LossHistogram, LossAndMetric, WeightsViolins +) +from .plot import ( + FeatureSpaceData, FeatureSpaceLines, ProbHistogramData, LossHistogramData, LossAndMetricData, WeightsViolinsData +) TRAINING_MODE = 1 TEST_MODE = 0 +ACTIVATIONS = ['softmax', 'relu', 'elu', 'tanh', 'sigmoid', 'hard_sigmoid', 'linear', 'softplus', 'softsign', 'selu'] + class Replay(object): """Creates an instance of Replay, to process information collected @@ -70,6 +76,10 @@ def __init__(self, replay_filename, group_name, model_filename=''): self.targets = self.group['targets'][:] self.n_epochs = self.group.attrs['n_epochs'] self.n_layers = self.group.attrs['n_layers'] + + # Generates ranges for the number of different weight arrays in each layer + self.n_weights = [range(len(self.group['layer{}'.format(l)])) for l in range(self.n_layers)] + # Retrieves weights as a list, each element being one epoch self.weights = self._retrieve_weights() @@ -92,26 +102,44 @@ def __init__(self, replay_filename, group_name, model_filename=''): outputs=[K.binary_crossentropy(self.model.targets[0], self.model.outputs[0])]) + __activation_tensors = list(filter(lambda t: t[1].op.type.lower() in ACTIVATIONS, + [(self.model.layers[i].name, self.model.layers[i].output) + for i in range(self.n_layers)])) + + __z_tensors = list(filter(lambda t: t[1].op.type in ['BiasAdd', 'MatMul'], + [(self.model.layers[i - (self.model.layers[i].input.op.type in + ['BiasAdd', 'MatMul'])].name, + self.model.layers[i].output.op.inputs[0]) for i in range(self.n_layers)])) + + self._activation_layers = ['inputs'] + list(map(lambda t: t[0], __activation_tensors)) + self._activation_tensors = self.model.inputs + list(map(lambda t: t[1], __activation_tensors)) + self._get_activations = K.function(inputs=[K.learning_phase()] + self.model.inputs + self._model_weights, + outputs=self._activation_tensors) + + self._z_layers = list(map(lambda t: t[0], __z_tensors)) + self._z_tensors = list(map(lambda t: t[1], __z_tensors)) + self._get_zvalues = K.function(inputs=[K.learning_phase()] + self.model.inputs + self._model_weights, + outputs=self._z_tensors) + # Attributes for the visualizations - Data self._feature_space_data = None self._loss_hist_data = None self._loss_and_metric_data = None self._prob_hist_data = None self._decision_boundary_data = None + self._weights_violins_data = None # Attributes for the visualizations - Plot objects self._feature_space_plot = None self._loss_hist_plot = None self._loss_and_metric_plot = None self._prob_hist_plot = None self._decision_boundary_plot = None + self._weights_violins_plot = None def _retrieve_weights(self): - # Generates ranges for the number of different weight arrays in each layer - n_weights = [range(len(self.group['layer{}'.format(l)])) - for l in range(self.n_layers)] # Retrieves weights for each layer and sequence of weights weights = [np.array(self.group['layer{}'.format(l)]['weights{}'.format(w)]) - for l, ws in enumerate(n_weights) + for l, ws in enumerate(self.n_weights) for w in ws] # Since initial weights are also saved, there are n_epochs + 1 elements in total return [[w[epoch] for w in weights] for epoch in range(self.n_epochs + 1)] @@ -202,6 +230,39 @@ def predict_proba(self, epoch_start=0, epoch_end=-1): probas = np.array(probas) return probas + def build_weights_violins(self, ax, epoch_start=0, epoch_end=-1): + """Builds a WeightsViolins object to be used for plotting and + animating. + + Parameters + ---------- + ax: AxesSubplot + Subplot of a Matplotlib figure. + epoch_start: int, optional + First epoch to consider. + epoch_end: int, optional + Last epoch to consider. + + Returns + ------- + weights_violins_plot: WeightsViolins + An instance of a WeightsViolins object to make plots and + animations. + """ + if epoch_end == -1: + epoch_end = self.n_epochs + epoch_end = min(epoch_end, self.n_epochs) + + names = [layer.name for layer, weights in zip(self.model.layers, self.weights[0]) if len(weights) in (1, 2)] + weights = [] + # For each epoch, uses the corresponding weights + for epoch in range(epoch_start, epoch_end + 1): + weights.append(list(filter(lambda weights: len(weights) in (1, 2), self.weights[epoch]))) + + self._weights_violins_data = WeightsViolinsData(names=names, weights=weights) + self._weights_violins_plot = WeightsViolins(ax).load_data(self._weights_violins_data) + return self._weights_violins_plot + def build_loss_histogram(self, ax, epoch_start=0, epoch_end=-1): """Builds a LossHistogram object to be used for plotting and animating. From c38b72cd111d6383c0e44c92d360cfda2ecdb1d8 Mon Sep 17 00:00:00 2001 From: dvgodoy Date: Thu, 31 May 2018 23:43:42 +0200 Subject: [PATCH 02/13] new datasets and plots --- deepreplay/datasets/ball.py | 47 ++++++++++++++++++++ deepreplay/datasets/hypercube.py | 37 ++++++++++++++++ deepreplay/plot.py | 50 +++++++++++---------- deepreplay/replay.py | 75 ++++++++++++++++++++++++++++---- setup.py | 2 +- 5 files changed, 179 insertions(+), 32 deletions(-) create mode 100644 deepreplay/datasets/ball.py create mode 100644 deepreplay/datasets/hypercube.py diff --git a/deepreplay/datasets/ball.py b/deepreplay/datasets/ball.py new file mode 100644 index 0000000..17d44b2 --- /dev/null +++ b/deepreplay/datasets/ball.py @@ -0,0 +1,47 @@ +import numpy as np + +def load_data(n_dims=10, radius=2.0, n_points=1000, only_sphere=False, shuffle=True, seed=13): + """ + + Parameters + ---------- + n_dims: int, optional + Number of dimensions of the n-ball. Default is 10. + radius: float, optional + Radius of the n-ball. Default is 2.0. + n_points: int, optional + Number of points in each parabola. Default is 1,000. + only_sphere: boolean + If True, generates a n-sphere, that is, a hollow n-ball. + Default is False. + shuffle: boolean, optional + If True, the points are shuffled. Default is True. + seed: int, optional + Random seed. Default is 13. + + Returns + ------- + X, y: tuple of ndarray + X is an array of shape (n_points, n_dims) containing the + points in the n-ball. + y is an array of shape (n_points, 1) containing the + classes of the samples. + """ + points = np.random.normal(size=(1000, n_dims)) + sphere = points / np.linalg.norm(points, axis=1).reshape(-1, 1) + if only_sphere: + X = sphere + else: + X = radius * sphere * np.random.uniform(size=(n_points, 1))**(1 / n_dims) + + y = (np.abs(np.sum(X, axis=1)) > (radius / 2.0)).astype(np.int) + + # But we must not feed the network with neatly organized inputs... + # so let's randomize them + if shuffle: + np.random.seed(seed) + shuffled = np.random.permutation(range(X.shape[0])) + X = X[shuffled] + y = y[shuffled].reshape(-1, 1) + + return (X, y) diff --git a/deepreplay/datasets/hypercube.py b/deepreplay/datasets/hypercube.py new file mode 100644 index 0000000..ee5710e --- /dev/null +++ b/deepreplay/datasets/hypercube.py @@ -0,0 +1,37 @@ +import itertools +import numpy as np + +def load_data(n_dims=10, vertices=(-1., 1.), shuffle=True, seed=13): + """ + + Parameters + ---------- + n_dims: int, optional + Number of dimensions of the hypercube. Default is 10. + edge: tuple of floats, optional + Two vertices of an edge. Default is (-1., 1.). + shuffle: boolean, optional + If True, the points are shuffled. Default is True. + seed: int, optional + Random seed. Default is 13. + + Returns + ------- + X, y: tuple of ndarray + X is an array of shape (2 ** n_dims, n_dims) containing the + vertices coordinates of the hypercube. + y is an array of shape (2 ** n_dims, 1) containing the + classes of the samples. + """ + X = np.array(list(itertools.product(vertices, repeat=n_dims))) + y = (np.sum(np.clip(X, a_min=0, a_max=1), axis=1) >= (n_dims / 2.0)).astype(np.int) + + # But we must not feed the network with neatly organized inputs... + # so let's randomize them + if shuffle: + np.random.seed(seed) + shuffled = np.random.permutation(range(X.shape[0])) + X = X[shuffled] + y = y[shuffled].reshape(-1, 1) + + return (X, y) diff --git a/deepreplay/plot.py b/deepreplay/plot.py index f1f2c04..8f9261f 100644 --- a/deepreplay/plot.py +++ b/deepreplay/plot.py @@ -7,7 +7,7 @@ import seaborn as sns from collections import namedtuple from matplotlib import animation -matplotlib.rcParams['animation.writer'] = 'ffmpeg' +matplotlib.rcParams['animation.writer'] = 'avconv' sns.set_style('white') FeatureSpaceData = namedtuple('FeatureSpaceData', ['line', 'bent_line', 'prediction', 'target']) @@ -15,7 +15,7 @@ LossAndMetricData = namedtuple('LossAndMetricData', ['loss', 'metric', 'metric_name']) ProbHistogramData = namedtuple('ProbHistogramData', ['prob', 'target']) LossHistogramData = namedtuple('LossHistogramData', ['loss']) -WeightsViolinsData = namedtuple('WeightsViolinsData', ['names', 'weights']) +LayerViolinsData = namedtuple('LayerViolinsData', ['names', 'values', 'layers']) def build_2d_grid(xlim, ylim, n_lines=11, n_points=1000): """Returns a 2D grid of boundaries given by `xlim` and `ylim`, @@ -592,17 +592,21 @@ def _update(i, lh, epoch_start=0): return lh.line -class WeightsViolins(Basic): - def __init__(self, ax): - super(WeightsViolins, self).__init__(ax) - self.weights = None +class LayerViolins(Basic): + def __init__(self, ax, title): + super(LayerViolins, self).__init__(ax) + self.values = None self.names = None - self._title = 'Weights' - - def load_data(self, weights_violins_data): - self.weights = weights_violins_data.weights - self.names = weights_violins_data.names - self.n_epochs = len(self.weights) + self._title = title + + def load_data(self, layer_violins_data): + self.values = layer_violins_data.values + self.names = layer_violins_data.names + self.layers = ['inputs'] + layer_violins_data.layers + self.palette = dict(zip(self.layers, sns.palettes.husl_palette(len(self.layers), .7))) + self.n_epochs = len(self.values) + _flat_values = np.array([np.concatenate([v.ravel() for v in epoch]) for epoch in self.values]) + self.ylim = [_flat_values.min(), _flat_values.max()] self._prepare_plot() return self @@ -610,18 +614,20 @@ def _prepare_plot(self): self.line = self.ax.plot([], []) @staticmethod - def _update(i, wv, epoch_start=0): + def _update(i, lv, epoch_start=0): epoch = i + epoch_start - df = pd.concat([pd.DataFrame(layer_weights[0].ravel(), - columns=[layer_name]).melt(var_name='layers', value_name='weights') - for layer_name, layer_weights in zip(wv.names, wv.weights[i])]) + df = pd.concat([pd.DataFrame(layer_values.ravel(), + columns=[layer_name]).melt(var_name='layers', value_name='values') + for layer_name, layer_values in zip(lv.names, lv.values[i])]) - sns.violinplot(data=df, x='layers', y='weights', ax=wv.ax) - wv.ax.set_xticklabels(wv.names) - wv.ax.set_xlabel('Layers') - wv.ax.set_ylabel('Weights') - wv.ax.set_title('{} - Epoch: {}'.format(wv.title[0], epoch)) + lv.ax.clear() + sns.violinplot(data=df, x='layers', y='values', ax=lv.ax, cut=0, palette=lv.palette, scale='width') + lv.ax.set_xticklabels(lv.names) + lv.ax.set_xlabel('Layers') + lv.ax.set_ylabel(lv._title) + lv.ax.set_ylim(lv.ylim) + lv.ax.set_title('{} - Epoch: {}'.format(lv.title[0], epoch)) - return wv.line + return lv.line diff --git a/deepreplay/replay.py b/deepreplay/replay.py index d3a63ad..f7c930b 100644 --- a/deepreplay/replay.py +++ b/deepreplay/replay.py @@ -4,10 +4,10 @@ import keras.backend as K from keras.models import load_model from .plot import ( - build_2d_grid, FeatureSpace, ProbabilityHistogram, LossHistogram, LossAndMetric, WeightsViolins + build_2d_grid, FeatureSpace, ProbabilityHistogram, LossHistogram, LossAndMetric, LayerViolins ) from .plot import ( - FeatureSpaceData, FeatureSpaceLines, ProbHistogramData, LossHistogramData, LossAndMetricData, WeightsViolinsData + FeatureSpaceData, FeatureSpaceLines, ProbHistogramData, LossHistogramData, LossAndMetricData, LayerViolinsData ) TRAINING_MODE = 1 @@ -116,8 +116,8 @@ def __init__(self, replay_filename, group_name, model_filename=''): self._get_activations = K.function(inputs=[K.learning_phase()] + self.model.inputs + self._model_weights, outputs=self._activation_tensors) - self._z_layers = list(map(lambda t: t[0], __z_tensors)) - self._z_tensors = list(map(lambda t: t[1], __z_tensors)) + self._z_layers = ['inputs'] + list(map(lambda t: t[0], __z_tensors)) + self._z_tensors = self.model.inputs + list(map(lambda t: t[1], __z_tensors)) self._get_zvalues = K.function(inputs=[K.learning_phase()] + self.model.inputs + self._model_weights, outputs=self._z_tensors) @@ -128,6 +128,8 @@ def __init__(self, replay_filename, group_name, model_filename=''): self._prob_hist_data = None self._decision_boundary_data = None self._weights_violins_data = None + self._activations_violins_data = None + self._zvalues_violins_data = None # Attributes for the visualizations - Plot objects self._feature_space_plot = None self._loss_hist_plot = None @@ -135,6 +137,8 @@ def __init__(self, replay_filename, group_name, model_filename=''): self._prob_hist_plot = None self._decision_boundary_plot = None self._weights_violins_plot = None + self._activations_violins_plot = None + self._zvalues_violins_plot = None def _retrieve_weights(self): # Retrieves weights for each layer and sequence of weights @@ -230,8 +234,61 @@ def predict_proba(self, epoch_start=0, epoch_end=-1): probas = np.array(probas) return probas + def build_outputs_violins(self, ax, before_activation=False, epoch_start=0, epoch_end=-1): + """Builds a LayerViolins object to be used for plotting and + animating. + + Parameters + ---------- + ax: AxesSubplot + Subplot of a Matplotlib figure. + before_activation: Boolean, optional + If True, returns Z-values, that is, before applying + the activation function. + epoch_start: int, optional + First epoch to consider. + epoch_end: int, optional + Last epoch to consider. + + Returns + ------- + activations_violins_plot: LayerViolins + An instance of a LayerViolins object to make plots and + animations. + """ + if epoch_end == -1: + epoch_end = self.n_epochs + epoch_end = min(epoch_end, self.n_epochs) + + if before_activation: + title = 'Z-values' + names = self._z_layers + else: + title = 'Activations' + names = self._activation_layers + outputs = [] + # For each epoch, uses the corresponding weights + for epoch in range(epoch_start, epoch_end + 1): + weights = self.weights[epoch] + inputs = [self.learning_phase, self.inputs] + weights + if before_activation: + outputs.append(self._get_zvalues(inputs=inputs)) + else: + outputs.append(self._get_activations(inputs=inputs)) + + layers = [layer.name for layer, weights in zip(self.model.layers, self.weights[0]) if len(weights) in (1, 2)] + data = LayerViolinsData(names=names, values=outputs, layers=layers) + plot = LayerViolins(ax, title).load_data(data) + if before_activation: + self._zvalues_violins_data = data + self._zvalues_violins_plot = plot + else: + self._activations_violins_data = data + self._activations_violins_plot = plot + return plot + def build_weights_violins(self, ax, epoch_start=0, epoch_end=-1): - """Builds a WeightsViolins object to be used for plotting and + """Builds a LayerViolins object to be used for plotting and animating. Parameters @@ -245,8 +302,8 @@ def build_weights_violins(self, ax, epoch_start=0, epoch_end=-1): Returns ------- - weights_violins_plot: WeightsViolins - An instance of a WeightsViolins object to make plots and + weights_violins_plot: LayerViolins + An instance of a LayerViolins object to make plots and animations. """ if epoch_end == -1: @@ -259,8 +316,8 @@ def build_weights_violins(self, ax, epoch_start=0, epoch_end=-1): for epoch in range(epoch_start, epoch_end + 1): weights.append(list(filter(lambda weights: len(weights) in (1, 2), self.weights[epoch]))) - self._weights_violins_data = WeightsViolinsData(names=names, weights=weights) - self._weights_violins_plot = WeightsViolins(ax).load_data(self._weights_violins_data) + self._weights_violins_data = LayerViolinsData(names=names, values=weights, layers=names) + self._weights_violins_plot = LayerViolins(ax, 'Weights').load_data(self._weights_violins_data) return self._weights_violins_plot def build_loss_histogram(self, ax, epoch_start=0, epoch_end=-1): diff --git a/setup.py b/setup.py index 2dacc30..e7cc00f 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ def readme(): return f.read() setup(name='deepreplay', - version='0.1.0a6', + version='0.1.1a1', install_requires=['matplotlib', 'numpy', 'h5py', 'seaborn', 'keras', 'scikit-learn'], description='"Hyper-parameters in Action!" visualizing tool for Keras models.', long_description=readme(), From dfc358e1831ac3597a020e2729a0e351db041284 Mon Sep 17 00:00:00 2001 From: dvgodoy Date: Sun, 3 Jun 2018 22:08:55 +0200 Subject: [PATCH 03/13] including gradients --- deepreplay/callbacks.py | 26 +++++++++++- deepreplay/plot.py | 8 ++-- deepreplay/replay.py | 77 ++++++++++++++++++++++++++++++++--- deepreplay/utils.py | 48 ++++++++++++++++++++++ examples/gradients_example.py | 51 +++++++++++++++++++++++ 5 files changed, 199 insertions(+), 11 deletions(-) create mode 100644 deepreplay/utils.py create mode 100644 examples/gradients_example.py diff --git a/deepreplay/callbacks.py b/deepreplay/callbacks.py index 87d87cd..51fc8af 100644 --- a/deepreplay/callbacks.py +++ b/deepreplay/callbacks.py @@ -2,6 +2,7 @@ import os import numpy as np import h5py +import keras.backend as K from keras.callbacks import Callback class ReplayData(Callback): @@ -31,7 +32,7 @@ class ReplayData(Callback): saved. If the informed group name already exists, it will throw an exception. """ - def __init__(self, inputs, targets, filename, group_name): + def __init__(self, inputs, targets, filename, group_name, model=None): super(ReplayData, self).__init__() self.handler = h5py.File('{}'.format(filename), 'a') self.inputs = inputs @@ -42,6 +43,16 @@ def __init__(self, inputs, targets, filename, group_name): self.group_name = group_name self.current_epoch = -1 self.n_epochs = 0 + if model is not None: + self.set_model(model) + self.set_params({ + 'epochs': 0, + 'samples': len(self.inputs), + 'batch_size': len(self.inputs), + }) + self.group_name = group_name + '_init' + self.on_train_begin() + self.group_name = group_name return def _append_weights(self): @@ -52,6 +63,13 @@ def _append_weights(self): for j, weights in enumerate(layer_weights): self.group['layer{}'.format(i)]['weights{}'.format(j)][self.current_epoch + 1] = weights + def get_lr(self): + optimizer = self.model.optimizer + return K.function(inputs=[], + outputs=[optimizer.lr * + (1. / (1. + optimizer.decay * K.cast(optimizer.iterations, + K.dtype(optimizer.decay))))])(inputs=[])[0] + def on_train_begin(self, logs={}): self.model.save(os.path.join(self.filepath, '{}_model.h5'.format(self.group_name))) self.n_epochs = self.params['epochs'] @@ -59,7 +77,8 @@ def on_train_begin(self, logs={}): self.group = self.handler.create_group(self.group_name) self.group.attrs['samples'] = self.params['samples'] self.group.attrs['batch_size'] = self.params['batch_size'] - self.group.attrs['n_batches'] = np.ceil(self.params['samples'] / self.params['batch_size']).astype(np.int) + self.group.attrs['n_batches'] = (self.params['samples'] + self.params['batch_size'] - 1) // \ + self.params['batch_size'] self.group.attrs['n_epochs'] = self.n_epochs self.group.attrs['n_layers'] = len(self.model.layers) try: @@ -81,6 +100,8 @@ def on_train_begin(self, logs={}): for metric in self.model.metrics: self.group.create_dataset(metric, shape=(self.n_epochs,), dtype='f') + self.group.create_dataset('lr', shape=(self.n_epochs,), dtype='f') + for i, layer in enumerate(self.model.layers): layer_grp = self.group.create_group('layer{}'.format(i)) layer_weights = layer.get_weights() @@ -97,6 +118,7 @@ def on_train_end(self, logs={}): def on_epoch_begin(self, epoch, logs={}): self.current_epoch = epoch + self.group['lr'][epoch] = self.get_lr() return def on_epoch_end(self, epoch, logs={}): diff --git a/deepreplay/plot.py b/deepreplay/plot.py index 8f9261f..11fa5a2 100644 --- a/deepreplay/plot.py +++ b/deepreplay/plot.py @@ -605,8 +605,6 @@ def load_data(self, layer_violins_data): self.layers = ['inputs'] + layer_violins_data.layers self.palette = dict(zip(self.layers, sns.palettes.husl_palette(len(self.layers), .7))) self.n_epochs = len(self.values) - _flat_values = np.array([np.concatenate([v.ravel() for v in epoch]) for epoch in self.values]) - self.ylim = [_flat_values.min(), _flat_values.max()] self._prepare_plot() return self @@ -615,18 +613,20 @@ def _prepare_plot(self): @staticmethod def _update(i, lv, epoch_start=0): + assert len(lv.names) == len(lv.values[i]), "Layer names and values have different lengths!" epoch = i + epoch_start df = pd.concat([pd.DataFrame(layer_values.ravel(), columns=[layer_name]).melt(var_name='layers', value_name='values') for layer_name, layer_values in zip(lv.names, lv.values[i])]) + df = df.query('layers != "{}"'.format(lv.names[-1])) lv.ax.clear() sns.violinplot(data=df, x='layers', y='values', ax=lv.ax, cut=0, palette=lv.palette, scale='width') - lv.ax.set_xticklabels(lv.names) + lv.ax.set_xticklabels(df.layers.unique()) lv.ax.set_xlabel('Layers') lv.ax.set_ylabel(lv._title) - lv.ax.set_ylim(lv.ylim) + lv.ax.set_ylim([df['values'].min(), df['values'].max()]) lv.ax.set_title('{} - Epoch: {}'.format(lv.title[0], epoch)) return lv.line diff --git a/deepreplay/replay.py b/deepreplay/replay.py index f7c930b..4450452 100644 --- a/deepreplay/replay.py +++ b/deepreplay/replay.py @@ -9,6 +9,9 @@ from .plot import ( FeatureSpaceData, FeatureSpaceLines, ProbHistogramData, LossHistogramData, LossAndMetricData, LayerViolinsData ) +from .utils import make_batches, slice_arrays +from itertools import groupby +from operator import itemgetter TRAINING_MODE = 1 TEST_MODE = 0 @@ -102,6 +105,13 @@ def __init__(self, replay_filename, group_name, model_filename=''): outputs=[K.binary_crossentropy(self.model.targets[0], self.model.outputs[0])]) + self.__trainable_weights = [w for layer in self.model.layers + for w in layer.trainable_weights if layer.trainable] + self.__trainable_gradients = self.model.optimizer.get_gradients(self.model.total_loss, self.__trainable_weights) + self._get_gradients = K.function(inputs=[K.learning_phase()] + self.model.inputs + self.model.targets + + self._model_weights + self.model.sample_weights, + outputs=self.__trainable_gradients) + __activation_tensors = list(filter(lambda t: t[1].op.type.lower() in ACTIVATIONS, [(self.model.layers[i].name, self.model.layers[i].output) for i in range(self.n_layers)])) @@ -121,6 +131,9 @@ def __init__(self, replay_filename, group_name, model_filename=''): self._get_zvalues = K.function(inputs=[K.learning_phase()] + self.model.inputs + self._model_weights, outputs=self._z_tensors) + self.weights_layers = [layer.name for layer, weights in zip(self.model.layers, self.n_weights) + if len(weights) in (1, 2)] + # Attributes for the visualizations - Data self._feature_space_data = None self._loss_hist_data = None @@ -130,6 +143,7 @@ def __init__(self, replay_filename, group_name, model_filename=''): self._weights_violins_data = None self._activations_violins_data = None self._zvalues_violins_data = None + self._gradients_data = None # Attributes for the visualizations - Plot objects self._feature_space_plot = None self._loss_hist_plot = None @@ -139,6 +153,33 @@ def __init__(self, replay_filename, group_name, model_filename=''): self._weights_violins_plot = None self._activations_violins_plot = None self._zvalues_violins_plot = None + self._gradients_plot = None + + def _make_batches(self, seed): + inputs = self.inputs[:] + targets = self.targets[:] + + np.random.seed(seed) + np.random.shuffle(inputs) + np.random.shuffle(targets) + num_training_samples = inputs.shape[0] + + batches = make_batches(num_training_samples, self.params['batch_size']) + index_array = np.arange(num_training_samples) + + inputs_batches = [] + targets_batches = [] + for batch_index, (batch_start, batch_end) in enumerate(batches): + batch_ids = index_array[batch_start:batch_end] + inputs_batch, targets_batch = slice_arrays([inputs, targets], batch_ids) + inputs_batches.append(inputs_batch) + targets_batches.append(targets_batch) + + return inputs_batches, targets_batches + + @staticmethod + def __assign_gradients_to_layers(layers, gradients): + return [list(list(zip(*g))[1]) for k, g in groupby(zip(layers, gradients), itemgetter(0))] def _retrieve_weights(self): # Retrieves weights for each layer and sequence of weights @@ -234,6 +275,27 @@ def predict_proba(self, epoch_start=0, epoch_end=-1): probas = np.array(probas) return probas + def build_gradients(self, ax, include_output=True, epoch_start=0, epoch_end=-1): + if epoch_end == -1: + epoch_end = self.n_epochs + epoch_end = min(epoch_end, self.n_epochs) + + gradient_names = [layer.name for layer in self.model.layers for _ in layer.trainable_weights if layer.trainable] + gradients = [] + # For each epoch, uses the corresponding weights + for epoch in range(epoch_start, epoch_end + 1): + weights = self.weights[epoch] + + # Sample weights fixed to one! + inputs = [self.learning_phase, self.inputs, self.targets] + weights + [np.ones(shape=self.inputs.shape[0])] + grad = [w for v in Replay.__assign_gradients_to_layers(gradient_names, self._get_gradients(inputs=inputs)) + for w in v] + gradients.append(grad) + + self._gradients_data = LayerViolinsData(names=gradient_names, values=gradients, layers=self.weights_layers) + self._gradients_plot = LayerViolins(ax, 'Gradients').load_data(self._gradients_data) + return self._gradients_plot + def build_outputs_violins(self, ax, before_activation=False, epoch_start=0, epoch_end=-1): """Builds a LayerViolins object to be used for plotting and animating. @@ -276,8 +338,7 @@ def build_outputs_violins(self, ax, before_activation=False, epoch_start=0, epoc else: outputs.append(self._get_activations(inputs=inputs)) - layers = [layer.name for layer, weights in zip(self.model.layers, self.weights[0]) if len(weights) in (1, 2)] - data = LayerViolinsData(names=names, values=outputs, layers=layers) + data = LayerViolinsData(names=names, values=outputs, layers=self.weights_layers) plot = LayerViolins(ax, title).load_data(data) if before_activation: self._zvalues_violins_data = data @@ -310,13 +371,19 @@ def build_weights_violins(self, ax, epoch_start=0, epoch_end=-1): epoch_end = self.n_epochs epoch_end = min(epoch_end, self.n_epochs) - names = [layer.name for layer, weights in zip(self.model.layers, self.weights[0]) if len(weights) in (1, 2)] + names = [layer.name for layer, weights in zip(self.model.layers, self.n_weights) for _ in weights + if len(weights) in (1, 2)] + n_weights = [len(weights) for layer, weights in zip(self.model.layers, self.n_weights) for _ in weights] + weights = [] # For each epoch, uses the corresponding weights for epoch in range(epoch_start, epoch_end + 1): - weights.append(list(filter(lambda weights: len(weights) in (1, 2), self.weights[epoch]))) + weights.append([w for w, n in zip(self.weights[epoch], n_weights) if n in (1, 2)]) + #weights.append(list(filter(lambda weights: len(weights) in (1, 2), self.weights[epoch]))) - self._weights_violins_data = LayerViolinsData(names=names, values=weights, layers=names) + self._weights_violins_data = LayerViolinsData(names=names, + values=weights, + layers=self.weights_layers) self._weights_violins_plot = LayerViolins(ax, 'Weights').load_data(self._weights_violins_data) return self._weights_violins_plot diff --git a/deepreplay/utils.py b/deepreplay/utils.py new file mode 100644 index 0000000..53b34ef --- /dev/null +++ b/deepreplay/utils.py @@ -0,0 +1,48 @@ +def make_batches(size, batch_size): + """Returns a list of batch indices (tuples of indices). + # Arguments + size: Integer, total size of the data to slice into batches. + batch_size: Integer, batch size. + # Returns + A list of tuples of array indices. + """ + num_batches = (size + batch_size - 1) // batch_size # round up + return [(i * batch_size, min(size, (i + 1) * batch_size)) + for i in range(num_batches)] + +def slice_arrays(arrays, start=None, stop=None): + """Slices an array or list of arrays. + This takes an array-like, or a list of + array-likes, and outputs: + - arrays[start:stop] if `arrays` is an array-like + - [x[start:stop] for x in arrays] if `arrays` is a list + Can also work on list/array of indices: `_slice_arrays(x, indices)` + # Arguments + arrays: Single array or list of arrays. + start: can be an integer index (start index) + or a list/array of indices + stop: integer (stop index); should be None if + `start` was a list. + # Returns + A slice of the array(s). + """ + if arrays is None: + return [None] + elif isinstance(arrays, list): + if hasattr(start, '__len__'): + # hdf5 datasets only support list objects as indices + if hasattr(start, 'shape'): + start = start.tolist() + return [None if x is None else x[start] for x in arrays] + else: + return [None if x is None else x[start:stop] for x in arrays] + else: + if hasattr(start, '__len__'): + if hasattr(start, 'shape'): + start = start.tolist() + return arrays[start] + elif hasattr(start, '__getitem__'): + return arrays[start:stop] + else: + return [None] + diff --git a/examples/gradients_example.py b/examples/gradients_example.py new file mode 100644 index 0000000..31489e8 --- /dev/null +++ b/examples/gradients_example.py @@ -0,0 +1,51 @@ +import numpy as np +from keras.models import Sequential +from keras.layers import Dense +from keras.initializers import glorot_uniform, normal +from deepreplay.datasets.ball import load_data +from deepreplay.callbacks import ReplayData +from deepreplay.replay import Replay +from deepreplay.plot import compose_plots + +import matplotlib.pyplot as plt + +group_name = 'gradients' + +X, y = load_data() + +n_layers = 5 +variance = 0.01 +activation = 'tanh' + +init = normal(mean=0, stddev=np.sqrt(variance)) +#init = VarianceScaling(scale=1.0, mode='fan_avg', distribution='uniform') +#init = glorot_normal() # sigmoid +#init = glorot_uniform() # tanh +#init = he_uniform() # relu +#init_name = 'He Uniform' + +model = Sequential() +model.add(Dense(units=100, input_dim=10, activation=activation, kernel_initializer=init, name='h1')) +for i in range(2, n_layers + 1): + model.add(Dense(units=100, activation=activation, kernel_initializer=init, name='h{}'.format(i))) +model.add(Dense(units=1, activation='sigmoid', kernel_initializer=init, name='o')) + +model.compile(loss='binary_crossentropy', optimizer='sgd', metrics=['acc']) + +replaydata = ReplayData(X, y, filename='gradients.h5', group_name=group_name, model=model) + +replay = Replay(replay_filename='gradients.h5', group_name=group_name + '_init') + +fig = plt.figure(figsize=(12, 6)) +ax_zvalues = plt.subplot2grid((2, 2), (0, 0)) +ax_weights = plt.subplot2grid((2, 2), (0, 1)) +ax_activations = plt.subplot2grid((2, 2), (1, 0)) +ax_gradients = plt.subplot2grid((2, 2), (1, 1)) + +zv = replay.build_outputs_violins(ax_zvalues, before_activation=True) +wv = replay.build_weights_violins(ax_weights) +av = replay.build_outputs_violins(ax_activations) +gv = replay.build_gradients(ax_gradients) + +sample_figure = compose_plots([zv, wv, av, gv], 0) +sample_figure.savefig('gradients.png', dpi=120, format='png') From 703005b6488d077f3737e658c034afb8fc9e79ac Mon Sep 17 00:00:00 2001 From: dvgodoy Date: Mon, 4 Jun 2018 22:57:11 +0200 Subject: [PATCH 04/13] select layers for plotting --- deepreplay/plot.py | 5 +- deepreplay/replay.py | 133 +++++++++++++++++++++++++++++----- examples/gradients_example.py | 27 ++++--- 3 files changed, 134 insertions(+), 31 deletions(-) diff --git a/deepreplay/plot.py b/deepreplay/plot.py index 11fa5a2..381a53a 100644 --- a/deepreplay/plot.py +++ b/deepreplay/plot.py @@ -15,7 +15,7 @@ LossAndMetricData = namedtuple('LossAndMetricData', ['loss', 'metric', 'metric_name']) ProbHistogramData = namedtuple('ProbHistogramData', ['prob', 'target']) LossHistogramData = namedtuple('LossHistogramData', ['loss']) -LayerViolinsData = namedtuple('LayerViolinsData', ['names', 'values', 'layers']) +LayerViolinsData = namedtuple('LayerViolinsData', ['names', 'values', 'layers', 'selected_layers']) def build_2d_grid(xlim, ylim, n_lines=11, n_points=1000): """Returns a 2D grid of boundaries given by `xlim` and `ylim`, @@ -603,6 +603,7 @@ def load_data(self, layer_violins_data): self.values = layer_violins_data.values self.names = layer_violins_data.names self.layers = ['inputs'] + layer_violins_data.layers + self.selected_layers = layer_violins_data.selected_layers self.palette = dict(zip(self.layers, sns.palettes.husl_palette(len(self.layers), .7))) self.n_epochs = len(self.values) self._prepare_plot() @@ -619,7 +620,7 @@ def _update(i, lv, epoch_start=0): df = pd.concat([pd.DataFrame(layer_values.ravel(), columns=[layer_name]).melt(var_name='layers', value_name='values') for layer_name, layer_values in zip(lv.names, lv.values[i])]) - df = df.query('layers != "{}"'.format(lv.names[-1])) + df = df[df.isin({'layers': lv.selected_layers}).values] lv.ax.clear() sns.violinplot(data=df, x='layers', y='values', ax=lv.ax, cut=0, palette=lv.palette, scale='width') diff --git a/deepreplay/replay.py b/deepreplay/replay.py index 4450452..a5f97bf 100644 --- a/deepreplay/replay.py +++ b/deepreplay/replay.py @@ -55,24 +55,41 @@ class Replay(object): animating; namedtuple containing information about classification probabilities and targets. + weights_violins: + + activations_violins: + + zvalues_violins: + + gradients_violins: + training_loss: ndarray An array of shape (n_epochs, ) with training loss as reported by Keras at the end of each epoch. + + learning_rate: ndarray + An array of shape (n_epochs, ) with learning rate as reported + by Keras at the beginning of each epoch. """ def __init__(self, replay_filename, group_name, model_filename=''): # Set learning phase to TEST self.learning_phase = TEST_MODE + # Loads ReplayData file + self.replay_data = h5py.File('{}'.format(replay_filename), 'r') + try: + self.group = self.replay_data[group_name] + except KeyError: + self.group = self.replay_data[group_name + '_init'] + group_name += '_init' + + self.group_name = group_name + # If not informed, defaults to '_model' suffix if model_filename == '': model_filename = '{}_model.h5'.format(group_name) - # Loads Keras model self.model = load_model(model_filename) - # Loads ReplayData file - self.replay_data = h5py.File('{}'.format(replay_filename), 'r') - self.group_name = group_name - self.group = self.replay_data[self.group_name] # Retrieves some basic information from the replay data self.inputs = self.group['inputs'][:] @@ -105,6 +122,8 @@ def __init__(self, replay_filename, group_name, model_filename=''): outputs=[K.binary_crossentropy(self.model.targets[0], self.model.outputs[0])]) + # Keras function to compute the gradients for trainable weights, given inputs, targets, weights and + # sample weights self.__trainable_weights = [w for layer in self.model.layers for w in layer.trainable_weights if layer.trainable] self.__trainable_gradients = self.model.optimizer.get_gradients(self.model.total_loss, self.__trainable_weights) @@ -112,25 +131,27 @@ def __init__(self, replay_filename, group_name, model_filename=''): self._model_weights + self.model.sample_weights, outputs=self.__trainable_gradients) + # Keras function to compute the activation values given inputs and weights __activation_tensors = list(filter(lambda t: t[1].op.type.lower() in ACTIVATIONS, [(self.model.layers[i].name, self.model.layers[i].output) for i in range(self.n_layers)])) - - __z_tensors = list(filter(lambda t: t[1].op.type in ['BiasAdd', 'MatMul'], - [(self.model.layers[i - (self.model.layers[i].input.op.type in - ['BiasAdd', 'MatMul'])].name, - self.model.layers[i].output.op.inputs[0]) for i in range(self.n_layers)])) - self._activation_layers = ['inputs'] + list(map(lambda t: t[0], __activation_tensors)) self._activation_tensors = self.model.inputs + list(map(lambda t: t[1], __activation_tensors)) self._get_activations = K.function(inputs=[K.learning_phase()] + self.model.inputs + self._model_weights, outputs=self._activation_tensors) + # Keras function to compute the Z values given inputs and weights + __z_tensors = list(filter(lambda t: t[1].op.type in ['BiasAdd', 'MatMul'], + [(self.model.layers[i - (self.model.layers[i].input.op.type in + ['BiasAdd', 'MatMul'])].name, + self.model.layers[i].output.op.inputs[0]) for i in range(self.n_layers)])) self._z_layers = ['inputs'] + list(map(lambda t: t[0], __z_tensors)) self._z_tensors = self.model.inputs + list(map(lambda t: t[1], __z_tensors)) self._get_zvalues = K.function(inputs=[K.learning_phase()] + self.model.inputs + self._model_weights, outputs=self._z_tensors) + # Gets names of all layers with arrays of weights of lengths 1 (no biases) or 2 (with biases) + # Layers without weights (e.g. Activation, BatchNorm) are not included self.weights_layers = [layer.name for layer, weights in zip(self.model.layers, self.n_weights) if len(weights) in (1, 2)] @@ -219,10 +240,30 @@ def loss_and_metric(self): def probability_histogram(self): return self._prob_hist_plot, self._prob_hist_data + @property + def weights_violins(self): + return self._weights_violins_plot, self._weights_violins_data + + @property + def activations_violins(self): + return self._activations_violins_plot, self._activations_violins_data + + @property + def zvalues_violins(self): + return self._zvalues_violins_plot, self._zvalues_violins_data + + @property + def gradients_violins(self): + return self._gradients_plot, self._gradients_data + @property def training_loss(self): return self.group['loss'][:] + @property + def learning_rate(self): + return self.group['lr'][:] + def get_training_metric(self, metric_name): """Returns corresponding metric as reported by Keras at the end of each epoch. @@ -275,7 +316,30 @@ def predict_proba(self, epoch_start=0, epoch_end=-1): probas = np.array(probas) return probas - def build_gradients(self, ax, include_output=True, epoch_start=0, epoch_end=-1): + def build_gradients(self, ax, layer_names=None, exclude_outputs=True, epoch_start=0, epoch_end=-1): + """Builds a LayerViolins object to be used for plotting and + animating. + + Parameters + ---------- + ax: AxesSubplot + Subplot of a Matplotlib figure. + layer_names: list of Strings, optional + If informed, plots only the listed layers. + exclude_outputs: boolean, optional + If True, excludes distribution of output layer. Default is True. + If `layer_names` is informed, `exclude_outputs` is ignored. + epoch_start: int, optional + First epoch to consider. + epoch_end: int, optional + Last epoch to consider. + + Returns + ------- + gradients_plot: LayerViolins + An instance of a LayerViolins object to make plots and + animations. + """ if epoch_end == -1: epoch_end = self.n_epochs epoch_end = min(epoch_end, self.n_epochs) @@ -292,11 +356,18 @@ def build_gradients(self, ax, include_output=True, epoch_start=0, epoch_end=-1): for w in v] gradients.append(grad) - self._gradients_data = LayerViolinsData(names=gradient_names, values=gradients, layers=self.weights_layers) + if layer_names is None: + layer_names = self.weights_layers + if exclude_outputs: + layer_names = layer_names[:-1] + + self._gradients_data = LayerViolinsData(names=gradient_names, values=gradients, layers=self.weights_layers, + selected_layers=layer_names) self._gradients_plot = LayerViolins(ax, 'Gradients').load_data(self._gradients_data) return self._gradients_plot - def build_outputs_violins(self, ax, before_activation=False, epoch_start=0, epoch_end=-1): + def build_outputs_violins(self, ax, before_activation=False, layer_names=None, include_inputs=True, + exclude_outputs=True, epoch_start=0, epoch_end=-1): """Builds a LayerViolins object to be used for plotting and animating. @@ -307,6 +378,13 @@ def build_outputs_violins(self, ax, before_activation=False, epoch_start=0, epoc before_activation: Boolean, optional If True, returns Z-values, that is, before applying the activation function. + layer_names: list of Strings, optional + If informed, plots only the listed layers. + include_inputs: boolean, optional + If True, includes distribution of inputs. Default is True. + exclude_outputs: boolean, optional + If True, excludes distribution of output layer. Default is True. + If `layer_names` is informed, `exclude_outputs` is ignored. epoch_start: int, optional First epoch to consider. epoch_end: int, optional @@ -314,7 +392,7 @@ def build_outputs_violins(self, ax, before_activation=False, epoch_start=0, epoc Returns ------- - activations_violins_plot: LayerViolins + activations_violins_plot/zvalues_violins_plot: LayerViolins An instance of a LayerViolins object to make plots and animations. """ @@ -338,7 +416,14 @@ def build_outputs_violins(self, ax, before_activation=False, epoch_start=0, epoc else: outputs.append(self._get_activations(inputs=inputs)) - data = LayerViolinsData(names=names, values=outputs, layers=self.weights_layers) + if layer_names is None: + layer_names = self.weights_layers + if exclude_outputs: + layer_names = layer_names[:-1] + if include_inputs: + layer_names = ['inputs'] + layer_names + + data = LayerViolinsData(names=names, values=outputs, layers=self.weights_layers, selected_layers=layer_names) plot = LayerViolins(ax, title).load_data(data) if before_activation: self._zvalues_violins_data = data @@ -348,7 +433,7 @@ def build_outputs_violins(self, ax, before_activation=False, epoch_start=0, epoc self._activations_violins_plot = plot return plot - def build_weights_violins(self, ax, epoch_start=0, epoch_end=-1): + def build_weights_violins(self, ax, layer_names=None, exclude_outputs=True, epoch_start=0, epoch_end=-1): """Builds a LayerViolins object to be used for plotting and animating. @@ -356,6 +441,11 @@ def build_weights_violins(self, ax, epoch_start=0, epoch_end=-1): ---------- ax: AxesSubplot Subplot of a Matplotlib figure. + layer_names: list of Strings, optional + If informed, plots only the listed layers. + exclude_outputs: boolean, optional + If True, excludes distribution of output layer. Default is True. + If `layer_names` is informed, `exclude_outputs` is ignored. epoch_start: int, optional First epoch to consider. epoch_end: int, optional @@ -379,11 +469,16 @@ def build_weights_violins(self, ax, epoch_start=0, epoch_end=-1): # For each epoch, uses the corresponding weights for epoch in range(epoch_start, epoch_end + 1): weights.append([w for w, n in zip(self.weights[epoch], n_weights) if n in (1, 2)]) - #weights.append(list(filter(lambda weights: len(weights) in (1, 2), self.weights[epoch]))) + + if layer_names is None: + layer_names = self.weights_layers + if exclude_outputs: + layer_names = layer_names[:-1] self._weights_violins_data = LayerViolinsData(names=names, values=weights, - layers=self.weights_layers) + layers=self.weights_layers, + selected_layers=layer_names) self._weights_violins_plot = LayerViolins(ax, 'Weights').load_data(self._weights_violins_data) return self._weights_violins_plot diff --git a/examples/gradients_example.py b/examples/gradients_example.py index 31489e8..ca7680d 100644 --- a/examples/gradients_example.py +++ b/examples/gradients_example.py @@ -1,7 +1,7 @@ import numpy as np from keras.models import Sequential from keras.layers import Dense -from keras.initializers import glorot_uniform, normal +from keras.initializers import glorot_uniform, normal, he_uniform from deepreplay.datasets.ball import load_data from deepreplay.callbacks import ReplayData from deepreplay.replay import Replay @@ -11,30 +11,36 @@ group_name = 'gradients' -X, y = load_data() +X, y = load_data(n_dims=10) n_layers = 5 variance = 0.01 activation = 'tanh' init = normal(mean=0, stddev=np.sqrt(variance)) +init_name = 'Normal' +init_parms = r'$\sigma = {}$'.format(np.sqrt(variance)) +init_title = '{} {}'.format(init_name, init_parms) #init = VarianceScaling(scale=1.0, mode='fan_avg', distribution='uniform') #init = glorot_normal() # sigmoid #init = glorot_uniform() # tanh #init = he_uniform() # relu #init_name = 'He Uniform' -model = Sequential() -model.add(Dense(units=100, input_dim=10, activation=activation, kernel_initializer=init, name='h1')) -for i in range(2, n_layers + 1): - model.add(Dense(units=100, activation=activation, kernel_initializer=init, name='h{}'.format(i))) -model.add(Dense(units=1, activation='sigmoid', kernel_initializer=init, name='o')) +def build_model(n_layers, input_dim, units, activation, initializer): + model = Sequential() + model.add(Dense(units=units, input_dim=input_dim, activation=activation, kernel_initializer=initializer, name='h1')) + for i in range(2, n_layers + 1): + model.add(Dense(units=units, activation=activation, kernel_initializer=initializer, name='h{}'.format(i))) + model.add(Dense(units=1, activation='sigmoid', kernel_initializer=initializer, name='o')) + model.compile(loss='binary_crossentropy', optimizer='sgd', metrics=['acc']) + return model -model.compile(loss='binary_crossentropy', optimizer='sgd', metrics=['acc']) +model = build_model(n_layers, 10, 100, activation, init) replaydata = ReplayData(X, y, filename='gradients.h5', group_name=group_name, model=model) -replay = Replay(replay_filename='gradients.h5', group_name=group_name + '_init') +replay = Replay(replay_filename='gradients.h5', group_name=group_name) fig = plt.figure(figsize=(12, 6)) ax_zvalues = plt.subplot2grid((2, 2), (0, 0)) @@ -42,10 +48,11 @@ ax_activations = plt.subplot2grid((2, 2), (1, 0)) ax_gradients = plt.subplot2grid((2, 2), (1, 1)) +layers = ['h1', 'h2', 'h3', 'h4', 'h5', 'o'] zv = replay.build_outputs_violins(ax_zvalues, before_activation=True) wv = replay.build_weights_violins(ax_weights) av = replay.build_outputs_violins(ax_activations) gv = replay.build_gradients(ax_gradients) -sample_figure = compose_plots([zv, wv, av, gv], 0) +sample_figure = compose_plots([zv, wv, av, gv], 0, title='Activation: {} - Initializer: {}'.format(activation, init_title)) sample_figure.savefig('gradients.png', dpi=120, format='png') From 83b88ad7b8f5996ddb8392fb6e118e2336486f13 Mon Sep 17 00:00:00 2001 From: dvgodoy Date: Tue, 12 Jun 2018 00:10:23 +0200 Subject: [PATCH 05/13] fixing merge layers --- deepreplay/datasets/ball.py | 15 +++++++++------ deepreplay/replay.py | 32 ++++++++++++++++++++++++++------ 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/deepreplay/datasets/ball.py b/deepreplay/datasets/ball.py index 17d44b2..93fccde 100644 --- a/deepreplay/datasets/ball.py +++ b/deepreplay/datasets/ball.py @@ -1,14 +1,12 @@ import numpy as np -def load_data(n_dims=10, radius=2.0, n_points=1000, only_sphere=False, shuffle=True, seed=13): +def load_data(n_dims=10, n_points=1000, only_sphere=False, shuffle=True, seed=13): """ Parameters ---------- n_dims: int, optional Number of dimensions of the n-ball. Default is 10. - radius: float, optional - Radius of the n-ball. Default is 2.0. n_points: int, optional Number of points in each parabola. Default is 1,000. only_sphere: boolean @@ -27,12 +25,17 @@ def load_data(n_dims=10, radius=2.0, n_points=1000, only_sphere=False, shuffle=T y is an array of shape (n_points, 1) containing the classes of the samples. """ - points = np.random.normal(size=(1000, n_dims)) - sphere = points / np.linalg.norm(points, axis=1).reshape(-1, 1) + radius = np.sqrt(n_dims) + points = np.random.normal(size=(n_points, n_dims)) + sphere = radius * points / np.linalg.norm(points, axis=1).reshape(-1, 1) if only_sphere: X = sphere else: - X = radius * sphere * np.random.uniform(size=(n_points, 1))**(1 / n_dims) + X = sphere * np.random.uniform(size=(n_points, 1))**(1 / n_dims) + + adjustment = 1 / np.std(X) + radius *= adjustment + X *= adjustment y = (np.abs(np.sum(X, axis=1)) > (radius / 2.0)).astype(np.int) diff --git a/deepreplay/replay.py b/deepreplay/replay.py index a5f97bf..1c2a293 100644 --- a/deepreplay/replay.py +++ b/deepreplay/replay.py @@ -140,13 +140,33 @@ def __init__(self, replay_filename, group_name, model_filename=''): self._get_activations = K.function(inputs=[K.learning_phase()] + self.model.inputs + self._model_weights, outputs=self._activation_tensors) + #__z_tensors = list(filter(lambda t: t[1].op.type in ['BiasAdd', 'MatMul'], + # [(self.model.layers[i - (self.model.layers[i].input.op.type in + # ['BiasAdd', 'MatMul'])].name, + # self.model.layers[i].output.op.inputs[0]) for i in range(self.n_layers)])) + #self._z_layers = ['inputs'] + list(map(lambda t: t[0], __z_tensors)) + #self._z_tensors = self.model.inputs + list(map(lambda t: t[1], __z_tensors)) + + self._z_layers = ['inputs'] + self._z_tensors = self.model.inputs[:] + for i in range(self.n_layers): + layer_inputs = self.model.layers[i].input + try: + layer_output_inputs = self.model.layers[i].output.op.inputs + except AttributeError: + layer_output_inputs = [] + + if not isinstance(layer_inputs, list): + layer_inputs = [layer_inputs] + + for layer_input in layer_inputs: + op_type = layer_input.op.type + for layer_output_input in layer_output_inputs: + if layer_output_input.op.type in ['BiasAdd', 'MatMul']: + self._z_tensors.append(layer_output_input) + self._z_layers.append(self.model.layers[i - (op_type in ['BiasAdd', 'MatMul'])].name) + # Keras function to compute the Z values given inputs and weights - __z_tensors = list(filter(lambda t: t[1].op.type in ['BiasAdd', 'MatMul'], - [(self.model.layers[i - (self.model.layers[i].input.op.type in - ['BiasAdd', 'MatMul'])].name, - self.model.layers[i].output.op.inputs[0]) for i in range(self.n_layers)])) - self._z_layers = ['inputs'] + list(map(lambda t: t[0], __z_tensors)) - self._z_tensors = self.model.inputs + list(map(lambda t: t[1], __z_tensors)) self._get_zvalues = K.function(inputs=[K.learning_phase()] + self.model.inputs + self._model_weights, outputs=self._z_tensors) From dfd6437386421c7df954c86647e9221bf2bc4eda Mon Sep 17 00:00:00 2001 From: dvgodoy Date: Wed, 13 Jun 2018 00:26:20 +0200 Subject: [PATCH 06/13] fix z tensors --- deepreplay/replay.py | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/deepreplay/replay.py b/deepreplay/replay.py index 1c2a293..63530a4 100644 --- a/deepreplay/replay.py +++ b/deepreplay/replay.py @@ -131,14 +131,12 @@ def __init__(self, replay_filename, group_name, model_filename=''): self._model_weights + self.model.sample_weights, outputs=self.__trainable_gradients) - # Keras function to compute the activation values given inputs and weights + """ __activation_tensors = list(filter(lambda t: t[1].op.type.lower() in ACTIVATIONS, [(self.model.layers[i].name, self.model.layers[i].output) for i in range(self.n_layers)])) self._activation_layers = ['inputs'] + list(map(lambda t: t[0], __activation_tensors)) self._activation_tensors = self.model.inputs + list(map(lambda t: t[1], __activation_tensors)) - self._get_activations = K.function(inputs=[K.learning_phase()] + self.model.inputs + self._model_weights, - outputs=self._activation_tensors) #__z_tensors = list(filter(lambda t: t[1].op.type in ['BiasAdd', 'MatMul'], # [(self.model.layers[i - (self.model.layers[i].input.op.type in @@ -165,10 +163,31 @@ def __init__(self, replay_filename, group_name, model_filename=''): if layer_output_input.op.type in ['BiasAdd', 'MatMul']: self._z_tensors.append(layer_output_input) self._z_layers.append(self.model.layers[i - (op_type in ['BiasAdd', 'MatMul'])].name) + """ + + __z_layers = np.array([i for i, layer in enumerate(self.model.layers) + if (layer.output.op.type in ['BiasAdd', 'MatMul', 'Add']) + or (layer.output.op.inputs[0].op.type in ['BiasAdd', 'MatMul', 'Add'])]) + __act_layers = np.array([i for i, layer in enumerate(self.model.layers) + if layer.output.op.type.lower() in ACTIVATIONS]) + __z_layers = [__z_layers[np.argmax(layer < __z_layers) - 1] for layer in __act_layers] + self.z_act_layers = [self.model.layers[i].name for i in __z_layers] + + self._z_layers = ['inputs'] + [self.model.layers[i].name for i in __z_layers] + self._z_tensors = self.model.inputs + [output + if output.op.type in ['BiasAdd', 'MatMul', 'Add'] + else output.op.inputs[0] + for output in [self.model.layers[i].output for i in __z_layers]] + + self._activation_layers = ['inputs'] + [self.model.layers[i].name for i in __act_layers] + self._activation_tensors = self.model.inputs + [self.model.layers[i].output for i in __act_layers] # Keras function to compute the Z values given inputs and weights self._get_zvalues = K.function(inputs=[K.learning_phase()] + self.model.inputs + self._model_weights, outputs=self._z_tensors) + # Keras function to compute the activation values given inputs and weights + self._get_activations = K.function(inputs=[K.learning_phase()] + self.model.inputs + self._model_weights, + outputs=self._activation_tensors) # Gets names of all layers with arrays of weights of lengths 1 (no biases) or 2 (with biases) # Layers without weights (e.g. Activation, BatchNorm) are not included @@ -437,13 +456,13 @@ def build_outputs_violins(self, ax, before_activation=False, layer_names=None, i outputs.append(self._get_activations(inputs=inputs)) if layer_names is None: - layer_names = self.weights_layers + layer_names = self.z_act_layers if exclude_outputs: layer_names = layer_names[:-1] if include_inputs: layer_names = ['inputs'] + layer_names - data = LayerViolinsData(names=names, values=outputs, layers=self.weights_layers, selected_layers=layer_names) + data = LayerViolinsData(names=names, values=outputs, layers=self.z_act_layers, selected_layers=layer_names) plot = LayerViolins(ax, title).load_data(data) if before_activation: self._zvalues_violins_data = data From 5fc91ceadb9d547e944fb4e545e64e609963649c Mon Sep 17 00:00:00 2001 From: dvgodoy Date: Wed, 13 Jun 2018 08:33:07 +0200 Subject: [PATCH 07/13] fix z tensors --- deepreplay/replay.py | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/deepreplay/replay.py b/deepreplay/replay.py index 63530a4..d94899e 100644 --- a/deepreplay/replay.py +++ b/deepreplay/replay.py @@ -16,6 +16,7 @@ TRAINING_MODE = 1 TEST_MODE = 0 ACTIVATIONS = ['softmax', 'relu', 'elu', 'tanh', 'sigmoid', 'hard_sigmoid', 'linear', 'softplus', 'softsign', 'selu'] +Z_OPS = ['BiasAdd', 'MatMul', 'Add'] class Replay(object): @@ -165,19 +166,33 @@ def __init__(self, replay_filename, group_name, model_filename=''): self._z_layers.append(self.model.layers[i - (op_type in ['BiasAdd', 'MatMul'])].name) """ - __z_layers = np.array([i for i, layer in enumerate(self.model.layers) - if (layer.output.op.type in ['BiasAdd', 'MatMul', 'Add']) - or (layer.output.op.inputs[0].op.type in ['BiasAdd', 'MatMul', 'Add'])]) + def get_z_op(layer): + op = layer.output.op + if op.type in Z_OPS: + return layer.output + else: + op_layer_name = op.name.split('/')[0] + for input in op.inputs: + input_layer_name = input.name.split('/')[0] + if (input.op.type in Z_OPS) and (op_layer_name == input_layer_name): + return input + return None + + #__z_layers = np.array([i for i, layer in enumerate(self.model.layers) + # if (layer.output.op.type in Z_OPS) + # or ((layer.output.op.inputs[0].op.type in Z_OPS) and + # (layer.output.op.name.split('/')[0] == layer.output.op.inputs[0].name.split('/')[0]))]) + __z_layers = np.array([i for i, layer in enumerate(self.model.layers) if get_z_op(layer) is not None]) __act_layers = np.array([i for i, layer in enumerate(self.model.layers) if layer.output.op.type.lower() in ACTIVATIONS]) - __z_layers = [__z_layers[np.argmax(layer < __z_layers) - 1] for layer in __act_layers] + __z_layers = np.array([__z_layers[np.argmax(layer < __z_layers) - 1] for layer in __act_layers]) self.z_act_layers = [self.model.layers[i].name for i in __z_layers] self._z_layers = ['inputs'] + [self.model.layers[i].name for i in __z_layers] - self._z_tensors = self.model.inputs + [output - if output.op.type in ['BiasAdd', 'MatMul', 'Add'] - else output.op.inputs[0] - for output in [self.model.layers[i].output for i in __z_layers]] + #self._z_tensors = self.model.inputs + [output if output.op.type in Z_OPS else output.op.inputs[0] + # for output in [self.model.layers[i].output for i in __z_layers]] + self._z_tensors = self.model.inputs + filter(lambda t: t is not None, + [get_z_op(self.model.layers[i]) for i in __z_layers]) self._activation_layers = ['inputs'] + [self.model.layers[i].name for i in __act_layers] self._activation_tensors = self.model.inputs + [self.model.layers[i].output for i in __act_layers] From f249f6b1868b1d0081370587004a44029f6b4255 Mon Sep 17 00:00:00 2001 From: dvgodoy Date: Wed, 13 Jun 2018 20:28:56 +0200 Subject: [PATCH 08/13] fix multiclass targets/fix callback test --- deepreplay/callbacks.py | 2 +- deepreplay/replay.py | 44 +----------------- tests/rawdata/hyperparms_in_action.h5 | Bin 55804 -> 56116 bytes .../part1_activation_functions_epoch01.h5 | Bin 20232 -> 20232 bytes .../part1_activation_functions_epoch02.h5 | Bin 20232 -> 20232 bytes .../part1_activation_functions_epoch03.h5 | Bin 20232 -> 20232 bytes .../part1_activation_functions_epoch04.h5 | Bin 20232 -> 20232 bytes .../part1_activation_functions_epoch05.h5 | Bin 20232 -> 20232 bytes .../part1_activation_functions_epoch06.h5 | Bin 20232 -> 20232 bytes .../part1_activation_functions_epoch07.h5 | Bin 20232 -> 20232 bytes .../part1_activation_functions_epoch08.h5 | Bin 20232 -> 20232 bytes .../part1_activation_functions_epoch09.h5 | Bin 20232 -> 20232 bytes .../part1_activation_functions_epoch10.h5 | Bin 20232 -> 20232 bytes .../part1_activation_functions_epoch11.h5 | Bin 20232 -> 20232 bytes .../part1_activation_functions_epoch12.h5 | Bin 20232 -> 20232 bytes .../part1_activation_functions_epoch13.h5 | Bin 20232 -> 20232 bytes .../part1_activation_functions_epoch14.h5 | Bin 20232 -> 20232 bytes .../part1_activation_functions_epoch15.h5 | Bin 20232 -> 20232 bytes .../part1_activation_functions_epoch16.h5 | Bin 20232 -> 20232 bytes .../part1_activation_functions_epoch17.h5 | Bin 20232 -> 20232 bytes .../part1_activation_functions_epoch18.h5 | Bin 20232 -> 20232 bytes .../part1_activation_functions_epoch19.h5 | Bin 20232 -> 20232 bytes .../part1_activation_functions_epoch20.h5 | Bin 20232 -> 20232 bytes .../part1_activation_functions_model.h5 | Bin 20232 -> 20232 bytes 24 files changed, 3 insertions(+), 43 deletions(-) diff --git a/deepreplay/callbacks.py b/deepreplay/callbacks.py index 51fc8af..bd5e9dc 100644 --- a/deepreplay/callbacks.py +++ b/deepreplay/callbacks.py @@ -36,7 +36,7 @@ def __init__(self, inputs, targets, filename, group_name, model=None): super(ReplayData, self).__init__() self.handler = h5py.File('{}'.format(filename), 'a') self.inputs = inputs - self.targets = targets.reshape(-1, 1) + self.targets = targets.reshape(len(targets), -1) self.filepath = os.path.split(filename)[0] self.filename = filename self.group = None diff --git a/deepreplay/replay.py b/deepreplay/replay.py index d94899e..26a56b5 100644 --- a/deepreplay/replay.py +++ b/deepreplay/replay.py @@ -132,40 +132,6 @@ def __init__(self, replay_filename, group_name, model_filename=''): self._model_weights + self.model.sample_weights, outputs=self.__trainable_gradients) - """ - __activation_tensors = list(filter(lambda t: t[1].op.type.lower() in ACTIVATIONS, - [(self.model.layers[i].name, self.model.layers[i].output) - for i in range(self.n_layers)])) - self._activation_layers = ['inputs'] + list(map(lambda t: t[0], __activation_tensors)) - self._activation_tensors = self.model.inputs + list(map(lambda t: t[1], __activation_tensors)) - - #__z_tensors = list(filter(lambda t: t[1].op.type in ['BiasAdd', 'MatMul'], - # [(self.model.layers[i - (self.model.layers[i].input.op.type in - # ['BiasAdd', 'MatMul'])].name, - # self.model.layers[i].output.op.inputs[0]) for i in range(self.n_layers)])) - #self._z_layers = ['inputs'] + list(map(lambda t: t[0], __z_tensors)) - #self._z_tensors = self.model.inputs + list(map(lambda t: t[1], __z_tensors)) - - self._z_layers = ['inputs'] - self._z_tensors = self.model.inputs[:] - for i in range(self.n_layers): - layer_inputs = self.model.layers[i].input - try: - layer_output_inputs = self.model.layers[i].output.op.inputs - except AttributeError: - layer_output_inputs = [] - - if not isinstance(layer_inputs, list): - layer_inputs = [layer_inputs] - - for layer_input in layer_inputs: - op_type = layer_input.op.type - for layer_output_input in layer_output_inputs: - if layer_output_input.op.type in ['BiasAdd', 'MatMul']: - self._z_tensors.append(layer_output_input) - self._z_layers.append(self.model.layers[i - (op_type in ['BiasAdd', 'MatMul'])].name) - """ - def get_z_op(layer): op = layer.output.op if op.type in Z_OPS: @@ -178,10 +144,6 @@ def get_z_op(layer): return input return None - #__z_layers = np.array([i for i, layer in enumerate(self.model.layers) - # if (layer.output.op.type in Z_OPS) - # or ((layer.output.op.inputs[0].op.type in Z_OPS) and - # (layer.output.op.name.split('/')[0] == layer.output.op.inputs[0].name.split('/')[0]))]) __z_layers = np.array([i for i, layer in enumerate(self.model.layers) if get_z_op(layer) is not None]) __act_layers = np.array([i for i, layer in enumerate(self.model.layers) if layer.output.op.type.lower() in ACTIVATIONS]) @@ -189,10 +151,8 @@ def get_z_op(layer): self.z_act_layers = [self.model.layers[i].name for i in __z_layers] self._z_layers = ['inputs'] + [self.model.layers[i].name for i in __z_layers] - #self._z_tensors = self.model.inputs + [output if output.op.type in Z_OPS else output.op.inputs[0] - # for output in [self.model.layers[i].output for i in __z_layers]] - self._z_tensors = self.model.inputs + filter(lambda t: t is not None, - [get_z_op(self.model.layers[i]) for i in __z_layers]) + self._z_tensors = self.model.inputs + list(filter(lambda t: t is not None, + [get_z_op(self.model.layers[i]) for i in __z_layers])) self._activation_layers = ['inputs'] + [self.model.layers[i].name for i in __act_layers] self._activation_tensors = self.model.inputs + [self.model.layers[i].output for i in __act_layers] diff --git a/tests/rawdata/hyperparms_in_action.h5 b/tests/rawdata/hyperparms_in_action.h5 index c60385832166e7c70185cce089bb3a6352812d48..2ad88c4cf25e906fddb30f01a722797b92a33ba5 100644 GIT binary patch delta 1425 zcmb`Fc~BEq9LKW@LV{wwjWlvq0fi!tS4gsXt5V7#POP@I93vGE=zvC@Y0)ZyFqVh` zg;#3rXa(CUQXSb+3xR#_#RCJH)F6lgT7t)vqSDl(I#{*P>}c!XeKYU){^qye@8|9A zUo~z&{^Yhhgr42SO`qUK)xr60!4l_ZqJPFHj`O4`7H%FUr#Fh=_roSr(wdfGM`%g| zKNU}-JmGiZK+jDXYqQp8tb*-gBjto?k^uLp)PN-DFlcKYOFXuX@D%yG>_p28OE_PN zSWpf)v%oJgLQvU8*MSmrxuCT+S3xe+&Bg!i)3ms>w$-&ug}#^_gIQ2hAg0rI4yBF~ z>T=jp@HS)(TLxJh3@=za>CZbMzr$Z(-_b|kd__n7OFIj{^m+*hJMBS|$3F}C9QTNV z+MQC#3o;ncHW~V!k;DEyi&Ehho54$S+4Z=Ff|(Mji=w(! zOZVD_uEDm6Y1YnhBPnRVu!Sza0vFi@RIPnC^|*#=c}}@3TLsH-7}1(Y$8@~x9JHf) z2U&t?wq39afw`C7@p5*!vX_ZoW)FTnogpC}=Wn&3{H_PMXl1c#rY0T-5ewp)9^oYm z52(&H#bLhIf-IXUq95O=8XgjdH+eOovP)j%Qky5!O2=X)XhQ8KDLL~-By;>~G+wx_ z37IU@iEL^f^Ls)RjxBFOA^Q2GKbm7=a{$iU-i#K)6f(B)j*c0qmSf9D=h6E59Fj9u zrd~a7Ccdq#>8&Z zJllZ=+DjXmTiwynqWd9Oao0fc`qVEJM~^1+ug_heoN%?6zvx-6JoIEIU#+{KtXScV z;$-QnB==qXvy<^ETZM*SIxbh$*JI$1EUr^+JmlcBZn!ZCg>GmA2xKG^4f!B5k%_xWvLbgdD~ z!XenRzaHx|tAQiSj_+4kLI2H1xT~WV%s4rYyjM=M`%XlVo}|(2ioAG|;_%b4hYf2; zNn$uVAzDZF-%_%MX+|>J6vM_e?jgRR%huOq-r3Hl--*})+NY^ zF`Okf>mtTxjT>72p9zyYWC7NxVW8V*I%rS@12Sg{xS!z?1WR2IS5R9UsTjaRNfFR6 z;&$W+U92J?PqZQMW+q?}P=av_B3gr6DJ}SGv|KaojdotbLQk9c5f@~ zc6oE(wlR}O+fjNr)6R?k>PJrha$*<u<;Hii zAvt`UHAkcOhH|5OKA#8U)t_6eRjRGlU>ZN!(isk8eu@^vwpq_8IFMC~9Qv$tI7$tZ z`X8h8e;N8bk>I^?cp%4ULBze93$qRJFePWS;Js_O+ThAn=2<#R;I5x( zDBSH0%WLHtJrDLC6+>DVZYhcTpm47omY)%8h#U$}OCYUL^j9mqp-?Mif=t{qWP)Yw_uMJ8;qRBQo z^}*=T8YT{!2iwEm76Dgj&@irofsBpk8AeE1j;C;|S|Mw6?|*1$HtwRFDmPrPjNKRN z>q|;19@ZE=Y*~INNO53C-ngM0&VXd}6|1Bvh2p^)lkI)Xrf*g0TN}{!_GdVEX`a+y z5s4>|2E;VJ#EMymq)m+xxLVtQ8n#em+o*b}z&iqO5HzBKD*|%qvV-g*cPz(ZqCF_q^G7PEjNB&X`HM!Wdb^Hh|MIo6sy6N1W;($lg>) zRQSy5b7+0_Hj+BRg-)2^kGp6C@?McoK6@BJ$2WW8@xv}4pO7QuSX>&NROf;P!Ix3` zCPJQ{Ev8$;hGUPwR;0?lN&+Gp=n2mDcsJFKGRmHikfv^0bK@C0A-;)XbM5tY&xIg= zlo@4AyN%u{N9cJMJ;9wQRWs_i`Zw}0yXl+b03c^r)PwJ%;2a;l+CCcebTKGn=_8c9 za=Jb5zd&azf+$FZ2y9D^3yH4g058Ez4Hat zbnkK9UG2H*%rQ50Qw@h|j`5R&HdnOO96#X|{79vy*p&D^!Axl^>pDF&p$tX6eCr1)z`w7?}%8Z&4o+glqcH5Dt^MI+Os4?5KB@u(_k!eKL;eA`YoC)*Bv z2gb|zW;Yaci*nqL`1e$X5c2UYnYB2mGjsG zvd%Zl*;992>2)rAlr&>2-D{qW=#V3HVbkx(`QZ&(yWkpnn&k-MLiyP9v@b|0li>N? z^TDK^L_FkNDj2QJ!OngV94xHH>1!*%y>&+Xw740(xbqUXw{?Pm+L7eTBCfpXl$dlZ z{z$$eJ(4UjyUEMc2`ZwA@sqoRsYv175P9~b95PK8E{~|+OI&>x%1u4LkiJMGnOijn zAB;u%#ARrI_jTl$KLlrd?}^!}IDB|xCJy8mV45`GKj+rrz%mn#&3}P|esv@s9nPe8 zpC>u;+Lyp50c6cQ3CZ@8lafS#Q0L?iPO<@F;20_f>Z4PE?^!>fESUlf6Mex==gFW( f>IL+H6Tq`o?%*Gh3%H*&1}rNX4s`IswPpVTA>fGB diff --git a/tests/rawdata/part1_activation_functions_epoch01.h5 b/tests/rawdata/part1_activation_functions_epoch01.h5 index 969f542e6148a9eca6b3ed75f26b743ac3a51e90..e7862a22508f4b1a024681c62ac647c63740db6d 100644 GIT binary patch delta 555 zcmeB}$JjBCaYG9SquJzEj{3<0oZ0%tsi`SSRthFYItog;`6;PDwo+PRUVLI%x>9W| zP%tks7c7{OnUa#4H;GGOas!v-bk`xKwHazvIcqvrXVM#CW34N@=zR?Us_U7T7qz#kpM3HrD66P0!_na zdq0n@x)}ol0~-*7z)J|lPz|)Zq$o2P0L<9j!r_`WaDdA`jR*UFufU)AoE zlALAx>vzhYyp4}`*Fvr8nmW&7Tea%;R8jVwcUz~L>r1fj)Oa$P~LumYF;SsKtOwhCeSc7px^CGbJT8 zPigW7F44&gxTPkua7j)s;gJ>w8c|Y|n3d7L!X1pn>$%&OP`zCwx+D@LrD?RxUkI3Zpypr-jC*PC+Rj# Ue&sU@%m&I7P5$NE1(j$30M%-`iU0rr diff --git a/tests/rawdata/part1_activation_functions_epoch02.h5 b/tests/rawdata/part1_activation_functions_epoch02.h5 index e31552db11c848121b73909f7f50cbc45d3b274d..e07753ba5690e4864b76243fe12ac258a4b72ec9 100644 GIT binary patch delta 559 zcmeB}$JjBCaYG9SquJzEj{3<0oZ0%tsi`SSRthFYItog;`6;PDwo+PRUVLI%x>9W| zP%tks7c7{OnUa#4H;GGOas!v-bk`xKwHazvIcqvrXVM#CW34N@=zR?Us_U7T7qz#kpM3HrD66P0!_na zdq0n@x)}ol0~-*7z)J|lPz|)Zq$o2PPCFA}k(W5pG&QINC z`EUKs?N8C<&Uqub_ZQT{L!For+8n21?zJ5ok3F1%s(8@*!98S z#jd5#l6G%?$h(1ka)C|{*p5xQIV=#%3JiVNAglxCJggAb1)CO#*yIgz9h)CGw{U>P YC+Rj#e&sU@%m&I7P5$NE1(j$30Qp_MOaK4? delta 540 zcmeB}$JjBCaYG9Sqv_;Uj(VZeyv&kfB`XCZ9R;QA)S|r9oXG)P@{=_=-Q;rfQ&NF4 zN@P~LumYF;SsKtOwhCeSc7px^CGbJT8 zPigW7F44&gxTPkua7j)s;gJ>w8c|Y|n3d7L!X1pn>$%&OP`zCwx+D@LrD?RxUkI3Zpypr-jC*9W| zP%tks7c7{OnUa#4H;GGOas!v-bk`xKwHazvIcqvrXVM#CW34N@=zR?Us_U7T7qz#kpM3HrD66P0!_na zdq0n@x)}ol0~-*7z)J|lPz|)Zq$o2P|?+S$%$XmL1x;x4A% z-@B503U+UP$h(1ka)C|{*p5xQIV=#%3JiVNAglxCJggAb1)CO#*yIgz9h)CGw{U>P YC+Rj#e&sU@%m&I7P5$NE1(j$30CQ2f(f|Me delta 540 zcmeB}$JjBCaYG9Sqv_;Uj(VZeyv&kfB`XCZ9R;QA)S|r9oXG)P@{=_=-Q;rfQ&NF4 zN@P~LumYF;SsKtOwhCeSc7px^CGbJT8 zPigW7F44&gxTPkua7j)s;gJ>w8c|Y|n3d7L!X1pn>$%&OP`zCwx+D@LrD?RxUkI3Zpypr-jC*GspR zTE*}0%r~#G`}e?Q?}3cuz2%G$`=iaaS*XiQ-rj#|_s*T)`Ye2BX6w--SL~QZ~xsJ^boLe}+ Z;*)fnCcpBT1!e9W| zP%tks7c7{OnUa#4H;GGOas!v-bk`xKwHazvIcqvrXVM#CW34N@=zR?Us_U7T7qz#kpM3HrD66P0!_na zdq0n@x)}ol0~-*7z)J|lPz|)Zq$o2Pb!>j%+`<7C YpQPI~`IXNsFdHaSH2Ifr7gVAF04Kq@B>(^b delta 540 zcmeB}$JjBCaYG9Sqv_;Uj(VZeyv&kfB`XCZ9R;QA)S|r9oXG)P@{=_=-Q;rfQ&NF4 zN@P~LumYF;SsKtOwhCeSc7px^CGbJT8 zPigW7F44&gxTPkua7j)s;gJ>w8c|Y|n3d7L!X1pn>$%&OP`zCwx+D@LrD?RxUkI3Zpypr-jC*9W| zP%tks7c7{OnUa#4H;GGOas!v-bk`xKwHazvIcqvrXVM#CW34N@=zR?Us_U7T7qz#kpM3HrD66P0!_na zdq0n@x)}ol0~-*7z)J|lPz|)Zq$o2Pb!>j%+`<7C YpQPI~`IXNsFdHaSH2Ifr7gVAF0CYjQI{*Lx delta 540 zcmeB}$JjBCaYG9Sqv_;Uj(VZeyv&kfB`XCZ9R;QA)S|r9oXG)P@{=_=-Q;rfQ&NF4 zN@P~LumYF;SsKtOwhCeSc7px^CGbJT8 zPigW7F44&gxTPkua7j)s;gJ>w8c|Y|n3d7L!X1pn>$%&OP`zCwx+D@LrD?RxUkI3Zpypr-jC*>|15q8PqF;FXYmg8FW;;r zI2Ty49kR9F{E&A8`{V+h99W| zP%tks7c7{OnUa#4H;GGOas!v-bk`xKwHazvIcqvrXVM#CW34N@=zR?Us_U7T7qz#kpM3HrD66P0!_na zdq0n@x)}ol0~-*7z)J|lPz|)Zq$o2PP~LumYF;SsKtOwhCeSc7px^CGbJT8 zPigW7F44&gxTPkua7j)s;gJ>w8c|Y|n3d7L!X1pn>$%&OP`zCwx+D@LrD?RxUkI3Zpypr-jC*C?Fql0nJ zwS;L_-`jcZQ=K^XwU(>xd&dN^f49Tboq=EGSp0sfyh|ut!}6u1o7MJ3o;x$lDy)wm zGPOQ^XovOYhrAouCl~1SfbH0%o5KRJtiaHR4Z=EL&cg~}U9f3^h)v!g*RlD5a|;Jp Ye3EX{9W| zP%tks7c7{OnUa#4H;GGOas!v-bk`xKwHazvIcqvrXVM#CW34N@=zR?Us_U7T7qz#kpM3HrD66P0!_na zdq0n@x)}ol0~-*7z)J|lPz|)Zq$o2P`a4xFdF7HZ#P&tqeOYcn!+TPBCBJ)xP+hrC|H6UH|7?vn*YywEK6$ z;@#%Cl6y8k69##nJf=vrVZ1M)Vj?E98TR6bt YlXROVzw(&{W&>r4Cjav7f=V<102rOQPXGV_ delta 540 zcmeB}$JjBCaYG9Sqv_;Uj(VZeyv&kfB`XCZ9R;QA)S|r9oXG)P@{=_=-Q;rfQ&NF4 zN@P~LumYF;SsKtOwhCeSc7px^CGbJT8 zPigW7F44&gxTPkua7j)s;gJ>w8c|Y|n3d7L!X1pn>$%&OP`zCwx+D@LrD?RxUkI3Zpypr-jC*9W| zP%tks7c7{OnUa#4H;GGOas!v-bk`xKwHazvIcqvrXVM#CW34N@=zR?Us_U7T7qz#kpM3HrD66P0!_na zdq0n@x)}ol0~-*7z)J|lPz|)Zq$o2Pwn>|o!|R-Eg2*fOfSrD*nTUo!_vDm zZRdjH^_H6-@@`2lYjYkK_wahUi-T) delta 540 zcmeB}$JjBCaYG9Sqv_;Uj(VZeyv&kfB`XCZ9R;QA)S|r9oXG)P@{=_=-Q;rfQ&NF4 zN@P~LumYF;SsKtOwhCeSc7px^CGbJT8 zPigW7F44&gxTPkua7j)s;gJ>w8c|Y|n3d7L!X1pn>$%&OP`zCwx+D@LrD?RxUkI3Zpypr-jC*33?oQvH zPwWe9W| zP%tks7c7{OnUa#4H;GGOas!v-bk`xKwHazvIcqvrXVM#CW34N@=zR?Us_U7T7qz#kpM3HrD66P0!_na zdq0n@x)}ol0~-*7z)J|lPz|)Zq$o2Pd#yAp02eQ+}^X#igW6|R2GPRyBptG$f`;1C_ZJe>vKk!rFDATPV3Gvi}y0`Ej?qn zcD2v5-?jN6?*{hC1v))oJ2vU&us|#;F!W)Aunw5>utHcDY+4{P~LumYF;SsKtOwhCeSc7px^CGbJT8 zPigW7F44&gxTPkua7j)s;gJ>w8c|Y|n3d7L!X1pn>$%&OP`zCwx+D@LrD?RxUkI3Zpypr-jC*yi|JKHL7K&<;JIYR3?BdM~vkXd)+iBYwW+5c|-qJgU zYgg?&`(2wK@@`2lYjYkK_wahL9n-j diff --git a/tests/rawdata/part1_activation_functions_epoch10.h5 b/tests/rawdata/part1_activation_functions_epoch10.h5 index 5b1f7aa391929b5f6a8c0e2db367635134cee5c8..020f0eb10ad3bbe7c82c8e903a4f6d77b056ff68 100644 GIT binary patch delta 559 zcmeB}$JjBCaYG9SquJzEj{3<0oZ0%tsi`SSRthFYItog;`6;PDwo+PRUVLI%x>9W| zP%tks7c7{OnUa#4H;GGOas!v-bk`xKwHazvIcqvrXVM#CW34N@=zR?Us_U7T7qz#kpM3HrD66P0!_na zdq0n@x)}ol0~-*7z)J|lPz|)Zq$o2PP~LumYF;SsKtOwhCeSc7px^CGbJT8 zPigW7F44&gxTPkua7j)s;gJ>w8c|Y|n3d7L!X1pn>$%&OP`zCwx+D@LrD?RxUkI3Zpypr-jC*A}tXJ$m65C)mH=cKI zmQ03?qTXctHFJ0EbJ%};-y;@?{VSWcS@Fsp-N_{Kch`kDXDhL9%XgkFh_qaNf40@H ziEnl#ypY_z`62HH_Q?f0JzzUF>E^INEGsbdVS}&^nDekgSQl(sAYzj@$aQRf;M~Fi Z7N4ZsH2Ia!EHE1=Q#ARPZx>Xe0RZAeyR!fQ diff --git a/tests/rawdata/part1_activation_functions_epoch11.h5 b/tests/rawdata/part1_activation_functions_epoch11.h5 index 2977f4deb3cbe14f2381cb606ec9f7e737bd7aae..75c0fad9858d36cf3a5e2ba92ccf73fae5e3e590 100644 GIT binary patch delta 559 zcmeB}$JjBCaYG9SquJzEj{3<0oZ0%tsi`SSRthFYItog;`6;PDwo+PRUVLI%x>9W| zP%tks7c7{OnUa#4H;GGOas!v-bk`xKwHazvIcqvrXVM#CW34N@=zR?Us_U7T7qz#kpM3HrD66P0!_na zdq0n@x)}ol0~-*7z)J|lPz|)Zq$o2P3kO(y Xl5W%FS3a}AY@kfhBWrIUBgh delta 540 zcmeB}$JjBCaYG9Sqv_;Uj(VZeyv&kfB`XCZ9R;QA)S|r9oXG)P@{=_=-Q;rfQ&NF4 zN@P~LumYF;SsKtOwhCeSc7px^CGbJT8 zPigW7F44&gxTPkua7j)s;gJ>w8c|Y|n3d7L!X1pn>$%&OP`zCwx+D@LrD?RxUkI3Zpypr-jC*@esr7N; z&z5(WeX-j7kaq+79W| zP%tks7c7{OnUa#4H;GGOas!v-bk`xKwHazvIcqvrXVM#CW34N@=zR?Us_U7T7qz#kpM3HrD66P0!_na zdq0n@x)}ol0~-*7z)J|lPz|)Zq$o2Pak!hk)& zA`Ld57oM?y_vHINtF_nm-C>2;ry}USbJqkd%Vbu~-S@S-thT>3-ev#vy5)k+SFQH? z>+EKjmbH8HL*5PSlM8fuz;)8CjxrGBP YK1sJ}@++TNU^Y;uX!0-LE~rEU07@>qDgXcg delta 540 zcmeB}$JjBCaYG9Sqv_;Uj(VZeyv&kfB`XCZ9R;QA)S|r9oXG)P@{=_=-Q;rfQ&NF4 zN@P~LumYF;SsKtOwhCeSc7px^CGbJT8 zPigW7F44&gxTPkua7j)s;gJ>w8c|Y|n3d7L!X1pn>$%&OP`zCwx+D@LrD?RxUkI3Zpypr-jC*7W`^Hr;( z{yMvVP0iZ9`62HH_Q?f0JzzUF>E^INEGsbdVS}&^nDekgSQl(sAYzj@$aQRf;M~Fi Z7N4ZsH2Ia!EHE1=Q#ARPZx>Xe0RZ@0y+{B6 diff --git a/tests/rawdata/part1_activation_functions_epoch13.h5 b/tests/rawdata/part1_activation_functions_epoch13.h5 index 2ad2359b910c7e966c7526c65f3ba830b93953f9..1757c592ff7795cc3fec0e3c4cb96c0e72a8f823 100644 GIT binary patch delta 559 zcmeB}$JjBCaYG9SquJzEj{3<0oZ0%tsi`SSRthFYItog;`6;PDwo+PRUVLI%x>9W| zP%tks7c7{OnUa#4H;GGOas!v-bk`xKwHazvIcqvrXVM#CW34N@=zR?Us_U7T7qz#kpM3HrD66P0!_na zdq0n@x)}ol0~-*7z)J|lPz|)Zq$o2PXxLFb)RADuIXJztZzOu zw7w+!#Cr2X-VN-N3v_zGc5KqkVS!jyVCcgJVI45%VTG_R*t9^zCU21I*!;k`g##=; XNw;b8E1y|lHc+N$@-N>ms6+z*!J515 delta 540 zcmeB}$JjBCaYG9Sqv_;Uj(VZeyv&kfB`XCZ9R;QA)S|r9oXG)P@{=_=-Q;rfQ&NF4 zN@P~LumYF;SsKtOwhCeSc7px^CGbJT8 zPigW7F44&gxTPkua7j)s;gJ>w8c|Y|n3d7L!X1pn>$%&OP`zCwx+D@LrD?RxUkI3Zpypr-jC*km8j zoc7yc^W^3Wd#8gU`(q}3+ZPFS$8p~qR;+7Sc5Qyex7#2!&FYqHq)$ diff --git a/tests/rawdata/part1_activation_functions_epoch14.h5 b/tests/rawdata/part1_activation_functions_epoch14.h5 index 7e5998340c01eced5de6febcf0e8a3d8dd03238e..25e901c1a047d376b0149bab06c22a40de1881b9 100644 GIT binary patch delta 559 zcmeB}$JjBCaYG9SquJzEj{3<0oZ0%tsi`SSRthFYItog;`6;PDwo+PRUVLI%x>9W| zP%tks7c7{OnUa#4H;GGOas!v-bk`xKwHazvIcqvrXVM#CW34N@=zR?Us_U7T7qz#kpM3HrD66P0!_na zdq0n@x)}ol0~-*7z)J|lPz|)Zq$o2PutHcDY+4{P~LumYF;SsKtOwhCeSc7px^CGbJT8 zPigW7F44&gxTPkua7j)s;gJ>w8c|Y|n3d7L!X1pn>$%&OP`zCwx+D@LrD?RxUkI3Zpypr-jC*}5M})8CjxrGBP YK1sJ}@++TNU^Y;uX!0-LE~rEU0HqwfCjbBd diff --git a/tests/rawdata/part1_activation_functions_epoch15.h5 b/tests/rawdata/part1_activation_functions_epoch15.h5 index dc5d1d2e59064ba4a9faf8b25196ea8938b20358..12839f5ca946d142c285535598139461b05816e3 100644 GIT binary patch delta 559 zcmeB}$JjBCaYG9SquJzEj{3<0oZ0%tsi`SSRthFYItog;`6;PDwo+PRUVLI%x>9W| zP%tks7c7{OnUa#4H;GGOas!v-bk`xKwHazvIcqvrXVM#CW34N@=zR?Us_U7T7qz#kpM3HrD66P0!_na zdq0n@x)}ol0~-*7z)J|lPz|)Zq$o2PRt^%QT)_fL* z)?W4ZtT#X8-M~J%K&J<6$0pqz7Kmj9hCXZ%)&X-KRtW2YO$$V9@&>t%%@3SgIKbkQ Xbekr>@|gu@17(UP|MKmEN;CifwfD9^ delta 540 zcmeB}$JjBCaYG9Sqv_;Uj(VZeyv&kfB`XCZ9R;QA)S|r9oXG)P@{=_=-Q;rfQ&NF4 zN@P~LumYF;SsKtOwhCeSc7px^CGbJT8 zPigW7F44&gxTPkua7j)s;gJ>w8c|Y|n3d7L!X1pn>$%&OP`zCwx+D@LrD?RxUkI3Zpypr-jC*vZAeU0aX(SZx=Jv;JYbX;+@mI%{SN zLu=Rid)AvD@@`2lYjYkK_wah&jq(1 diff --git a/tests/rawdata/part1_activation_functions_epoch16.h5 b/tests/rawdata/part1_activation_functions_epoch16.h5 index 1940d459942f37b69b3174407a8ddb127a591301..4e7660942bb306735ee6b1c9f08df885836219b9 100644 GIT binary patch delta 559 zcmeB}$JjBCaYG9SquJzEj{3<0oZ0%tsi`SSRthFYItog;`6;PDwo+PRUVLI%x>9W| zP%tks7c7{OnUa#4H;GGOas!v-bk`xKwHazvIcqvrXVM#CW34N@=zR?Us_U7T7qz#kpM3HrD66P0!_na zdq0n@x)}ol0~-*7z)J|lPz|)Zq$o2PP~LumYF;SsKtOwhCeSc7px^CGbJT8 zPigW7F44&gxTPkua7j)s;gJ>w8c|Y|n3d7L!X1pn>$%&OP`zCwx+D@LrD?RxUkI3Zpypr-jC*ZVLR_|tE<+47$=JT$} z-FtW6sFT^V`62HH_Q?f0JzzUF>E^INEGsbdVS}&^nDekgSQl(sAYzj@$aQRf;M~Fi Z7N4ZsH2Ia!EHE1=Q#ARPZx>Xe0RR9My(j9W| zP%tks7c7{OnUa#4H;GGOas!v-bk`xKwHazvIcqvrXVM#CW34N@=zR?Us_U7T7qz#kpM3HrD66P0!_na zdq0n@x)}ol0~-*7z)J|lPz|)Zq$o2PutHcDY+4{P~LumYF;SsKtOwhCeSc7px^CGbJT8 zPigW7F44&gxTPkua7j)s;gJ>w8c|Y|n3d7L!X1pn>$%&OP`zCwx+D@LrD?RxUkI3Zpypr-jC*)8CjxrGBP YK1sJ}@++TNU^Y;uX!0-LE~rEU0590Ll>h($ diff --git a/tests/rawdata/part1_activation_functions_epoch18.h5 b/tests/rawdata/part1_activation_functions_epoch18.h5 index b424afd4386f697c0ceb419cf7cb9b20b0a07cfb..12f749ae1e573ec2e99a1886893c25ff3b479de6 100644 GIT binary patch delta 559 zcmeB}$JjBCaYG9SquJzEj{3<0oZ0%tsi`SSRthFYItog;`6;PDwo+PRUVLI%x>9W| zP%tks7c7{OnUa#4H;GGOas!v-bk`xKwHazvIcqvrXVM#CW34N@=zR?Us_U7T7qz#kpM3HrD66P0!_na zdq0n@x)}ol0~-*7z)J|lPz|)Zq$o2P{baoV6bHmUpSvPf8)~NPEdIP;m#9&&)yWlnyXPi$StV2!@3duf z++Ee!wR`hJ-VN-N3v_zGc5KqkVS!jyVCcgJVI45%VTG_R*t9^zCU21I*!;k`g##=; XNw;b8E1y|lHc+N$@-N>ms6+z*vunA` delta 540 zcmeB}$JjBCaYG9Sqv_;Uj(VZeyv&kfB`XCZ9R;QA)S|r9oXG)P@{=_=-Q;rfQ&NF4 zN@P~LumYF;SsKtOwhCeSc7px^CGbJT8 zPigW7F44&gxTPkua7j)s;gJ>w8c|Y|n3d7L!X1pn>$%&OP`zCwx+D@LrD?RxUkI3Zpypr-jC*+T*kzSmS-jJV z(Q$WiW7qD@4|zARPcG2u0o$=jH-`mcS%IMs8-#VhoQD;{x?s}+5u3b0u4D59=N1mI Y_$1w?$*+87f!RQrqRGE}yPy&c0A0zub^rhX diff --git a/tests/rawdata/part1_activation_functions_epoch19.h5 b/tests/rawdata/part1_activation_functions_epoch19.h5 index daa8400960dc3372ffd48f1e826319df4c3157e0..a928f360c3fa64ad32ba015eea26d134fccdc173 100644 GIT binary patch delta 559 zcmeB}$JjBCaYG9SquJzEj{3<0oZ0%tsi`SSRthFYItog;`6;PDwo+PRUVLI%x>9W| zP%tks7c7{OnUa#4H;GGOas!v-bk`xKwHazvIcqvrXVM#CW34N@=zR?Us_U7T7qz#kpM3HrD66P0!_na zdq0n@x)}ol0~-*7z)J|lPz|)Zq$o2P*v=W@7hq;Xk8R& zVI6+)r1j>9yc^gj7wGhW?bxK7!ve9az|e;c!a88i!wO+tuxWvaP2M2avH5{>3kO(y Xl5W%FS3a}AY@kfhBWr?uxj4 delta 540 zcmeB}$JjBCaYG9Sqv_;Uj(VZeyv&kfB`XCZ9R;QA)S|r9oXG)P@{=_=-Q;rfQ&NF4 zN@P~LumYF;SsKtOwhCeSc7px^CGbJT8 zPigW7F44&gxTPkua7j)s;gJ>w8c|Y|n3d7L!X1pn>$%&OP`zCwx+D@LrD?RxUkI3Zpypr-jC*)8CjxrGBP YK1sJ}@++TNU^Y;uX!0-LE~rEU0E70q`~Uy| diff --git a/tests/rawdata/part1_activation_functions_epoch20.h5 b/tests/rawdata/part1_activation_functions_epoch20.h5 index 4d44cb92148fe6114a2bf755b61352e3a39a4c2b..a00ad7e3a58357cedab41b8c0add3789fb1df642 100644 GIT binary patch delta 559 zcmeB}$JjBCaYG9SquJzEj{3<0oZ0%tsi`SSRthFYItog;`6;PDwo+PRUVLI%x>9W| zP%tks7c7{OnUa#4H;GGOas!v-bk`xKwHazvIcqvrXVM#CW34N@=zR?Us_U7T7qz#kpM3HrD66P0!_na zdq0n@x)}ol0~-*7z)J|lPz|)Zq$o2PP~LumYF;SsKtOwhCeSc7px^CGbJT8 zPigW7F44&gxTPkua7j)s;gJ>w8c|Y|n3d7L!X1pn>$%&OP`zCwx+D@LrD?RxUkI3Zpypr-jC*glZNjHZDVp)Nq4;zGaz?_E_!n$D70uh_ML9S!-1LqbF Zu=ph1rpd2-W`Ws2nWD+Re7m3$4FK}$z>NR^ diff --git a/tests/rawdata/part1_activation_functions_model.h5 b/tests/rawdata/part1_activation_functions_model.h5 index 5f16fd702bcb69a844f6f9e9ae3babdb162ad6b1..ee5cd31ecb864095967d7d1ed3741426918ae7c7 100644 GIT binary patch delta 119 zcmeB}$JjBCaYG9SquJ(G4tEwHb(ZJ8Ad5?aV)W(>az1=u)+XH?7Kl)Rp${8`b-Rm1qC}D8n~? From 0c47c364d9224e4db41cd1170641ce817e2abc8c Mon Sep 17 00:00:00 2001 From: dvgodoy Date: Thu, 14 Jun 2018 23:53:38 +0200 Subject: [PATCH 09/13] no bias in weights/gradients --- deepreplay/replay.py | 55 +++++++++++++++++++++++++++-------- examples/gradients_example.py | 51 ++++++++++++++++++++++++++++++-- requirements.txt | 2 +- 3 files changed, 92 insertions(+), 16 deletions(-) diff --git a/deepreplay/replay.py b/deepreplay/replay.py index 26a56b5..d442de4 100644 --- a/deepreplay/replay.py +++ b/deepreplay/replay.py @@ -126,7 +126,8 @@ def __init__(self, replay_filename, group_name, model_filename=''): # Keras function to compute the gradients for trainable weights, given inputs, targets, weights and # sample weights self.__trainable_weights = [w for layer in self.model.layers - for w in layer.trainable_weights if layer.trainable] + for w in layer.trainable_weights + if layer.trainable and ('bias' not in w.op.name)] self.__trainable_gradients = self.model.optimizer.get_gradients(self.model.total_loss, self.__trainable_weights) self._get_gradients = K.function(inputs=[K.learning_phase()] + self.model.inputs + self.model.targets + self._model_weights + self.model.sample_weights, @@ -270,6 +271,26 @@ def zvalues_violins(self): def gradients_violins(self): return self._gradients_plot, self._gradients_data + @staticmethod + def __calc_std(values): + return np.array([[layer.std() for layer in epoch] for epoch in values]) + + @property + def weights_std(self): + std = None + if self._weights_violins_data is not None: + weights = self._weights_violins_data.values + std = Replay.__calc_std(weights) + return std + + @property + def gradients_std(self): + std = None + if self._gradients_data is not None: + gradients = self._gradients_data.values + std = Replay.__calc_std(gradients) + return std + @property def training_loss(self): return self.group['loss'][:] @@ -358,7 +379,8 @@ def build_gradients(self, ax, layer_names=None, exclude_outputs=True, epoch_star epoch_end = self.n_epochs epoch_end = min(epoch_end, self.n_epochs) - gradient_names = [layer.name for layer in self.model.layers for _ in layer.trainable_weights if layer.trainable] + gradient_names = [layer.name for layer in self.model.layers for w in layer.trainable_weights + if layer.trainable and ('bias' not in w.op.name)] gradients = [] # For each epoch, uses the corresponding weights for epoch in range(epoch_start, epoch_end + 1): @@ -377,11 +399,14 @@ def build_gradients(self, ax, layer_names=None, exclude_outputs=True, epoch_star self._gradients_data = LayerViolinsData(names=gradient_names, values=gradients, layers=self.weights_layers, selected_layers=layer_names) - self._gradients_plot = LayerViolins(ax, 'Gradients').load_data(self._gradients_data) + if ax is None: + self._gradients_plot = None + else: + self._gradients_plot = LayerViolins(ax, 'Gradients').load_data(self._gradients_data) return self._gradients_plot - def build_outputs_violins(self, ax, before_activation=False, layer_names=None, include_inputs=True, - exclude_outputs=True, epoch_start=0, epoch_end=-1): + def build_outputs(self, ax, before_activation=False, layer_names=None, include_inputs=True, + exclude_outputs=True, epoch_start=0, epoch_end=-1): """Builds a LayerViolins object to be used for plotting and animating. @@ -438,7 +463,10 @@ def build_outputs_violins(self, ax, before_activation=False, layer_names=None, i layer_names = ['inputs'] + layer_names data = LayerViolinsData(names=names, values=outputs, layers=self.z_act_layers, selected_layers=layer_names) - plot = LayerViolins(ax, title).load_data(data) + if ax is None: + plot = None + else: + plot = LayerViolins(ax, title).load_data(data) if before_activation: self._zvalues_violins_data = data self._zvalues_violins_plot = plot @@ -447,7 +475,7 @@ def build_outputs_violins(self, ax, before_activation=False, layer_names=None, i self._activations_violins_plot = plot return plot - def build_weights_violins(self, ax, layer_names=None, exclude_outputs=True, epoch_start=0, epoch_end=-1): + def build_weights(self, ax, layer_names=None, exclude_outputs=True, epoch_start=0, epoch_end=-1): """Builds a LayerViolins object to be used for plotting and animating. @@ -475,14 +503,14 @@ def build_weights_violins(self, ax, layer_names=None, exclude_outputs=True, epoc epoch_end = self.n_epochs epoch_end = min(epoch_end, self.n_epochs) - names = [layer.name for layer, weights in zip(self.model.layers, self.n_weights) for _ in weights - if len(weights) in (1, 2)] - n_weights = [len(weights) for layer, weights in zip(self.model.layers, self.n_weights) for _ in weights] + names = [layer.name for layer, weights in zip(self.model.layers, self.n_weights) if len(weights) in (1, 2)] + n_weights = [(i, len(weights)) for layer, weights in zip(self.model.layers, self.n_weights) for i in weights] weights = [] # For each epoch, uses the corresponding weights for epoch in range(epoch_start, epoch_end + 1): - weights.append([w for w, n in zip(self.weights[epoch], n_weights) if n in (1, 2)]) + # takes only the weights (i == 0), not the biases (i == 1) + weights.append([w for w, (i, n) in zip(self.weights[epoch], n_weights) if (n in (1, 2)) and (i == 0)]) if layer_names is None: layer_names = self.weights_layers @@ -493,7 +521,10 @@ def build_weights_violins(self, ax, layer_names=None, exclude_outputs=True, epoc values=weights, layers=self.weights_layers, selected_layers=layer_names) - self._weights_violins_plot = LayerViolins(ax, 'Weights').load_data(self._weights_violins_data) + if ax is None: + self._weights_violins_plot = None + else: + self._weights_violins_plot = LayerViolins(ax, 'Weights').load_data(self._weights_violins_data) return self._weights_violins_plot def build_loss_histogram(self, ax, epoch_start=0, epoch_end=-1): diff --git a/examples/gradients_example.py b/examples/gradients_example.py index ca7680d..214df33 100644 --- a/examples/gradients_example.py +++ b/examples/gradients_example.py @@ -49,10 +49,55 @@ def build_model(n_layers, input_dim, units, activation, initializer): ax_gradients = plt.subplot2grid((2, 2), (1, 1)) layers = ['h1', 'h2', 'h3', 'h4', 'h5', 'o'] -zv = replay.build_outputs_violins(ax_zvalues, before_activation=True) -wv = replay.build_weights_violins(ax_weights) -av = replay.build_outputs_violins(ax_activations) +zv = replay.build_outputs(ax_zvalues, before_activation=True) +wv = replay.build_weights(ax_weights) +av = replay.build_outputs(ax_activations) gv = replay.build_gradients(ax_gradients) sample_figure = compose_plots([zv, wv, av, gv], 0, title='Activation: {} - Initializer: {}'.format(activation, init_title)) sample_figure.savefig('gradients.png', dpi=120, format='png') + + +# Showdown +input_dim = 10 + +results = {} +pairs = [('tanh', 'glorot_normal'), + ('tanh', 'glorot_uniform'), + ('relu', 'he_normal'), + ('relu', 'he_uniform')] + +X, y = load_data(n_dims=input_dim) +y0 = np.zeros_like(y) +y1 = np.ones_like(y) +yr = np.random.randint(2, size=y.shape) + +ys = [y, y0, y1, yr] + +for y, yname in zip(ys, ['y', 'y0', 'y1', 'yr']): + results.update({yname: {}}) + for n_layers in [5, 20, 50, 100]: + results[yname].update({n_layers: {}}) + units = [100] * n_layers + for pair in pairs: + activation, init_key = pair + group_name = '{}_{}_{}_{}'.format(activation, init_key, n_layers, yname) + key_name = '{}_{}'.format(activation, init_key) + print(group_name) + + init = initializers[init_key]['init'] + + model = build_model(n_layers, + input_dim=input_dim, + units=units, + activation=activation, + initializer=init) + + replaydata = ReplayData(X, y, filename='showdown.h5', group_name=group_name, model=model) + replay = Replay(replay_filename='showdown.h5', group_name=group_name) + replay.build_weights(None) + replay.build_gradients(None) + + gstd = replay.gradients_std[0][:-1] + wstd = replay.weights_std[0][:-1] + results[yname][n_layers].update({key_name: {'gradient': gstd, 'weights': wstd, 'ratio': gstd / wstd}}) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 516da19..544e093 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ numpy>=1.14 tensorflow==1.5.0 -keras>=2.0.7 +keras==2.2.0 scikit-learn>=0.18 h5py matplotlib From 823ce33e49f381339efb433ad3c7711c824bbc7b Mon Sep 17 00:00:00 2001 From: dvgodoy Date: Fri, 15 Jun 2018 00:11:58 +0200 Subject: [PATCH 10/13] some examples --- examples/gradients_example.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/examples/gradients_example.py b/examples/gradients_example.py index 214df33..2b70343 100644 --- a/examples/gradients_example.py +++ b/examples/gradients_example.py @@ -100,4 +100,30 @@ def build_model(n_layers, input_dim, units, activation, initializer): gstd = replay.gradients_std[0][:-1] wstd = replay.weights_std[0][:-1] - results[yname][n_layers].update({key_name: {'gradient': gstd, 'weights': wstd, 'ratio': gstd / wstd}}) \ No newline at end of file + results[yname][n_layers].update({key_name: {'gradient': gstd, 'weights': wstd, 'ratio': gstd / wstd}}) + +def plot(y=None, l=None, act=None, dist=None): + if y is None: + ys = ['y', 'y0', 'y1', 'yr'] + else: + ys = [y] + if l is None: + ls = [5, 20, 50, 100] + else: + ls = [l] + if act is None: + acts = ['tanh', 'relu'] + else: + acts = [act] + if dist is None: + dists = ['normal', 'uniform'] + else: + dists = [dist] + return [((y, l, act, dist) , results[y][l]['{}_{}_{}'.format(act, 'glorot' if act == 'tanh' else 'he', dist)]['ratio']) + for y in ys for l in ls for act in acts for dist in dists] + +sub = plot(y='y', act='tanh', dist='uniform') + +for n, v in sub: + plt.plot(v, label=n) +plt.legend() \ No newline at end of file From 68d0c01c30713260b8cd7db31cdcd7291ccd0234 Mon Sep 17 00:00:00 2001 From: dvgodoy Date: Sat, 16 Jun 2018 14:10:32 +0200 Subject: [PATCH 11/13] fixing tests/example --- deepreplay/callbacks.py | 8 ++ deepreplay/datasets/ball.py | 9 +- deepreplay/replay.py | 30 +++- deepreplay/utils.py | 13 +- docs/source/deepreplay.datasets.rst | 16 +++ docs/source/deepreplay.rst | 8 ++ examples/gradients_example.py | 129 ------------------ examples/part2_initializers.py | 55 -------- examples/part2_weight_initializers.py | 94 +++++++++++++ tests/rawdata/hyperparms_in_action.h5 | Bin 56116 -> 56116 bytes .../part1_activation_functions_epoch01.h5 | Bin 20232 -> 20232 bytes .../part1_activation_functions_epoch02.h5 | Bin 20232 -> 20232 bytes .../part1_activation_functions_epoch03.h5 | Bin 20232 -> 20232 bytes .../part1_activation_functions_epoch04.h5 | Bin 20232 -> 20232 bytes .../part1_activation_functions_epoch05.h5 | Bin 20232 -> 20232 bytes .../part1_activation_functions_epoch06.h5 | Bin 20232 -> 20232 bytes .../part1_activation_functions_epoch07.h5 | Bin 20232 -> 20232 bytes .../part1_activation_functions_epoch08.h5 | Bin 20232 -> 20232 bytes .../part1_activation_functions_epoch09.h5 | Bin 20232 -> 20232 bytes .../part1_activation_functions_epoch10.h5 | Bin 20232 -> 20232 bytes .../part1_activation_functions_epoch11.h5 | Bin 20232 -> 20232 bytes .../part1_activation_functions_epoch12.h5 | Bin 20232 -> 20232 bytes .../part1_activation_functions_epoch13.h5 | Bin 20232 -> 20232 bytes .../part1_activation_functions_epoch14.h5 | Bin 20232 -> 20232 bytes .../part1_activation_functions_epoch15.h5 | Bin 20232 -> 20232 bytes .../part1_activation_functions_epoch16.h5 | Bin 20232 -> 20232 bytes .../part1_activation_functions_epoch17.h5 | Bin 20232 -> 20232 bytes .../part1_activation_functions_epoch18.h5 | Bin 20232 -> 20232 bytes .../part1_activation_functions_epoch19.h5 | Bin 20232 -> 20232 bytes .../part1_activation_functions_epoch20.h5 | Bin 20232 -> 20232 bytes .../part1_activation_functions_model.h5 | Bin 20232 -> 20232 bytes 31 files changed, 167 insertions(+), 195 deletions(-) delete mode 100644 examples/gradients_example.py delete mode 100644 examples/part2_initializers.py create mode 100644 examples/part2_weight_initializers.py diff --git a/deepreplay/callbacks.py b/deepreplay/callbacks.py index bd5e9dc..6d20ca2 100644 --- a/deepreplay/callbacks.py +++ b/deepreplay/callbacks.py @@ -31,6 +31,14 @@ class ReplayData(Callback): Group inside the HDF5 file where the information is to be saved. If the informed group name already exists, it will throw an exception. + model: Keras Model, optional + If provided, it will set the model directly to the callback + instance and execute `on_train_begin` method to initialize + all variables and create the corresponding group in the HDF5 + file. + This is intended to be used for analyzing the initial conditions + of the model without ever calling its `fit` function, where + the callback is usually called. """ def __init__(self, inputs, targets, filename, group_name, model=None): super(ReplayData, self).__init__() diff --git a/deepreplay/datasets/ball.py b/deepreplay/datasets/ball.py index 93fccde..a64ba14 100644 --- a/deepreplay/datasets/ball.py +++ b/deepreplay/datasets/ball.py @@ -1,6 +1,6 @@ import numpy as np -def load_data(n_dims=10, n_points=1000, only_sphere=False, shuffle=True, seed=13): +def load_data(n_dims=10, n_points=1000, classif_radius_fraction=0.5, only_sphere=False, shuffle=True, seed=13): """ Parameters @@ -9,6 +9,11 @@ def load_data(n_dims=10, n_points=1000, only_sphere=False, shuffle=True, seed=13 Number of dimensions of the n-ball. Default is 10. n_points: int, optional Number of points in each parabola. Default is 1,000. + classif_radius_fraction: float, optional + Points farther away from the center than + `classification_radius_fraction * ball radius` are + considered to be positive cases. The remaining + points are the negative cases. only_sphere: boolean If True, generates a n-sphere, that is, a hollow n-ball. Default is False. @@ -37,7 +42,7 @@ def load_data(n_dims=10, n_points=1000, only_sphere=False, shuffle=True, seed=13 radius *= adjustment X *= adjustment - y = (np.abs(np.sum(X, axis=1)) > (radius / 2.0)).astype(np.int) + y = (np.abs(np.sum(X, axis=1)) > (radius * classif_radius_fraction)).astype(np.int) # But we must not feed the network with neatly organized inputs... # so let's randomize them diff --git a/deepreplay/replay.py b/deepreplay/replay.py index d442de4..e0823df 100644 --- a/deepreplay/replay.py +++ b/deepreplay/replay.py @@ -16,13 +16,12 @@ TRAINING_MODE = 1 TEST_MODE = 0 ACTIVATIONS = ['softmax', 'relu', 'elu', 'tanh', 'sigmoid', 'hard_sigmoid', 'linear', 'softplus', 'softsign', 'selu'] -Z_OPS = ['BiasAdd', 'MatMul', 'Add'] +Z_OPS = ['BiasAdd', 'MatMul', 'Add', 'Sub', 'Mul', 'Maximum', 'Minimum', 'RealDiv', 'ExpandDims'] class Replay(object): """Creates an instance of Replay, to process information collected - by the callback and generate data to feed the supported visualiza- - tions. + by the callback and generate data to feed the supported visualizations. Parameters ---------- @@ -56,13 +55,30 @@ class Replay(object): animating; namedtuple containing information about classification probabilities and targets. - weights_violins: + weights_violins: (LayerViolins, LayerViolinsData) + LayerViolins object to be used for plotting and animating; + namedtuple containing information about weights values + per layer. - activations_violins: + activations_violins: (LayerViolins, LayerViolinsData) + LayerViolins object to be used for plotting and animating; + namedtuple containing information about activation values + per layer. - zvalues_violins: + zvalues_violins: (LayerViolins, LayerViolinsData) + LayerViolins object to be used for plotting and animating; + namedtuple containing information about Z-values per layer. - gradients_violins: + gradients_violins: (LayerViolins, LayerViolinsData) + LayerViolins object to be used for plotting and animating; + namedtuple containing information about gradient values + per layer. + + weights_std: ndarray + Standard deviation of the weights per layer. + + gradients_std: ndarray + Standard deivation of the gradients per layer. training_loss: ndarray An array of shape (n_epochs, ) with training loss as reported diff --git a/deepreplay/utils.py b/deepreplay/utils.py index 53b34ef..361f3f7 100644 --- a/deepreplay/utils.py +++ b/deepreplay/utils.py @@ -1,5 +1,11 @@ +# These functions were extracted from Keras to assure there will +# be no break in compatibility with DeepReplay's code. + def make_batches(size, batch_size): - """Returns a list of batch indices (tuples of indices). + """Function extracted from Keras - check keras.engine.training_utils + for the original version. + + Returns a list of batch indices (tuples of indices). # Arguments size: Integer, total size of the data to slice into batches. batch_size: Integer, batch size. @@ -11,7 +17,10 @@ def make_batches(size, batch_size): for i in range(num_batches)] def slice_arrays(arrays, start=None, stop=None): - """Slices an array or list of arrays. + """ Function extracted from Keras - check keras.utils.generic_utils + for the original version. + + Slices an array or list of arrays. This takes an array-like, or a list of array-likes, and outputs: - arrays[start:stop] if `arrays` is an array-like diff --git a/docs/source/deepreplay.datasets.rst b/docs/source/deepreplay.datasets.rst index ecfffe7..9e3130b 100644 --- a/docs/source/deepreplay.datasets.rst +++ b/docs/source/deepreplay.datasets.rst @@ -4,6 +4,22 @@ deepreplay\.datasets package Submodules ---------- +deepreplay\.datasets\.ball module +--------------------------------- + +.. automodule:: deepreplay.datasets.ball + :members: + :undoc-members: + :show-inheritance: + +deepreplay\.datasets\.hypercube module +-------------------------------------- + +.. automodule:: deepreplay.datasets.hypercube + :members: + :undoc-members: + :show-inheritance: + deepreplay\.datasets\.parabola module ------------------------------------- diff --git a/docs/source/deepreplay.rst b/docs/source/deepreplay.rst index 5ae24e6..dc6665a 100644 --- a/docs/source/deepreplay.rst +++ b/docs/source/deepreplay.rst @@ -35,6 +35,14 @@ deepreplay\.replay module :undoc-members: :show-inheritance: +deepreplay\.utils module +------------------------ + +.. automodule:: deepreplay.utils + :members: + :undoc-members: + :show-inheritance: + Module contents --------------- diff --git a/examples/gradients_example.py b/examples/gradients_example.py deleted file mode 100644 index 2b70343..0000000 --- a/examples/gradients_example.py +++ /dev/null @@ -1,129 +0,0 @@ -import numpy as np -from keras.models import Sequential -from keras.layers import Dense -from keras.initializers import glorot_uniform, normal, he_uniform -from deepreplay.datasets.ball import load_data -from deepreplay.callbacks import ReplayData -from deepreplay.replay import Replay -from deepreplay.plot import compose_plots - -import matplotlib.pyplot as plt - -group_name = 'gradients' - -X, y = load_data(n_dims=10) - -n_layers = 5 -variance = 0.01 -activation = 'tanh' - -init = normal(mean=0, stddev=np.sqrt(variance)) -init_name = 'Normal' -init_parms = r'$\sigma = {}$'.format(np.sqrt(variance)) -init_title = '{} {}'.format(init_name, init_parms) -#init = VarianceScaling(scale=1.0, mode='fan_avg', distribution='uniform') -#init = glorot_normal() # sigmoid -#init = glorot_uniform() # tanh -#init = he_uniform() # relu -#init_name = 'He Uniform' - -def build_model(n_layers, input_dim, units, activation, initializer): - model = Sequential() - model.add(Dense(units=units, input_dim=input_dim, activation=activation, kernel_initializer=initializer, name='h1')) - for i in range(2, n_layers + 1): - model.add(Dense(units=units, activation=activation, kernel_initializer=initializer, name='h{}'.format(i))) - model.add(Dense(units=1, activation='sigmoid', kernel_initializer=initializer, name='o')) - model.compile(loss='binary_crossentropy', optimizer='sgd', metrics=['acc']) - return model - -model = build_model(n_layers, 10, 100, activation, init) - -replaydata = ReplayData(X, y, filename='gradients.h5', group_name=group_name, model=model) - -replay = Replay(replay_filename='gradients.h5', group_name=group_name) - -fig = plt.figure(figsize=(12, 6)) -ax_zvalues = plt.subplot2grid((2, 2), (0, 0)) -ax_weights = plt.subplot2grid((2, 2), (0, 1)) -ax_activations = plt.subplot2grid((2, 2), (1, 0)) -ax_gradients = plt.subplot2grid((2, 2), (1, 1)) - -layers = ['h1', 'h2', 'h3', 'h4', 'h5', 'o'] -zv = replay.build_outputs(ax_zvalues, before_activation=True) -wv = replay.build_weights(ax_weights) -av = replay.build_outputs(ax_activations) -gv = replay.build_gradients(ax_gradients) - -sample_figure = compose_plots([zv, wv, av, gv], 0, title='Activation: {} - Initializer: {}'.format(activation, init_title)) -sample_figure.savefig('gradients.png', dpi=120, format='png') - - -# Showdown -input_dim = 10 - -results = {} -pairs = [('tanh', 'glorot_normal'), - ('tanh', 'glorot_uniform'), - ('relu', 'he_normal'), - ('relu', 'he_uniform')] - -X, y = load_data(n_dims=input_dim) -y0 = np.zeros_like(y) -y1 = np.ones_like(y) -yr = np.random.randint(2, size=y.shape) - -ys = [y, y0, y1, yr] - -for y, yname in zip(ys, ['y', 'y0', 'y1', 'yr']): - results.update({yname: {}}) - for n_layers in [5, 20, 50, 100]: - results[yname].update({n_layers: {}}) - units = [100] * n_layers - for pair in pairs: - activation, init_key = pair - group_name = '{}_{}_{}_{}'.format(activation, init_key, n_layers, yname) - key_name = '{}_{}'.format(activation, init_key) - print(group_name) - - init = initializers[init_key]['init'] - - model = build_model(n_layers, - input_dim=input_dim, - units=units, - activation=activation, - initializer=init) - - replaydata = ReplayData(X, y, filename='showdown.h5', group_name=group_name, model=model) - replay = Replay(replay_filename='showdown.h5', group_name=group_name) - replay.build_weights(None) - replay.build_gradients(None) - - gstd = replay.gradients_std[0][:-1] - wstd = replay.weights_std[0][:-1] - results[yname][n_layers].update({key_name: {'gradient': gstd, 'weights': wstd, 'ratio': gstd / wstd}}) - -def plot(y=None, l=None, act=None, dist=None): - if y is None: - ys = ['y', 'y0', 'y1', 'yr'] - else: - ys = [y] - if l is None: - ls = [5, 20, 50, 100] - else: - ls = [l] - if act is None: - acts = ['tanh', 'relu'] - else: - acts = [act] - if dist is None: - dists = ['normal', 'uniform'] - else: - dists = [dist] - return [((y, l, act, dist) , results[y][l]['{}_{}_{}'.format(act, 'glorot' if act == 'tanh' else 'he', dist)]['ratio']) - for y in ys for l in ls for act in acts for dist in dists] - -sub = plot(y='y', act='tanh', dist='uniform') - -for n, v in sub: - plt.plot(v, label=n) -plt.legend() \ No newline at end of file diff --git a/examples/part2_initializers.py b/examples/part2_initializers.py deleted file mode 100644 index 6d65706..0000000 --- a/examples/part2_initializers.py +++ /dev/null @@ -1,55 +0,0 @@ -from keras.layers import Dense, Activation, BatchNormalization -from keras.models import Sequential -from keras.optimizers import SGD -from keras.initializers import glorot_normal, normal -from deepreplay.datasets.parabola import load_data -from deepreplay.callbacks import ReplayData - -X, y = load_data() - -sgd = SGD(lr=0.05) - -def basic_model(activation, initializers): - model = Sequential() - model.add(Dense(units=2, - input_dim=2, - kernel_initializer=initializers[0], - activation=activation, - name='hidden')) - - model.add(Dense(units=1, - kernel_initializer=initializers[1], - activation='sigmoid', - name='output')) - return model - -def bn_model(activation, initializers): - model = Sequential() - model.add(Dense(units=2, - input_dim=2, - kernel_initializer=initializers[0], - name='hidden_linear')) - model.add(BatchNormalization(name='hidden_bn')) - model.add(Activation(activation, name='hidden_activation')) - - model.add(Dense(units=1, - kernel_initializer=initializers[1], - activation='sigmoid', - name='output')) - return model - - -for seed in range(100): - print('Using seed {}') - replay = ReplayData(X, y, filename='part2_relu.h5', group_name='seed{:03}'.format(seed)) - - glorot_initializer = glorot_normal(seed=seed) - normal_initializer = normal(seed=42) - - model = basic_model('relu', [glorot_initializer, normal_initializer]) - - model.compile(loss='binary_crossentropy', - optimizer=sgd, - metrics=['acc']) - - model.fit(X, y, epochs=150, batch_size=16, callbacks=[replay]) diff --git a/examples/part2_weight_initializers.py b/examples/part2_weight_initializers.py new file mode 100644 index 0000000..678a737 --- /dev/null +++ b/examples/part2_weight_initializers.py @@ -0,0 +1,94 @@ +from keras.initializers import normal, glorot_normal, glorot_uniform, he_normal, he_uniform +from keras.layers import Dense +from keras.models import Sequential + +from deepreplay.callbacks import ReplayData +from deepreplay.datasets.ball import load_data +from deepreplay.plot import compose_plots +from deepreplay.replay import Replay + +from matplotlib import pyplot as plt + +# Model builder function +def build_model(n_layers, input_dim, units, activation, initializer): + if isinstance(units, list): + assert len(units) == n_layers + else: + units = [units] * n_layers + + model = Sequential() + # Adds first hidden layer with input_dim parameter + model.add(Dense(units=units[0], + input_dim=input_dim, + activation=activation, + kernel_initializer=initializer, + name='h1')) + + # Adds remaining hidden layers + for i in range(2, n_layers + 1): + model.add(Dense(units=units[i-1], + activation=activation, + kernel_initializer=initializer, + name='h{}'.format(i))) + + # Adds output layer + model.add(Dense(units=1, activation='sigmoid', kernel_initializer=initializer, name='o')) + # Compiles the model + model.compile(loss='binary_crossentropy', optimizer='sgd', metrics=['acc']) + return model + +X, y = load_data(n_dims=10) + +filename = 'part2_weight_initializers.h5' + +# Examples of different initializers + +# Uses normal initializer +# activation = 'sigmoid' +# initializer = normal(mean=0, stddev=0.01, seed=13) +# title = r'Activation: sigmoid - Initializer: $\sigma = 0.01$' +# group_name = 'sigmoid_stdev_0.01' + +# Users Glorot Uniform +# activation = 'tanh' +# initializer = glorot_uniform(seed=13) +# title = 'Activation: Tanh - Initializer: Glorot Uniform' +# group_name = 'tanh_glorot_uniform' + +# Uses He Uniform +activation = 'relu' +initializer = he_uniform(seed=13) +title = 'Activation: ReLU - Initializer: He Uniform' +group_name = 'relu_he_uniform' + +# Builds BLOCK model +model = build_model(n_layers=5, input_dim=10, units=100, + activation=activation, initializer=initializer) + +# Since we only need initial weights, we don't even need to train the model! +# We still use the ReplayData callback, but we can pass the model as argument instead +replaydata = ReplayData(X, y, filename=filename, group_name=group_name, model=model) + +# Now we feed the data to the actual Replay object so we can build the visualizations +replay = Replay(replay_filename=filename, group_name=group_name) + +# Using subplot2grid to assemble a complex figure... +fig = plt.figure(figsize=(12, 6)) +ax_zvalues = plt.subplot2grid((2, 2), (0, 0)) +ax_weights = plt.subplot2grid((2, 2), (0, 1)) +ax_activations = plt.subplot2grid((2, 2), (1, 0)) +ax_gradients = plt.subplot2grid((2, 2), (1, 1)) + +wv = replay.build_weights(ax_weights) +gv = replay.build_gradients(ax_gradients) +# Z-values +zv = replay.build_outputs(ax_zvalues, before_activation=True, + exclude_outputs=True, include_inputs=False) +# Activations +av = replay.build_outputs(ax_activations, exclude_outputs=True, include_inputs=False) + +# Finally, we use compose_plots to update all +# visualizations at once +fig = compose_plots([zv, wv, av, gv], + epoch=0, title=title) +fig.savefig('part2.png', format='png', dpi=120) \ No newline at end of file diff --git a/tests/rawdata/hyperparms_in_action.h5 b/tests/rawdata/hyperparms_in_action.h5 index 2ad88c4cf25e906fddb30f01a722797b92a33ba5..b979796a7f09354cfda1f5d38b7e02075c3d0baa 100644 GIT binary patch delta 1087 zcmZXQdrXsO7{v=k3I!dv(1PQR86r1jU<%*&Jp~+P8OUW0klTuYI)sUeT9Ntbz~Ti2 zRwPD^>&60RGa>?A3g7#_m61&dAQQGu$Y9(IR2NVt1`(Y&_m}6dC%COvTxbf-)+Mnl3 zDm~)p=reMhv^;^R1&vgFQMEp5 z!I7eA^mOJNG1K@mw=@yBTxmd^@;Xw&$(i}qU|irbi&iVz$md5Z81*r4thSj)A=mB_ zXTuGqP~wa`bqff`KO~QR{$R?ESYh*+C3HZ>*Oe)ZV6>Ny_ZstX@-{17Q<50!G>=eg zv@veE|E@05l?{LgrGKIhy$Q}6@X|e!M8RRtKD4lGisR=4byco%P!9ftcuF&z7{ch* zBpiks1wnz|nd6{KGF@7W5(>l^1=k93j3iDsu%Q}wZ=~sFwPw^IZHngmwk(-SE6~rj zwFuPO%G=s@%K~ADKl7q}E$dkmQsFD&@}i!EWJeTpF{8@R9V2S)gO7g;RVQ5JQVY$g z=La27;I;}XbNV7@H-DV^+cASXvHc45(6g4C{Aid;iG9iaA{5hXmk3=o1kt&xY2-Q= zPv>unL|&aK^r3~%P>FXAJ#ILKs+O|p5hX?&Zz||sdq)5UT#*+I+1Eal&jxI z(i=BR?{t=u+Kw&K{nx6w!^am4ne-K*w!KwxyDDL$J2e}u-iPu5Y z4{lIf<^;aWVz6ZFfOqO07!X=O>y#-t`|yGP>r3W*)HCM5ZX*!gTLTUz_HbQh4R@Y; z!0@6UxSZSor^5n(7H)>IqW9recNkbDOWeR?SOP&AA<*_A1#B+$D$Ib|5kOxjh3rs( Fe*w=I`4<2H delta 1087 zcmZXQdrXsO7{$2>D_A!!3bF=7k&E1e+}eKMQ-Kk#MuIX2G%{~P0tmP`G(e%mz_74E z$Ba%Us6lNYv;tz`d*8Rfuw2SS#S1qnIHO$0Krw;9Hk|(~p1+>+J9%=F(|AX}@s7Sc zo24I|XxAWO9T2S_QV}adD(E+p#(jU*=wBbNqmxyp$a#;Fi$6kV&;JjpO061xlo$kl z_&ivq5K6Th3>(&df3Ew!W~{$S|D$l=zi_=;R1_6~3$oRyVE7F#Ib6#3+82Qxh#D!n z-r?w=EBv0WaLhETk@}1-5pV9~n>dBzQ^sAW;;}J#JYYmU(hI{p=t57FR^+!G0aX1+ z2o6f?LQ3^6!r7Tm{TUUE!>YQFlXwrA3sF#Exq$P}b)z7ukeGMQh^S?`KUOFAqLVGT zB-flnAM^9V({vwlI#y01XTxcs#sS-Z`T)7|ZW5U=k4|alU=!{D5}zVuv8S4T6k>+$ ze4e7b;%CGoTupDYHpFGR!>FKcf%tUK(Z@k+++mbh8%a?|y;)&N5dS zJQO*=#9j@0{_GX9)7q-;2?5YO1})=h#FeZtcCx&#cihsW0@YsA7GT0oy5-))x6s(3E7OE!vJl zskzf!^hv-0>gwyS(3snoR7Za$>Y!4nKE30NC{;1uYQ6OmwY+kaX?|%#r*l{+!!MiG z84pIlyGdW|zJY9JM`mP93Ff#<;X-8# z7N^KTo7#^Tt2Hq9$2&Yc_!>OgEJ$pXp5StmCmD}76&%WsAVRIJNN}U*I4O&B7i{f;fLOU-I8<+ipfz@kdebF?2*Wkz+7G05HZZO- zk7v}~{b*Xs?&ndrcW-{kyMcXjfld$DivK#fEMP{yp)VVlvEQ7R70ftq(+U=uyk4$z g^8@D=4v^^N|2oZ+U-`@ev)9WNPyXfG1r=`u0P%lWQUCw| delta 207 zcmeB}$JjBCaYGXav!R~Z=2i|@7BJ&1&wW8=mjuPl>*Wrz1{el-?^D}LS_TN{nmW&7Tea%;R8jVwcUz~L>r1fj)Oa$< zd`+g|?%MjK-5cU=?cV&5cLV$60-YYP75{Z|S-^~ZLti#9W4}2sE0}TKrWGtQdA(fc g<_FF#93auj|8<%tzw(&{X0Mkkp8U(V3o70S04Z%#wg3PC diff --git a/tests/rawdata/part1_activation_functions_epoch02.h5 b/tests/rawdata/part1_activation_functions_epoch02.h5 index e07753ba5690e4864b76243fe12ac258a4b72ec9..d21d0c3f2955fe0e81e49cc177f00862d056d9d3 100644 GIT binary patch delta 207 zcmeB}$JjBCaYGXavyq;`=2i|@7BJ&1&wWAW-d8G{*UKGb4G@e-+o#0-cyElO@cyY{ z_iUoJt=g6BK6&TM6jQrLPbBy5{baQF7Xt+B_OmfJZ+f=bv3~kaeTG^KwP2~8E0~xq zHZTS3+V%O>E~f7(yEi}N-M~J%K&J<6#ebb#7BC~<(3cI&*l*6u3TB+QX$6Z+UN6_V g`GIo_2S{}Cf1T#ZuY6{K+3V$sC;#&8f{Hf+05winNdN!< delta 207 zcmeB}$JjBCaYGXav!R~Z=2i|@7BJ&1&wW8=mjuPl>*Wrz2An$KwQtY%<9m-+GVX5@ zJ!&feg}XOD*Wrz25@V5?JMRwv3KfKuKic9 zu-ksxetK7?`2C%Q<~4Q<4_)@2$w=N?&d2};(PrB$G-W1lpLA;X&fVYoECOa`?QG{W zv^X3;aTinX?_Ehg1-myt;2sOg^>Xa_^!EGMrhgYSh2-nSE8kZWmV7p?VooZ zF#BCR(duLP^qmG$T{|~F*Wrz23%(H+-GfnV(+z`68n!R zTiTkRoxhujtH9FUdWGGhlgszc`*3IP7e)p!nB$^rnRQZQhyOyYT^A?lSnB?rz5Um_ z4zmfKO;%Im>vu9RMDN`Ekaq+7>E diff --git a/tests/rawdata/part1_activation_functions_epoch05.h5 b/tests/rawdata/part1_activation_functions_epoch05.h5 index 3271e196bfcf903ac493245f271dc6f1c7157d90..8586b81d96a7627823ba723020c36ae93ac1591f 100644 GIT binary patch delta 207 zcmeB}$JjBCaYGXavyq;`=2i|@7BJ&1&wWAW-d8G{*UKGb4e;qo*cbly@m|e2j{CXq zO|aFs`?p(KWr~$z-ETWHhOc`!{gT~R$ix5!g0aq4yBocB>c#W!njXk&dA)XlrR>Ws zJG6CwTW#x~X%#GIZN2#+?*{hC1v))oEB@=`vVa-+hQ4fI#(r~NRxsndO)FSr@_M<> g%@3SgI6$J4|LZhQe&sU@%w8{7Jo%Sz7gW3v0FCHYZ~y=R delta 207 zcmeB}$JjBCaYGXav!R~Z=2i|@7BJ&1&wW8=*967Q>*Wrz2CTp1w$C~5#NM69wf4)X zHrh69xVih{CUq;hw|DLK1mD^#IEi~-Ark`_sPXApnXWL|xjW&{&R6sPS^O5BV#&OB z@ecVf->lR*7g%v0vbEm)kaq+78F~9l1ycJgC2jY;V`2b g%@3SgI6$J4|LZhQe&sU@%w8{7Jo%Sz7gW3v0PBEP3;+NC delta 207 zcmeB}$JjBCaYGXav!R~Z=2i|@7BJ&1&wW8=*967Q>*Wrz2E6xm+2^_7#9oai$Nlfz zHrl@JVBFJ`FwN>~JFk6=6X(A6a9rdv%}P#&R^zO{CKLoi!)rq@~Nbo)#^o_ zJJZc7tWO>?wLWubhxO)%yc^gj7wGhWt@y8#%K~QP8~U<=8T-w7S;36+HmzWh$?N4h gH$QN0;Q)zF{;$(K`IXNsFnhgR@#J5=T~P5x00K`@kpKVy diff --git a/tests/rawdata/part1_activation_functions_epoch07.h5 b/tests/rawdata/part1_activation_functions_epoch07.h5 index 611601af647d00e877bed1506a58c6b2b2e0f6d9..6bb5946df3293d982e781589b442a44492f6ed08 100644 GIT binary patch delta 207 zcmeB}$JjBCaYGXavyq;`=2i|@7BJ&1&wWAWzE>)n*UKGb4aoHi+xKPJ)4iK_R_;G( z$zqq{T(igbew+2OJx=yll>GKZS5@rGW@Z2b?M6<^$IQP?3tm3mCC*Wrz26)!l?`v#5xz~Ae?0(^+ zZ*14BiP)n*UKGb4S1O9yU$Jh#oq5P`}T8e z)3r;ww_uNJ@KtN3o!R#8GaC0LKAy7g1v3K}%;Dy-UKysbYj-l+uKmBnEN89kFgshj zcRSy^*_Q6Q89Rf`R#|R-$h(1ka)C|{*oyx;xh!BtzM(H0n6clSmle!7Z_^4EnY>=E hbMph|77mc;*Wrz2K?G!vF~`+>Am^8>h{+O z%h-wC>DzOOeSvlGhd_JvzZv^h6}9er!OQ>#%=`XYJM)Vd>qOwsXPpddtlZc{i|6F3{-#Tk&5fmj%qoH}qu#GxnSFvVs}sZCb%1lh@02 gZhqk0!T}PU{9mVe@++TNVD@^s;>o{!yP)EY053*U#Q*>R diff --git a/tests/rawdata/part1_activation_functions_epoch09.h5 b/tests/rawdata/part1_activation_functions_epoch09.h5 index d06ac36fd56949d0cb526ab912ac4b38aff0c2b6..6eea3591733cb650116f7524928577598c993735 100644 GIT binary patch delta 207 zcmeB}$JjBCaYGXavyq;`=2i|@7BJ&1&wWAWzE>)n*UKGb4LG>Xd0$W4tG!1O7VmG~ zlVG6Zc^}Y5jPILBc^Ix_vm4yKeL|^{4uz#e!qj{VAu5h_bOBdCio%sh_ zEOzNLSh;_d*mYPibl2vGyc^gj7wGhWt@y8#%K~QP8~U<=8T-w7S;36+HmzWh$?N4h gH$QN0;Q)zF{;$(K`IXNsFnhgR@#J5=T~P5x01!J_E&u=k delta 207 zcmeB}$JjBCaYGXav!R~Z=2i|@7BJ&1&wW8=*967Q>*Wrz2BexB?t5)_cCSY2r2Rf` zz3i6#?%uPZ{=7Bs=~{ci?LGUfIH&GQWnln=-Hq=oWYr{h6rZx#^*JNV(mFkEr*&tT z#e13emYy+OyV~d3@7ny3cLV$60-YYP75{Z|S-^~ZLti#9W4}2sE0}TKrWGtQdA(fc g<_FF#93auj|8<%tzw(&{X0Mkkp8U(V3o70S0J88<%>V!Z diff --git a/tests/rawdata/part1_activation_functions_epoch10.h5 b/tests/rawdata/part1_activation_functions_epoch10.h5 index 020f0eb10ad3bbe7c82c8e903a4f6d77b056ff68..380ebe275e484294cb20b83786068357ce89b063 100644 GIT binary patch delta 207 zcmeB}$JjBCaYGXavyq;`=2i|@7BJ&1&wWAWzE>)n*UKGb4fwp(Y+r%=`@OssyY}Z* z_uBm_mE3C_(rZ(3WU>9TX=nEF)!*9ph=lCMey9*dG{al?&VGaR<{`E?OeVt z)AH)44OZ%FKJH>WA-;R_L*5PSlM8fuz*hX%$z=gE@(q32z>NLoysTixd7D*Wrz2E?9G+o!^Raj%Z`iv6o% z8|)Uw^X~PM$*_^pn`}RQ?yh}K`;YH?#KHgu%bK=XeV0AD^S|)lU3=r4t;D}A-?^(G z(sKI!*;X$mzS$N3LUQ-!hrAouCl~1SfUWqilgk2TzzFko9MgX^UQ~>}0 diff --git a/tests/rawdata/part1_activation_functions_epoch11.h5 b/tests/rawdata/part1_activation_functions_epoch11.h5 index 75c0fad9858d36cf3a5e2ba92ccf73fae5e3e590..eac92cf868b96dc576ea5de5507a7e45e106cbc1 100644 GIT binary patch delta 207 zcmeB}$JjBCaYGXavyq;`=2i|@7BJ&1&wWAWzE>)n*UKGb4Omd8wQqsY=e<^IF720l zwZSg&#jHIdpBrrK3l7<5F+SaQoBjH}cvc25_?D(@CC`$+b8_W_U5)wTR?+Xwtae`w z+*Wrz2IL%)*~e{lWp7jQ{{5d% zEwqbf>EHA3ptns^&IbDjxfk}G@;tOJo|OR%e9lT*8L@}$Y)n*UKGb4Jfgc+jr!^x4ruU-|YXS zbHnbfRNS7%$ayxKuiv*X5o6zf`{(O@cUT#~;LA3xo$^l%EgMd2?0zgc!ODA?!LH*+ zo?435KeOUqtFyb~T-xr<4|zARPcG2uVeSW7@n0vG1*Wrz23&R%+NWrGeQ!h8js5yw z$L-E84A>Ja(qQv>;TijPPrmQ7T6=BZ9aaV~P!V+Bxod)!WiqSg?)%zZR@>hi@3Mb- z-EzU^t5$pcb#^mM%i6v9A@2tE$ptz+U@QLX;TR1?XlmF{9Pk!Yy3(Q_GS3LQbZx>X&5da#WQ^5cL diff --git a/tests/rawdata/part1_activation_functions_epoch13.h5 b/tests/rawdata/part1_activation_functions_epoch13.h5 index 1757c592ff7795cc3fec0e3c4cb96c0e72a8f823..d371c19e3e0ba726083f6fab10c37ea6186276bf 100644 GIT binary patch delta 207 zcmeB}$JjBCaYGXavyq;`=2i|@7BJ&1&wWAW{#PoS*UKGb4QQ_v*vIMndvCEj^MUpX zM*BA^)Al^^zHJkK;*WiAv%>xeakl-DYz$x!Kl`Os^LD;nn_c*K%WloF5;)^#%`u~M zSBBX^>qmKN)(Tm-tT#X8-M~J%K&J<6#ebb#7BC~<(3cI&*l*6u3TB+QX$6Z+UN6_V g`GIo_2S{}Cf1T#ZuY6{K+3V$sC;#&8f{Hf+05>UC0RR91 delta 207 zcmeB}$JjBCaYGXav!R~Z=2i|@7BJ&1&wW8=w**Wrz1}I3e?VHGWdvEjFPy0uw&0kza2KWZ@#eCKPa+4aniSak!%cLaKiV7)sNLIyEeb#+by1&W_3%_$-2+5 zb=UN+Bi1*c8CqYGePX@&A@2tE$ptz+U@QLX;TR1?XlmF{9Pk!Yy3(Q_GS3LQbZx>X&5de-sRD%Ej diff --git a/tests/rawdata/part1_activation_functions_epoch14.h5 b/tests/rawdata/part1_activation_functions_epoch14.h5 index 25e901c1a047d376b0149bab06c22a40de1881b9..89aa8b5ab7bf883dede829a35a9bc3fe0d8792e9 100644 GIT binary patch delta 207 zcmeB}$JjBCaYGXavyq;`=2i|@7BJ&1&wWAW{#PoS*UKGb4Y+@nd0##g!@h_Rfdd=E zCG4a8|L-}@C1Il+FJ9W{LgR*ciZo@!xtYfqgu?+;{)kwX>o{!yP)EY07bc3MF0Q* delta 207 zcmeB}$JjBCaYGXav!R~Z=2i|@7BJ&1&wW8=w**Wrz2K3(jvUk0~{k<{0YzOv# z<+CqZ_Fzx$V`kg11xyZewi@iWTEe^k8XE%`+_ze5HEllUuFX1ZyS1kGTdBw1wpv)S zZ0GDBIo3|Qig#9r9<$o~kaq+7 g%@3SgI6$J4|LZhQe&sU@%w8{7Jo%Sz7gW3v0AeCmlmGw# delta 207 zcmeB}$JjBCaYGXav!R~Z=2i|@7BJ&1&wW8=w**Wrz2K0$O*xP9Gc<=uT;R7s} z)a-q`nD-XiE7?Ab7IcVj^xm)XP-=fLI|CRPU;1Yy(dfGCeBtF?D~|eDZ5E5OerdaD zSAozvYd#A@Yp?oy)|(&lZeX8Wpwk1k;=fKV3z(5_=*tFX>^J9S1vAduw1Pz@ub1oG g{J^<|10*{6zfSYyS3a}A?DcZRlYar-0}*co0G|I&)&Kwi diff --git a/tests/rawdata/part1_activation_functions_epoch16.h5 b/tests/rawdata/part1_activation_functions_epoch16.h5 index 4e7660942bb306735ee6b1c9f08df885836219b9..53464dcc9a0b0fed81b1a6386d45f2bf093390df 100644 GIT binary patch delta 207 zcmeB}$JjBCaYGXavyq;`=2i|@7BJ&1&wWAW{#PoS*UKGb4S3yhckiL=Ec0zJ6Yr9wBL!j-t1)2_fmKE;j2r}4zft>*igg?JAm;S11$+|^sxB2`&t2uR>ckfje zu)h87{jSGRdw2UY%k0_wkaq+7*Wrz20ZdRv-jSO=X(_&$R9Ws zYj4l_Kz^^qBRkvJ2l5WfJJR+u%v9Tdft>*iTrWM@URkDT>F{4-cl5+QE2FyAyT7w= zS?^u*c~^h;-rd*hWcF-+$h(1ka)C|{*oyx;xh!BtzM(H0n6clSmle!7Z_^4EnY>=E hbMph|77mc;ENYo{dy};JRzJ6ySZ{vFyMcXjfld$DivK#fEMP{yp)VVlvEQ7R70ftq(+U=uyk4$z g^8@D=4v^^N|2oZ+U-`@ev)9WNPyXfG1r=`u0LQ&o7XSbN delta 207 zcmeB}$JjBCaYGXav!R~Z=2i|@7BJ&1&wW8=_XNew>*Wrz2JAhyXYa3lulHK@Xdh@| zi?HwEk>9Jd&%<_Nl%9i_TmAl3zjXHdaxj3w28jojy8f(|+Fy?DjZ0~ zUEz;;tiSFrvg)o8qjv-@Lq;Cj(uTPb_e)^ z%I(X(W$ta(O0}JK$i<;(&4m3Tr8fIdaWH@Z`yUCbM~>5X&UrI+*PZL0R%`tDcmH0{ zVa0BozVpR#%iW)!x9r~hkaq+7*Wrz2K;)pac^DHyS)YJW(Vqw z3+&&OS?%Sp47Hv6)Y_p#vuFR*PsaOCaWH^^&s`C#4K-7C7Jpu_OVlXX>f{Q(-E$MW ztP(1VciJ*K?yhR=+P(Q9?*{hC1v))oEB@=`vVa-+hQ4fI#(r~NRxsndO)FSr@_M<> g%@3SgI6$J4|LZhQe&sU@%w8{7Jo%Sz7gW3v0E)y?V*mgE diff --git a/tests/rawdata/part1_activation_functions_epoch19.h5 b/tests/rawdata/part1_activation_functions_epoch19.h5 index a928f360c3fa64ad32ba015eea26d134fccdc173..712050cb421a3f4f306ef9395b0aa6502b328f70 100644 GIT binary patch delta 207 zcmeB}$JjBCaYGXavyq;`=2i|@7BJ&1&wWAW39nQ(ua`T>8Zf0||6b!Iocl~?yB~J0}Aey#K7fOXnH8)vhydc5-W-whVaSZOv5m zXxFR>CD!#E7S{HfN3AzM*Wrz25eZhX0L(vhrQ8J4hK$d zZMI+ZB69D2-EdozogNMzN2l-4^R?LT&dC48c-myZ?D-ju6@3i0S7K? zPqe?+wr{WDrV85^j4=)`C6@2caChH-gp&abj_OtK`uah{^7j>n-8zqZteCa?b{TJA zw_;YfVAWj6v^#d)-(8y@@@`*Wrz2Am05wYRy2qyy=tT>pytK!2)3*`*v-EUt`u$uq4d6&5E zKg+uoH?7JKG4A%L_`GZLL*5PSlM8fuz*hX%$z=gE@(q32z>NLoysTixd7DwZH*UKGb<-aVSzEASxqrIo}xc6_p zw{Ua4N-YOi_P^J9S1vAduw1Pz@ub1oG{J^;dtYPwho#x4}d}e`E R040hi|MKkuv;XTf0st+&J5>Mx delta 138 zcmeB}$JjBCaYGXav!R~Z=2i|@7BJ&1&wW8=mjuPl>*Wrz@^|=p?`y6(zPI`ImwmlS zeVgl5YB|8N|8;U%z>IuDUp6pfzd0`}m~q~w6)ZA&ydM0L?fwLI3~& From 916db2e7034f6ec49bdb9a95bea094dd96ae6526 Mon Sep 17 00:00:00 2001 From: dvgodoy Date: Sat, 16 Jun 2018 14:17:44 +0200 Subject: [PATCH 12/13] test --- tests/deepreplay/test_callbacks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/deepreplay/test_callbacks.py b/tests/deepreplay/test_callbacks.py index 8c44288..fd3b1c8 100644 --- a/tests/deepreplay/test_callbacks.py +++ b/tests/deepreplay/test_callbacks.py @@ -71,6 +71,8 @@ def test_weights(replay_data, training_data): layers = filter(lambda key: 'layer' in key, replay_data.keys()) for layer in layers: for weight in replay_data[layer].keys(): + print(replay_data[layer][weight]) + print(training_data[layer][weight]) npt.assert_allclose(replay_data[layer][weight], training_data[layer][weight], atol=1e-3) def test_metrics(replay_data, training_data, model_data): From 62c4446a8393f61b08c8a977c991658819e1691a Mon Sep 17 00:00:00 2001 From: dvgodoy Date: Sat, 16 Jun 2018 14:25:58 +0200 Subject: [PATCH 13/13] test --- tests/deepreplay/test_callbacks.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/deepreplay/test_callbacks.py b/tests/deepreplay/test_callbacks.py index fd3b1c8..8c44288 100644 --- a/tests/deepreplay/test_callbacks.py +++ b/tests/deepreplay/test_callbacks.py @@ -71,8 +71,6 @@ def test_weights(replay_data, training_data): layers = filter(lambda key: 'layer' in key, replay_data.keys()) for layer in layers: for weight in replay_data[layer].keys(): - print(replay_data[layer][weight]) - print(training_data[layer][weight]) npt.assert_allclose(replay_data[layer][weight], training_data[layer][weight], atol=1e-3) def test_metrics(replay_data, training_data, model_data):