diff --git a/egs/wsj/s5/steps/libs/nnet3/report/log_parse.py b/egs/wsj/s5/steps/libs/nnet3/report/log_parse.py index d962e8839dd..afa75eb0296 100755 --- a/egs/wsj/s5/steps/libs/nnet3/report/log_parse.py +++ b/egs/wsj/s5/steps/libs/nnet3/report/log_parse.py @@ -34,12 +34,17 @@ "value-avg=\[.*=\((.+)\), mean=([0-9\.\-e]+), stddev=([0-9\.e\-]+)\].*", "deriv-avg=\[.*=\((.+)\), mean=([0-9\.\-e]+), stddev=([0-9\.e\-]+)\]"]) - g_normal_nonlin_regex_pattern = ''.join([".*progress.([0-9]+).log:component name=(.+) ", "type=(.*)Component,.*", "value-avg=\[.*=\((.+)\), mean=([0-9\.\-e]+), stddev=([0-9\.e\-]+)\].*", "deriv-avg=\[.*=\((.+)\), mean=([0-9\.\-e]+), stddev=([0-9\.e\-]+)\]"]) +g_normal_nonlin_regex_pattern_with_oderiv = ''.join([".*progress.([0-9]+).log:component name=(.+) ", + "type=(.*)Component,.*", + "value-avg=\[.*=\((.+)\), mean=([0-9\.\-e]+), stddev=([0-9\.e\-]+)\].*", + "deriv-avg=\[.*=\((.+)\), mean=([0-9\.\-e]+), stddev=([0-9\.e\-]+)\].*", + "oderiv-rms=\[.*=\((.+)\), mean=([0-9\.\-e]+), stddev=([0-9\.e\-]+)\]"]) + class KaldiLogParseException(Exception): """ An Exception class that throws an error when there is an issue in parsing the log files. Extend this class if more granularity is needed. @@ -54,10 +59,12 @@ def __init__(self, message = None): # This function is used to fill stats_per_component_per_iter table with the # results of regular expression. + def fill_nonlin_stats_table_with_regex_result(groups, gate_index, stats_table): iteration = int(groups[0]) component_name = groups[1] component_type = groups[2] + # for value-avg value_percentiles = groups[3+gate_index*6] value_mean = float(groups[4+gate_index*6]) value_stddev = float(groups[5+gate_index*6]) @@ -66,6 +73,7 @@ def fill_nonlin_stats_table_with_regex_result(groups, gate_index, stats_table): value_5th = float(value_percentiles_split[4]) value_50th = float(value_percentiles_split[6]) value_95th = float(value_percentiles_split[9]) + # for deriv-avg deriv_percentiles = groups[6+gate_index*6] deriv_mean = float(groups[7+gate_index*6]) deriv_stddev = float(groups[8+gate_index*6]) @@ -74,29 +82,68 @@ def fill_nonlin_stats_table_with_regex_result(groups, gate_index, stats_table): deriv_5th = float(deriv_percentiles_split[4]) deriv_50th = float(deriv_percentiles_split[6]) deriv_95th = float(deriv_percentiles_split[9]) - try: - if stats_table[component_name]['stats'].has_key(iteration): - stats_table[component_name]['stats'][iteration].extend( - [value_mean, value_stddev, - deriv_mean, deriv_stddev, - value_5th, value_50th, value_95th, - deriv_5th, deriv_50th, deriv_95th]) - else: - stats_table[component_name]['stats'][iteration] = [ - value_mean, value_stddev, - deriv_mean, deriv_stddev, - value_5th, value_50th, value_95th, - deriv_5th, deriv_50th, deriv_95th] - except KeyError: - stats_table[component_name] = {} - stats_table[component_name]['type'] = component_type - stats_table[component_name]['stats'] = {} - stats_table[component_name][ - 'stats'][iteration] = [value_mean, value_stddev, - deriv_mean, deriv_stddev, - value_5th, value_50th, value_95th, - deriv_5th, deriv_50th, deriv_95th] + if len(groups) <= 9: + try: + if stats_table[component_name]['stats'].has_key(iteration): + stats_table[component_name]['stats'][iteration].extend( + [value_mean, value_stddev, + deriv_mean, deriv_stddev, + value_5th, value_50th, value_95th, + deriv_5th, deriv_50th, deriv_95th]) + else: + stats_table[component_name]['stats'][iteration] = [ + value_mean, value_stddev, + deriv_mean, deriv_stddev, + value_5th, value_50th, value_95th, + deriv_5th, deriv_50th, deriv_95th] + except KeyError: + stats_table[component_name] = {} + stats_table[component_name]['type'] = component_type + stats_table[component_name]['stats'] = {} + stats_table[component_name][ + 'stats'][iteration] = [value_mean, value_stddev, + deriv_mean, deriv_stddev, + value_5th, value_50th, value_95th, + deriv_5th, deriv_50th, deriv_95th] + else: + #for oderiv-rms + oderiv_percentiles = groups[9+gate_index*6] + oderiv_mean = float(groups[10+gate_index*6]) + oderiv_stddev = float(groups[11+gate_index*6]) + oderiv_percentiles_split = re.split(',| ',oderiv_percentiles) + assert len(oderiv_percentiles_split) == 13 + oderiv_5th = float(oderiv_percentiles_split[4]) + oderiv_50th = float(oderiv_percentiles_split[6]) + oderiv_95th = float(oderiv_percentiles_split[9]) + try: + if stats_table[component_name]['stats'].has_key(iteration): + stats_table[component_name]['stats'][iteration].extend( + [value_mean, value_stddev, + deriv_mean, deriv_stddev, + oderiv_mean, oderiv_stddev, + value_5th, value_50th, value_95th, + deriv_5th, deriv_50th, deriv_95th, + oderiv_5th, oderiv_50th, oderiv_95th]) + else: + stats_table[component_name]['stats'][iteration] = [ + value_mean, value_stddev, + deriv_mean, deriv_stddev, + oderiv_mean, oderiv_stddev, + value_5th, value_50th, value_95th, + deriv_5th, deriv_50th, deriv_95th, + oderiv_5th, oderiv_50th, oderiv_95th] + except KeyError: + stats_table[component_name] = {} + stats_table[component_name]['type'] = component_type + stats_table[component_name]['stats'] = {} + stats_table[component_name][ + 'stats'][iteration] = [value_mean, value_stddev, + deriv_mean, deriv_stddev, + oderiv_mean, oderiv_stddev, + value_5th, value_50th, value_95th, + deriv_5th, deriv_50th, deriv_95th, + oderiv_5th, oderiv_50th, oderiv_95th] def parse_progress_logs_for_nonlinearity_stats(exp_dir): @@ -116,11 +163,18 @@ def parse_progress_logs_for_nonlinearity_stats(exp_dir): stats_per_component_per_iter = {} progress_log_lines = common_lib.get_command_stdout( - 'grep -e "value-avg.*deriv-avg" {0}'.format(progress_log_files), + 'grep -e "value-avg.*deriv-avg.*oderiv" {0}'.format(progress_log_files), require_zero_status = False) - parse_regex = re.compile(g_normal_nonlin_regex_pattern) - + if progress_log_lines: + # cases with oderiv-rms + parse_regex = re.compile(g_normal_nonlin_regex_pattern_with_oderiv) + else: + # cases with only value-avg and deriv-avg + progress_log_lines = common_lib.get_command_stdout( + 'grep -e "value-avg.*deriv-avg" {0}'.format(progress_log_files), + require_zero_status = False) + parse_regex = re.compile(g_normal_nonlin_regex_pattern) for line in progress_log_lines.split("\n"): mat_obj = parse_regex.search(line) @@ -333,7 +387,6 @@ def get_train_times(exp_dir): train_times[iter] = max(values) return train_times - def parse_prob_logs(exp_dir, key='accuracy', output="output"): train_prob_files = "%s/log/compute_prob_train.*.log" % (exp_dir) valid_prob_files = "%s/log/compute_prob_valid.*.log" % (exp_dir) @@ -456,7 +509,6 @@ def parse_rnnlm_prob_logs(exp_dir, key='objf'): - def generate_acc_logprob_report(exp_dir, key="accuracy", output="output"): try: times = get_train_times(exp_dir) @@ -488,4 +540,4 @@ def generate_acc_logprob_report(exp_dir, key="accuracy", output="output"): total_time += times[iter] report.append("Total training time is {0}\n".format( str(datetime.timedelta(seconds=total_time)))) - return ["\n".join(report), times, data] + return ["\n".join(report), times, data] \ No newline at end of file diff --git a/egs/wsj/s5/steps/nnet3/report/generate_plots.py b/egs/wsj/s5/steps/nnet3/report/generate_plots.py index b645b5cafd0..0e336cdbc11 100755 --- a/egs/wsj/s5/steps/nnet3/report/generate_plots.py +++ b/egs/wsj/s5/steps/nnet3/report/generate_plots.py @@ -239,55 +239,114 @@ def insert_a_column_legend(legend_handle, legend_label, lp, mp, hp, # This function is used to plot a normal nonlinearity component or a gate of lstmp def plot_a_nonlin_component(fig, dirs, stat_tables_per_component_per_dir, component_name, common_prefix, prefix_length, component_type, - start_iter, gate_index=0): + start_iter, gate_index=0, with_oderiv=0): fig.clf() index = 0 legend_handle = [extra, extra, extra, extra] legend_label = ["", '5th percentile', '50th percentile', '95th percentile'] - for dir in dirs: - color_val = g_plot_colors[index] - index += 1 - try: - iter_stats = (stat_tables_per_component_per_dir[dir][component_name]) - except KeyError: - # this component is not available in this network so lets - # not just plot it + if not with_oderiv: + for dir in dirs: + color_val = g_plot_colors[index] + index += 1 + try: + iter_stats = (stat_tables_per_component_per_dir[dir][component_name]) + except KeyError: + # this component is not available in this network so lets + # not just plot it + insert_a_column_legend(legend_handle, legend_label, lp, mp, hp, + dir, prefix_length, index+1) + continue + + data = np.array(iter_stats) + data = data[data[:, 0] >= start_iter, :] + + ax = plt.subplot(211) + lp, = ax.plot(data[:, 0], data[:, gate_index*10+5], color=color_val, + linestyle='--') + mp, = ax.plot(data[:, 0], data[:, gate_index*10+6], color=color_val, + linestyle='-') + hp, = ax.plot(data[:, 0], data[:, gate_index*10+7], color=color_val, + linestyle='--') insert_a_column_legend(legend_handle, legend_label, lp, mp, hp, dir, prefix_length, index+1) - continue - data = np.array(iter_stats) - data = data[data[:, 0] >= start_iter, :] - ax = plt.subplot(211) - lp, = ax.plot(data[:, 0], data[:, gate_index*10+5], color=color_val, - linestyle='--') - mp, = ax.plot(data[:, 0], data[:, gate_index*10+6], color=color_val, - linestyle='-') - hp, = ax.plot(data[:, 0], data[:, gate_index*10+7], color=color_val, - linestyle='--') - insert_a_column_legend(legend_handle, legend_label, lp, mp, hp, - dir, prefix_length, index+1) - - ax.set_ylabel('Value-{0}'.format(component_type)) - ax.grid(True) - - ax = plt.subplot(212) - lp, = ax.plot(data[:, 0], data[:, gate_index*10+8], color=color_val, - linestyle='--') - mp, = ax.plot(data[:, 0], data[:, gate_index*10+9], color=color_val, - linestyle='-') - hp, = ax.plot(data[:, 0], data[:, gate_index*10+10], color=color_val, - linestyle='--') - ax.set_xlabel('Iteration') - ax.set_ylabel('Derivative-{0}'.format(component_type)) - ax.grid(True) - - lgd = plt.legend(legend_handle, legend_label, loc='lower center', - bbox_to_anchor=(0.5 , -0.5 + len(dirs) * -0.2), - ncol=4, handletextpad = -2, title="[1]:{0}".format(common_prefix), - borderaxespad=0.) - plt.grid(True) + ax.set_ylabel('Value-{0}'.format(component_type)) + ax.grid(True) + + ax = plt.subplot(212) + lp, = ax.plot(data[:, 0], data[:, gate_index*10+8], color=color_val, + linestyle='--') + mp, = ax.plot(data[:, 0], data[:, gate_index*10+9], color=color_val, + linestyle='-') + hp, = ax.plot(data[:, 0], data[:, gate_index*10+10], color=color_val, + linestyle='--') + ax.set_xlabel('Iteration') + ax.set_ylabel('Derivative-{0}'.format(component_type)) + ax.grid(True) + + lgd = plt.legend(legend_handle, legend_label, loc='lower center', + bbox_to_anchor=(0.5 , -0.5 + len(dirs) * -0.2), + ncol=4, handletextpad = -2, title="[1]:{0}".format(common_prefix), + borderaxespad=0.) + plt.grid(True) + + else: + for dir in dirs: + color_val = g_plot_colors[index] + index += 1 + try: + iter_stats = (stat_tables_per_component_per_dir[dir][component_name]) + except KeyError: + # this component is not available in this network so lets + # not just plot it + insert_a_column_legend(legend_handle, legend_label, lp, mp, hp, + dir, prefix_length, index+1) + continue + + data = np.array(iter_stats) + data = data[data[:, 0] >= start_iter, :] + ax = plt.subplot(311) + lp, = ax.plot(data[:, 0], data[:, gate_index*10+7], color=color_val, + linestyle='--') + mp, = ax.plot(data[:, 0], data[:, gate_index*10+8], color=color_val, + linestyle='-') + hp, = ax.plot(data[:, 0], data[:, gate_index*10+9], color=color_val, + linestyle='--') + insert_a_column_legend(legend_handle, legend_label, lp, mp, hp, + dir, prefix_length, index+1) + + ax.set_ylabel('Value-{0}'.format(component_type)) + ax.grid(True) + + ax = plt.subplot(312) + lp, = ax.plot(data[:, 0], data[:, gate_index*10+10], color=color_val, + linestyle='--') + mp, = ax.plot(data[:, 0], data[:, gate_index*10+11], color=color_val, + linestyle='-') + hp, = ax.plot(data[:, 0], data[:, gate_index*10+12], color=color_val, + linestyle='--') + ax.set_ylabel('Derivative-{0}'.format(component_type)) + ax.grid(True) + + ax = plt.subplot(313) + lp, = ax.plot(data[:, 0], data[:, gate_index*10+13], color=color_val, + linestyle='--') + mp, = ax.plot(data[:, 0], data[:, gate_index*10+14], color=color_val, + linestyle='-') + hp, = ax.plot(data[:, 0], data[:, gate_index*10+15], color=color_val, + linestyle='--') + ax.set_xlabel('Iteration') + ax.set_ylabel('Oderivative-{0}'.format(component_type)) + ax.grid(True) + + plt.subplots_adjust(top=0.8, hspace = 1.0, bottom = -0.2) + lgd = plt.legend(legend_handle, legend_label, loc='lower center', + bbox_to_anchor=(0.5 , -1.5 + len(dirs) * -0.2), + ncol=4, handletextpad = -2, title="[1]:{0}".format(common_prefix), + borderaxespad=0.) + plt.grid(True) + return lgd @@ -308,6 +367,7 @@ def generate_nonlin_stats_plots(exp_dir, output_dir, plot, comparison_dir=None, dirs = [exp_dir] + comparison_dir index = 0 stats_per_dir = {} + with_oderiv = 0 for dir in dirs: stats_per_component_per_iter = ( @@ -316,9 +376,11 @@ def generate_nonlin_stats_plots(exp_dir, output_dir, plot, comparison_dir=None, if len(stats_per_component_per_iter[key]['stats']) == 0: logger.warning("Couldn't find any rows for the" "nonlin stats plot, not generating it") + stats_per_dir[dir] = stats_per_component_per_iter # convert the nonlin stats into tables stat_tables_per_component_per_dir = {} + for dir in dirs: stats_per_component_per_iter = stats_per_dir[dir] component_names = stats_per_component_per_iter.keys() @@ -334,13 +396,23 @@ def generate_nonlin_stats_plots(exp_dir, output_dir, plot, comparison_dir=None, iter_stats.append([iter] + comp_stats[iter]) stat_tables_per_component[component_name] = iter_stats stat_tables_per_component_per_dir[dir] = stat_tables_per_component - + if len(comp_stats[iter]) == 15: + with_oderiv = 1 main_stat_tables = stat_tables_per_component_per_dir[exp_dir] + for component_name in main_stat_tables.keys(): # this is the main experiment directory with open("{dir}/nonlinstats_{comp_name}.log".format( dir=output_dir, comp_name=component_name), "w") as f: - f.write("Iteration\tValueMean\tValueStddev\tDerivMean\tDerivStddev\t" + if with_oderiv: + # with oderiv-rms + f.write("Iteration\tValueMean\tValueStddev\tDerivMean\tDerivStddev\tOderivMean\tOderivStddev\t" + "Value_5th\tValue_50th\tValue_95th\t" + "Deriv_5th\tDeriv_50th\tDeriv_95th\t" + "Oderiv_5th\tOderiv_50th\tOderiv_95th\n") + else: + # without oderiv-rms + f.write("Iteration\tValueMean\tValueStddev\tDerivMean\tDerivStddev\t" "Value_5th\tValue_50th\tValue_95th\t" "Deriv_5th\tDeriv_50th\tDeriv_95th\n") iter_stat_report = [] @@ -378,7 +450,7 @@ def generate_nonlin_stats_plots(exp_dir, output_dir, plot, comparison_dir=None, component_type = 'Lstm-' + g_lstm_gate[i] lgd = plot_a_nonlin_component(fig, dirs, stat_tables_per_component_per_dir, component_name, - common_prefix, prefix_length, component_type, start_iter, i) + common_prefix, prefix_length, component_type, start_iter, i, with_oderiv) fig.suptitle("Per-dimension average-(value, derivative) percentiles for " "{component_name}-{gate}".format(component_name=component_name, gate=g_lstm_gate[i])) comp_name = latex_compliant_name(component_name) @@ -395,8 +467,12 @@ def generate_nonlin_stats_plots(exp_dir, output_dir, plot, comparison_dir=None, component_type = stats_per_dir[exp_dir][component_name]['type'] lgd = plot_a_nonlin_component(fig, dirs, stat_tables_per_component_per_dir,component_name, - common_prefix, prefix_length, component_type, start_iter, 0) - fig.suptitle("Per-dimension average-(value, derivative) percentiles for " + common_prefix, prefix_length, component_type, start_iter, 0, with_oderiv) + if with_oderiv: + fig.suptitle("Per-dimension average-(value, derivative) and rms-oderivative percentiles for " + "{component_name}".format(component_name=component_name)) + else: + fig.suptitle("Per-dimension average-(value, derivative) percentiles for " "{component_name}".format(component_name=component_name)) comp_name = latex_compliant_name(component_name) figfile_name = '{dir}/nonlinstats_{comp_name}.pdf'.format( @@ -404,10 +480,16 @@ def generate_nonlin_stats_plots(exp_dir, output_dir, plot, comparison_dir=None, fig.savefig(figfile_name, bbox_extra_artists=(lgd,), bbox_inches='tight') if latex_report is not None: - latex_report.add_figure( - figfile_name, - "Per-dimension average-(value, derivative) percentiles for " - "{0}".format(component_name)) + if with_oderiv: + latex_report.add_figure( + figfile_name, + "Per-dimension average-(value, derivative) and rms-oderivative percentiles for " + "{0}".format(component_name)) + else: + latex_report.add_figure( + figfile_name, + "Per-dimension average-(value, derivative) percentiles for " + "{0}".format(component_name))