Adding dropout schedule option to nnet3 #1248
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,236 @@ | ||
|
|
||
|
|
||
| # Copyright 2016 Vimal Manohar | ||
| # Apache 2.0 | ||
|
|
||
| """This module contains methods related to scheduling dropout. | ||
| """ | ||
|
|
||
| import logging | ||
|
|
||
| logger = logging.getLogger(__name__) | ||
| logger.addHandler(logging.NullHandler()) | ||
|
|
||
|
|
||
| def _parse_dropout_option(dropout_option): | ||
| """Parses the string option to --trainer.dropout-schedule and | ||
| returns a list of dropout schedules for different component name patterns. | ||
| Calls _parse_dropout_string() function for each component name pattern | ||
| in the option. | ||
|
|
||
| Arguments: | ||
| dropout_option: The string option passed to --trainer.dropout-schedule. | ||
| See its help for details. | ||
| num_archive_to_process: See _parse_dropout_string() for details. | ||
|
|
||
| Returns a list of (component_name, dropout_schedule) tuples, | ||
| where dropout_schedule is itself a list of | ||
| (data_fraction, dropout_proportion) tuples. | ||
| A data fraction of 0 corresponds to beginning of training | ||
| and 1 corresponds to all data. | ||
| """ | ||
| components = dropout_option.strip().split(' ') | ||
| dropout_schedule = [] | ||
| for component in components: | ||
| parts = component.split('=') | ||
|
|
||
| if len(parts) == 2: | ||
| component_name = parts[0] | ||
| this_dropout_str = parts[1] | ||
| elif len(parts) == 1: | ||
| component_name = '*' | ||
| this_dropout_str = parts[0] | ||
| else: | ||
| raise Exception("The dropout schedule must be specified in the " | ||
| "format 'pattern1=func1 patter2=func2' where " | ||
| "the pattern can be omitted for a global function " | ||
| "for all components.\n" | ||
| "Got {0} in {1}".format(component, dropout_option)) | ||
|
|
||
| this_dropout_values = _parse_dropout_string(this_dropout_str) | ||
| dropout_schedule.append((component_name, this_dropout_values)) | ||
|
|
||
| logger.info("Dropout schedules for component names is as follows:") | ||
| logger.info("<component-name-pattern>: [(num_archives_processed), " | ||
| "(dropout_proportion) ...]") | ||
| for name, schedule in dropout_schedule: | ||
| logger.info("{0}: {1}".format(name, schedule)) | ||
|
|
||
| return dropout_schedule | ||
|
|
||
|
|
||
| def _parse_dropout_string(dropout_str): | ||
| """Parses the dropout schedule from the string corresponding to a | ||
| single component in --trainer.dropout-schedule. | ||
| This is a module-internal function called by parse_dropout_function(). | ||
|
|
||
| Arguments: | ||
| dropout_str: Specifies dropout schedule for a particular component | ||
| name pattern. | ||
| See help for the option --trainer.dropout-schedule. | ||
|
|
||
| Returns a list of (data_fraction_processed, dropout_proportion) tuples | ||
| sorted in descending order of num_archives_processed. | ||
| A data fraction of 1 corresponds to all data. | ||
| """ | ||
| dropout_values = [] | ||
| parts = dropout_str.strip().split(',') | ||
|
|
||
| try: | ||
| if len(parts) < 2: | ||
| raise Exception("dropout proportion string must specify " | ||
| "at least the start and end dropouts") | ||
|
|
||
| # Starting dropout proportion | ||
| dropout_values.append((0, float(parts[0]))) | ||
| for i in range(1, len(parts) - 1): | ||
| value_x_pair = parts[i].split('@') | ||
| if len(value_x_pair) == 1: | ||
| # Dropout proportion at half of training | ||
| dropout_proportion = float(value_x_pair[0]) | ||
| data_fraction = 0.5 | ||
| else: | ||
| assert len(value_x_pair) == 2 | ||
|
|
||
| dropout_proportion = float(value_x_pair[0]) | ||
| data_fraction = float(value_x_pair[1]) | ||
|
|
||
| if (data_fraction < dropout_values[-1][0] | ||
| or data_fraction > 1.0): | ||
| logger.error( | ||
| "Failed while parsing value %s in dropout-schedule. " | ||
| "dropout-schedule must be in incresing " | ||
| "order of data fractions.", value_x_pair) | ||
| raise ValueError | ||
|
|
||
| dropout_values.append((data_fraction, float(dropout_proportion))) | ||
|
|
||
| dropout_values.append((1.0, float(parts[-1]))) | ||
| except Exception: | ||
| logger.error("Unable to parse dropout proportion string %s. " | ||
| "See help for option " | ||
| "--trainer.dropout-schedule.", dropout_str) | ||
| raise | ||
|
|
||
| # reverse sort so that its easy to retrieve the dropout proportion | ||
| # for a particular data fraction | ||
| dropout_values.reverse() | ||
| for data_fraction, proportion in dropout_values: | ||
| assert data_fraction <= 1.0 and data_fraction >= 0.0 | ||
| assert proportion <= 1.0 and proportion >= 0.0 | ||
|
|
||
| return dropout_values | ||
|
|
||
|
|
||
| def _get_component_dropout(dropout_schedule, data_fraction): | ||
| """Retrieve dropout proportion from schedule when data_fraction | ||
| proportion of data is seen. This value is obtained by using a | ||
| piecewise linear function on the dropout schedule. | ||
| This is a module-internal function called by _get_dropout_proportions(). | ||
|
|
||
| See help for --trainer.dropout-schedule for how the dropout value | ||
| is obtained from the options. | ||
|
|
||
| Arguments: | ||
| dropout_schedule: A list of (data_fraction, dropout_proportion) values | ||
| sorted in descending order of data_fraction. | ||
| data_fraction: The fraction of data seen until this stage of | ||
| training. | ||
| """ | ||
| if data_fraction == 0: | ||
| # Dropout at start of the iteration is in the last index of | ||
| # dropout_schedule | ||
| assert dropout_schedule[-1][0] == 0 | ||
| return dropout_schedule[-1][1] | ||
| try: | ||
| # Find lower bound of the data_fraction. This is the | ||
| # lower end of the piecewise linear function. | ||
| (dropout_schedule_index, initial_data_fraction, | ||
| initial_dropout) = next((i, tup[0], tup[1]) | ||
| for i, tup in enumerate(dropout_schedule) | ||
| if tup[0] <= data_fraction) | ||
| except StopIteration: | ||
| raise RuntimeError( | ||
| "Could not find data_fraction in dropout schedule " | ||
| "corresponding to data_fraction {0}.\n" | ||
| "Maybe something wrong with the parsed " | ||
| "dropout schedule {1}.".format(data_fraction, dropout_schedule)) | ||
|
|
||
| if dropout_schedule_index == 0: | ||
| assert dropout_schedule[0][0] == 1 and data_fraction == 1 | ||
| return dropout_schedule[0][1] | ||
|
|
||
| # The upper bound of data_fraction is at the index before the | ||
| # lower bound. | ||
| final_data_fraction, final_dropout = dropout_schedule[ | ||
| dropout_schedule_index - 1] | ||
|
|
||
| if final_data_fraction == initial_data_fraction: | ||
| assert data_fraction == initial_data_fraction | ||
| return initial_dropout | ||
|
|
||
| assert (data_fraction >= initial_data_fraction | ||
| and data_fraction < final_data_fraction) | ||
|
|
||
| return ((data_fraction - initial_data_fraction) | ||
| * (final_dropout - initial_dropout) | ||
| / (final_data_fraction - initial_data_fraction) | ||
| + initial_dropout) | ||
|
|
||
|
|
||
| def _get_dropout_proportions(dropout_schedule, data_fraction): | ||
| """Returns dropout proportions based on the dropout_schedule for the | ||
| fraction of data seen at this stage of training. | ||
| Returns None if dropout_schedule is None. | ||
|
|
||
|
Contributor
There was a problem hiding this comment. Can you please give a couple of examples of what this function might return for different inputs, covering different types of input? e.g. (and this will be wrong): IMO it's always a good idea for this type of code to give such examples, it will
Contributor
There was a problem hiding this comment. ... actually, here's an idea (this is similar to something I did in the xconfig code), and have it called directly from
Contributor
Author
There was a problem hiding this comment. Ok I added self_test. Should it be called every time the module is imported on only when run? |
||
| Calls _get_component_dropout() for the different component name patterns | ||
| in dropout_schedule. | ||
|
|
||
| Arguments: | ||
| dropout_schedule: Value for the --trainer.dropout-schedule option. | ||
| See help for --trainer.dropout-schedule. | ||
| data_fraction: The fraction of data seen until this stage of | ||
| training. | ||
| """ | ||
| if dropout_schedule is None: | ||
| return None | ||
| dropout_schedule = _parse_dropout_option(dropout_schedule) | ||
| dropout_proportions = [] | ||
| for component_name, component_dropout_schedule in dropout_schedule: | ||
| dropout_proportions.append( | ||
| (component_name, _get_component_dropout( | ||
| component_dropout_schedule, data_fraction))) | ||
| return dropout_proportions | ||
|
|
||
|
|
||
| def get_dropout_edit_string(dropout_schedule, data_fraction, iter_): | ||
| """Return an nnet3-copy --edits line to modify raw_model_string to | ||
| set dropout proportions according to dropout_proportions. | ||
|
|
||
| Arguments: | ||
| dropout_schedule: Value for the --trainer.dropout-schedule option. | ||
| See help for --trainer.dropout-schedule. | ||
|
|
||
| See ReadEditConfig() in nnet3/nnet-utils.h to see how | ||
| set-dropout-proportion directive works. | ||
| """ | ||
|
|
||
| if dropout_schedule is None: | ||
| return "" | ||
|
|
||
| dropout_proportions = _get_dropout_proportions( | ||
| dropout_schedule, data_fraction) | ||
|
|
||
| edit_config_lines = [] | ||
| dropout_info = [] | ||
|
|
||
| for component_name, dropout_proportion in dropout_proportions: | ||
| edit_config_lines.append( | ||
| "set-dropout-proportion name={0} proportion={1}".format( | ||
| component_name, dropout_proportion)) | ||
| dropout_info.append("pattern/dropout-proportion={0}/{1}".format( | ||
| component_name, dropout_proportion)) | ||
|
|
||
| logger.info("On iteration %d, %s", iter_, ', '.join(dropout_info)) | ||
| return ("""nnet3-copy --edits='{edits}' - - |""".format( | ||
| edits=";".join(edit_config_lines))) | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I think it would be better if you just imported get_dropout_edit_string, because that's the only function we need from there, and if you just import the one function it's clear that that's the only one that's the real interface. You could rename all the others with underscores at the start of their names (assuming they really are internal to the module and assuming that's what the Google style guide recommends in such circumstances).