From 98304246eb3eaddd06ad920ead17611df7087074 Mon Sep 17 00:00:00 2001 From: climbfuji Date: Tue, 13 Aug 2019 07:58:29 -0600 Subject: [PATCH 1/6] scripts/metavar.py: intent and optional are only for scheme variables, not for type definitions/module variables --- scripts/metavar.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/scripts/metavar.py b/scripts/metavar.py index 5a3b86f1..dcace0dc 100755 --- a/scripts/metavar.py +++ b/scripts/metavar.py @@ -381,8 +381,6 @@ class Var(object): default_fn_in=default_kind_val), VariableProperty('state_variable', bool, optional_in=True, default_in=False), - VariableProperty('optional', bool, - optional_in=True, default_in=False), VariableProperty('constant', bool, optional_in=True, default_in=False), VariableProperty('allocatable', bool, @@ -392,7 +390,9 @@ class Var(object): default_in='timestep')] # __var_props contains properties which are not in __spec_props - __var_props = [VariableProperty('intent', str, + __var_props = [VariableProperty('optional', bool, + optional_in=True, default_in=False), + VariableProperty('intent', str, valid_values_in=['in', 'out', 'inout'])] @@ -708,11 +708,13 @@ def print_debug(self): type = {type} * dimensions = {dimensions} * kind = {kind} * - intent = {intent} - optional = {optional} - ''' +''' + if 'intent' in self.__spec_propdict.keys(): + str += ' intent = {intent}\n' + if 'optional' in self.__spec_propdict.keys(): + str += ' optional = {optional}\n' if self._context is not None: - str = str + '\n context = {}'.format(self._context) + str += ' context = {}'.format(self._context) # End if return str.format(**self._prop_dict) From 6c26eae30f74b9a1548d316588f9c850eff8f0b8 Mon Sep 17 00:00:00 2001 From: climbfuji Date: Tue, 13 Aug 2019 07:58:59 -0600 Subject: [PATCH 2/6] scripts/metadata_table.py, scripts/metadata2html.py: recording development work for generating html table from metadata table --- scripts/metadata2html.py | 37 +++++++++++++++++++++++++++++++++++++ scripts/metadata_table.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100755 scripts/metadata2html.py diff --git a/scripts/metadata2html.py b/scripts/metadata2html.py new file mode 100755 index 00000000..9e518150 --- /dev/null +++ b/scripts/metadata2html.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python + +import argparse +import os +import sys + +from metadata_table import MetadataHeader + +parser = argparse.ArgumentParser() +parser.add_argument('--metafile', '-m', action='store', help='name of metadata file to convert', required=True) + +attributes = [ 'standard_name', 'long_name', 'units', 'local_name', 'type', 'dimensions', 'kind', 'intent', 'optional' ] + +def parse_arguments(): + """Parse command line arguments.""" + args = parser.parse_args() + filename = args.metafile + return filename + +def convert_to_html(filename_in): + """Convert a metadata file into html (one html file for each table)""" + if not os.path.isfile(filename_in): + raise Exception("Metadata file {} not found".format(filename_in)) + filename_out = filename_in.replace(".meta", ".html") + metadata_headers = MetadataHeader.parse_metadata_file(filename_in) + + for metadata_header in metadata_headers: + print metadata_header + metadata_header.to_html(filename_out, attributes) + +def main(): + filename = parse_arguments() + print filename + convert_to_html(filename) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/scripts/metadata_table.py b/scripts/metadata_table.py index 0c92cadc..85503ca9 100755 --- a/scripts/metadata_table.py +++ b/scripts/metadata_table.py @@ -372,6 +372,42 @@ def variable_list(self): "Return an ordered list of the header's variables" return self._variables.variable_list() +# DH* + def to_html(self, filename, props): + HTML = """ + + +My First HTML + + + + +{header}{contents}
+ + +""" + # Write table header + header = "" + for prop in props: + header += "{}".format(prop) + header += "\n" + # Write table contents, one row per variable + contents = "" + for var in self._variables.variable_list(): + row = "" + for prop in props: + value = var.get_prop_value(prop) + if value is None: + value = "n/a" + row += "{}".format(value) + row += "\n" + contents += row + with open(filename,"w") as f: + f.writelines(HTML.format(header=header, contents=contents)) + print(filename) + raise Exception +# *DH + def get_var(self, standard_name=None, intent=None): if standard_name is not None: var = self._variables.find_variable(standard_name) From b3c96ec4a4085f664398f092ace9356e99531636 Mon Sep 17 00:00:00 2001 From: climbfuji Date: Thu, 15 Aug 2019 15:27:58 -0600 Subject: [PATCH 3/6] scripts/metadata_table.py, scripts/metadata2html.py: initial working version of metadata-to-html converter --- scripts/metadata2html.py | 29 +++++++++++++++++--------- scripts/metadata_table.py | 44 ++++++++++++++++++++++----------------- 2 files changed, 44 insertions(+), 29 deletions(-) diff --git a/scripts/metadata2html.py b/scripts/metadata2html.py index 9e518150..fe96b20d 100755 --- a/scripts/metadata2html.py +++ b/scripts/metadata2html.py @@ -1,15 +1,21 @@ #!/usr/bin/env python import argparse +import logging import os import sys +# CCPP framework imports +from parse_tools import init_log, set_log_level from metadata_table import MetadataHeader parser = argparse.ArgumentParser() -parser.add_argument('--metafile', '-m', action='store', help='name of metadata file to convert', required=True) +parser.add_argument('--metafile', '-m', action='store', + help='name of metadata file to convert', + required=True) -attributes = [ 'standard_name', 'long_name', 'units', 'local_name', 'type', 'dimensions', 'kind', 'intent', 'optional' ] +attributes = [ 'standard_name', 'long_name', 'units', 'local_name', + 'type', 'dimensions', 'kind', 'intent', 'optional' ] def parse_arguments(): """Parse command line arguments.""" @@ -17,21 +23,24 @@ def parse_arguments(): filename = args.metafile return filename -def convert_to_html(filename_in): +def convert_to_html(filename_in, logger): """Convert a metadata file into html (one html file for each table)""" if not os.path.isfile(filename_in): raise Exception("Metadata file {} not found".format(filename_in)) - filename_out = filename_in.replace(".meta", ".html") + logger.info("Converting file {} to HTML".format(filename_in)) metadata_headers = MetadataHeader.parse_metadata_file(filename_in) - for metadata_header in metadata_headers: - print metadata_header - metadata_header.to_html(filename_out, attributes) + filename_out = metadata_header.to_html(attributes) + if filename_out: + logger.info(" ... wrote {}".format(filename_out)) def main(): + # Initialize logging + logger = init_log('metadata2html') + set_log_level(logger, logging.INFO) + # Convert metadata file filename = parse_arguments() - print filename - convert_to_html(filename) + convert_to_html(filename, logger) if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/scripts/metadata_table.py b/scripts/metadata_table.py index 85503ca9..5cf03abb 100755 --- a/scripts/metadata_table.py +++ b/scripts/metadata_table.py @@ -175,6 +175,19 @@ class MetadataHeader(ParseSource): __blank_line__ = re.compile(r"\s*[#;]") + __html_template__ = """ + + +My First HTML + + + + +{header}{contents}
+ + +""" + def __init__(self, parse_object=None, title=None, type_in=None, module=None, var_dict=None, logger=None): @@ -372,20 +385,11 @@ def variable_list(self): "Return an ordered list of the header's variables" return self._variables.variable_list() -# DH* - def to_html(self, filename, props): - HTML = """ - - -My First HTML - - - - -{header}{contents}
- - -""" + def to_html(self, props): + """Write html file for metadata table and return filename. + Skip metadata headers without variables""" + if not self._variables.variable_list(): + return None # Write table header header = "" for prop in props: @@ -397,16 +401,18 @@ def to_html(self, filename, props): row = "" for prop in props: value = var.get_prop_value(prop) - if value is None: + # Pretty-print for dimensions + if prop == 'dimensions': + value = '(' + ','.join(value) + ')' + elif value is None: value = "n/a" row += "{}".format(value) row += "\n" contents += row + filename = self.title + '.html' with open(filename,"w") as f: - f.writelines(HTML.format(header=header, contents=contents)) - print(filename) - raise Exception -# *DH + f.writelines(self.__html_template__.format(header=header, contents=contents)) + return filename def get_var(self, standard_name=None, intent=None): if standard_name is not None: From eec13217acc60c5114cf8b99a824e2b777ea9fec Mon Sep 17 00:00:00 2001 From: climbfuji Date: Thu, 15 Aug 2019 15:41:35 -0600 Subject: [PATCH 4/6] scripts/metadata_table.py, scripts/metadata2html.py: write html files to specified output directory --- scripts/metadata2html.py | 14 +++++++++----- scripts/metadata_table.py | 7 ++++--- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/scripts/metadata2html.py b/scripts/metadata2html.py index fe96b20d..f8d8e3b6 100755 --- a/scripts/metadata2html.py +++ b/scripts/metadata2html.py @@ -13,6 +13,9 @@ parser.add_argument('--metafile', '-m', action='store', help='name of metadata file to convert', required=True) +parser.add_argument('--outputdir', '-o', action='store', + help='directory where to write the html files', + required=True) attributes = [ 'standard_name', 'long_name', 'units', 'local_name', 'type', 'dimensions', 'kind', 'intent', 'optional' ] @@ -21,16 +24,17 @@ def parse_arguments(): """Parse command line arguments.""" args = parser.parse_args() filename = args.metafile - return filename + outdir = args.outputdir + return (filename, outdir) -def convert_to_html(filename_in, logger): +def convert_to_html(filename_in, outdir, logger): """Convert a metadata file into html (one html file for each table)""" if not os.path.isfile(filename_in): raise Exception("Metadata file {} not found".format(filename_in)) logger.info("Converting file {} to HTML".format(filename_in)) metadata_headers = MetadataHeader.parse_metadata_file(filename_in) for metadata_header in metadata_headers: - filename_out = metadata_header.to_html(attributes) + filename_out = metadata_header.to_html(outdir, attributes) if filename_out: logger.info(" ... wrote {}".format(filename_out)) @@ -39,8 +43,8 @@ def main(): logger = init_log('metadata2html') set_log_level(logger, logging.INFO) # Convert metadata file - filename = parse_arguments() - convert_to_html(filename, logger) + (filename, outdir) = parse_arguments() + convert_to_html(filename, outdir, logger) if __name__ == '__main__': main() diff --git a/scripts/metadata_table.py b/scripts/metadata_table.py index 5cf03abb..ee13d50f 100755 --- a/scripts/metadata_table.py +++ b/scripts/metadata_table.py @@ -91,6 +91,7 @@ # Python library imports from __future__ import print_function +import os import re # CCPP framework imports from metavar import Var, VarDictionary @@ -385,7 +386,7 @@ def variable_list(self): "Return an ordered list of the header's variables" return self._variables.variable_list() - def to_html(self, props): + def to_html(self, outdir, props): """Write html file for metadata table and return filename. Skip metadata headers without variables""" if not self._variables.variable_list(): @@ -403,13 +404,13 @@ def to_html(self, props): value = var.get_prop_value(prop) # Pretty-print for dimensions if prop == 'dimensions': - value = '(' + ','.join(value) + ')' + value = '(' + ', '.join(value) + ')' elif value is None: value = "n/a" row += "{}".format(value) row += "\n" contents += row - filename = self.title + '.html' + filename = os.path.join(outdir, self.title + '.html') with open(filename,"w") as f: f.writelines(self.__html_template__.format(header=header, contents=contents)) return filename From 577ffaf5b4bdf985bc222c954d973ca357ebaee2 Mon Sep 17 00:00:00 2001 From: climbfuji Date: Thu, 15 Aug 2019 17:19:34 -0600 Subject: [PATCH 5/6] scripts/convert_metadata*.py: don't add htmlinclude statement for blank metadata tables --- scripts/convert_metadata.py | 5 +++-- scripts/convert_metadata_schemes_using_typedef_dims.py | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/convert_metadata.py b/scripts/convert_metadata.py index 47da4042..34082f77 100755 --- a/scripts/convert_metadata.py +++ b/scripts/convert_metadata.py @@ -328,8 +328,6 @@ def convert_file(filename_in, filename_out, metadata_filename_out, model, logger # logger.info('Found old metadata table, {}, on line {}'.format(table_name, lindex+1)) # The header line is not modified file.write(line+"\n") - # Write an include line for the metadata table - file.write('!! \htmlinclude {}.html\n'.format(table_name)) # Create the table start section mdtable = MetadataTable(table_name, current_module) mdconfig.append(mdtable) @@ -339,6 +337,9 @@ def convert_file(filename_in, filename_out, metadata_filename_out, model, logger dim_names = [__not_found__]*15 # Do not work on a blank table if len(words) > 1: + # Write an include line for the metadata table + file.write('!! \htmlinclude {}.html\n'.format(table_name)) + # table_header = [x.strip() for x in words[1:-1]] for ind in xrange(len(table_header)): header_locs[table_header[ind]] = ind diff --git a/scripts/convert_metadata_schemes_using_typedef_dims.py b/scripts/convert_metadata_schemes_using_typedef_dims.py index e6d3f8bb..e04b9447 100755 --- a/scripts/convert_metadata_schemes_using_typedef_dims.py +++ b/scripts/convert_metadata_schemes_using_typedef_dims.py @@ -207,8 +207,6 @@ def convert_file(filename_in, filename_out, metadata_filename_out, typedef_dimen # logger.info('Found old metadata table, {}, on line {}'.format(table_name, lindex+1)) # The header line is not modified file.write(line+"\n") - # Write an include line for the metadata table - file.write('!! \htmlinclude {}.html\n'.format(table_name)) # Create the table start section mdtable = MetadataTable(table_name, current_module) mdconfig.append(mdtable) @@ -218,6 +216,9 @@ def convert_file(filename_in, filename_out, metadata_filename_out, typedef_dimen dim_names = [__not_found__]*15 # Do not work on a blank table if len(words) > 1: + # Write an include line for the metadata table + file.write('!! \htmlinclude {}.html\n'.format(table_name)) + # table_header = [x.strip() for x in words[1:-1]] for ind in xrange(len(table_header)): header_locs[table_header[ind]] = ind From 844fed01654d89d512cd45ee33d41dc558898649 Mon Sep 17 00:00:00 2001 From: climbfuji Date: Fri, 16 Aug 2019 12:24:50 -0600 Subject: [PATCH 6/6] scripts/metadata_table.py: replace dummy title with name of argument table plus 'argument table', i.e. 'ozphys_run argument table' --- scripts/metadata_table.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/metadata_table.py b/scripts/metadata_table.py index ee13d50f..7775ba3e 100755 --- a/scripts/metadata_table.py +++ b/scripts/metadata_table.py @@ -179,7 +179,7 @@ class MetadataHeader(ParseSource): __html_template__ = """ -My First HTML +{title} @@ -412,7 +412,8 @@ def to_html(self, outdir, props): contents += row filename = os.path.join(outdir, self.title + '.html') with open(filename,"w") as f: - f.writelines(self.__html_template__.format(header=header, contents=contents)) + f.writelines(self.__html_template__.format(title=self.title + ' argument table', + header=header, contents=contents)) return filename def get_var(self, standard_name=None, intent=None):