diff --git a/scripts/ccpp_prebuild.py b/scripts/ccpp_prebuild.py index 74dc1891..01dbaf42 100755 --- a/scripts/ccpp_prebuild.py +++ b/scripts/ccpp_prebuild.py @@ -3,6 +3,7 @@ # Standard modules import argparse import collections +import filecmp import importlib import itertools import logging @@ -114,10 +115,10 @@ def import_config(configfile, builddir): # To handle new metadata: import DDT references (if exist) try: config['typedefs_new_metadata'] = ccpp_prebuild_config.TYPEDEFS_NEW_METADATA - logging.info("Found TYPEDEFS_NEW_METADATA dictionary in config, assume at least some data is in new metadata formet") + logging.info("Found TYPEDEFS_NEW_METADATA dictionary in config, assume at least some data is in new metadata format") except AttributeError: config['typedefs_new_metadata'] = None - logging.info("Could not find TYPEDEFS_NEW_METADATA dictionary in config, assume all data is in old metadata formet") + logging.info("Could not find TYPEDEFS_NEW_METADATA dictionary in config, assume all data is in old metadata format") return(success, config) @@ -141,6 +142,9 @@ def clean_files(config, static): logging.info('Performing clean ....') # Create list of files to remove, use wildcards where necessary files_to_remove = [ + config['typedefs_makefile'], + config['typedefs_cmakefile'], + config['typedefs_sourcefile'], config['schemes_makefile'], config['schemes_cmakefile'], config['schemes_sourcefile'], @@ -153,6 +157,7 @@ def clean_files(config, static): if static: files_to_remove.append(os.path.join(config['caps_dir'], 'ccpp_*_cap.F90')) files_to_remove.append(os.path.join(config['static_api_dir'], '{api}.F90'.format(api=CCPP_STATIC_API_MODULE))) + files_to_remove.append(config['static_api_srcfile']) else: files_to_remove.append(os.path.join(config['caps_dir'], '*_cap.F90')) for target_file in config['target_files']: @@ -679,16 +684,32 @@ def generate_typedefs_makefile(metadata_define, typedefs_makefile, typedefs_cmak logging.info('Generating typedefs makefile/cmakefile snippet ...') # Write the Fortran modules without path - the build system knows where they are makefile = TypedefsMakefile() - makefile.filename = typedefs_makefile + makefile.filename = typedefs_makefile + '.tmp' cmakefile = TypedefsCMakefile() - cmakefile.filename = typedefs_cmakefile + cmakefile.filename = typedefs_cmakefile + '.tmp' sourcefile = TypedefsSourcefile() - sourcefile.filename = typedefs_sourcefile + sourcefile.filename = typedefs_sourcefile + '.tmp' makefile.write(typedefs) cmakefile.write(typedefs) sourcefile.write(typedefs) + if os.path.isfile(typedefs_makefile) and \ + filecmp.cmp(typedefs_makefile, makefile.filename): + os.remove(makefile.filename) + os.remove(cmakefile.filename) + os.remove(sourcefile.filename) + else: + if os.path.isfile(typedefs_makefile): + os.remove(typedefs_makefile) + if os.path.isfile(typedefs_cmakefile): + os.remove(typedefs_cmakefile) + if os.path.isfile(typedefs_sourcefile): + os.remove(typedefs_sourcefile) + os.rename(makefile.filename, typedefs_makefile) + os.rename(cmakefile.filename, typedefs_cmakefile) + os.rename(sourcefile.filename, typedefs_sourcefile) + # logging.info('Added {0} typedefs to {1}, {2}, {3}'.format( - len(typedefs), makefile.filename, cmakefile.filename, sourcefile.filename)) + len(typedefs), typedefs_makefile, typedefs_cmakefile, typedefs_sourcefile)) return success def generate_schemes_makefile(schemes, schemes_makefile, schemes_cmakefile, schemes_sourcefile): @@ -696,18 +717,34 @@ def generate_schemes_makefile(schemes, schemes_makefile, schemes_cmakefile, sche logging.info('Generating schemes makefile/cmakefile snippet ...') success = True makefile = SchemesMakefile() - makefile.filename = schemes_makefile + makefile.filename = schemes_makefile + '.tmp' cmakefile = SchemesCMakefile() - cmakefile.filename = schemes_cmakefile + cmakefile.filename = schemes_cmakefile + '.tmp' sourcefile = SchemesSourcefile() - sourcefile.filename = schemes_sourcefile + sourcefile.filename = schemes_sourcefile + '.tmp' # Generate list of schemes with absolute path schemes_with_abspath = [ os.path.abspath(scheme) for scheme in schemes ] makefile.write(schemes_with_abspath) cmakefile.write(schemes_with_abspath) sourcefile.write(schemes_with_abspath) + if os.path.isfile(schemes_makefile) and \ + filecmp.cmp(schemes_makefile, makefile.filename): + os.remove(makefile.filename) + os.remove(cmakefile.filename) + os.remove(sourcefile.filename) + else: + if os.path.isfile(schemes_makefile): + os.remove(schemes_makefile) + if os.path.isfile(schemes_cmakefile): + os.remove(schemes_cmakefile) + if os.path.isfile(schemes_sourcefile): + os.remove(schemes_sourcefile) + os.rename(makefile.filename, schemes_makefile) + os.rename(cmakefile.filename, schemes_cmakefile) + os.rename(sourcefile.filename, schemes_sourcefile) + # logging.info('Added {0} schemes to {1}, {2}, {3}'.format( - len(schemes_with_abspath), makefile.filename, cmakefile.filename, sourcefile.filename)) + len(schemes_with_abspath), schemes_makefile, schemes_cmakefile, schemes_sourcefile)) return success def generate_caps_makefile(caps, caps_makefile, caps_cmakefile, caps_sourcefile, caps_dir): @@ -715,18 +752,34 @@ def generate_caps_makefile(caps, caps_makefile, caps_cmakefile, caps_sourcefile, logging.info('Generating caps makefile/cmakefile snippet ...') success = True makefile = CapsMakefile() - makefile.filename = caps_makefile + makefile.filename = caps_makefile + '.tmp' cmakefile = CapsCMakefile() - cmakefile.filename = caps_cmakefile + cmakefile.filename = caps_cmakefile + '.tmp' sourcefile = CapsSourcefile() - sourcefile.filename = caps_sourcefile + sourcefile.filename = caps_sourcefile + '.tmp' # Generate list of caps with absolute path caps_with_abspath = [ os.path.abspath(os.path.join(caps_dir, cap)) for cap in caps ] makefile.write(caps_with_abspath) cmakefile.write(caps_with_abspath) sourcefile.write(caps_with_abspath) + if os.path.isfile(caps_makefile) and \ + filecmp.cmp(caps_makefile, makefile.filename): + os.remove(makefile.filename) + os.remove(cmakefile.filename) + os.remove(sourcefile.filename) + else: + if os.path.isfile(caps_makefile): + os.remove(caps_makefile) + if os.path.isfile(caps_cmakefile): + os.remove(caps_cmakefile) + if os.path.isfile(caps_sourcefile): + os.remove(caps_sourcefile) + os.rename(makefile.filename, caps_makefile) + os.rename(cmakefile.filename, caps_cmakefile) + os.rename(sourcefile.filename, caps_sourcefile) + # logging.info('Added {0} auto-generated caps to {1} and {2}, {3}'.format( - len(caps_with_abspath), makefile.filename, cmakefile.filename, sourcefile.filename)) + len(caps_with_abspath), caps_makefile, caps_cmakefile, caps_sourcefile)) return success def main(): diff --git a/scripts/mkstatic.py b/scripts/mkstatic.py index 1095400b..ed6fd6fd 100755 --- a/scripts/mkstatic.py +++ b/scripts/mkstatic.py @@ -4,6 +4,7 @@ import collections import copy import getopt +import filecmp import logging import os import sys @@ -203,6 +204,7 @@ def __init__(self, **kwargs): self._subroutines = None self._suites = [] self._directory = '.' + self._update_api = True for key, value in kwargs.items(): setattr(self, "_"+key, value) @@ -224,6 +226,15 @@ def directory(self): def directory(self, value): self._directory = value + @property + def update_api(self): + '''Get the update_api flag.''' + return self._update_api + + @update_api.setter + def update_api(self, value): + self._update_api = value + @property def module(self): '''Get the module name of the API.''' @@ -342,7 +353,21 @@ def write(self): filepath = os.path.split(self.filename)[0] if filepath and not os.path.isdir(filepath): os.makedirs(filepath) - f = open(self.filename, 'w') + # If the file exists, write to temporary file first and compare them: + # - if identical, delete the temporary file and keep the existing one + # and set the API update flag to false + # - if different, replace existing file with temporary file and set + # the API update flag to true (default value) + # - always replace the file if any of the suite caps has changed + # If the file does not exist, write the API an set the flag to true + if os.path.isfile(self.filename) and \ + not any([suite.update_cap for suite in suites]): + write_to_test_file = True + test_filename = self.filename + '.test' + f = open(test_filename, 'w') + else: + write_to_test_file = False + f = open(self.filename, 'w') else: f = sys.stdout f.write(API.header.format(module=self._module, @@ -352,6 +377,21 @@ def write(self): f.write(Suite.footer.format(module=self._module)) if (f is not sys.stdout): f.close() + # See comment above on updating the API or not + if write_to_test_file: + if filecmp.cmp(self.filename, test_filename): + # Files are equal, delete the test API and set update flag to False + os.remove(test_filename) + self.update_api = False + else: + # Files are different, replace existing API with + # the test API and set update flag to True + # Python 3 only: os.replace(test_filename, self.filename) + os.remove(self.filename) + os.rename(test_filename, self.filename) + self.update_api = True + else: + self.update_api = True return def write_sourcefile(self, source_filename): @@ -359,7 +399,18 @@ def write_sourcefile(self, source_filename): filepath = os.path.split(source_filename)[0] if filepath and not os.path.isdir(filepath): os.makedirs(filepath) - f = open(source_filename, 'w') + # If the file exists, write to temporary file first and compare them: + # - if identical, delete the temporary file and keep the existing one + # - if different, replace existing file with temporary file + # - however, always replace the file if the API update flag is true + if os.path.isfile(source_filename) and not self.update_api: + write_to_test_file = True + test_filename = source_filename + '.test' + f = open(test_filename, 'w') + else: + write_to_test_file = False + f = open(source_filename, 'w') + # Contents of shell/source file contents = """# The CCPP static API is defined here. # # This file is auto-generated using ccpp_prebuild.py @@ -369,8 +420,19 @@ def write_sourcefile(self, source_filename): """.format(filename=os.path.abspath(os.path.join(self.directory,self.filename))) f.write(contents) f.close() + # See comment above on updating the API or not + if write_to_test_file: + if filecmp.cmp(source_filename, test_filename): + # Files are equal, delete the test file + os.remove(test_filename) + else: + # Files are different, replace existing file + # Python 3 only: os.replace(test_filename, source_filename) + os.remove(source_filename) + os.rename(test_filename, source_filename) return success + class Suite(object): header=''' @@ -426,6 +488,7 @@ class Suite(object): def __init__(self, **kwargs): self._name = None + self._filename = sys.stdout self._sdf_name = None self._all_schemes_called = None self._all_subroutines_called = None @@ -434,6 +497,7 @@ def __init__(self, **kwargs): self._subroutines = None self._parents = { ccpp_stage : {} for ccpp_stage in CCPP_STAGES } self._arguments = { ccpp_stage : [] for ccpp_stage in CCPP_STAGES } + self._update_cap = True for key, value in kwargs.items(): setattr(self, "_"+key, value) @@ -451,6 +515,24 @@ def sdf_name(self): def sdf_name(self, value): self._sdf_name = value + @property + def filename(self): + '''Get the filename of write the output to.''' + return self._filename + + @filename.setter + def filename(self, value): + self._filename = value + + @property + def update_cap(self): + '''Get the update_cap flag.''' + return self._update_cap + + @update_cap.setter + def update_cap(self, value): + self._update_cap = value + def parse(self): '''Parse the suite definition file.''' success = True @@ -572,7 +654,7 @@ def write(self, metadata_request, metadata_define, arguments): (calling the group caps one after another)""" # Set name of module and filename of cap self._module = 'ccpp_{suite_name}_cap'.format(suite_name=self._name) - self._filename = '{module_name}.F90'.format(module_name=self._module) + self.filename = '{module_name}.F90'.format(module_name=self._module) # Init self._subroutines = [] # Write group caps and generate module use statements; combine the argument lists @@ -627,8 +709,26 @@ def write(self, metadata_request, metadata_define, arguments): body=body) # Write cap to stdout or file - if (self._filename is not sys.stdout): - f = open(self._filename, 'w') + if (self.filename is not sys.stdout): + filepath = os.path.split(self.filename)[0] + if filepath and not os.path.isdir(filepath): + os.makedirs(filepath) + # If the file exists, write to temporary file first and compare them: + # - if identical, delete the temporary file and keep the existing one + # and set the suite cap update flag to false + # - if different, replace existing file with temporary file and set + # the suite cap update flag to true (default value) + # - however, if any of the group caps has changed, rewrite the suite + # cap as well and set the suite cap update flag to true + # If the file does not exist, write the cap an set the flag to true + if os.path.isfile(self.filename) and \ + not any([group.update_cap for group in self._groups]): + write_to_test_file = True + test_filename = self.filename + '.test' + f = open(test_filename, 'w') + else: + write_to_test_file = False + f = open(self.filename, 'w') else: f = sys.stdout f.write(Suite.header.format(module=self._module, @@ -638,9 +738,25 @@ def write(self, metadata_request, metadata_define, arguments): f.write(Suite.footer.format(module=self._module)) if (f is not sys.stdout): f.close() + # See comment above on updating the suite cap or not + if write_to_test_file: + if filecmp.cmp(self.filename, test_filename): + # Files are equal, delete the test cap + # and set update flag to False + os.remove(test_filename) + self.update_cap = False + else: + # Files are different, replace existing cap + # with test cap and set flag to True + # Python 3 only: os.replace(test_filename, self.filename) + os.remove(self.filename) + os.rename(test_filename, self.filename) + self.update_cap = True + else: + self.update_cap = True # Create list of all caps generated (for groups and suite) - self._caps = [ self._filename ] + self._caps = [ self.filename ] for group in self._groups: self._caps.append(group.filename) @@ -735,7 +851,7 @@ class Group(object): def __init__(self, **kwargs): self._name = '' self._suite = None - self._filename = 'sys.stdout' + self._filename = sys.stdout self._init = False self._finalize = False self._module = None @@ -743,6 +859,7 @@ def __init__(self, **kwargs): self._pset = None self._parents = { ccpp_stage : {} for ccpp_stage in CCPP_STAGES } self._arguments = { ccpp_stage : [] for ccpp_stage in CCPP_STAGES } + self._update_cap = True for key, value in kwargs.items(): setattr(self, "_"+key, value) @@ -989,7 +1106,22 @@ def write(self, metadata_request, metadata_define, arguments): # Write output to stdout or file if (self.filename is not sys.stdout): - f = open(self.filename, 'w') + filepath = os.path.split(self.filename)[0] + if filepath and not os.path.isdir(filepath): + os.makedirs(filepath) + # If the file exists, write to temporary file first and compare them: + # - if identical, delete the temporary file and keep the existing one + # and set the group cap update flag to false + # - if different, replace existing file with temporary file and set + # the group cap update flag to true (default value) + # If the file does not exist, write the cap an set the flag to true + if os.path.isfile(self.filename): + write_to_test_file = True + test_filename = self.filename + '.test' + f = open(test_filename, 'w') + else: + write_to_test_file = False + f = open(self.filename, 'w') else: f = sys.stdout f.write(Group.header.format(group=self._name, @@ -1000,7 +1132,22 @@ def write(self, metadata_request, metadata_define, arguments): f.write(Group.footer.format(module=self._module)) if (f is not sys.stdout): f.close() - + # See comment above on updating the group cap or not + if write_to_test_file: + if filecmp.cmp(self.filename, test_filename): + # Files are equal, delete the test cap + # and set update flag to False + os.remove(test_filename) + self.update_cap = False + else: + # Files are different, replace existing cap + # with test cap and set flag to True + # Python 3 only: os.replace(test_filename, self.filename) + os.remove(self.filename) + os.rename(test_filename, self.filename) + self.update_cap = True + else: + self.update_cap = True return @property @@ -1021,6 +1168,15 @@ def filename(self): def filename(self, value): self._filename = value + @property + def update_cap(self): + '''Get the update_cap flag.''' + return self._update_cap + + @update_cap.setter + def update_cap(self, value): + self._update_cap = value + @property def init(self): '''Get the init flag.'''