diff --git a/.travis.yml b/.travis.yml index 9f9d576fde..acd989ec7e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -88,31 +88,14 @@ install: ln -s $(pwd)/iris-test-data-${IRIS_TEST_DATA_SUFFIX} iris-test-data; fi -# prepare iris build directory - - if [[ $TEST_TARGET -ne 'coding' ]]; then - IRIS=$(ls -d1 build/lib*/iris); - mkdir $IRIS/etc; - else - IRIS=lib/iris; - fi - # set config paths - - SITE_CFG=$IRIS/etc/site.cfg + - SITE_CFG=lib/iris/etc/site.cfg - echo "[Resources]" > $SITE_CFG - echo "test_data_dir = $(pwd)/iris-test-data/test_data" >> $SITE_CFG - echo "doc_dir = $(pwd)/docs/iris" >> $SITE_CFG - echo "[System]" >> $SITE_CFG - echo "udunits2_path = $PREFIX/lib/libudunits2.so" >> $SITE_CFG - # The coding standards tests expect all the standard names and PyKE - # modules to be present. - - if [[ $TEST_TARGET == 'coding' ]]; then - python setup.py std_names; - PYTHONPATH=lib python setup.py pyke_rules; - fi - -# iris - - python setup.py --quiet build - python setup.py --quiet install script: @@ -160,5 +143,7 @@ script: fi - if [[ $TEST_TARGET == 'coding' ]]; then cd $INSTALL_DIR; + python setup.py pyke_rules; + python setup.py std_names; python setup.py test --coding-tests; fi diff --git a/MANIFEST.in b/MANIFEST.in index 8d6f86d989..ff8e330fd5 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,20 +2,18 @@ include CHANGES COPYING COPYING.LESSER INSTALL # Files from setup.py package_data that are not automatically added to source distributions -recursive-include lib/iris/tests/results *.cml *.cdl *.txt *.xml -recursive-include lib/iris/etc *.txt +recursive-include lib/iris/tests/results *.cml *.cdl *.txt *.xml *.json +recursive-include lib/iris/etc * include lib/iris/fileformats/_pyke_rules/*.k?b include lib/iris/tests/stock*.npz # File required to build docs recursive-include docs Makefile *.js *.png *.py *.rst -recursive-exclude docs/iris/build * +prune docs/iris/build # Files required to build std_names module include tools/generate_std_names.py include etc/cf-standard-name-table.xml -# Extension source -recursive-include src *.c *.cc *.cpp *.h - - +global-exclude *.pyc +global-exclude __pycache__ diff --git a/setup.py b/setup.py index 6cd19e5cb1..702e59eb82 100644 --- a/setup.py +++ b/setup.py @@ -1,22 +1,16 @@ from __future__ import print_function -import contextlib -from distutils.command import build_ext, build_py -from distutils.core import setup, Command -from distutils.sysconfig import get_config_var +from contextlib import contextmanager +from distutils.core import Command from distutils.util import convert_path -import fnmatch -import multiprocessing import os +from shutil import copyfile import sys +import textwrap -import numpy as np -import setuptools - -# Add full path so Python doesn't load any __init__.py in the intervening -# directories, thereby saving setup.py from additional dependencies. -sys.path.append('lib/iris/tests/runner') -from _runner import TestRunner +from setuptools import setup +from setuptools.command.develop import develop as develop_cmd +from setuptools.command.build_py import build_py # Returns the package and all its sub-packages @@ -38,7 +32,7 @@ def find_package_tree(root_path, root_package): if dir_names: prefix = dir_path.split(os.path.sep)[root_count:] packages.extend(['.'.join([root_package] + prefix + [dir_name]) - for dir_name in dir_names]) + for dir_name in dir_names]) return packages @@ -55,24 +49,34 @@ def file_walk_relative(top, remove=''): yield os.path.join(root, file).replace(remove, '') -def std_name_cmd(target_dir): - script_path = os.path.join('tools', 'generate_std_names.py') - xml_path = os.path.join('etc', 'cf-standard-name-table.xml') - module_path = os.path.join(target_dir, 'iris', 'std_names.py') - cmd = (sys.executable, script_path, xml_path, module_path) - return cmd +@contextmanager +def temporary_path(directory): + """ + Context manager that adds and subsequently removes the given directory + to sys.path + + """ + sys.path.insert(0, directory) + try: + yield + finally: + del sys.path[0] + + +# Add full path so Python doesn't load any __init__.py in the intervening +# directories, thereby saving setup.py from additional dependencies. +with temporary_path('lib/iris/tests/runner'): + from _runner import TestRunner # noqa: -class SetupTestRunner(TestRunner, setuptools.Command): +class SetupTestRunner(TestRunner, Command): pass -class CleanSource(Command): - """ - Removes orphaned pyc/pyo files from the sources. +class BaseCommand(Command): + """A valid no-op command for setuptools & distutils.""" - """ - description = 'clean orphaned pyc/pyo files from sources' + description = 'A no-op command.' user_options = [] def initialize_options(self): @@ -81,6 +85,13 @@ def initialize_options(self): def finalize_options(self): pass + def run(self): + pass + + +class CleanSource(BaseCommand): + description = 'clean orphaned pyc/pyo files from the source directory' + def run(self): for root_path, dir_names, file_names in os.walk('lib'): for file_name in file_names: @@ -92,92 +103,85 @@ def run(self): os.remove(compiled_path) -class MakeStdNames(Command): - """ - Generates the CF standard name module containing mappings from - CF standard name to associated metadata. - - """ - description = "generate CF standard name module" - user_options = [] +def compile_pyke_rules(cmd, directory): + # Call out to the python executable to pre-compile the Pyke rules. + # Significant effort was put in to trying to get these to compile + # within this build process but there was no obvious way of finding + # a workaround to the issue presented in + # https://github.com/SciTools/iris/issues/2481. - def initialize_options(self): - pass + shelled_code = textwrap.dedent("""\ - def finalize_options(self): - pass + import os - def run(self): - cmd = std_name_cmd('lib') - self.spawn(cmd) + # Monkey patch the load method to avoid "ModuleNotFoundError: No module + # named 'iris.fileformats._pyke_rules.compiled_krb'". In this instance + # we simply don't want the knowledge engine, so we turn the load method + # into a no-op. + from pyke.target_pkg import target_pkg + target_pkg.load = lambda *args, **kwargs: None + # Compile the rules by hand, without importing iris. That way we can + # avoid the need for all of iris' dependencies being installed. + os.chdir(os.path.join('{bld_dir}', 'iris', 'fileformats', '_pyke_rules')) + + # Import pyke *after* changing directory. Without this we get the compiled + # rules in the wrong place. Identified in + # https://github.com/SciTools/iris/pull/2891#issuecomment-341404187 + from pyke import knowledge_engine + knowledge_engine.engine('') -class MakePykeRules(Command): - """ - Compile the PyKE CF-NetCDF loader rule base. + """.format(bld_dir=directory)).split('\n') + shelled_code = '; '.join( + [line for line in shelled_code + if not line.strip().startswith('#') and line.strip()]) + args = [sys.executable, '-c', shelled_code] + cmd.spawn(args) - """ - description = "compile CF-NetCDF loader rule base" - user_options = [] - def initialize_options(self): - pass +def copy_copyright(cmd, directory): + # Copy the COPYRIGHT information into the package root + iris_build_dir = os.path.join(directory, 'iris') + for fname in ['COPYING', 'COPYING.LESSER']: + copyfile(fname, os.path.join(iris_build_dir, fname)) - def finalize_options(self): - pass - @staticmethod - def _pyke_rule_compile(): - """Compile the PyKE rule base.""" - from pyke import knowledge_engine - import iris.fileformats._pyke_rules - knowledge_engine.engine(iris.fileformats._pyke_rules) +def build_std_names(cmd, directory): + # Call out to tools/generate_std_names.py to build std_names module. - def run(self): - # Compile the PyKE rules. - MakePykeRules._pyke_rule_compile() + script_path = os.path.join('tools', 'generate_std_names.py') + xml_path = os.path.join('etc', 'cf-standard-name-table.xml') + module_path = os.path.join(directory, 'iris', 'std_names.py') + args = (sys.executable, script_path, xml_path, module_path) + cmd.spawn(args) -class MissingHeaderError(Exception): - """ - Raised when one or more files do not have the required copyright - and licence header. +def custom_cmd(command_to_override, functions, help_doc=""): """ - pass - + Allows command specialisation to include calls to the given functions. -class BuildPyWithExtras(build_py.build_py): """ - Adds the creation of the CF standard names module and compilation - of the PyKE rules to the standard "build_py" command. + class ExtendedCommand(command_to_override): + description = help_doc or command_to_override.description - """ - @contextlib.contextmanager - def temporary_path(self): - """ - Context manager that adds and subsequently removes the build - directory to the beginning of the module search path. - - """ - sys.path.insert(0, self.build_lib) - try: - yield - finally: - del sys.path[0] + def run(self): + # Run the original command first to make sure all the target + # directories are in place. + command_to_override.run(self) - def run(self): - # Run the main build command first to make sure all the target - # directories are in place. - build_py.build_py.run(self) + # build_lib is defined if we are building the package. Otherwise + # we want to to the work in-place. + dest = getattr(self, 'build_lib', None) + if dest is None: + print(' [Running in-place]') + # Pick the source dir instead (currently in the sub-dir "lib") + dest = 'lib' - # Now build the std_names module. - cmd = std_name_cmd(self.build_lib) - self.spawn(cmd) + for func in functions: + func(self, dest) - # Compile the PyKE rules. - with self.temporary_path(): - MakePykeRules._pyke_rule_compile() + return ExtendedCommand def extract_version(): @@ -193,24 +197,33 @@ def extract_version(): return version +custom_commands = { + 'test': SetupTestRunner, + 'develop': custom_cmd( + develop_cmd, [build_std_names, compile_pyke_rules]), + 'build_py': custom_cmd( + build_py, + [build_std_names, compile_pyke_rules, copy_copyright]), + 'std_names': + custom_cmd(BaseCommand, [build_std_names], + help_doc="generate CF standard name module"), + 'pyke_rules': + custom_cmd(BaseCommand, [compile_pyke_rules], + help_doc="compile CF-NetCDF loader rules"), + 'clean_source': CleanSource, + } + + setup( name='Iris', version=extract_version(), url='http://scitools.org.uk/iris/', author='UK Met Office', - + author_email='scitools-iris-dev@googlegroups.com', packages=find_package_tree('lib/iris', 'iris'), package_dir={'': 'lib'}, - package_data={ - 'iris': list(file_walk_relative('lib/iris/etc', remove='lib/iris/')) + \ - list(file_walk_relative('lib/iris/tests/results', - remove='lib/iris/')) + \ - ['fileformats/_pyke_rules/*.k?b'] + \ - ['tests/stock*.npz'] - }, - data_files=[('iris', ['CHANGES', 'COPYING', 'COPYING.LESSER'])], + include_package_data=True, tests_require=['nose'], - cmdclass={'test': SetupTestRunner, 'build_py': BuildPyWithExtras, - 'std_names': MakeStdNames, 'pyke_rules': MakePykeRules, - 'clean_source': CleanSource}, -) + cmdclass=custom_commands, + zip_safe=False, + )