diff --git a/.travis.yml b/.travis.yml index acd989ec7e..84179b87b9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,80 +24,97 @@ env: - secure: "N9/qBUT5CqfC7KQBDy5mIWZcGNuUJk3e/qmKJpotWYV+zwOI4GghJsRce6nFnlRiwl65l5oBEcvf3+sBvUfbZqh7U0MdHpw2tHhr2FSCmMB3bkvARZblh9M37f4da9G9VmRkqnyBM5G5TImXtoq4dusvNWKvLW0qETciaipq7ws=" git: + # We need a deep clone so that we can compute the age of the files using their git history. depth: 10000 install: - - export IRIS_TEST_DATA_REF="2f3a6bcf25f81bd152b3d66223394074c9069a96" - - export IRIS_TEST_DATA_SUFFIX=$(echo "${IRIS_TEST_DATA_REF}" | sed "s/^v//") + - > + export IRIS_TEST_DATA_REF="2f3a6bcf25f81bd152b3d66223394074c9069a96"; + export IRIS_TEST_DATA_SUFFIX=$(echo "${IRIS_TEST_DATA_REF}" | sed "s/^v//"); # Install miniconda # ----------------- - - export CONDA_BASE=https://repo.continuum.io/miniconda/Miniconda - - if [[ "$TRAVIS_PYTHON_VERSION" == 2* ]]; then - wget ${CONDA_BASE}2-latest-Linux-x86_64.sh -O miniconda.sh; + - > + echo 'Installing miniconda'; + export CONDA_BASE=https://repo.continuum.io/miniconda/Miniconda; + if [[ "$TRAVIS_PYTHON_VERSION" == 2* ]]; then + wget --quiet ${CONDA_BASE}2-latest-Linux-x86_64.sh -O miniconda.sh; else - wget ${CONDA_BASE}3-latest-Linux-x86_64.sh -O miniconda.sh; - fi - - bash miniconda.sh -b -p $HOME/miniconda - - export PATH="$HOME/miniconda/bin:$PATH" + wget --quiet ${CONDA_BASE}3-latest-Linux-x86_64.sh -O miniconda.sh; + fi; + bash miniconda.sh -b -p $HOME/miniconda; + export PATH="$HOME/miniconda/bin:$PATH"; # Create the basic testing environment # ------------------------------------ # Explicitly add defaults channel, see https://github.com/conda/conda/issues/2675 - - conda config --add channels defaults - - conda config --set always_yes yes --set changeps1 no - - conda config --set show_channel_urls True - - conda update --quiet conda - - ENV_NAME='test-environment' - - conda create --quiet -n $ENV_NAME python=$TRAVIS_PYTHON_VERSION - - source activate $ENV_NAME + - > + echo 'Configure conda and create an environment'; + conda config --set always_yes yes --set changeps1 no; + conda config --set show_channel_urls True; + conda config --add channels conda-forge; + conda update --quiet conda; + ENV_NAME='test-environment'; + conda create --quiet -n $ENV_NAME python=$TRAVIS_PYTHON_VERSION pip; + source activate $ENV_NAME; # Customise the testing environment # --------------------------------- - - conda config --add channels conda-forge - - if [[ "$TEST_MINIMAL" == true ]]; then - conda install --quiet --file minimal-conda-requirements.txt; - else - if [[ "$TRAVIS_PYTHON_VERSION" == 3* ]]; then - sed -e '/esmpy/d' -e 's/#.\+$//' conda-requirements.txt | xargs conda install --quiet; - else - conda install --quiet --file conda-requirements.txt; - fi - fi - - # JUST FOR NOW : Install latest version of iris-grib. - # TODO : remove when this release is available on conda-forge. - - if [[ "$TEST_MINIMAL" != true ]]; then - pip install git+https://github.com/SciTools/iris-grib.git@v0.11.0 ; - fi + - > + echo 'Install Iris dependencies'; + CONDA_REQS_FLAGS=""; + CONDA_REQS_GROUPS="test"; + if [[ "$TRAVIS_PYTHON_VERSION" == 2* ]]; then + CONDA_REQS_FLAGS="${CONDA_REQS_FLAGS} --py2"; + fi; + if [[ "$TEST_MINIMAL" != true ]]; then + CONDA_REQS_GROUPS="${CONDA_REQS_GROUPS} all"; + fi; + if [[ "${TEST_TARGET}" == 'doctest' ]]; then + CONDA_REQS_GROUPS="${CONDA_REQS_GROUPS} docs"; + fi; + CONDA_REQS_FILE=conda-requirements.txt; + python requirements/gen_conda_requirements.py ${CONDA_REQS_FLAGS} --groups ${CONDA_REQS_GROUPS} > ${CONDA_REQS_FILE}; + cat ${CONDA_REQS_FILE}; + conda install --quiet --file ${CONDA_REQS_FILE}; - PREFIX=$HOME/miniconda/envs/$ENV_NAME # Output debug info - - conda list - - conda info -a + - > + conda list; + conda info -a; # Pre-load Natural Earth data to avoid multiple, overlapping downloads. # i.e. There should be no DownloadWarning reports in the log. - python -c 'import cartopy; cartopy.io.shapereader.natural_earth()' # iris test data - - if [[ "$TEST_MINIMAL" != true ]]; then - wget -O iris-test-data.zip https://github.com/SciTools/iris-test-data/archive/${IRIS_TEST_DATA_REF}.zip; + - > + if [[ "$TEST_MINIMAL" != true ]]; then + wget --quiet -O iris-test-data.zip https://github.com/SciTools/iris-test-data/archive/${IRIS_TEST_DATA_REF}.zip; unzip -q iris-test-data.zip; - ln -s $(pwd)/iris-test-data-${IRIS_TEST_DATA_SUFFIX} iris-test-data; + mv "iris-test-data-${IRIS_TEST_DATA_SUFFIX}" iris-test-data; fi # set config paths - - SITE_CFG=lib/iris/etc/site.cfg - - echo "[Resources]" > $SITE_CFG - - echo "test_data_dir = $(pwd)/iris-test-data/test_data" >> $SITE_CFG - - echo "doc_dir = $(pwd)/docs/iris" >> $SITE_CFG - - echo "[System]" >> $SITE_CFG - - echo "udunits2_path = $PREFIX/lib/libudunits2.so" >> $SITE_CFG + - > + SITE_CFG=lib/iris/etc/site.cfg; + echo "[Resources]" > $SITE_CFG; + echo "test_data_dir = $(pwd)/iris-test-data/test_data" >> $SITE_CFG; + echo "doc_dir = $(pwd)/docs/iris" >> $SITE_CFG; + echo "[System]" >> $SITE_CFG; + echo "udunits2_path = $PREFIX/lib/libudunits2.so" >> $SITE_CFG; - python setup.py --quiet install + # JUST FOR NOW : Install latest version of iris-grib. + # TODO : remove when iris doesn't do an integration test requiring iris-grib. + - if [[ "$TEST_MINIMAL" != true && ${TRAVIS_PYTHON_VERSION} == 2* ]]; then + conda install python-ecmwf_grib; + pip install git+https://github.com/SciTools/iris-grib.git@v0.11.0; + fi + script: - if [[ $TEST_TARGET == 'default' ]]; then python -m iris.tests.runner --default-tests --system-tests --print-failed-images --num-processors=3; diff --git a/conda-requirements.txt b/conda-requirements.txt deleted file mode 100644 index 9ada018681..0000000000 --- a/conda-requirements.txt +++ /dev/null @@ -1,38 +0,0 @@ -# Use this file to create a conda environment using: -# conda create -n --file conda-requirements.txt - -# Mandatory dependencies -cartopy -matplotlib<1.9 -netcdf4 -numpy -pyke -udunits2 -cf_units -dask>=0.15.0 - -# Iris build dependencies -setuptools - -# Iris testing/documentation dependencies -mock -nose -pep8 -sphinx -iris-sample-data -filelock -imagehash -requests - -# Optional iris dependencies -esmpy>=7.0 -gdal -libmo_unpack -mo_pack -nc_time_axis -pandas -python-stratify -pyugrid - -# Iris extensions (i.e. key tools that depend on Iris) -# iris_grib>=0.12 diff --git a/minimal-conda-requirements.txt b/minimal-conda-requirements.txt deleted file mode 100644 index 4ba37a39c6..0000000000 --- a/minimal-conda-requirements.txt +++ /dev/null @@ -1,27 +0,0 @@ -# Use this file to create a conda environment using: -# conda create -n --file minimal-conda-requirements.txt - -# Mandatory dependencies -cartopy -matplotlib<1.9 -netcdf4 -numpy -pyke -udunits2 -cf_units -dask>=0.15.0 - -# Iris build dependencies -setuptools - -# Iris testing/documentation dependencies -mock -nose -pep8 -sphinx -filelock -imagehash -requests - -# Optional iris dependencies -mo_pack diff --git a/requirements/all.txt b/requirements/all.txt new file mode 100644 index 0000000000..eb0354fc80 --- /dev/null +++ b/requirements/all.txt @@ -0,0 +1,17 @@ +# Dependencies for a feature complete installation +# ------------------------------------------------ + +# esmpy regridding not available through pip. +#conda: esmpy>=7.0 (only python=2) +gdal +mo_pack +nc_time_axis +pandas +stratify #conda: python-stratify +pyugrid + +#conda: graphviz + +# Iris sample data is not available through pip. It can be installed from +# https://github.com/SciTools/iris-sample-data/archive/master.zip +#conda: iris-sample-data diff --git a/requirements/core.txt b/requirements/core.txt new file mode 100644 index 0000000000..d2540da9f1 --- /dev/null +++ b/requirements/core.txt @@ -0,0 +1,12 @@ +# Absolute minimal dependencies for iris +# -------------------------------------- + +# Without these, iris won't even import. + +cartopy +matplotlib<1.9 +netcdf4 +numpy +# pyke (not pip installable) #conda: pyke +cf_units +dask>=0.15.0 diff --git a/requirements/docs.txt b/requirements/docs.txt new file mode 100644 index 0000000000..6966869c70 --- /dev/null +++ b/requirements/docs.txt @@ -0,0 +1 @@ +sphinx diff --git a/requirements/extensions.txt b/requirements/extensions.txt new file mode 100644 index 0000000000..64a28e91f2 --- /dev/null +++ b/requirements/extensions.txt @@ -0,0 +1,8 @@ +# Iris extensions (i.e. key tools that depend on Iris) +# ---------------------------------------------------- + +# Note: pip can handle the circularity of these extensions, but conda will +# struggle. To install these extensions, ensure iris[core] has been installed +# first. + +iris_grib;python_version<"3" #conda: diff --git a/requirements/gen_conda_requirements.py b/requirements/gen_conda_requirements.py new file mode 100644 index 0000000000..a6a5fd84aa --- /dev/null +++ b/requirements/gen_conda_requirements.py @@ -0,0 +1,91 @@ +# (C) British Crown Copyright 2017, Met Office +# +# This file is part of Iris. +# +# Iris is free software: you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by the +# Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Iris is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Iris. If not, see . + +import argparse +import os.path + + +REQS_DIR = os.path.dirname(__file__) +CONDA_PATTERN = '#conda:' + + +def read_conda_reqs(fname, options): + lines = [] + with open(fname, 'r') as fh: + for line in fh: + line = line.strip() + if '#conda:' in line: + line_start = line.index(CONDA_PATTERN) + len(CONDA_PATTERN) + line = line[line_start:].strip() + if 'only python=2' in line: + if 'python=2' in options: + line = line.replace('(only python=2)', '') + lines.append(line) + else: + continue + else: + lines.append(line) + else: + lines.append(line) + return lines + + +def compute_requirements(requirement_names=('core', ), options=None): + conda_reqs_lines = [] + + if 'python=2' in options: + conda_reqs_lines.append('python=2.*') + else: + conda_reqs_lines.append('# Python 3 conda configuration') + + for req_name in requirement_names: + fname = os.path.join(REQS_DIR, '{}.txt'.format(req_name)) + if not os.path.exists(fname): + raise RuntimeError('Unable to find the requirements file for {} ' + 'in {}'.format(req_name, fname)) + conda_reqs_lines.extend(read_conda_reqs(fname, options)) + conda_reqs_lines.append('') + + return conda_reqs_lines + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--verbosity", help="increase output verbosity") + parser.add_argument( + "--groups", nargs='*', default=[], + help=("Gather requirements for these given named groups " + "(as found in the requirements/ folder)")) + parser.add_argument( + "--py2", action="store_true", + help="Build the conda requirements for a python 2 installation") + + args = parser.parse_args() + + requirement_names = args.groups + requirement_names.insert(0, 'core') + requirement_names.insert(0, 'setup') + + options = [] + if args.py2: + options.append('python=2') + + print('\n'.join(compute_requirements(requirement_names, options))) + + +if __name__ == '__main__': + main() diff --git a/requirements/setup.txt b/requirements/setup.txt new file mode 100644 index 0000000000..552f54318c --- /dev/null +++ b/requirements/setup.txt @@ -0,0 +1,6 @@ +# Dependencies necessary to run setup.py of iris +# ---------------------------------------------- + +setuptools +# pyke (not pip installable) #conda: pyke +six diff --git a/requirements/test.txt b/requirements/test.txt new file mode 100644 index 0000000000..26a00bcacc --- /dev/null +++ b/requirements/test.txt @@ -0,0 +1,9 @@ +# Dependencies needed to run the iris tests +#------------------------------------------ + +mock +nose +pep8 +filelock +imagehash +requests diff --git a/setup.py b/setup.py index 702e59eb82..2b7787effc 100644 --- a/setup.py +++ b/setup.py @@ -1,14 +1,13 @@ from __future__ import print_function from contextlib import contextmanager -from distutils.core import Command from distutils.util import convert_path import os from shutil import copyfile import sys import textwrap -from setuptools import setup +from setuptools import setup, Command from setuptools.command.develop import develop as develop_cmd from setuptools.command.build_py import build_py @@ -68,6 +67,22 @@ def temporary_path(directory): with temporary_path('lib/iris/tests/runner'): from _runner import TestRunner # noqa: +SETUP_DIR = os.path.dirname(__file__) + +def pip_requirements(name): + fname = os.path.join(SETUP_DIR, 'requirements', '{}.txt'.format(name)) + if not os.path.exists(fname): + raise RuntimeError('Unable to find the {} requirements file at {}' + ''.format(name, fname)) + reqs = [] + with open(fname, 'r') as fh: + for line in fh: + line = line.strip() + if not line or line.startswith('#'): + continue + reqs.append(line) + return reqs + class SetupTestRunner(TestRunner, Command): pass @@ -186,8 +201,7 @@ def run(self): def extract_version(): version = None - fdir = os.path.dirname(__file__) - fnme = os.path.join(fdir, 'lib', 'iris', '__init__.py') + fnme = os.path.join(SETUP_DIR, 'lib', 'iris', '__init__.py') with open(fnme) as fd: for line in fd: if (line.startswith('__version__')): @@ -214,8 +228,11 @@ def extract_version(): } +pypi_name = 'scitools-iris' + + setup( - name='Iris', + name=pypi_name, version=extract_version(), url='http://scitools.org.uk/iris/', author='UK Met Office', @@ -223,7 +240,16 @@ def extract_version(): packages=find_package_tree('lib/iris', 'iris'), package_dir={'': 'lib'}, include_package_data=True, - tests_require=['nose'], cmdclass=custom_commands, + zip_safe=False, - ) + + setup_requires=pip_requirements('setup'), + install_requires=pip_requirements('core'), + tests_require=['{}[test]'.format(pypi_name)], + extras_require = { + 'test': pip_requirements('test'), + 'all': pip_requirements('all'), + 'extensions': pip_requirements('extensions'), + }, +)