Improved the way requirements are handled, ensuring that pip gets all the information it needs.

.travis.yml

@@ -55,16 +55,34 @@ install:
   # Customise the testing environment
   # ---------------------------------
   - conda config --add channels conda-forge
-  - if [[ "$TEST_MINIMAL" == true ]]; then
-      conda install --quiet --file minimal-conda-requirements.txt;
+  - >
+    if [[ "$TEST_MINIMAL" == true ]]; then
+      CONDA_REQS_FILE=conda-requirements.txt
+      if [[ "$TRAVIS_PYTHON_VERSION" == 3* ]]; then
+          python requirements/gen_conda_requirements.py --groups test > ${CONDA_REQS_FILE};
+      else
+          python requirements/gen_conda_requirements.py --py2 --groups test > ${CONDA_REQS_FILE};
+      fi
+      cat ${CONDA_REQS_FILE};
+      conda install --quiet --file ${CONDA_REQS_FILE};
     else
+      CONDA_REQS_FILE=conda-requirements.txt
       if [[ "$TRAVIS_PYTHON_VERSION" == 3* ]]; then
-        sed -e '/esmpy/d' -e 's/#.\+$//' conda-requirements.txt | xargs conda install --quiet;
+          python requirements/gen_conda_requirements.py --groups test full > ${CONDA_REQS_FILE};
       else
-        conda install --quiet --file conda-requirements.txt;
+          python requirements/gen_conda_requirements.py --py2 --groups test full > ${CONDA_REQS_FILE};
       fi
+      cat ${CONDA_REQS_FILE};
+      conda install --quiet --file ${CONDA_REQS_FILE};
     fi
 
+
+  - >
+    if [[ "${TEST_TARGET}" == 'doctest' ]]; then
+        conda install --quiet --file requirements/docs.txt
+    fi
+
+
   # JUST FOR NOW : Install latest version of iris-grib.
   # TODO : remove when this release is available on conda-forge.
   - if [[ "$TEST_MINIMAL" != true ]]; then

requirements/core.txt

@@ -0,0 +1,12 @@
+# Absolute minimal dependencies for iris
+# --------------------------------------
+
+# Without these, iris won't even import.
+
+cartopy
+matplotlib<1.9
+netcdf4
+numpy
+pyke
+cf_units
+dask>=0.15.0

requirements/docs.txt

@@ -0,0 +1 @@
+sphinx

requirements/full.txt

@@ -0,0 +1,26 @@
+# Dependencies for a feature complete installation
+# ------------------------------------------------
+
+# esmpy regridding not available through pip.
+#conda: esmpy>=7.0  (only python=2)
+
+gdal
+mo_pack
+nc_time_axis
+pandas
+stratify  #conda: python-stratify
+pyugrid
+
+# Iris sample data is not available through pip. It can be installed from
+# https://github.com/SciTools/iris-sample-data/archive/master.zip
+#conda: iris-sample-data
+
+
+# Iris extensions (i.e. key tools that depend on Iris)
+# ----------------------------------------------------
+
+# Note: pip can handle the circularity of these extensions, but conda will
+# struggle. To install these extensions, ensure iris[core] has been installed
+# first.
+
+iris_grib;python_version<"3"  #conda:

requirements/gen_conda_requirements.py

@@ -0,0 +1,91 @@
+# (C) British Crown Copyright 2017, Met Office
+#
+# This file is part of Iris.
+#
+# Iris is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the
+# Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Iris is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with Iris.  If not, see <http://www.gnu.org/licenses/>.
+
+import argparse
+import os.path
+
+
+REQS_DIR = os.path.dirname(__file__)
+CONDA_PATTERN = '#conda:'
+
+
+def read_conda_reqs(fname, options):
+    lines = []
+    with open(fname, 'r') as fh:
+        for line in fh:
+            line = line.strip()
+            if '#conda:' in line:
+                line_start = line.index(CONDA_PATTERN) + len(CONDA_PATTERN)
+                line = line[line_start:].strip()
+                if 'only python=2' in line:
+                    if 'python=2' in options:
+                        line = line.replace('(only python=2)', '')
+                        lines.append(line)
+                    else:
+                        continue
+                else:
+                    lines.append(line)
+            else:
+                lines.append(line)
+    return lines
+
+
+def compute_requirements(requirement_names=('core', ), options=None):
+    conda_reqs_lines = []
+
+    if 'python=2' in options:
+        conda_reqs_lines.append('python=2.*')
+    else:
+        conda_reqs_lines.append('# Python 3 conda configuration')
+
+    for req_name in requirement_names:
+        fname = os.path.join(REQS_DIR, '{}.txt'.format(req_name))
+        if not os.path.exists(fname):
+            raise RuntimeError('Unable to find the requirements file for {} '
+                               'in {}'.format(req_name, fname))
+        conda_reqs_lines.extend(read_conda_reqs(fname, options))
+        conda_reqs_lines.append('')
+
+    return conda_reqs_lines
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--verbosity", help="increase output verbosity")
+    parser.add_argument(
+            "--groups", nargs='*', default=[],
+            help=("Gather requirements for these given named groups "
+                  "(as found in the requirements/ folder)"))
+    parser.add_argument(
+            "--py2", action="store_true",
+            help="Build the conda requirements for a python 2 installation")
+
+    args = parser.parse_args()
+
+    requirement_names = args.groups
+    requirement_names.insert(0, 'core')
+    requirement_names.insert(0, 'setup')
+
+    options = []
+    if args.py2:
+        options.append('python=2')
+
+    print('\n'.join(compute_requirements(requirement_names, options)))
+
+
+if __name__ == '__main__':
+    main()

requirements/setup.txt

@@ -0,0 +1,5 @@
+# Dependencies necessary to run setup.py of iris
+# ----------------------------------------------
+
+setuptools
+numpy

requirements/test.txt

@@ -0,0 +1,9 @@
+# Dependencies needed to run the iris tests
+#------------------------------------------
+
+mock
+nose
+pep8
+filelock
+imagehash
+requests

setup.py

@@ -1,14 +1,13 @@
 from __future__ import print_function
 
 from contextlib import contextmanager
-from distutils.core import Command
 from distutils.util import convert_path
 import os
 from shutil import copyfile
 import sys
 import textwrap
 
-from setuptools import setup
+from setuptools import setup, Command
 from setuptools.command.develop import develop as develop_cmd
 from setuptools.command.build_py import build_py
 
@@ -68,6 +67,22 @@ def temporary_path(directory):
 with temporary_path('lib/iris/tests/runner'):
     from _runner import TestRunner  # noqa:
 
+SETUP_DIR = os.path.dirname(__file__)
+
+def requirements(name):
+    fname = os.path.join(SETUP_DIR, 'requirements', '{}.txt'.format(name))
+    if not os.path.exists(fname):
+        raise RuntimeError('Unable to find the {} requirements file at {}'
+                           ''.format(name, fname))
+    reqs = []
+    with open(fname, 'r') as fh:
+        for line in fh:
+            line = line.strip()
+            if not line or line.startswith('#'):
+                continue
+            reqs.append(line)
+    return reqs
+
 
 class SetupTestRunner(TestRunner, Command):
     pass
@@ -186,8 +201,7 @@ def run(self):
 
 def extract_version():
     version = None
-    fdir = os.path.dirname(__file__)
-    fnme = os.path.join(fdir, 'lib', 'iris', '__init__.py')
+    fnme = os.path.join(SETUP_DIR, 'lib', 'iris', '__init__.py')
     with open(fnme) as fd:
         for line in fd:
             if (line.startswith('__version__')):
@@ -214,16 +228,27 @@ def extract_version():
     }
 
 
+pypi_name = 'scitools-iris'
+
+
 setup(
-    name='Iris',
+    name=pypi_name,
     version=extract_version(),
     url='http://scitools.org.uk/iris/',
     author='UK Met Office',
     author_email='[email protected]',
     packages=find_package_tree('lib/iris', 'iris'),
     package_dir={'': 'lib'},
     include_package_data=True,
-    tests_require=['nose'],
     cmdclass=custom_commands,
+
     zip_safe=False,
-   )
+
+    setup_requires=requirements('setup'),
+    install_requires=requirements('core'),
+    tests_require=['{}[test]'.format(pypi_name)],
+    extras_require = {
+                      'test': requirements('test'),
+                      'all': requirements('full'),
+                      },
+)