Merge branch 'master' into 2d-coord

* master: (21 commits)
  xarray.backends refactor (pydata#2261)
  Fix indexing error for data loaded with open_rasterio (pydata#2456)
  Properly support user-provided norm. (pydata#2443)
  pep8speaks (pydata#2462)
  isort (pydata#2469)
  tests shoudn't need to pass for a PR (pydata#2471)
  Replace the last of unittest with pytest (pydata#2467)
  Add python_requires to setup.py (pydata#2465)
  Update whats-new.rst (pydata#2466)
  Clean up _parse_array_of_cftime_strings (pydata#2464)
  plot.contour: Don't make cmap if colors is a single color. (pydata#2453)
  np.AxisError was added in numpy 1.13 (pydata#2455)
  Add CFTimeIndex.shift (pydata#2431)
  Fix FutureWarning in CFTimeIndex.date_type (pydata#2448)
  fix:2445 (pydata#2446)
  Enable use of cftime.datetime coordinates with differentiate and interp (pydata#2434)
  restore ddof support in std (pydata#2447)
  Future warning for default reduction dimension of groupby (pydata#2366)
  Remove incorrect statement about "drop" in the text docs (pydata#2439)
  Use profile mechanism, not no-op mutation (pydata#2442)
  ...

Author: dcherian

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,4 +1,3 @@
  - [ ] Closes #xxxx (remove if there is no corresponding issue, which should only be the case for minor changes)
  - [ ] Tests added (for all bug fixes or enhancements)
- - [ ] Tests passed (for all non-documentation changes)
  - [ ] Fully documented, including `whats-new.rst` for all changes and `api.rst` for new API (remove if this change should not be visible to users, e.g., if it is an internal clean-up, or if this is part of a larger project that will be documented later)

.pep8speaks.yml

@@ -0,0 +1,11 @@
+# File : .pep8speaks.yml
+
+scanner:
+    diff_only: True  # If True, errors caused by only the patch are shown
+
+pycodestyle:
+    max-line-length: 79
+    ignore:  # Errors and warnings to ignore
+        - E402,  # module level import not at top of file
+        - E731,  # do not assign a lambda expression, use a def
+        - W503   # line break before binary operator

.stickler.yml

@@ -1,5 +1,5 @@
 # Based on http://conda.pydata.org/docs/travis.html
-language: python
+language: minimal
 sudo: false # use container based build
 notifications:
   email: false
@@ -10,74 +10,48 @@ branches:
 matrix:
   fast_finish: true
   include:
-  - python: 2.7
-    env: CONDA_ENV=py27-min
-  - python: 2.7
-    env: CONDA_ENV=py27-cdat+iris+pynio
-  - python: 3.5
-    env: CONDA_ENV=py35
-  - python: 3.6
-    env: CONDA_ENV=py36
-  - python: 3.6  # TODO: change this to 3.7 once https://github.com/travis-ci/travis-ci/issues/9815 is fixed
-    env: CONDA_ENV=py37
-  - python: 3.6
-    env:
+  - env: CONDA_ENV=py27-min
+  - env: CONDA_ENV=py27-cdat+iris+pynio
+  - env: CONDA_ENV=py35
+  - env: CONDA_ENV=py36
+  - env: CONDA_ENV=py37
+  - env:
     - CONDA_ENV=py36
     - EXTRA_FLAGS="--run-flaky --run-network-tests"
-  - python: 3.6
-    env: CONDA_ENV=py36-netcdf4-dev
+  - env: CONDA_ENV=py36-netcdf4-dev
     addons:
       apt_packages:
         - libhdf5-serial-dev
         - netcdf-bin
         - libnetcdf-dev
-  - python: 3.6
-    env: CONDA_ENV=py36-dask-dev
-  - python: 3.6
-    env: CONDA_ENV=py36-pandas-dev
-  - python: 3.6
-    env: CONDA_ENV=py36-bottleneck-dev
-  - python: 3.6
-    env: CONDA_ENV=py36-condaforge-rc
-  - python: 3.6
-    env: CONDA_ENV=py36-pynio-dev
-  - python: 3.6
-    env: CONDA_ENV=py36-rasterio-0.36
-  - python: 3.6
-    env: CONDA_ENV=py36-zarr-dev
-  - python: 3.5
-    env: CONDA_ENV=docs
-  - python: 3.6
-    env: CONDA_ENV=py36-hypothesis
+  - env: CONDA_ENV=py36-dask-dev
+  - env: CONDA_ENV=py36-pandas-dev
+  - env: CONDA_ENV=py36-bottleneck-dev
+  - env: CONDA_ENV=py36-condaforge-rc
+  - env: CONDA_ENV=py36-pynio-dev
+  - env: CONDA_ENV=py36-rasterio-0.36
+  - env: CONDA_ENV=py36-zarr-dev
+  - env: CONDA_ENV=docs
+  - env: CONDA_ENV=py36-hypothesis
+
   allow_failures:
-  - python: 3.6
-    env:
+  - env:
     - CONDA_ENV=py36
     - EXTRA_FLAGS="--run-flaky --run-network-tests"
-  - python: 3.6
-    env: CONDA_ENV=py36-netcdf4-dev
+  - env: CONDA_ENV=py36-netcdf4-dev
     addons:
       apt_packages:
         - libhdf5-serial-dev
         - netcdf-bin
         - libnetcdf-dev
-  - python: 3.6
-    env: CONDA_ENV=py36-pandas-dev
-  - python: 3.6
-    env: CONDA_ENV=py36-bottleneck-dev
-  - python: 3.6
-    env: CONDA_ENV=py36-condaforge-rc
-  - python: 3.6
-    env: CONDA_ENV=py36-pynio-dev
-  - python: 3.6
-    env: CONDA_ENV=py36-zarr-dev
+  - env: CONDA_ENV=py36-pandas-dev
+  - env: CONDA_ENV=py36-bottleneck-dev
+  - env: CONDA_ENV=py36-condaforge-rc
+  - env: CONDA_ENV=py36-pynio-dev
+  - env: CONDA_ENV=py36-zarr-dev
 
 before_install:
-  - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then
-      wget http://repo.continuum.io/miniconda/Miniconda-3.16.0-Linux-x86_64.sh -O miniconda.sh;
-    else
-      wget http://repo.continuum.io/miniconda/Miniconda3-3.16.0-Linux-x86_64.sh -O miniconda.sh;
-    fi
+  - wget http://repo.continuum.io/miniconda/Miniconda3-3.16.0-Linux-x86_64.sh -O miniconda.sh;
   - bash miniconda.sh -b -p $HOME/miniconda
   - export PATH="$HOME/miniconda/bin:$PATH"
   - hash -r
@@ -97,6 +71,8 @@ install:
   - python xarray/util/print_versions.py
 
 script:
+  - which python
+  - python --version
   - python -OO -c "import xarray"
   - if [[ "$CONDA_ENV" == "docs" ]]; then
       conda install -c conda-forge sphinx sphinx_rtd_theme sphinx-gallery numpydoc;

.travis.yml

@@ -64,6 +64,7 @@
         "scipy": [""],
         "bottleneck": ["", null],
         "dask": [""],
+        "distributed": [""],
     },
 
 

asv_bench/asv.conf.json

@@ -1,11 +1,13 @@
 from __future__ import absolute_import, division, print_function
 
+import os
+
 import numpy as np
 import pandas as pd
 
 import xarray as xr
 
-from . import randn, randint, requires_dask
+from . import randint, randn, requires_dask
 
 try:
     import dask
@@ -14,6 +16,9 @@
     pass
 
 
+os.environ['HDF5_USE_FILE_LOCKING'] = 'FALSE'
+
+
 class IOSingleNetCDF(object):
     """
     A few examples that benchmark reading/writing a single netCDF file with
@@ -405,3 +410,39 @@ def time_open_dataset_scipy_with_time_chunks(self):
         with dask.set_options(get=dask.multiprocessing.get):
             xr.open_mfdataset(self.filenames_list, engine='scipy',
                               chunks=self.time_chunks)
+
+
+def create_delayed_write():
+    import dask.array as da
+    vals = da.random.random(300, chunks=(1,))
+    ds = xr.Dataset({'vals': (['a'], vals)})
+    return ds.to_netcdf('file.nc', engine='netcdf4', compute=False)
+
+
+class IOWriteNetCDFDask(object):
+    timeout = 60
+    repeat = 1
+    number = 5
+
+    def setup(self):
+        requires_dask()
+        self.write = create_delayed_write()
+
+    def time_write(self):
+        self.write.compute()
+
+
+class IOWriteNetCDFDaskDistributed(object):
+    def setup(self):
+        try:
+            import distributed
+        except ImportError:
+            raise NotImplementedError
+        self.client = distributed.Client()
+        self.write = create_delayed_write()
+
+    def cleanup(self):
+        self.client.shutdown()
+
+    def time_write(self):
+        self.write.compute()

asv_bench/benchmarks/dataset_io.py

@@ -1,6 +1,7 @@
 from __future__ import absolute_import, division, print_function
 
 import numpy as np
+
 import xarray as xr
 
 from . import requires_dask

asv_bench/benchmarks/unstacking.py

@@ -151,3 +151,5 @@
    plot.FacetGrid.set_titles
    plot.FacetGrid.set_ticks
    plot.FacetGrid.map
+
+   CFTimeIndex.shift

doc/api-hidden.rst

@@ -624,3 +624,6 @@ arguments for the ``from_store`` and ``dump_to_store`` Dataset methods:
    backends.H5NetCDFStore
    backends.PydapDataStore
    backends.ScipyDataStore
+   backends.FileManager
+   backends.CachingFileManager
+   backends.DummyFileManager

doc/api.rst

@@ -408,13 +408,6 @@ operations keep around coordinates:
     list(ds[['x']])
     list(ds.drop('temperature'))
 
-If a dimension name is given as an argument to ``drop``, it also drops all
-variables that use that dimension:
-
-.. ipython:: python
-
-    list(ds.drop('time'))
-
 As an alternate to dictionary-like modifications, you can use
 :py:meth:`~xarray.Dataset.assign` and :py:meth:`~xarray.Dataset.assign_coords`.
 These methods return a new dataset with additional (or replaced) or values: