Merge pull request #33 from NREL/gb/parallax_correct

Gb/parallax correct

Author: grantbuster

.gitignore

@@ -1,6 +1,9 @@
 # test plots from pytest
 tests/data/test_plots/*
 
+# mac stuff
+.DS_Store
+
 # Dask stuff:
 dask-worker-space/
 

.pylintrc

@@ -55,6 +55,7 @@ confidence=
 # no Warning level messages displayed, use"--disable=all --enable=classes
 # --disable=W"
 disable=
+        consider-using-f-string,
         # Defaults
         print-statement,
         parameter-unpacking,

README.rst

@@ -21,25 +21,27 @@ documentation can be found `here <https://github.com/NREL/nsrdb/tree/master/nsrd
 Installation
 ============
 
-1. Use conda (anaconda or miniconda with python 3.7) to create an nsrdb
+#. Use conda (anaconda or miniconda with python 3.7) to create an nsrdb
    environment: ``conda create --name nsrdb python=3.7``
-2. Activate your new conda env: ``conda activate nsrdb``
-3. Navigate to the nsrdb directory that contains setup.py and run:
+#. Activate your new conda env: ``conda activate nsrdb``
+#. There is a known dependency issue between h5py and netCDF4. it is
+   recommended that you install ``conda install netCDF4`` before pip installing
+   the nsrdb code base.
+#. You will have to install the REST2 code base (the private NREL-REST2 repo is
+   `here <https://github.com/NREL/rest2>`_) BEFORE installing the nsrdb code
+   base.
+#. Navigate to the nsrdb directory that contains setup.py and run:
    ``pip install -e .``
-4. There is a known dependency issue between h5py and netCDF4. If you get an
-   error using the netCDF4 module, try installing from ``conda`` instead of
-   ``pip``
-
-    a. ``pip uninstall netCDF4``
-    b. ``conda install netCDF4``
-
-5. Test your installation:
+#. If running with the MLClouds model, you will have to install this code base
+   (the private MLClouds repo is `here <https://github.com/NREL/mlclouds>`_)
+   AFTER installing the nsrdb code base.
+#. Test your installation:
 
     a. Start ipython and test the following import:
        ``from nsrdb.data_model import DataModel``
     b. Navigate to the tests/ directory and run the command: ``pytest``
 
-6. If you are a developer, also run `pre-commit install` in the directory
+#. If you are a developer, also run `pre-commit install` in the directory
    containing .pre-commit-config.yaml.
 
 NOTE: If you are trying to run the ``albedo`` sub-module you will need to
@@ -143,6 +145,14 @@ NSRDB Versions
       - Enabled cloud solar shading coordinate adjustment by default, enabled
         MLClouds machine learning gap fill method for missing cloud properties
         (cloud fill flag #7)
+    * - 3.2.1
+      - 1/12/2021
+      - 2021
+      - Implemented an algorithm to re-map the parallax and shading corrected
+        cloud coordinates to the nominal GOES coordinate system. This fixes the
+        issue of PC cloud coordinates conflicting with clearsky coordinates.
+        This also fixes the strange pattern that was found in the long term
+        means generated from PC data.
 
 *Note: The “Data Years” column shows which years of NSRDB data were updated at
 the time of version release. However, each NSRDB file should be checked for the

nsrdb/all_sky/all_sky.py

@@ -314,7 +314,7 @@ def all_sky_h5_parallel(f_source, rows=slice(None), cols=slice(None),
 
     with Handler(f_source) as res:
         data_shape = res.shape
-        missing = [arg not in res.dsets for arg in ALL_SKY_ARGS]
+        missing = [arg for arg in ALL_SKY_ARGS if arg not in res.dsets]
         if any(missing):
             msg = ('Cannot run all_sky, missing datasets {} from source: {}'
                    .format(missing, f_source))

nsrdb/cli.py

@@ -907,6 +907,7 @@ def eagle(ctx, alloc, memory, walltime, feature, stdout_path):
     else:
         cmd = ("python -c '{import_str};{f}({a})'"
                .format(import_str=import_str, f=fun_str, a=arg_str))
+        slurm_id = None
         out = slurm_manager.sbatch(cmd,
                                    alloc=alloc,
                                    memory=memory,
@@ -916,13 +917,15 @@ def eagle(ctx, alloc, memory, walltime, feature, stdout_path):
                                    stdout_path=stdout_path)[0]
 
         if out:
+            slurm_id = out
             msg = ('Kicked off job "{}" (SLURM jobid #{}) on Eagle.'
-                   .format(name, out))
-            Status.add_job(
-                out_dir, command, name, replace=True,
-                job_attrs={'job_id': out,
-                           'hardware': 'eagle',
-                           'out_dir': out_dir})
+                   .format(name, slurm_id))
+
+        Status.add_job(
+            out_dir, command, name, replace=True,
+            job_attrs={'job_id': slurm_id,
+                       'hardware': 'eagle',
+                       'out_dir': out_dir})
 
     click.echo(msg)
     logger.info(msg)

nsrdb/data_model/albedo.py

@@ -11,7 +11,7 @@
 import pandas as pd
 
 from nsrdb.data_model.base_handler import AncillaryVarHandler
-from nsrdb.file_handlers.filesystem import NSRDBFileSystem as NFS
+from nsrdb.file_handlers.file_system import NSRDBFileSystem as NFS
 
 logger = logging.getLogger(__name__)
 

nsrdb/data_model/asymmetry.py

@@ -5,7 +5,7 @@
 import os
 
 from nsrdb.data_model.base_handler import AncillaryVarHandler
-from nsrdb.file_handlers.filesystem import NSRDBFileSystem as NFS
+from nsrdb.file_handlers.file_system import NSRDBFileSystem as NFS
 
 logger = logging.getLogger(__name__)
 

nsrdb/data_model/base_handler.py

@@ -7,7 +7,7 @@
 from warnings import warn
 
 from nsrdb import DATADIR, DEFAULT_VAR_META
-from nsrdb.file_handlers.filesystem import NSRDBFileSystem as NFS
+from nsrdb.file_handlers.file_system import NSRDBFileSystem as NFS
 
 logger = logging.getLogger(__name__)