DAS-2214 Add NSIDC IceSAT2 Regression Tests

.github/workflows/build-all-images.yml

@@ -29,6 +29,9 @@ jobs:
           -
             image: "n2z"
             notebook: "N2Z_Regression.ipynb"
+          -
+            image: "nsidc-icesat2"
+            notebook: "NSIDC-ICESAT2_Regression.ipynb"
           -
             image: "regridder"
             notebook: "Regridder_Regression.ipynb"

CHANGELOG.md

@@ -5,14 +5,20 @@ versioning. Rather than a static releases, this repository contains of a number
 of regression tests that are each semi-independent.  This CHANGELOG file should be used
 to document pull requests to this repository.
 
-## 2024-08-30
+## 2024-09-04 ([#92](https://github.com/nasa/harmony-regression-tests/pull/92))
+
+- Adds NSIDC ICESat2 Regression test suite.
+
+- Adds `shared_utils` functionality. This directory contains routines that are commonly used in regression tests and limits code duplication. To include the `shared_utils` directory in your docker container, update the Makefile to add a shared_utils build arg. E.g. `--build-arg shared_utils=true` (see the nsidc-icesat2-image target)
+
+## 2024-08-30 ([#94](https://github.com/nasa/harmony-regression-tests/pull/94))
 
 Add regression test for net2cog
 
 ## 2024-08-05 ([#86](https://github.com/nasa/harmony-regression-tests/pull/86))
 
-Adds this file to capture changes to the repository.
+- Adds this file to capture changes to the repository.
 
-Adds pre-commit.ci behavior to the repository. This setup ensures consistent code style, catches common errors, and maintains file hygiene across the project.
+- Adds pre-commit.ci behavior to the repository. This setup ensures consistent code style, catches common errors, and maintains file hygiene across the project.
 
-Updates the base image for all regression tests to `mambaorg/micromamba:1.5.8-jammy`
+- Updates the base image for all regression tests to `mambaorg/micromamba:1.5.8-jammy`

README.md

@@ -90,12 +90,15 @@ environment before installing from the environment.yml.
    is simplest to use the same string for the subdirectory name and the suite
    name.
 1. Update the `test/Makefile` to be able to build a Docker image for the new
-   test suite:
+   test suite optionally including the shared utility directory:
 
    ```
    <new-suite-name>-image
-       docker build -t ghcr.io/nasa/regression-tests-<new-suite-name>:latest -f ./Dockerfile --build-arg notebook=<new-test-notebook-name> --build-arg sub_dir=<new-suite-subdirectory> .
+       docker build -t ghcr.io/nasa/regression-tests-<new-suite-name>:latest -f ./Dockerfile --build-arg notebook=<new-test-notebook-name> --build-arg sub_dir=<new-suite-subdirectory> [--build-arg shared_utils=true] .
    ```
+
+1. If you would like to use shared utilities to help ease the coding you can include a build arg in
+
 1. Update the `make images` rule to include building the new image.
 
    ```

script/test-in-bamboo.sh

@@ -42,7 +42,7 @@ echo "harmony host url: ${harmony_host_url}"
 ## e.g. if REGRESSION_TESTS_N2Z_IMAGE environment was set, the value would be used instead of the default.
 
 image_names=()
-all_tests=(harmony harmony-regression hoss hga n2z swath-projector trajectory-subsetter variable-subsetter regridder hybig geoloco net2cog)
+all_tests=(harmony harmony-regression hoss hga n2z nsidc-icesat2 swath-projector trajectory-subsetter variable-subsetter regridder hybig geoloco net2cog)
 for image in "${all_tests[@]}"; do
     image_names+=($(image_name "$image" true))
 done

test/Dockerfile

@@ -3,9 +3,11 @@ USER root
 
 ARG sub_dir
 ARG notebook
+ARG shared_utils=false
 ENV env_sub_dir=$sub_dir
 ENV env_notebook=$notebook
 
+
 ENV AWS_SECRET_ACCESS_KEY=no-access
 ENV AWS_ACCESS_KEY_ID=no-access
 ENV AWS_SESSION_TOKEN=no-access
@@ -20,9 +22,24 @@ COPY build-netrc.sh notebook-entrypoint.sh ./
 RUN mkdir ./${sub_dir}
 COPY ${sub_dir}/environment.yaml ./${sub_dir}
 
+# Shared util temporary solution: because datatree is not released we are using
+# a pinned version from the git repo and need to have git installed to install
+# xarray
+RUN if [ "$shared_utils" = "true" ]; then \
+        apt-get update && apt-get install -y git; \
+    fi
+
 RUN micromamba create -y -f ${sub_dir}/environment.yaml \
     && micromamba clean --all --force-pkgs-dirs --yes
 
+# Include shared utility functions if requested.  This is a bit awkward, it
+# always copies the shared utils directory to the image, but then deletes it if
+# you didn't want it.
+COPY shared_utils ./shared_utils
+RUN if [ "$shared_utils" = "false" ]; then \
+        rm -rf ./shared_utils; \
+    fi
+
 COPY ${sub_dir} ./${sub_dir}
 
 ENTRYPOINT /bin/bash ./notebook-entrypoint.sh

test/Makefile

@@ -1,37 +1,65 @@
 harmony-image: Dockerfile harmony/environment.yaml
-	docker build -t ghcr.io/nasa/regression-tests-harmony:latest -f ./Dockerfile --build-arg notebook=Harmony.ipynb --build-arg sub_dir=harmony .
+	docker build -t ghcr.io/nasa/regression-tests-harmony:latest -f ./Dockerfile \
+	--build-arg notebook=Harmony.ipynb --build-arg sub_dir=harmony .
 
 harmony-regression-image: Dockerfile harmony-regression/environment.yaml
-	docker build -t ghcr.io/nasa/regression-tests-harmony-regression:latest -f ./Dockerfile --build-arg notebook=HarmonyRegression.ipynb --build-arg sub_dir=harmony-regression .
+	docker build -t ghcr.io/nasa/regression-tests-harmony-regression:latest -f ./Dockerfile \
+	--build-arg notebook=HarmonyRegression.ipynb --build-arg sub_dir=harmony-regression .
 
 hga-image: Dockerfile hga/environment.yaml
-	docker build -t ghcr.io/nasa/regression-tests-hga:latest -f ./Dockerfile --build-arg notebook=HGA_Regression.ipynb --build-arg sub_dir=hga .
+	docker build -t ghcr.io/nasa/regression-tests-hga:latest -f ./Dockerfile \
+	--build-arg notebook=HGA_Regression.ipynb --build-arg sub_dir=hga .
 
 hoss-image: Dockerfile hoss/environment.yaml
-	docker build -t ghcr.io/nasa/regression-tests-hoss:latest -f ./Dockerfile --build-arg notebook=HOSS_Regression.ipynb --build-arg sub_dir=hoss .
+	docker build -t ghcr.io/nasa/regression-tests-hoss:latest -f ./Dockerfile \
+	--build-arg notebook=HOSS_Regression.ipynb --build-arg sub_dir=hoss .
 
 hybig-image: Dockerfile hybig/environment.yaml
-	docker build -t ghcr.io/nasa/regression-tests-hybig:latest -f ./Dockerfile --build-arg notebook=HyBIG_Regression.ipynb --build-arg sub_dir=hybig .
+	docker build -t ghcr.io/nasa/regression-tests-hybig:latest -f ./Dockerfile \
+	--build-arg notebook=HyBIG_Regression.ipynb --build-arg sub_dir=hybig .
 
 n2z-image: Dockerfile n2z/environment.yaml
-	docker build -t ghcr.io/nasa/regression-tests-n2z:latest -f ./Dockerfile --build-arg notebook=N2Z_Regression.ipynb --build-arg sub_dir=n2z .
+	docker build -t ghcr.io/nasa/regression-tests-n2z:latest -f ./Dockerfile \
+	--build-arg notebook=N2Z_Regression.ipynb --build-arg sub_dir=n2z .
+
+nsidc-icesat2-image: Dockerfile nsidc-icesat2/environment.yaml
+	docker build -t ghcr.io/nasa/regression-tests-nsidc-icesat2:latest -f ./Dockerfile \
+	--build-arg notebook=NSIDC-ICESAT2_Regression.ipynb --build-arg sub_dir=nsidc-icesat2 --build-arg shared_utils=true .
 
 regridder-image: Dockerfile regridder/environment.yaml
-	docker build -t ghcr.io/nasa/regression-tests-regridder:latest -f ./Dockerfile --build-arg notebook=Regridder_Regression.ipynb --build-arg sub_dir=regridder .
+	docker build -t ghcr.io/nasa/regression-tests-regridder:latest -f ./Dockerfile \
+	--build-arg notebook=Regridder_Regression.ipynb --build-arg sub_dir=regridder .
 
 swath-projector-image: Dockerfile swath-projector/environment.yaml
-	docker build -t ghcr.io/nasa/regression-tests-swath-projector:latest -f ./Dockerfile --build-arg notebook=SwathProjector_Regression.ipynb --build-arg sub_dir=swath-projector .
+	docker build -t ghcr.io/nasa/regression-tests-swath-projector:latest -f ./Dockerfile \
+	--build-arg notebook=SwathProjector_Regression.ipynb --build-arg sub_dir=swath-projector .
 
 trajectory-subsetter-image: Dockerfile trajectory-subsetter/environment.yaml
-	docker build -t ghcr.io/nasa/regression-tests-trajectory-subsetter:latest -f ./Dockerfile --build-arg notebook=TrajectorySubsetter_Regression.ipynb --build-arg sub_dir=trajectory-subsetter .
+	docker build -t ghcr.io/nasa/regression-tests-trajectory-subsetter:latest -f ./Dockerfile \
+	--build-arg notebook=TrajectorySubsetter_Regression.ipynb --build-arg sub_dir=trajectory-subsetter .
 
 variable-subsetter-image: Dockerfile variable-subsetter/environment.yaml
-	docker build -t ghcr.io/nasa/regression-tests-variable-subsetter:latest -f ./Dockerfile --build-arg notebook=VariableSubsetter_Regression.ipynb --build-arg sub_dir=variable-subsetter .
+	docker build -t ghcr.io/nasa/regression-tests-variable-subsetter:latest -f ./Dockerfile \
+	--build-arg notebook=VariableSubsetter_Regression.ipynb --build-arg sub_dir=variable-subsetter .
 
 geoloco-image: Dockerfile geoloco/environment.yaml
-	docker build -t ghcr.io/nasa/regression-tests-geoloco:latest -f ./Dockerfile --build-arg notebook=Geoloco_Regression.ipynb --build-arg sub_dir=geoloco .
+	docker build -t ghcr.io/nasa/regression-tests-geoloco:latest -f ./Dockerfile \
+	--build-arg notebook=Geoloco_Regression.ipynb --build-arg sub_dir=geoloco .
 
 net2cog-image: Dockerfile net2cog/environment.yaml
-	docker build -t ghcr.io/nasa/regression-tests-net2cog:latest -f ./Dockerfile --build-arg notebook=net2cog_Regression.ipynb --build-arg sub_dir=net2cog .
+	docker build -t ghcr.io/nasa/regression-tests-net2cog:latest -f ./Dockerfile \
+	--build-arg notebook=net2cog_Regression.ipynb --build-arg sub_dir=net2cog .
 
-images: harmony-image harmony-regression-image hga-image hoss-image hybig-image n2z-image regridder-image swath-projector-image trajectory-subsetter-image variable-subsetter-image geoloco-image net2cog-image
+images: harmony-image \
+	harmony-regression-image \
+	hga-image \
+	hoss-image \
+	hybig-image \
+	n2z-image \
+	nsidc-icesat2-image \
+	regridder-image \
+	swath-projector-image \
+	trajectory-subsetter-image \
+	variable-subsetter-image \
+	geoloco-image \
+	net2cog-image