Switch to openeo-processes-dask engine

Dockerfile

@@ -5,6 +5,9 @@ ENV DEBIAN_FRONTEND=noninteractive \
     LANG=C.UTF-8 \
     TINI_VERSION=v0.19.0
 
+ENV PATH="/root/miniconda3/bin:${PATH}"
+ARG PATH="/root/miniconda3/bin:${PATH}"
+
 ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini
 RUN chmod +x /tini
 
@@ -14,31 +17,49 @@ RUN apt-get update && \
       git \
       wget \
       ffmpeg \
-      libsm6 \ 
-      libxext6 
+      libsm6 \
+      libxext6 \
+      libopengl0 \
+      libegl1
+
+# Install miniconda
+RUN wget \
+    https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \
+    && mkdir /root/.conda \
+    && bash Miniconda3-latest-Linux-x86_64.sh -b \
+    && rm -f Miniconda3-latest-Linux-x86_64.sh 
+RUN conda --version
+
+# RUN pip install --extra-index-url="https://packages.dea.ga.gov.au" \
+#   odc-ui \
+#   odc-stac \
+#   odc-stats \
+#   odc-algo \
+#   odc-io \
+#   odc-cloud[ASYNC] \
+#   odc-dscache \
+#   odc-index
+
+COPY ./environment.yml /
 
-COPY requirements.txt /conf/
+RUN conda env create -f /environment.yml
 
-RUN pip install --no-cache-dir --requirement /conf/requirements.txt
-RUN pip install --extra-index-url="https://packages.dea.ga.gov.au" \
-  odc-ui \
-  odc-stac \
-  odc-stats \
-  odc-algo \
-  odc-io \
-  odc-cloud[ASYNC] \
-  odc-dscache \
-  odc-index
+RUN git clone https://github.com/clausmichele/odc-tools.git
+RUN conda run -n openeo_odc_driver pip install odc-tools/apps/dc_tools
 
-RUN git clone https://github.com/Open-EO/openeo-pg-parser-python.git
-RUN cd openeo-pg-parser-python && pip install .
+# RUN pip install --requirement /requirements.txt
 
-RUN git clone https://github.com/SARScripts/openeo_odc_driver.git -b dev
+ADD "https://www.random.org/cgi-bin/randbyte?nbytes=10&format=h" skipcache
 
-WORKDIR /
+COPY . /openeo_odc_driver
 
-ENTRYPOINT ["/tini", "--"]
+# WORKDIR /
+
+# ENTRYPOINT ["/tini", "--"]
+
+ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/root/miniconda3/envs/openeo_odc_driver/lib/"
 
 WORKDIR /openeo_odc_driver/openeo_odc_driver/
 
-CMD ["gunicorn","-c","gunicorn.conf.py","odc_backend:app"]
+CMD ["conda", "run", "-n", "openeo_odc_driver", "--live-stream","gunicorn","-c","gunicorn.conf.py","odc_backend:app"]
+

Dockerfile_explorer

@@ -22,4 +22,4 @@ WORKDIR /
 
 ENTRYPOINT ["/tini", "--"]
 
-CMD ["gunicorn","-b 0.0.0.0:9000","-w 1","cubedash:app"]
+CMD ["gunicorn","-b 0.0.0.0:9000","-w 1","cubedash:create_app()"]

Makefile

@@ -14,41 +14,36 @@ setup: build up init product index explorer ## Run a full local/development setu
 update: build up ## Run a full local/development setup
 
 up: ## 1. Bring up your Docker environment
-	docker-compose up -d postgres
-	docker-compose run checkdb
-	docker-compose up -d explorer
-	docker-compose up -d openeo_odc_driver
+	docker compose up -d postgres
+	docker compose run checkdb
+	docker compose up -d explorer
+	docker compose up -d openeo_odc_driver
 
 init: ## 2. Prepare the database
-	docker-compose exec -T openeo_odc_driver datacube -v system init
+	docker compose exec -T openeo_odc_driver conda run -n openeo_odc_driver datacube -v system init
 
 product: ## 3. Add a product definition for Sentinel-2
-	docker-compose exec -T openeo_odc_driver wget https://raw.githubusercontent.com/digitalearthafrica/config/master/products/esa_s2_l2a.odc-product.yaml
-	docker-compose exec -T openeo_odc_driver datacube product add esa_s2_l2a.odc-product.yaml
+	docker compose exec -T openeo_odc_driver wget https://gist.githubusercontent.com/clausmichele/f95d687134bbaa6cb2fb7681513ce00b/raw/d92454b0970a56f1e64bee094f4ec5e636757303/esa_s2_l2a.odc-product.yaml
+	docker compose exec -T openeo_odc_driver conda run -n openeo_odc_driver datacube product add esa_s2_l2a.odc-product.yaml
 
 index: ## 4. Index some data (Change extents with BBOX='<left>,<bottom>,<right>,<top>')
-	docker-compose exec -T openeo_odc_driver bash -c \
-		"stac-to-dc \
-			--bbox='$(BBOX)' \
-			--catalog-href='https://earth-search.aws.element84.com/v0/' \
-			--collections='sentinel-s2-l2a-cogs' \
-			--datetime='2015-06-01/2023-07-01'"
+	docker compose exec -T openeo_odc_driver conda run -n openeo_odc_driver stac-to-dc --bbox='11,45,12,46' --catalog-href='https://earth-search.aws.element84.com/v1/' --collections='sentinel-2-l2a' --datetime='2015-01-01/2024-05-06'
 
 explorer: ## 5. Prepare the explorer
-	docker-compose exec -T explorer cubedash-gen --init --all
+	docker compose exec -T explorer cubedash-gen --init --all
 
 down: ## Bring down the system
-	docker-compose down
+	docker compose down
 
 build: ## Rebuild the base image
-	docker-compose pull
-	docker-compose build
+	docker compose pull
+	docker compose build
 
 shell: ## Start an interactive shell
-	docker-compose exec openeo_odc_driver bash
+	docker compose exec openeo_odc_driver bash
 
 clean: ## Delete everything
-	docker-compose down --rmi all -v
+	docker compose down --rmi all -v
 
 logs: ## Show the logs from the stack
-	docker-compose logs --follow
+	docker compose logs --follow

README.md

@@ -1,18 +1,16 @@
-# OpenEO ODC Driver
-OpenEO processing engine written in Python based on OpenDataCube, Xarray and Dask.
-
-**Please note: this project is still under development and many changes will occur in the next months.**
-
-Currently it is based on openEO processes implemented in this repository together with the openEO process graph parser from https://github.com/Open-EO/openeo-pg-parser-python
-
-The next phase of the project will use the process implementations available at [openeo-processes-dask](https://github.com/Open-EO/openeo-processes-dask/) and the parser [openeo-pg-parser-networkx](https://github.com/Open-EO/openeo-pg-parser-networkx).
-
 <p float="center">
   <img src="https://avatars.githubusercontent.com/u/23743223?s=200&v=4" width="200" hspace="20"/>
   <img src="https://avatars.githubusercontent.com/u/26125288?s=200&v=4" width="200" hspace="20"/>
   <img src="https://avatars.githubusercontent.com/u/63704085?s=200&v=4" width="200" hspace="20"/>  
 </p>
 
+# OpenEO ODC Driver
+OpenEO processing engine written in Python based on OpenDataCube, Xarray and Dask.
+
+Based on openEO processes implemented at [openeo-processes-dask](https://github.com/Open-EO/openeo-processes-dask/) and the parser [openeo-pg-parser-networkx](https://github.com/Open-EO/openeo-pg-parser-networkx).
+
+The `load_collection` process implementation is not part of the openeo-processes-dask repo but it's included here, where it leverages OpenDataCube for data loading: https://github.com/SARScripts/openeo_odc_driver/blob/037a5592c05044d4451d3896c0455414f34dd8f4/openeo_odc_driver/processing.py#L38
+
 # Docker deployment:
 
 Based on the [cube-in-a-box](https://github.com/opendatacube/cube-in-a-box) project and on the Docker image from [andriyreznik](https://github.com/andriyreznik): https://github.com/andriyreznik/docker-python-gdal
@@ -23,7 +21,7 @@ It also requires `make` to use the Makefile. If you don't have it and/or you can
 
 ## Step 1: Clone the repository
 ```sh
-git clone https://github.com/SARScripts/openeo_odc_driver.git -b dev
+git clone https://github.com/SARScripts/openeo_odc_driver.git -b dask_processes
 cd openeo_odc_driver
 ```
 
@@ -87,113 +85,3 @@ python tests/test_process_graph.py ./tests/process_graphs/NDVI_Bolzano_median.js
 ![image](https://user-images.githubusercontent.com/31700619/220927309-cd4be598-4f93-43cf-ac17-d6dbaa1a2bc3.png)
 
 ![image](https://user-images.githubusercontent.com/31700619/220927197-5fccca3a-fff4-4311-9c99-af7c6c4d08f4.png)
-
-# Local installation instructions:
-
-## Step 1: Clone the repository
-```sh
-git clone https://github.com/SARScripts/openeo_odc_driver.git -b dev
-cd openeo_odc_driver
-```
-
-## Step 2: Prepare the python environment
-
-New ad-hoc conda environment:
-```sh
-conda env create -f environment.yml
-conda activate openeo_odc_driver
-```
-
-## Step 3:
-Modify the `config.py` file with your system's details:
-1. Set the datacube-explorer address. For local deployment it should be `http://0.0.0.0:9000` and for the Docker deployment `http://explorer:9000`.
-```python
-DATACUBE_EXPLORER_ENDPOINT = "http://0.0.0.0:9000"
-```
-
-2. Set the OpenDatCube config file `.datacube.conf` path or leave it to None if ENV variables are set (like in the Docker deployment).
-```python
-OPENDATACUBE_CONFIG_FILE = ~/.datacube.conf # or None
-```
-
-3. Set the result folder path to write output files. If this application is used together with the [openeo-sping-driver](https://github.com/Open-EO/openeo-spring-driver), used for serving the full openEO API, this folder should be the same as the one set in `application.properties` for `org.openeo.tmp.dir`, so that the `openeo-spring-driver` can read the result directly from there.
-```python
-RESULT_FOLDER_PATH = ''
-```
-
-4. The `OPENEO_PROCESSES` variable is used to retrieve the list of available openEO processes. It can be the path to a json file, a dictionaty or an http address. See [here](https://github.com/Open-EO/openeo-pg-parser-python/blob/798668e461ec2a0d3153873413afb0a76a72b61a/src/openeo_pg_parser/translate.py#L263) for detailed info. The dault value is the /processes endpoint of the Eurac openEO back-end.
-```python
-OPENEO_PROCESSES = "https://openeo.eurac.edu/processes"
-```
-
-The other config parameters could be looked at later on and are not affecting the `openeo_odc_driver` functionality. In the config.py file there are comments explaining their usage.
-
-## Step 4: Start the web server:
-```sh
-gunicorn -c gunicorn.conf.py odc_backend:app
-```
-
-# Implemented OpenEO processes (to be updated)
-## aggregate & resample
-- resample_cube_temporal
-- resample_cube_spatial
-- aggregate_spatial
-- aggregate_spatial_window
-- aggregate_temporal_period
-## arrays
-- array_element
-- array_interpolate_linear
-## comparison
-- if
-- lt
-- lte
-- gt
-- gte
-- eq
-- neq
-## cubes
-- load_collection
-- save_result (PNG,GTIFF,NETCDF,JSON)
-- reduce_dimension
-- add_dimension
-- apply_dimension
-- filter_bands
-- filter_temporal
-- filter_spatial
-- filter_bbox
-- rename_labels
-- merge_cubes
-- apply
-- fit_curve
-- predict_curve
-- resample_cube_spatial
-- resample_cube_temporal 
-## logic
-- and
-- or
-## masks
-- mask
-## math
-- multiply
-- divide
-- subtract
-- add
-- sum
-- product
-- sqrt
-- normalized_difference
-- min
-- max
-- mean
-- median
-- power
-- absolute
-- linear_scale_range
-- log
-- ln
-- quantiles
-- clip
-## experimental processes (SAR2Cube)
-- coherence
-- geocoding
-- radar_mask

docker-compose.yml

@@ -44,7 +44,8 @@ services:
     ports:
       - 5001:5000
     volumes:
-      - ~/tmp/:/tmp/
+      - /mnt/CEPH_OPENEO/data/odc-driver/:/data/odc-driver/
+      - /mnt/CEPH_OPENEO/logs/odc-driver/:/logs/odc-driver/
     restart: always
 
   checkdb:

environment.yml

@@ -3,6 +3,7 @@ channels:
 - conda-forge
 dependencies:
 - python=3.10
+- libgcc=5.2.0
 - datacube
 - numpy
 - pandas
@@ -25,21 +26,10 @@ dependencies:
 - gunicorn
 - flask
 - opencv
-- rpy2
-- r-base
-- r-stars
-- r-lubridate
-- r-jsonlite
-- r-curl
-- r-caret
-- r-randomforest
 - pip
 - pip:
-    - --extra-index-url https://packages.dea.ga.gov.au
-    - openeo
-    - openeo_r_udf
+    - openeo[localprocessing]
     - dea-tools
-    - odc-algo
-    - datacube-explorer
-    - git+https://github.com/Open-EO/openeo-pg-parser-python.git
-
+    - raster2stac
+    - ujson
+    - rio-stac