From 0dac3f3440417d42642a2df81ade7ee79018cb7d Mon Sep 17 00:00:00 2001 From: smohiudd Date: Wed, 8 Feb 2023 12:33:44 -0700 Subject: [PATCH 1/8] include lambda for vector ingest --- deploy/cdk/lambda_stack.py | 15 ++++++++ deploy/cdk/queue_stack.py | 18 ++++++++++ deploy/cdk/step_function_stack.py | 57 ++++++++++++++++++++++++++++--- deploy/config.py | 4 ++- 4 files changed, 89 insertions(+), 5 deletions(-) diff --git a/deploy/cdk/lambda_stack.py b/deploy/cdk/lambda_stack.py index 80519c55..89470a8c 100644 --- a/deploy/cdk/lambda_stack.py +++ b/deploy/cdk/lambda_stack.py @@ -57,6 +57,15 @@ def __init__(self, app, construct_id, **kwargs) -> None: }, ) + # Ingest Vector + self.vector_lambda = self._lambda( + f"{construct_id}-vector-fn", + "../lambdas/submit-vector", + env={ + "VECTOR_SECRET_NAME": config.VECTOR_SECRET_NAME, + } + ) + # Proxy lambda to trigger cogify step function self.trigger_cogify_lambda = self._python_lambda( f"{construct_id}-trigger-cogify-fn", @@ -68,6 +77,12 @@ def __init__(self, app, construct_id, **kwargs) -> None: f"{construct_id}-trigger-ingest-fn", "../lambdas/proxy" ) + # Proxy lambda to trigger vector step function + self.trigger_vector_lambda = self._python_lambda( + f"{construct_id}-trigger-vector-fn", + "../lambdas/proxy", + ) + # Builds stac self.build_stac_lambda = self._lambda( f"{construct_id}-build-stac-fn", diff --git a/deploy/cdk/queue_stack.py b/deploy/cdk/queue_stack.py index bbae7a31..c01fb546 100644 --- a/deploy/cdk/queue_stack.py +++ b/deploy/cdk/queue_stack.py @@ -56,6 +56,24 @@ def __init__( ) ) + self.vector_queue = self._queue( + f"{construct_id}-vector-queue", + visibility_timeout=900, + dead_letter_queue=sqs.DeadLetterQueue( + max_receive_count=5, + queue=self._queue(f"{construct_id}-vector-dlq"), + ), + ) + + lambda_stack.trigger_cogify_lambda.add_event_source( + lambda_event_sources.SqsEventSource( + self.vector_queue, + batch_size=1, + max_batching_window=core.Duration.seconds(20), + report_batch_item_failures=True, + ) + ) + def _queue( self, name, visibility_timeout=30, dead_letter_queue=None, retention_days=4 ): diff --git a/deploy/cdk/step_function_stack.py b/deploy/cdk/step_function_stack.py index 4033ed21..9ab9c98f 100644 --- a/deploy/cdk/step_function_stack.py +++ b/deploy/cdk/step_function_stack.py @@ -31,6 +31,11 @@ def __init__( lambda_stack=lambda_stack, queue_stack=queue_stack, ) + self.vector_workflow = self._vector_workflow( + lambda_stack=lambda_stack, + queue_stack=queue_stack, + ) + self.publication_workflow = self._publication_workflow( lambda_stack=lambda_stack, ) @@ -77,8 +82,22 @@ def _discovery_workflow( queue=queue_stack.stac_ready_queue, ) - maybe_cogify = ( - stepfunctions.Choice(self, "Cogify?") + enqueue_vector_task = self._sqs_task( + "Send to vector queue", + queue=queue_stack.vector_queue, + ) + + vector_or_cogify = ( + stepfunctions.Choice(self, "Vector or Cogify?") + .when( + stepfunctions.Condition.boolean_equals("$.Payload.vector", True), + stepfunctions.Map( + self, + "Run queueing to vector queue", + max_concurrency=100, + items_path=stepfunctions.JsonPath.string_at("$.Payload.objects"), + ).iterator(enqueue_vector_task), + ) .when( stepfunctions.Condition.boolean_equals("$.Payload.cogify", True), stepfunctions.Map( @@ -102,11 +121,11 @@ def _discovery_workflow( stepfunctions.Choice(self, "Discovery Choice (CMR or S3)") .when( stepfunctions.Condition.string_equals("$.discovery", "s3"), - s3_discovery_task.next(maybe_cogify), + s3_discovery_task.next(vector_or_cogify), ) .when( stepfunctions.Condition.string_equals("$.discovery", "cmr"), - cmr_discovery_task.next(maybe_cogify), + cmr_discovery_task.next(vector_or_cogify), ) .otherwise(stepfunctions.Fail(self, "Discovery Type not supported")) ) @@ -148,6 +167,36 @@ def _cogify_workflow( definition=cogify_workflow, ) + def _vector_workflow( + self, + lambda_stack: "LambdaStack", + queue_stack: "QueueStack", + ) -> stepfunctions.StateMachine: + vector_task = self._lambda_task( + "Ingest Vector", + lambda_stack.vector_lambda, + ) + + # enqueue_task = self._sqs_task( + # "Send cogified to stac-ready queue", + # queue=queue_stack.stac_ready_queue, + # input_path="$.Payload", + # ) + + vector_workflow = stepfunctions.Map( + self, + "Run vector ingest", + max_concurrency=100, + items_path=stepfunctions.JsonPath.string_at("$"), + ).iterator(vector_task) + + return stepfunctions.StateMachine( + self, + f"vector-sf", + state_machine_name=f"{self.stack_name}-cogify", + definition=vector_workflow, + ) + def _publication_workflow( self, lambda_stack: "LambdaStack", diff --git a/deploy/config.py b/deploy/config.py index 37546e93..3028fe5c 100644 --- a/deploy/config.py +++ b/deploy/config.py @@ -9,7 +9,7 @@ EARTHDATA_USERNAME = os.environ.get("EARTHDATA_USERNAME", "XXXX") EARTHDATA_PASSWORD = os.environ.get("EARTHDATA_PASSWORD", "XXXX") -APP_NAME = "veda-data-pipelines" +APP_NAME = "veda-data-pipeline-vector-test" VEDA_DATA_BUCKET = "climatedashboard-data" VEDA_EXTERNAL_BUCKETS = ["nasa-maap-data-store", "covid-eo-blackmarble"] MCP_BUCKETS = { @@ -19,3 +19,5 @@ # This should throw if it is not provided EXTERNAL_ROLE_ARN = os.environ["EXTERNAL_ROLE_ARN"] + +VECTOR_SECRET_NAME = os.environ["VECTOR_SECRET_NAME"] From eb0be62b165eea3b27524b19e206f5c7fa689af6 Mon Sep 17 00:00:00 2001 From: smohiudd Date: Wed, 8 Feb 2023 12:34:31 -0700 Subject: [PATCH 2/8] vector ingest step function --- .gitignore | 2 + data/collections/eis_fire_fireline.json | 39 ++++++++++ env.sample.sh | 6 ++ lambdas/cogify/Dockerfile | 4 +- lambdas/submit-vector/Dockerfile | 18 +++++ lambdas/submit-vector/handler.py | 96 +++++++++++++++++++++++++ lambdas/submit-vector/requirements.txt | 1 + 7 files changed, 165 insertions(+), 1 deletion(-) create mode 100644 data/collections/eis_fire_fireline.json create mode 100644 lambdas/submit-vector/Dockerfile create mode 100644 lambdas/submit-vector/handler.py create mode 100644 lambdas/submit-vector/requirements.txt diff --git a/.gitignore b/.gitignore index 494cb13f..cff76769 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ __pycache__ *.hdf *.hdf5 *.tif +*.fgb *.nc *.HDF5 *.h5 @@ -35,6 +36,7 @@ cdk.context.json # Avoid checking in filled out env file env.sh +.env .hypothesis Makefile diff --git a/data/collections/eis_fire_fireline.json b/data/collections/eis_fire_fireline.json new file mode 100644 index 00000000..a59a4b81 --- /dev/null +++ b/data/collections/eis_fire_fireline.json @@ -0,0 +1,39 @@ +{ + "id": "eis_fire_fireline", + "type": "Collection", + "links":[ + { + "rel": "child", + "type": "application/json", + "href": "https://pzh7fedpc0.execute-api.us-west-2.amazonaws.com/collections/public.eis_fire_fireline" + } + ], + "title":"EIS Fireline dataset", + "extent":{ + "spatial": { + "bbox": [ + [ + -123.99217987060547, + 31.054306030273438, + -101.49222564697266, + 49.461177825927734 + ] + ], + "crs": "http://www.opengis.net/def/crs/EPSG/0/4326" + }, + "temporal": { + "interval": [ + [ + "2019-05-01T00:00:00+00:00", + "2020-10-31T12:00:00+00:00" + ] + ], + "trs": "http://www.opengis.net/def/uom/ISO-8601/0/Gregorian" + } + }, + "license":"public-domain", + "description": "EIS Fireline dataset", + "stac_version": "1.0.0", + "dashboard:is_periodic": true, + "dashboard:time_density": "day" +} diff --git a/env.sample.sh b/env.sample.sh index 8fa58ec5..c0fe03d0 100755 --- a/env.sample.sh +++ b/env.sample.sh @@ -8,6 +8,9 @@ devPGSecret=veda-backend-uah-dev/pgstac/621feede stagePGSecret=delta-backend-stagingv2/pgstac/5d4eb447 +devVectorDBSecret=dev/tifeatures-timvt/Features_DB_for_EIS_Fires/3A8D1807 +stageVectorDBSecret= + devVPCid=vpc-0512162c42da5e645 stageVPCid=vpc-09d7998dbf340fcb7 @@ -29,6 +32,7 @@ else if [[ $1 = 'staging' ]] then cognitoAppSecret=$stageCognitoAppSecret + vectordbSecret=$stageVectorDBSecret pgSecret=$stagePGSecret vpcId=$stageVPCid sgId=$stageSGid @@ -36,6 +40,7 @@ else else cognitoAppSecret=$devCognitoAppSecret pgSecret=$devPGSecret + vectordbSecret=$devVectorDBSecret vpcId=$devVPCid sgId=$devSGid stacIngestorUrl=$devStacIngestorUrl @@ -52,6 +57,7 @@ else export VPC_ID=$vpcId export SECURITY_GROUP_ID=$sgId export SECRET_NAME=$pgSecret + export VECTOR_SECRET_NAME=$vectordbSecret export STAC_INGESTOR_URL=$stacIngestorUrl diff --git a/lambdas/cogify/Dockerfile b/lambdas/cogify/Dockerfile index 73a8d9f3..9aa0e82b 100644 --- a/lambdas/cogify/Dockerfile +++ b/lambdas/cogify/Dockerfile @@ -1,4 +1,6 @@ -FROM python:3.9-slim-bullseye +ARG PYTHON_VERSION=3.9 + +FROM --platform=linux/amd64 public.ecr.aws/lambda/python:${PYTHON_VERSION} WORKDIR /app diff --git a/lambdas/submit-vector/Dockerfile b/lambdas/submit-vector/Dockerfile new file mode 100644 index 00000000..1cb276e6 --- /dev/null +++ b/lambdas/submit-vector/Dockerfile @@ -0,0 +1,18 @@ +FROM ghcr.io/lambgeo/lambda-gdal:3.5-python3.9 + +WORKDIR /app + +RUN pip install --upgrade pip +COPY requirements.txt requirements.txt +RUN pip install -r requirements.txt + +RUN find . -type f -name '*.pyc' | while read f; do n=$(echo $f | sed 's/__pycache__\///' | sed 's/.cpython-[2-3][0-9]//'); cp $f $n; done; +RUN find . -type d -a -name '__pycache__' -print0 | xargs -0 rm -rf +RUN find . -type f -a -name '*.py' -print0 | xargs -0 rm -f +RUN find . -type d -a -name 'tests' -print0 | xargs -0 rm -rf +RUN echo "Remove lambda python packages" +RUN rm -rdf ./numpy/doc/ +RUN rm -rdf ./stack +RUN rm -rdf ./docutils* + +COPY handler.py handler.py \ No newline at end of file diff --git a/lambdas/submit-vector/handler.py b/lambdas/submit-vector/handler.py new file mode 100644 index 00000000..787b15a7 --- /dev/null +++ b/lambdas/submit-vector/handler.py @@ -0,0 +1,96 @@ +import boto3 +import os +import subprocess +import json +from urllib.parse import urlparse + +s3 = boto3.client( + "s3", +) + +def download_file(file_uri: str): + + url_parse = urlparse(file_uri) + + bucket = url_parse.netloc + path = url_parse.path[1:] + filename = url_parse.path.split('/')[-1] + + s3.download_file(bucket, path, filename) + + print(f"downloaded {filename}") + + return filename + +def get_connection_string(secret: dict) -> str: + + return f"PG:host={secret['host']} dbname={secret['dbname']} user={secret['username']} password={secret['password']}" + +def get_secret(secret_name: str) -> None: + """Retrieve secrets from AWS Secrets Manager + + Args: + secret_name (str): name of aws secrets manager secret containing database connection secrets + profile_name (str, optional): optional name of aws profile for use in debugger only + + Returns: + secrets (dict): decrypted secrets in dict + """ + + # Create a Secrets Manager client + session = boto3.session.Session(region_name="us-west-2") + client = session.client(service_name="secretsmanager") + + # In this sample we only handle the specific exceptions for the 'GetSecretValue' API. + # See https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_GetSecretValue.html + # We rethrow the exception by default. + + get_secret_value_response = client.get_secret_value(SecretId=secret_name) + + # Decrypts secret using the associated KMS key. + # Depending on whether the secret is a string or binary, one of these fields will be populated. + if "SecretString" in get_secret_value_response: + return json.loads(get_secret_value_response["SecretString"]) + else: + return json.loads(base64.b64decode(get_secret_value_response["SecretBinary"])) + + +def load_to_featuresdb(filename: str, collection: str): + secret_name = os.environ.get("VECTOR_SECRET_NAME") + + con_secrets = get_secret(secret_name) + connection = get_connection_string(con_secrets) + + print(f"running ogr2ogr import for collection: {collection}") + + subprocess.run([ + "ogr2ogr", + "-f", + "PostgreSQL", + connection, + "-t_srs", + "EPSG:4326", + filename, + "-nln", + collection, + "-append", + "-update", + "-progress" + ]) + + +def handler(event, context): + href = event["href"] + collection = event["collection"] + + downloaded_filename = download_file(href) + + load_to_featuresdb(downloaded_filename, collection ) + + +if __name__ == "__main__": + sample_event = { + "collection": "eis_fire_newfirepix_2", + "href": "s3://covid-eo-data/fireline/newfirepix.fgb" + } + handler(sample_event, {}) diff --git a/lambdas/submit-vector/requirements.txt b/lambdas/submit-vector/requirements.txt new file mode 100644 index 00000000..1db657b6 --- /dev/null +++ b/lambdas/submit-vector/requirements.txt @@ -0,0 +1 @@ +boto3 \ No newline at end of file From 7c92499f8a1cd9acedc18c9be0d3428a2f4fc76b Mon Sep 17 00:00:00 2001 From: smohiudd Date: Wed, 8 Feb 2023 12:56:47 -0700 Subject: [PATCH 3/8] change back app name --- deploy/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/config.py b/deploy/config.py index 3028fe5c..8c541c58 100644 --- a/deploy/config.py +++ b/deploy/config.py @@ -9,7 +9,7 @@ EARTHDATA_USERNAME = os.environ.get("EARTHDATA_USERNAME", "XXXX") EARTHDATA_PASSWORD = os.environ.get("EARTHDATA_PASSWORD", "XXXX") -APP_NAME = "veda-data-pipeline-vector-test" +APP_NAME = "veda-data-pipelines" VEDA_DATA_BUCKET = "climatedashboard-data" VEDA_EXTERNAL_BUCKETS = ["nasa-maap-data-store", "covid-eo-blackmarble"] MCP_BUCKETS = { From f8e3cc70ea17cdac3f11d45b681cae69b9986502 Mon Sep 17 00:00:00 2001 From: Slesa Adhikari Date: Fri, 10 Feb 2023 12:09:16 -0600 Subject: [PATCH 4/8] Fix issues with vector addition --- deploy/app.py | 5 +++++ deploy/cdk/queue_stack.py | 2 +- deploy/cdk/step_function_stack.py | 4 +--- lambdas/s3-discovery/handler.py | 2 ++ 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/deploy/app.py b/deploy/app.py index eeaba96b..1b138521 100644 --- a/deploy/app.py +++ b/deploy/app.py @@ -40,6 +40,7 @@ # Need to build arn manually otherwise it'll result in cyclic dependency cogify_arn = step_function_stack.build_arn(env_details, "cogify") pub_arn = step_function_stack.build_arn(env_details, "publication") +vector_arn = step_function_stack.build_arn(env_details, "vector-ingest") lambda_stack.grant_execution_privileges( lambda_function=lambda_stack.trigger_cogify_lambda, @@ -49,5 +50,9 @@ lambda_function=lambda_stack.trigger_ingest_lambda, workflow_arn=pub_arn, ) +lambda_stack.grant_execution_privileges( + lambda_function=lambda_stack.trigger_vector_lambda, + workflow_arn=pub_arn, +) app.synth() diff --git a/deploy/cdk/queue_stack.py b/deploy/cdk/queue_stack.py index c01fb546..dcc9f412 100644 --- a/deploy/cdk/queue_stack.py +++ b/deploy/cdk/queue_stack.py @@ -65,7 +65,7 @@ def __init__( ), ) - lambda_stack.trigger_cogify_lambda.add_event_source( + lambda_stack.trigger_vector_lambda.add_event_source( lambda_event_sources.SqsEventSource( self.vector_queue, batch_size=1, diff --git a/deploy/cdk/step_function_stack.py b/deploy/cdk/step_function_stack.py index 9ab9c98f..3bc3f30e 100644 --- a/deploy/cdk/step_function_stack.py +++ b/deploy/cdk/step_function_stack.py @@ -33,7 +33,6 @@ def __init__( ) self.vector_workflow = self._vector_workflow( lambda_stack=lambda_stack, - queue_stack=queue_stack, ) self.publication_workflow = self._publication_workflow( @@ -170,7 +169,6 @@ def _cogify_workflow( def _vector_workflow( self, lambda_stack: "LambdaStack", - queue_stack: "QueueStack", ) -> stepfunctions.StateMachine: vector_task = self._lambda_task( "Ingest Vector", @@ -193,7 +191,7 @@ def _vector_workflow( return stepfunctions.StateMachine( self, f"vector-sf", - state_machine_name=f"{self.stack_name}-cogify", + state_machine_name=f"{self.stack_name}-vector-ingest", definition=vector_workflow, ) diff --git a/lambdas/s3-discovery/handler.py b/lambdas/s3-discovery/handler.py index fa654d70..29f276e4 100644 --- a/lambdas/s3-discovery/handler.py +++ b/lambdas/s3-discovery/handler.py @@ -49,6 +49,7 @@ def handler(event, context): files_objs = [] cogify = event.pop("cogify", False) + vector = event.pop("vector", False) collection = event.get("collection", prefix.rstrip("/")) for filename in filenames: files_objs.append( @@ -61,6 +62,7 @@ def handler(event, context): ) return { "cogify": cogify, + "vector": vector, "objects": files_objs, } From 3d337fc7ace94676dfd81a7b57b41dd012b4ab08 Mon Sep 17 00:00:00 2001 From: Slesa Adhikari Date: Thu, 9 Mar 2023 11:39:50 -0600 Subject: [PATCH 5/8] Add new datasets --- data/datasets/barc-thomasfire.json | 33 +++++++++++++++++++++ data/datasets/conus-reach.json | 34 ++++++++++++++++++++++ data/datasets/disalexi-etsuppression.json | 33 +++++++++++++++++++++ data/datasets/frp-max-thomasfire.json | 34 ++++++++++++++++++++++ data/datasets/houston-landcover.json | 33 +++++++++++++++++++++ data/datasets/houston-lst-day.json | 34 ++++++++++++++++++++++ data/datasets/houston-lst-diff.json | 33 +++++++++++++++++++++ data/datasets/houston-lst-night.json | 34 ++++++++++++++++++++++ data/datasets/houston-ndvi.json | 35 +++++++++++++++++++++++ data/datasets/lis-da-gpp-trend.json | 34 ++++++++++++++++++++++ data/datasets/lis-da-tws-trend.json | 35 +++++++++++++++++++++++ data/datasets/lis-etsuppression.json | 32 +++++++++++++++++++++ data/datasets/lisDA_Evap.json | 32 +++++++++++++++++++++ data/datasets/lisDA_GPP.json | 32 +++++++++++++++++++++ data/datasets/lisDA_GWS.json | 32 +++++++++++++++++++++ data/datasets/lisDA_Qs.json | 32 +++++++++++++++++++++ data/datasets/lisDA_Qsb.json | 32 +++++++++++++++++++++ data/datasets/lisDA_SWE.json | 32 +++++++++++++++++++++ data/datasets/lisDA_Streamflow.json | 33 +++++++++++++++++++++ data/datasets/lisDA_TWS.json | 33 +++++++++++++++++++++ data/datasets/lisDA_TotalPrecip.json | 33 +++++++++++++++++++++ data/datasets/listveg-suppression.json | 33 +++++++++++++++++++++ 22 files changed, 728 insertions(+) create mode 100644 data/datasets/barc-thomasfire.json create mode 100644 data/datasets/conus-reach.json create mode 100644 data/datasets/disalexi-etsuppression.json create mode 100644 data/datasets/frp-max-thomasfire.json create mode 100644 data/datasets/houston-landcover.json create mode 100644 data/datasets/houston-lst-day.json create mode 100644 data/datasets/houston-lst-diff.json create mode 100644 data/datasets/houston-lst-night.json create mode 100644 data/datasets/houston-ndvi.json create mode 100644 data/datasets/lis-da-gpp-trend.json create mode 100644 data/datasets/lis-da-tws-trend.json create mode 100644 data/datasets/lis-etsuppression.json create mode 100644 data/datasets/lisDA_Evap.json create mode 100644 data/datasets/lisDA_GPP.json create mode 100644 data/datasets/lisDA_GWS.json create mode 100644 data/datasets/lisDA_Qs.json create mode 100644 data/datasets/lisDA_Qsb.json create mode 100644 data/datasets/lisDA_SWE.json create mode 100644 data/datasets/lisDA_Streamflow.json create mode 100644 data/datasets/lisDA_TWS.json create mode 100644 data/datasets/lisDA_TotalPrecip.json create mode 100644 data/datasets/listveg-suppression.json diff --git a/data/datasets/barc-thomasfire.json b/data/datasets/barc-thomasfire.json new file mode 100644 index 00000000..224b258f --- /dev/null +++ b/data/datasets/barc-thomasfire.json @@ -0,0 +1,33 @@ +{ + "collection": "barc-thomasfire", + "title": "Burn Area Reflectance Classification for Thomas Fire", + "description": "Burn Area Reflectance Classification (BARC) from the Burned Area Emergency Response (BAER) program for Thomas fire, 2017", + "license": "CC0-1.0", + "is_periodic": false, + "time_density": null, + "spatial_extent": { + "xmin": -119.73, + "ymin": 34.19, + "xmax": -118.88, + "ymax": 34.73 + }, + "temporal_extent": { + "startdate": "2017-12-01T00:00:00Z", + "enddate": "2017-12-01T23:59:59Z" + }, + "sample_files": [ + "s3://veda-data-store-staging/EIS/COG/Fire-Hydro/thomas_fire_barc_201712.cog.tiff" + ], + "discovery_items": [ + { + "discovery": "s3", + "cogify": false, + "upload": false, + "dry_run": false, + "prefix": "EIS/COG/Fire-Hydro/", + "bucket": "veda-data-store-staging", + "filename_regex": "(.*)thomas_fire_barc(.*).tiff$", + "date_range": "month" + } + ] +} diff --git a/data/datasets/conus-reach.json b/data/datasets/conus-reach.json new file mode 100644 index 00000000..2650cc56 --- /dev/null +++ b/data/datasets/conus-reach.json @@ -0,0 +1,34 @@ +{ + "collection": "conus-reach", + "title": "Stream network across the Contiguous United States", + "description": "This dataset describes the Stream network across the Contiguous United States delineated using Soil and Water Assessment Tool.", + "license": "CC0-1.0", + "is_periodic": false, + "time_density": null, + "spatial_extent": { + "xmin": -127.78, + "ymin": 23.36, + "xmax": -65.61, + "ymax": 51.45 + }, + "temporal_extent": { + "startdate": "2020-01-01T00:00:00Z", + "enddate": "2020-01-01T23:59:59Z" + }, + "sample_files": [ + "s3://veda-data-store-staging/EIS/COG/Water-Quality/CONUS_Reach_2020.cog.tif" + ], + "discovery_items": [ + { + "discovery": "s3", + "cogify": false, + "upload": false, + "dry_run": false, + "prefix": "EIS/COG/Water-Quality/", + "bucket": "veda-data-store-staging", + "filename_regex": "(.*)CONUS_Reach(.*).cog.tif$", + "datetime_range": "year" + } + ], + "data_type": "cog" +} diff --git a/data/datasets/disalexi-etsuppression.json b/data/datasets/disalexi-etsuppression.json new file mode 100644 index 00000000..1eb77ad5 --- /dev/null +++ b/data/datasets/disalexi-etsuppression.json @@ -0,0 +1,33 @@ +{ + "collection": "disalexi-etsuppression", + "title": "Change in ET for fires using DisALEXI model", + "description": "Change in ET using DisALEXI model of OpenET observations for 2017-20 fires, calculated as the difference of ET in the immediate post-fire water year from ET in the immediate pre-fire water year. The difference is normalized by pre-fire ET and negative values denote vegetation disturbance induced by fire or by a climatological anomaly resulting in the decline in ET.", + "license": "CC0-1.0", + "is_periodic": true, + "time_density": "year", + "spatial_extent": { + "xmin": -125.35, + "ymin": 29.83, + "xmax": -108.12, + "ymax": 49.04 + }, + "temporal_extent": { + "startdate": "2017-10-01T00:00:00Z", + "enddate": "2020-10-01T00:00:00Z" + }, + "sample_files": [ + "s3://veda-data-store-staging/EIS/COG/Fire-Hydro/disalexi_etsuppression_2020.cog.tiff" + ], + "discovery_items": [ + { + "discovery": "s3", + "cogify": false, + "upload": false, + "dry_run": false, + "prefix": "EIS/COG/Fire-Hydro/", + "bucket": "veda-data-store-staging", + "filename_regex": "(.*)disalexi_etsuppression_(.*).cog.tiff$", + "datetime_range": "year" + } + ] + } diff --git a/data/datasets/frp-max-thomasfire.json b/data/datasets/frp-max-thomasfire.json new file mode 100644 index 00000000..5f3ed490 --- /dev/null +++ b/data/datasets/frp-max-thomasfire.json @@ -0,0 +1,34 @@ +{ + "collection": "frp-max-thomasfire", + "title": "Maximum Fire Radiative Power for Thomas Fire", + "description": "Maximum Fire Radiative Power per 12hr fire line segment for Thomas fire of 2017", + "license": "CC0-1.0", + "is_periodic": false, + "time_density": null, + "spatial_extent": { + "xmin": -119.73, + "ymin": 34.19, + "xmax": -118.88, + "ymax": 34.73 + }, + "temporal_extent": { + "startdate": "2017-12-01T00:00:00Z", + "enddate": "2017-12-01T23:59:59Z" + }, + "sample_files": [ + "s3://veda-data-store-staging/EIS/COG/Fire-Hydro/thomas_fire_frpmax_201712.cog.tiff" + ], + "discovery_items": [ + { + "discovery": "s3", + "cogify": false, + "upload": false, + "dry_run": false, + "prefix": "EIS/COG/Fire-Hydro/", + "bucket": "veda-data-store-staging", + "filename_regex": "(.*)thomas_fire_frpmax(.*).tiff$", + "datetime_range": "month" + } + ], + "data_type": "cog" +} diff --git a/data/datasets/houston-landcover.json b/data/datasets/houston-landcover.json new file mode 100644 index 00000000..092b28bc --- /dev/null +++ b/data/datasets/houston-landcover.json @@ -0,0 +1,33 @@ +{ + "collection": "houston-landcover", + "title": "Houston Land Cover", + "description": "MODIS-derived land use-land cover over the Houston metro area (zoomed-in extent). Displays colors 0-255 from a lookup table for land cover type (ex: reds = urbanized).", + "license": "CC0-1.0", + "is_periodic": false, + "time_density": null, + "spatial_extent": { + "xmin": -96.00, + "ymin": 29.80, + "xmax": -94.29, + "ymax": 30.40 + }, + "temporal_extent": { + "startdate": "2001-01-01T00:00:00Z", + "enddate": "2019-12-31T23:59:59Z" + }, + "sample_files": [ + "s3://veda-data-store-staging/houston-landcover/HOUS_DFAL_Z_f_landcover_2001.tif" + ], + "discovery_items": [ + { + "discovery": "s3", + "cogify": false, + "upload": false, + "dry_run": false, + "prefix": "houston-landcover/", + "bucket": "veda-data-store-staging", + "filename_regex": "(.*)HOUS_DFAL_Z_f(.*).tif$", + "datetime_range": "year" + } + ] +} diff --git a/data/datasets/houston-lst-day.json b/data/datasets/houston-lst-day.json new file mode 100644 index 00000000..92443019 --- /dev/null +++ b/data/datasets/houston-lst-day.json @@ -0,0 +1,34 @@ +{ + "collection": "houston-lst-day", + "title": "Houston land surface temperature during daytime - decadal average", + "description": "MODIS-calculated Land Surface Temperature (LST) Daytime over the Houston metro area, decadally averaged.", + "license": "CC0-1.0", + "is_periodic": false, + "time_density": "year", + "spatial_extent": { + "xmax": -94.00, + "ymax": 31.00, + "xmin": -97.00, + "ymin": 28.00 + }, + "temporal_extent": { + "startdate": "2000-01-01T00:00:00Z", + "enddate": "2019-12-31T23:59:59Z" + }, + "sample_files": [ + "s3://veda-data-store-staging/houston-lst-day/LST_Day_1km_2000_2009_cog.tif", + "s3://veda-data-store-staging/houston-lst-day/LST_Day_1km_2010_2019_cog.tif" + ], + "discovery_items": [ + { + "discovery": "s3", + "cogify": false, + "upload": false, + "dry_run": false, + "prefix": "houston-lst-day/", + "bucket": "veda-data-store-staging", + "filename_regex": "^(.*)LST_Day_1km(.*).tif$", + "datetime_range": "year" + } + ] +} diff --git a/data/datasets/houston-lst-diff.json b/data/datasets/houston-lst-diff.json new file mode 100644 index 00000000..680f5962 --- /dev/null +++ b/data/datasets/houston-lst-diff.json @@ -0,0 +1,33 @@ +{ + "collection": "houston-lst-diff", + "title": "Houston LST (Diff)", + "description": "Changes in decadally averaged land surface temperature (LST, daytime) in the Houston metro area. The higher the value, the larger the increase.", + "license": "CC0-1.0", + "is_periodic": false, + "time_density": null, + "spatial_extent": { + "xmax": -94.00, + "ymax": 31.00, + "xmin": -97.00, + "ymin": 28.00 + }, + "temporal_extent": { + "startdate": "2000-01-01T00:00:00Z", + "enddate": "2019-12-31T23:59:59Z" + }, + "sample_files": [ + "s3://climatedashboard-data/houston/houston-lst-diff_2000_2019.tif" + ], + "discovery_items": [ + { + "discovery": "s3", + "cogify": false, + "upload": true, + "dry_run": false, + "prefix": "houston", + "bucket": "climatedashboard-data", + "filename_regex": "^(.*)houston-lst-diff(.*).tif$", + "datetime_range": "year" + } + ] +} diff --git a/data/datasets/houston-lst-night.json b/data/datasets/houston-lst-night.json new file mode 100644 index 00000000..60d3d763 --- /dev/null +++ b/data/datasets/houston-lst-night.json @@ -0,0 +1,34 @@ +{ + "collection": "houston-lst-night", + "title": "Houston land surface temperature at night time - decadal average", + "description": "MODIS-calculated Land Surface Temperature (LST) Nighttime over the Houston metro area, decadally averaged.", + "license": "CC0-1.0", + "is_periodic": false, + "time_density": "year", + "spatial_extent": { + "xmax": -94.00, + "ymax": 31.00, + "xmin": -97.00, + "ymin": 28.00 + }, + "temporal_extent": { + "startdate": "2000-01-01T00:00:00Z", + "enddate": "2019-12-31T23:59:59Z" + }, + "sample_files": [ + "s3://veda-data-store-staging/houston-lst-night/LST_Night_1km_2010_2019_cog.tif", + "s3://veda-data-store-staging/houston-lst-night/LST_Night_1km_2000_2009_cog.tif" + ], + "discovery_items": [ + { + "discovery": "s3", + "cogify": false, + "upload": false, + "dry_run": false, + "prefix": "houston-lst-night/", + "bucket": "veda-data-store-staging", + "filename_regex": "^(.*)LST_Night_1km(.*).tif$", + "datetime_range": "year" + } + ] +} diff --git a/data/datasets/houston-ndvi.json b/data/datasets/houston-ndvi.json new file mode 100644 index 00000000..a50d8f16 --- /dev/null +++ b/data/datasets/houston-ndvi.json @@ -0,0 +1,35 @@ +{ + "collection": "houston-ndvi", + "title": "Houston NDVI: decadal average", + "description": "MODIS-calculated Normalized Difference Vegation Index (NDVI) over the Houston metro area, decadally averaged.", + "license": "CC0-1.0", + "is_periodic": true, + "time_density": "year", + "spatial_extent": { + "xmax": -94.00, + "ymax": 31.00, + "xmin": -97.00, + "ymin": 28.00 + }, + "temporal_extent": { + "startdate": "2000-01-01T00:00:00Z", + "enddate": "2019-12-31T23:59:59Z" + }, + "sample_files": [ + "s3://veda-data-store-staging/houston-ndvi/NDVI_2000_2009_cog.tif", + "s3://veda-data-store-staging/houston-ndvi/NDVI_2010_2019_cog.tif" + ], + "discovery_items": [ + { + "discovery": "s3", + "cogify": false, + "upload": false, + "dry_run": false, + "prefix": "houston-ndvi/", + "bucket": "veda-data-store-staging", + "filename_regex": "^(.*)NDVI_(.*).tif$", + "datetime_range": "year" + } + ], + "data_type": "cog" +} diff --git a/data/datasets/lis-da-gpp-trend.json b/data/datasets/lis-da-gpp-trend.json new file mode 100644 index 00000000..740462ad --- /dev/null +++ b/data/datasets/lis-da-gpp-trend.json @@ -0,0 +1,34 @@ +{ + "collection": "lis-global-da-gpp-trend", + "title": "Gross Primary Productivity Trend - LIS 10km Global DA", + "description": "Gridded trend in gross primary productivity (theil-sen slope estimation in gC m-2 yr-1) from 10km global LIS with assimilation", + "license": "CC0-1.0", + "is_periodic": false, + "time_density": null, + "spatial_extent": { + "xmin": -179.95, + "ymin": -59.45, + "xmax": 179.95, + "ymax": 83.55 + }, + "temporal_extent": { + "startdate": "2003-01-01T00:00:00Z", + "enddate": "2021-12-31T23:59:59Z" + }, + "sample_files": [ + "s3://veda-data-store-staging/EIS/COG/LIS_GLOBAL_DA/DA_Trends/DAGPP_STL_based_trendv2_2003_2021.cog.tif" + ], + "discovery_items": [ + { + "discovery": "s3", + "cogify": false, + "upload": false, + "dry_run": false, + "prefix": "EIS/COG/LIS_GLOBAL_DA/DA_Trends/", + "bucket": "veda-data-store-staging", + "filename_regex": "(.*)DAGPP_STL_based_trendv2_2003_2021.cog.tif$", + "datetime_range": "year" + } + ], + "data_type": "cog" + } diff --git a/data/datasets/lis-da-tws-trend.json b/data/datasets/lis-da-tws-trend.json new file mode 100644 index 00000000..eec65fc4 --- /dev/null +++ b/data/datasets/lis-da-tws-trend.json @@ -0,0 +1,35 @@ +{ + "collection": "lis-global-da-tws-trend", + "title": "Terrestrial Water Storage Trend - LIS 10km Global DA", + "description": "Gridded trend in terrestrial water storage (theil-sen slope estimation in mm yr-1) from 10km global LIS with assimilation", + "license": "CC0-1.0", + "is_periodic": false, + "time_density": null, + "spatial_extent": { + "xmin": -179.95, + "ymin": -59.45, + "xmax": 179.95, + "ymax": 83.55 + }, + "temporal_extent": { + "startdate": "2003-01-01T00:00:00Z", + "enddate": "2021-12-31T23:59:59Z" + }, + "sample_files": [ + "s3://veda-data-store-staging/EIS/COG/LIS_GLOBAL_DA/DA_Trends/DATWS_STL_based_trendv2_2003_2021.cog.tif" + ], + "discovery_items": [ + { + "collection": "lis-global-da-tws-trend-airflow", + "discovery": "s3", + "cogify": false, + "upload": false, + "dry_run": false, + "prefix": "EIS/COG/LIS_GLOBAL_DA/DA_Trends/", + "bucket": "veda-data-store-staging", + "filename_regex": "(.*)DATWS_STL_based_trendv2_2003_2021.cog.tif$", + "datetime_range": "year" + } + ], + "data_type": "cog" +} diff --git a/data/datasets/lis-etsuppression.json b/data/datasets/lis-etsuppression.json new file mode 100644 index 00000000..7e86d594 --- /dev/null +++ b/data/datasets/lis-etsuppression.json @@ -0,0 +1,32 @@ +{ + "collection": "lis-etsuppression", + "title": "Change in ET for 2020 fires using LIS outputs", + "description": "Change in ET for 2020 fires using model outputs from Land Information System (LIS) framework that synthesizes multiple remote sensing observations within the Noah-MP land surface model. Change is calculated as the difference of ET in the immediate post-fire water year from that in the immediate pre-fire water year. The difference is normalized by pre-fire ET and negative values denote vegetation disturbance induced by fire or by a climatological anomaly resulting in the decline in ET.", + "license": "CC0-1.0", + "is_periodic": false, + "time_density": null, + "spatial_extent": { + "xmin": -125.04, + "ymin": 31.97, + "xmax": -109.40, + "ymax": 49.28 + }, + "temporal_extent": { + "startdate": "2020-01-01T00:00:00Z", + "enddate": "2020-01-01T23:59:59Z" + }, + "sample_files": [ + "s3://veda-data-store-staging/EIS/COG/Fire-Hydro/lis_etsuppression_2020.cog.tiff" + ], + "discovery_items": [ + { + "discovery": "s3", + "cogify": false, + "upload": false, + "dry_run": false, + "prefix": "EIS/COG/Fire-Hydro/", + "bucket": "veda-data-store-staging", + "filename_regex": "(.*)lis_etsuppression_(.*).cog.tiff$" + } + ] +} diff --git a/data/datasets/lisDA_Evap.json b/data/datasets/lisDA_Evap.json new file mode 100644 index 00000000..36c4bce0 --- /dev/null +++ b/data/datasets/lisDA_Evap.json @@ -0,0 +1,32 @@ +{ + "collection": "lis-global-da-evap", + "title": "Evapotranspiration - LIS 10km Global DA", + "description": "Gridded total evapotranspiration (in kg m-2 s-1) from 10km global LIS with assimilation", + "license": "CC0-1.0", + "is_periodic": true, + "time_density": "day", + "spatial_extent": { + "xmin": -179.95, + "ymin": -59.45, + "xmax": 179.95, + "ymax": 83.55 + }, + "temporal_extent": { + "startdate": "2002-08-02T00:00:00Z", + "enddate": "2021-12-01T00:00:00Z" + }, + "sample_files": [ + "s3://veda-data-store-staging/EIS/COG/LIS_GLOBAL_DA/Evap/LIS_Evap_200208020000.d01.cog.tif" + ], + "discovery_items": [ + { + "discovery": "s3", + "cogify": false, + "upload": false, + "dry_run": false, + "prefix": "EIS/COG/LIS_GLOBAL_DA/Evap/", + "bucket": "veda-data-store-staging", + "filename_regex": "(.*)LIS_Evap_(.*).tif$" + } + ] +} diff --git a/data/datasets/lisDA_GPP.json b/data/datasets/lisDA_GPP.json new file mode 100644 index 00000000..2cce4e2e --- /dev/null +++ b/data/datasets/lisDA_GPP.json @@ -0,0 +1,32 @@ +{ + "collection": "lis-global-da-gpp", + "title": "Gross Primary Productivity - LIS 10km Global DA", + "description": "Gridded gross primary productivity (in g m-2 s-1) from 10km global LIS with assimilation", + "license": "CC0-1.0", + "is_periodic": true, + "time_density": "day", + "spatial_extent": { + "xmin": -179.95, + "ymin": -59.45, + "xmax": 179.95, + "ymax": 83.55 + }, + "temporal_extent": { + "startdate": "2002-08-02T00:00:00Z", + "enddate": "2021-12-01T00:00:00Z" + }, + "sample_files": [ + "s3://veda-data-store-staging/EIS/COG/LIS_GLOBAL_DA/GPP/LIS_GPP_200208020000.d01.cog.tif" + ], + "discovery_items": [ + { + "discovery": "s3", + "cogify": false, + "upload": false, + "dry_run": false, + "prefix": "EIS/COG/LIS_GLOBAL_DA/GPP/", + "bucket": "veda-data-store-staging", + "filename_regex": "(.*)LIS_GPP_(.*).tif$" + } + ] +} diff --git a/data/datasets/lisDA_GWS.json b/data/datasets/lisDA_GWS.json new file mode 100644 index 00000000..b21b3374 --- /dev/null +++ b/data/datasets/lisDA_GWS.json @@ -0,0 +1,32 @@ +{ + "collection": "lis-global-da-gws", + "title": "Groundwater Storage - LIS 10km Global DA", + "description": "Gridded groundwater storage (in mm) from 10km global LIS with assimilation", + "license": "CC0-1.0", + "is_periodic": true, + "time_density": "day", + "spatial_extent": { + "xmin": -179.95, + "ymin": -59.45, + "xmax": 179.95, + "ymax": 83.55 + }, + "temporal_extent": { + "startdate": "2002-08-02T00:00:00Z", + "enddate": "2021-12-01T00:00:00Z" + }, + "sample_files": [ + "s3://veda-data-store-staging/EIS/COG/LIS_GLOBAL_DA/GWS/LIS_GWS_200208020000.d01.cog.tif" + ], + "discovery_items": [ + { + "discovery": "s3", + "cogify": false, + "upload": false, + "dry_run": false, + "prefix": "EIS/COG/LIS_GLOBAL_DA/GWS/", + "bucket": "veda-data-store-staging", + "filename_regex": "(.*)LIS_GWS(.*).tif$" + } + ] +} diff --git a/data/datasets/lisDA_Qs.json b/data/datasets/lisDA_Qs.json new file mode 100644 index 00000000..a468e49b --- /dev/null +++ b/data/datasets/lisDA_Qs.json @@ -0,0 +1,32 @@ +{ + "collection": "lis-global-da-qs", + "title": "Surface runoff - LIS 10km Global DA", + "description": "Gridded surface runoff (in kg m-2 s-1) from 10km global LIS with assimilation", + "license": "CC0-1.0", + "is_periodic": true, + "time_density": "day", + "spatial_extent": { + "xmin": -179.95, + "ymin": -59.45, + "xmax": 179.95, + "ymax": 83.55 + }, + "temporal_extent": { + "startdate": "2002-08-02T00:00:00Z", + "enddate": "2021-12-01T00:00:00Z" + }, + "sample_files": [ + "s3://veda-data-store-staging/EIS/COG/LIS_GLOBAL_DA/Qs/LIS_Qs_200208020000.d01.cog.tif" + ], + "discovery_items": [ + { + "discovery": "s3", + "cogify": false, + "upload": false, + "dry_run": false, + "prefix": "EIS/COG/LIS_GLOBAL_DA/Qs/", + "bucket": "veda-data-store-staging", + "filename_regex": "(.*)LIS_Qs_(.*).tif$" + } + ] +} diff --git a/data/datasets/lisDA_Qsb.json b/data/datasets/lisDA_Qsb.json new file mode 100644 index 00000000..5a1294a0 --- /dev/null +++ b/data/datasets/lisDA_Qsb.json @@ -0,0 +1,32 @@ +{ + "collection": "lis-global-da-qsb", + "title": "Subsurface Runoff - LIS 10km Global DA", + "description": "Gridded subsurface runoff (in km m-2 s-1) from 10km global LIS with assimilation", + "license": "CC0-1.0", + "is_periodic": true, + "time_density": "day", + "spatial_extent": { + "xmin": -179.95, + "ymin": -59.45, + "xmax": 179.95, + "ymax": 83.55 + }, + "temporal_extent": { + "startdate": "2002-08-02T00:00:00Z", + "enddate": "2021-12-01T00:00:00Z" + }, + "sample_files": [ + "s3://veda-data-store-staging/EIS/COG/LIS_GLOBAL_DA/Qsb/LIS_Qsb_200208020000.d01.cog.tif" + ], + "discovery_items": [ + { + "discovery": "s3", + "cogify": false, + "upload": false, + "dry_run": false, + "prefix": "EIS/COG/LIS_GLOBAL_DA/Qsb/", + "bucket": "veda-data-store-staging", + "filename_regex": "(.*)LIS_Qsb_(.*).tif$" + } + ] +} diff --git a/data/datasets/lisDA_SWE.json b/data/datasets/lisDA_SWE.json new file mode 100644 index 00000000..c47ebcc6 --- /dev/null +++ b/data/datasets/lisDA_SWE.json @@ -0,0 +1,32 @@ +{ + "collection": "lis-global-da-swe", + "title": "Snow Water Equivalent - LIS 10km Global DA", + "description": "Gridded snow water equivalent (in kg m-2) from 10km global LIS with assimilation", + "license": "CC0-1.0", + "is_periodic": true, + "time_density": "day", + "spatial_extent": { + "xmin": -179.95, + "ymin": -59.45, + "xmax": 179.95, + "ymax": 83.55 + }, + "temporal_extent": { + "startdate": "2002-08-02T00:00:00Z", + "enddate": "2021-12-01T00:00:00Z" + }, + "sample_files": [ + "s3://veda-data-store-staging/EIS/COG/LIS_GLOBAL_DA/SWE/LIS_SWE_200208020000.d01.cog.tif" + ], + "discovery_items": [ + { + "discovery": "s3", + "cogify": false, + "upload": false, + "dry_run": false, + "prefix": "EIS/COG/LIS_GLOBAL_DA/SWE/", + "bucket": "veda-data-store-staging", + "filename_regex": "(.*)LIS_SWE_(.*).tif$", + } + ] +} diff --git a/data/datasets/lisDA_Streamflow.json b/data/datasets/lisDA_Streamflow.json new file mode 100644 index 00000000..e2c887bf --- /dev/null +++ b/data/datasets/lisDA_Streamflow.json @@ -0,0 +1,33 @@ +{ + "collection": "lis-global-da-streamflow", + "title": "Streamflow - LIS 10km Global DA", + "description": "Routed streamflow (in m3 s-1) from 10km global LIS+HyMAP with assimilation", + "license": "CC0-1.0", + "is_periodic": true, + "time_density": "day", + "spatial_extent": { + "xmin": -179.95, + "ymin": -59.45, + "xmax": 179.95, + "ymax": 83.55 + }, + "temporal_extent": { + "startdate": "2002-08-01T00:00:00Z", + "enddate": "2019-01-01T00:00:00Z" + }, + "sample_files": [ + "s3://veda-data-store-staging/EIS/COG/LIS_GLOBAL_DA/Streamflow/LIS_Streamflow_200208020000.d01.cog.tif" + ], + "discovery_items": [ + { + "discovery": "s3", + "cogify": false, + "upload": false, + "dry_run": false, + "prefix": "EIS/COG/LIS_GLOBAL_DA/Streamflow/", + "bucket": "veda-data-store-staging", + "filename_regex": "(.*)LIS_Streamflow_(.*).tif$", + "cpu_count": 20 + } + ] +} diff --git a/data/datasets/lisDA_TWS.json b/data/datasets/lisDA_TWS.json new file mode 100644 index 00000000..b7cd1adf --- /dev/null +++ b/data/datasets/lisDA_TWS.json @@ -0,0 +1,33 @@ +{ + "collection": "lis-global-da-tws", + "title": "Terrestrial Water Storage - LIS 10km Global DA", + "description": "Gridded terrestrial water storage (in mm) from 10km global LIS with assimilation", + "license": "CC0-1.0", + "is_periodic": true, + "time_density": "day", + "spatial_extent": { + "xmin": -179.95, + "ymin": -59.45, + "xmax": 179.95, + "ymax": 83.55 + }, + "temporal_extent": { + "startdate": "2002-08-02T00:00:00Z", + "enddate": "2021-12-01T00:00:00Z" + }, + "sample_files": [ + "s3://veda-data-store-staging/EIS/COG/LIS_GLOBAL_DA/TWS/LIS_TWS_200208020000.d01.cog.tif" + ], + "discovery_items": [ + { + "discovery": "s3", + "cogify": false, + "upload": false, + "dry_run": false, + "prefix": "EIS/COG/LIS_GLOBAL_DA/TWS/", + "bucket": "veda-data-store-staging", + "filename_regex": "(.*)LIS_TWS_(.*).tif$", + "datetime_range": "day" + } + ] +} diff --git a/data/datasets/lisDA_TotalPrecip.json b/data/datasets/lisDA_TotalPrecip.json new file mode 100644 index 00000000..1e00933f --- /dev/null +++ b/data/datasets/lisDA_TotalPrecip.json @@ -0,0 +1,33 @@ +{ + "collection": "lis-global-da-totalprecip", + "title": "Total Precipitation - LIS 10km Global DA", + "description": "Gridded total precipitation (in kg m-2 s-1) from 10km global LIS with assimilation", + "license": "CC0-1.0", + "is_periodic": true, + "time_density": "day", + "spatial_extent": { + "xmin": -179.95, + "ymin": -59.45, + "xmax": 179.95, + "ymax": 83.55 + }, + "temporal_extent": { + "startdate": "2002-08-02T00:00:00Z", + "enddate": "2021-12-01T00:00:00Z" + }, + "sample_files": [ + "s3://veda-data-store-staging/EIS/COG/LIS_GLOBAL_DA/TotalPrecip/LIS_TotalPrecip_200208020000.d01.cog.tif" + ], + "discovery_items": [ + { + "discovery": "s3", + "cogify": false, + "upload": false, + "dry_run": false, + "prefix": "EIS/COG/LIS_GLOBAL_DA/TotalPrecip/", + "bucket": "veda-data-store-staging", + "filename_regex": "(.*)LIS_TotalPrecip_(.*).tif$", + "cpu_count": 20 + } + ] +} diff --git a/data/datasets/listveg-suppression.json b/data/datasets/listveg-suppression.json new file mode 100644 index 00000000..2ecaae55 --- /dev/null +++ b/data/datasets/listveg-suppression.json @@ -0,0 +1,33 @@ +{ + "collection": "lis-tvegsuppression", + "title": "Change in transpiration for 2020 fires using LIS outputs", + "description": "Change in vegetation transpiration for 2020 fires using model outputs from Land Information System (LIS) framework that synthesizes multiple remote sensing observations within the Noah-MP land surface model. Change is calculated as the difference of transpiration in the immediate post-fire water year from that in the immediate pre-fire water year. The difference is normalized by pre-fire transpiration and negative values denote vegetation disturbance induced by fire or by a climatological anomaly resulting in the decline in transpiration.", + "license": "CC0-1.0", + "is_periodic": false, + "time_density": null, + "spatial_extent": { + "xmin": -125.04, + "ymin": 31.97, + "xmax": -109.40, + "ymax": 49.28 + }, + "temporal_extent": { + "startdate": "2020-01-01T00:00:00Z", + "enddate": "2020-01-01T23:59:59Z" + }, + "sample_files": [ + "s3://veda-data-store-staging/EIS/COG/Fire-Hydro/lis_tvegsuppression_2020.cog.tiff" + ], + "discovery_items": [ + { + "discovery": "s3", + "cogify": false, + "upload": false, + "dry_run": false, + "prefix": "EIS/COG/Fire-Hydro/", + "bucket": "veda-data-store-staging", + "filename_regex": "(.*)lis_tvegsuppression_(.*).cog.tiff$", + "date_range": "year" + } + ] +} From d402c9d2638a40d65990d930b2afd4961f90411b Mon Sep 17 00:00:00 2001 From: Slesa Adhikari Date: Fri, 10 Mar 2023 10:59:23 -0600 Subject: [PATCH 6/8] Fix typos --- data/datasets/barc-thomasfire.json | 2 +- data/datasets/lisDA_SWE.json | 2 +- data/datasets/listveg-suppression.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/data/datasets/barc-thomasfire.json b/data/datasets/barc-thomasfire.json index 224b258f..0998ef27 100644 --- a/data/datasets/barc-thomasfire.json +++ b/data/datasets/barc-thomasfire.json @@ -27,7 +27,7 @@ "prefix": "EIS/COG/Fire-Hydro/", "bucket": "veda-data-store-staging", "filename_regex": "(.*)thomas_fire_barc(.*).tiff$", - "date_range": "month" + "datetime_range": "month" } ] } diff --git a/data/datasets/lisDA_SWE.json b/data/datasets/lisDA_SWE.json index c47ebcc6..2f25bd4c 100644 --- a/data/datasets/lisDA_SWE.json +++ b/data/datasets/lisDA_SWE.json @@ -26,7 +26,7 @@ "dry_run": false, "prefix": "EIS/COG/LIS_GLOBAL_DA/SWE/", "bucket": "veda-data-store-staging", - "filename_regex": "(.*)LIS_SWE_(.*).tif$", + "filename_regex": "(.*)LIS_SWE_(.*).tif$" } ] } diff --git a/data/datasets/listveg-suppression.json b/data/datasets/listveg-suppression.json index 2ecaae55..41e5f32d 100644 --- a/data/datasets/listveg-suppression.json +++ b/data/datasets/listveg-suppression.json @@ -27,7 +27,7 @@ "prefix": "EIS/COG/Fire-Hydro/", "bucket": "veda-data-store-staging", "filename_regex": "(.*)lis_tvegsuppression_(.*).cog.tiff$", - "date_range": "year" + "datetime_range": "year" } ] } From bf60a62685cf22092bbf38f57cfe579e144eb9c5 Mon Sep 17 00:00:00 2001 From: Slesa Adhikari Date: Fri, 10 Mar 2023 11:57:10 -0600 Subject: [PATCH 7/8] Remove collection from discovery_items --- data/datasets/lis-da-tws-trend.json | 1 - 1 file changed, 1 deletion(-) diff --git a/data/datasets/lis-da-tws-trend.json b/data/datasets/lis-da-tws-trend.json index eec65fc4..ed3d45ad 100644 --- a/data/datasets/lis-da-tws-trend.json +++ b/data/datasets/lis-da-tws-trend.json @@ -20,7 +20,6 @@ ], "discovery_items": [ { - "collection": "lis-global-da-tws-trend-airflow", "discovery": "s3", "cogify": false, "upload": false, From ea96374aaf7b3f4d72f3656f57514d4a879cc681 Mon Sep 17 00:00:00 2001 From: Slesa Adhikari Date: Tue, 21 Mar 2023 13:59:09 -0500 Subject: [PATCH 8/8] Update files to newer version --- data/datasets/lis-da-gpp-trend.json | 2 +- data/datasets/lis-da-tws-trend.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/data/datasets/lis-da-gpp-trend.json b/data/datasets/lis-da-gpp-trend.json index 740462ad..c21193a2 100644 --- a/data/datasets/lis-da-gpp-trend.json +++ b/data/datasets/lis-da-gpp-trend.json @@ -26,7 +26,7 @@ "dry_run": false, "prefix": "EIS/COG/LIS_GLOBAL_DA/DA_Trends/", "bucket": "veda-data-store-staging", - "filename_regex": "(.*)DAGPP_STL_based_trendv2_2003_2021.cog.tif$", + "filename_regex": "(.*)DAGPP_STL_based_trend_v3_2003_2021.cog.tif$", "datetime_range": "year" } ], diff --git a/data/datasets/lis-da-tws-trend.json b/data/datasets/lis-da-tws-trend.json index ed3d45ad..b6f8660e 100644 --- a/data/datasets/lis-da-tws-trend.json +++ b/data/datasets/lis-da-tws-trend.json @@ -26,7 +26,7 @@ "dry_run": false, "prefix": "EIS/COG/LIS_GLOBAL_DA/DA_Trends/", "bucket": "veda-data-store-staging", - "filename_regex": "(.*)DATWS_STL_based_trendv2_2003_2021.cog.tif$", + "filename_regex": "(.*)DATWS_STL_based_trend_v3_2003_2021.cog.tif$", "datetime_range": "year" } ],