Cmr-10030: Ingesting and Searching for Visualizations

Generics.md

@@ -12,7 +12,8 @@ If adding a new document, you will need to update the defconf variable by either
              :data-quality-summary ["1.0.0"]
              :order-option ["1.0.0"]
              :service-entry ["1.0.0"]
-             :service-option ["1.0.0"]}
+             :service-option ["1.0.0"]
+			 :visualization ["1.0.0"]}
    		:parser #(json/parse-string % true)})
 
 When setting in an ENV or in AWS, use the JSON format:
@@ -21,7 +22,8 @@ When setting in an ENV or in AWS, use the JSON format:
     \"data-quality-summary\": [\"1.0.0\"],
     \"order-option\": [\"1.0.0\"],
     \"service-entry\": [\"1.0.0\"],
-    \"service-option\": [\"1.0.0\"]}"
+    \"service-option\": [\"1.0.0\"]
+	\"visualization\": [\"1.0.0\"]}"
 
 Each setting consists of a key, which is the name for the Generic which must be unique, and a list of version numbers. These values *must* match parts of a file system path under "schemas". For example, the order-option value must resolve to:
 

common-lib/src/cmr/common/config.clj

@@ -234,7 +234,8 @@
              :variable-draft ["1.0.0"]
              :data-quality-summary-draft ["1.0.0"]
              :order-option-draft ["1.0.0"]
-             :grid-draft ["1.0.0"]}
+             :grid-draft ["1.0.0"]
+             :visualization ["1.0.0"]}
    :parser #(json/parse-string % true)})
 
 (defconfig approved-pipeline-documentation

ingest-app/src/cmr/ingest/api/generic_documents.clj

@@ -52,12 +52,13 @@
       (errors/throw-service-error
        :invalid-data
        (format "The %s schema is currently disabled and cannot be ingested." (util/html-escape schema)))
-      (if-some [schema-file (gconfig/read-schema-specification schema version)]
-        (let [schema-obj (js-validater/json-string->json-schema schema-file)]
-          (js-validater/validate-json schema-obj raw-json true))
-        (errors/throw-service-error
-         :invalid-data
-         (format "While the [%s] schema with version [%s] is approved, it cannot be found." (util/html-escape schema) (util/html-escape version)))))))
+      (let [schema-path (format "schemas/%s/v%s/schema.json" (name schema) version)
+            schema-obj (js-validater/parse-json-schema-from-path schema-path)]
+        (if schema-obj
+          (js-validater/validate-json schema-obj raw-json true)
+          (errors/throw-service-error
+           :invalid-data
+           (format "While the [%s] schema with version [%s] is approved, it cannot be found." (util/html-escape schema) (util/html-escape version))))))))
 
 (defn- concept-type->singular
   "Common task to convert concepts from their public URL form to their internal

metadata-db-app/int-test/cmr/metadata_db/int_test/concepts/utils/generic.clj

@@ -56,3 +56,35 @@
                            :extra-fields extra-fields}
                           (dissoc attributes :extra-fields))]
     (concepts/create-any-concept provider-id concept-type uniq-num attributes)))
+
+(defmethod concepts/get-sample-metadata :visualization
+  [_concept-type]
+  (json/generate-string
+   {:Id "MODIS_Combined_L3_IGBP_Land_Cover_Type_Annual",
+    :VisualizationType "tiles",
+    :Name "Land Cover Type (L3, IGBP, Annual, Best Available, MODIS, Aqua+Terra)",
+    :What {:whattile1 "a", :whattile2 "b"},
+    :How {:howtile1 "a", :howtile2 "b"},
+    :ConceptIds [{:type "STD",
+                  :value "C186286578-LPDAAC_ECS",
+                  :shortName "MCD12Q1",
+                  :title "MODIS/Terra+Aqua Land Cover Type Yearly L3 Global 500m SIN Grid V006",
+                  :version "006",
+                  :dataCenter "LPDAAC_ECS"}],
+    :MetadataSpecification {:URL "https://cdn.earthdata.nasa.gov/generics/visualization/v1.0.0",
+                            :Name "Visualization",
+                            :Version "1.0.0"}}))
+
+(defmethod concepts/create-concept :visualization
+  [concept-type & args]
+  (let [[provider-id uniq-num attributes] (concepts/parse-create-concept-args concept-type args)
+        native-id (str "vsl-native" uniq-num)
+        extra-fields (merge {:document-name "vsl-docname"
+                             :schema "visualization"}
+                            (:extra-fields attributes))
+        attributes (merge {:user-id (str "user" uniq-num)
+                           :format "application/json"
+                           :native-id native-id
+                           :extra-fields extra-fields}
+                          (dissoc attributes :extra-fields))]
+    (concepts/create-any-concept provider-id concept-type uniq-num attributes)))

schemas/resources/schemas/validate.py

@@ -54,7 +54,7 @@ def test_schema(schema_name):
 def main():
   ret = test_schema("index")
 
-  for i in ["data-quality-summary", "grid", "order-option", "service-entry", "service-option"]:
+  for i in ["data-quality-summary", "grid", "order-option", "visualization"]:
     ret = ret + test_schema(i)
     ret = ret + test_index(i)
 

schemas/resources/schemas/visualization/CHANGELOG.md

@@ -0,0 +1,35 @@
+# visualization schema change Log
+
+## [1.0.0]
+- 2024-??-??
+Initial release in the CMR.
+
+----
+Using version 
+https://git.earthdata.nasa.gov/projects/VISLABS/repos/metadata-mapping/browse/umm/visualization/v0.10.1 as of 08/29/2024
+
+Changes to main.json:
+- Changed the name of Identifier to Id
+- Added Name to the schema. - Not sure if generics needs this - cant I just copy the title to Name?
+- Changed ScienceKeywords array minItems from 0 to 1
+- Fixed the reference of SpatialExtentType to spatial-temporal-extent.json
+- Moved the reference of TemporalExtentType to spatial-temporal-extent.json
+- ConceptIds moved additionalProperties: false up to the type: object declaration
+- MetadataSpecification is above the what and how.
+- required 
+  - Identifier changed name to id
+  - Name has been added - can this be Title?
+  - ConceptIds was added
+  - MetadataSpecification was added
+- In the allOf section the What refernces are different - they don't look correct in the given schema
+- In the allOf section the How refernces are different - they don't look correct in the given schema
+- The rest is just different.
+
+Added the UMM-C 1.18.1 umm-cmn-json-schema.json Spatial, Temporal, and supporting elements to spatial-temporal-extent.json
+Removed the older umm-cmn-json-schema.json. 
+
+
+
+Copyright © 2024-2024 United States Government as represented by the
+Administrator of the National Aeronautics and Space Administration. All Rights
+Reserved.

schemas/resources/schemas/visualization/v1.0.0/index.json

@@ -0,0 +1,74 @@
+{
+  "MetadataSpecification": {
+    "URL": "https://cdn.earthdata.nasa.gov/generic/index/v0.0.1",
+    "Name": "Generic-Index",
+    "Version": "0.0.1"
+  },
+  "Generic": {
+    "Name": "Visualization",
+    "Version": "1.0.0"
+  },
+  "SubConceptType": "VIS",
+  "IndexSetup" : {
+    "index": {
+      "number_of_shards": 3,
+      "number_of_replicas": 1,
+      "refresh_interval": "1s"
+    }
+  },
+  "Indexes":
+  [
+    {
+      "Description": "Identifier",
+      "Field": ".Identifier",
+      "Name": "Id",
+      "Mapping": "string"
+    },
+    {
+      "Description": "Identifier",
+      "Field": ".Identifier",
+      "Name": "Identifier",
+      "Mapping": "string"
+    },
+    {
+      "Description": "Schema Name as the Name field",
+      "Field": ".Name",
+      "Name": "Name",
+      "Mapping": "string"
+    },
+    {
+      "Description": "Schema Title as the Title field",
+      "Field": ".Title",
+      "Name": "Title",
+      "Mapping": "string"
+    },
+    {
+      "Description": "VisualizationType",
+      "Field": ".VisualizationType",
+      "Name": "Visualization-Type",
+      "Mapping": "string"
+    },
+    {    
+      "Description": "Visualization Source ConceptIds",
+      "Field": ".ConceptIds",
+      "Name": "Concept-Ids",
+      "Mapping": "string",
+      "Indexer": "simple-array-field",
+      "Configuration": {"sub-fields": ["value", "shortName"]}
+    },
+    {    
+      "Description": "Visualization Source ConceptIds in keywords",
+      "Field": ".ConceptIds",
+      "Name": "keyword",
+      "Mapping": "string",
+      "Indexer": "simple-array-field",
+      "Configuration": {"sub-fields": ["value"]}
+    },
+    {
+      "Description": "Identifier with the keywords",
+      "Field": ".Identifier",
+      "Name": "keyword",
+      "Mapping": "string"
+    }
+  ]
+}

schemas/resources/schemas/visualization/v1.0.0/ingest.md

@@ -0,0 +1,85 @@
+## <a name="visualization"></a> Visualization
+
+#### <a name="provider-info-visualization"></a> /providers/&lt;provider-id&gt;/visualizations/&lt;native-id&gt;
+
+### <a name="create-update-visualization"></a> Create / Update a Visualization
+
+Visualization concepts can be created or updated by sending an HTTP PUT with the metadata to the URL `%CMR-ENDPOINT%/providers/<provider-id>/visualizations/<native-id>`. The response will include the [concept id](#concept-id) and the [revision id](#revision-id). The contents of the metadata is passed in the body of the request.
+
+```
+    curl -XPOST \
+        -H "Cmr-Pretty: true" \
+        -H "Content-Type:application/vnd.nasa.cmr.umm+json" \
+        -H "Authorization: Bearer XXXX" \
+        "%CMR-ENDPOINT%/providers/PROV1/visualizations/sampleNativeId" \
+        -d @sampleVisualization.json
+```
+
+#### Successful Response in XML
+
+```
+    <?xml version="1.0" encoding="UTF-8"?>
+    <result>
+        <concept-id>VIS1200000000-PROV1</concept-id>
+        <revision-id>1</revision-id>
+        <warnings></warnings>
+        <existing-errors></existing-errors>
+    </result>
+```
+Subsequent ingests to the Visualization record will result in updates to it's metadata as well as increment the revision-id of the record.
+
+#### Successful Response in JSON
+
+By passing the option `-H "Accept: application/json"` to `curl`, one may
+get a JSON response:
+
+```
+    {"concept-id":"VIS1200000000-PROV1","revision-id":1,"warnings":null,"existing-errors":null}
+```
+
+### <a name="delete-visualization"></a> Delete a Visualization
+
+Visualization metadata can be deleted by sending an HTTP DELETE to the URL `%CMR-ENDPOINT%/providers/<provider-id>/visualizations/<native-id>`. The response will include the [concept id](#concept-id) and the [revision id](#revision-id) of the tombstone.
+
+```
+    curl -XDELETE \
+        -H "Cmr-Pretty: true" \
+        -H "Authorization: Bearer XXXX" \
+        %CMR-ENDPOINT%/providers/PROV1/visualizations/sampleNative23Id"
+```
+
+#### Successful Response in XML
+
+```
+    <?xml version="1.0" encoding="UTF-8"?>
+    <result>
+        <concept-id>VIS1200000000-PROV1</concept-id>
+        <revision-id>2</revision-id>
+    </result>
+```
+
+#### Successful Response in JSON
+
+```
+    {"concept-id":"VIS1200000000-PROV1","revision-id":2,"warnings":null,"existing-errors":null}
+```
+
+Attempting to delete an already deleted record will return
+the following error message
+
+#### Unsuccessful Response in XML
+
+```
+    <?xml version="1.0" encoding="UTF-8"?>
+    <errors>
+        <error>Concept with native-id [sampleNative23Id] and concept-id [VIS1200000000-PROV1] is already deleted.</error>
+    </errors>
+```
+
+#### Unsuccessful Response in JSON
+
+```
+    "errors": [
+            "Concept with native-id [sampleNative23Id] and concept-id [VIS1200000000-PROV1] is already deleted."
+        ]
+```