Skip to content

Commit

Permalink
CMR-9341: Added UMM-G v1.6.5
Browse files Browse the repository at this point in the history
  • Loading branch information
siwei xu authored and siwei xu committed Jul 31, 2023
1 parent 4a363d3 commit 963e3b4
Show file tree
Hide file tree
Showing 8 changed files with 1,471 additions and 7 deletions.
2 changes: 1 addition & 1 deletion ingest-app/src/cmr/ingest/config.clj
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
(defconfig granule-umm-version
"Defines the latest granule umm version accepted by ingest - it's the latest official version.
This environment variable needs to be manually set when newer UMM version becomes official"
{:default "1.6.4"})
{:default "1.6.5"})

(defconfig variable-umm-version
"Defines the latest variable umm version accepted by ingest - it's the latest official version.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -680,7 +680,7 @@
:identifiers [:identifier "a very nice granule :)"
:identifier-type "a type of identifier"]})})
(data-core/item->concept {:format :umm-json
:version "1.6.4"}))
:version "1.6.5"}))
{:keys [status] :as response} (ingest/ingest-concept granule)]
(is (#{200 201} status) (pr-str response))))

Expand Down
1,310 changes: 1,310 additions & 0 deletions umm-spec-lib/resources/json-schemas/granule/umm/v1.6.5/umm-g-json-schema.json

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
{
"$schema": "http://json-schema.org/draft-04/schema#",
"definitions": {
"ItemType": {
"type": "object",
"additionalProperties": false,
"description": "Represents a single item found in search results. It contains some metadata about the item found along with the UMM representing the item. umm won't be present if the item represents a tombstone.",
"properties": {
"meta": {
"$ref": "#/definitions/MetaType"
},
"umm": {
"$ref": "umm-g-json-schema.json"
}
},
"required": ["meta"]
},
"MetaType": {
"type": "object",
"additionalProperties": false,
"description": "CMR level metadata about the item found. This represents data not found in the actual metadata.",
"properties": {
"provider-id": {
"description": "The identity of the provider in the CMR",
"type": "string",
"minLength": 1,
"pattern": "[A-Z0-9_]+"
},
"concept-type": {
"description": "The type of item found.",
"type": "string",
"enum": ["granule"]
},
"native-id": {
"description": "The id used by the provider to identify this item during ingest.",
"type": "string",
"minLength": 1
},
"concept-id": {
"description": "The concept id of the item found.",
"$ref": "#/definitions/ConceptIdType"
},
"revision-id": {
"description": "A number >= 1 that indicates which revision of the item.",
"type": "number"
},
"revision-date": {
"description": "The date this revision was created. This would be the creation or update date of the item in the CMR.",
"format": "date-time",
"type": "string"
},
"format": {
"description": "The mime type of the original metadata",
"type": "string",
"minLength": 1
}
},
"required": ["provider-id", "concept-type", "native-id", "concept-id", "revision-id", "revision-date", "format"]
},
"ConceptIdType": {
"description": "The concept id of a concept.",
"type": "string",
"minLength": 4,
"pattern": "[A-Z]+\\d+-[A-Z0-9_]+"
}
},
"title": "UMM JSON Granule Search Results",
"type": "object",
"additionalProperties": false,
"properties": {
"hits": {
"description": "The total number of items that matched the search.",
"type": "number"
},
"took": {
"description": "How long the search took in milliseconds from the time the CMR received the request until it had generated the response. This does not include network traffic time to send the request or return the response.",
"type": "number"
},
"items": {
"description": "The list of items matching the result in this page.",
"type": "array",
"items": {
"$ref": "#/definitions/ItemType"
},
"minItems": 0
}
},
"required": ["hits", "took", "items"]
}
24 changes: 24 additions & 0 deletions umm-spec-lib/src/cmr/umm_spec/migration/version/granule.clj
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,15 @@
(assoc related-url :Type "GET DATA")
related-url))

(defn truncate-pge-version-size
"This function truncates the PGEVersion size to 10 characters, if it has more
than 10 characters."
[pge-version-class]
(when-let [pge-version (:PGEVersion pge-version-class)]
(if (> (count pge-version) 10)
(assoc pge-version-class :PGEVersion (subs pge-version 0 10))
pge-version-class)))

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;; Granule Migration Implementations

Expand Down Expand Up @@ -259,3 +268,18 @@
[context granule & _]
(-> granule
(m-spec/update-version :granule "1.6.4")))

;; v1.6.5 migrations

(defmethod interface/migrate-umm-version [:granule "1.6.5" "1.6.4"]
;; truncate PGEVersion to 10 characters if it has more than 10
;; characters long.
[context g & _]
(-> g
(update :PGEVersionClass truncate-pge-version-size)
(m-spec/update-version :granule "1.6.4")))

(defmethod interface/migrate-umm-version [:granule "1.6.4" "1.6.5"]
[context g & _]
(-> g
(m-spec/update-version :granule "1.6.5")))
4 changes: 2 additions & 2 deletions umm-spec-lib/src/cmr/umm_spec/umm_g/granule.clj
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,9 @@

(def ^:private umm-g-metadata-specification
"Defines the current UMM-G MetadataSpecification"
{:URL "https://cdn.earthdata.nasa.gov/umm/granule/v1.6.4"
{:URL "https://cdn.earthdata.nasa.gov/umm/granule/v1.6.5"
:Name "UMM-G"
:Version "1.6.4"})
:Version "1.6.5"})

(defn- get-date-by-type
"Returns the date of the given type from the given provider dates"
Expand Down
2 changes: 1 addition & 1 deletion umm-spec-lib/src/cmr/umm_spec/versioning.clj
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
"1.15" "1.15.1" "1.15.2" "1.15.3" "1.15.4" "1.15.5"
"1.16" "1.16.1" "1.16.2" "1.16.3" "1.16.4" "1.16.5" "1.16.6" "1.16.7"
"1.17.0" "1.17.1" "1.17.2" "1.17.3"]
:granule ["1.4" "1.5" "1.6" "1.6.1" "1.6.2" "1.6.3" "1.6.4"]
:granule ["1.4" "1.5" "1.6" "1.6.1" "1.6.2" "1.6.3" "1.6.4" "1.6.5"]
:variable ["1.0" "1.1" "1.2" "1.3" "1.4" "1.5" "1.6" "1.7" "1.8" "1.8.1" "1.8.2" "1.9.0"]
:service ["1.0" "1.1" "1.2" "1.3" "1.3.1" "1.3.2" "1.3.3" "1.3.4" "1.4" "1.4.1" "1.5.0" "1.5.1"]
:tool ["1.0" "1.1" "1.1.1" "1.2.0"]
Expand Down
45 changes: 43 additions & 2 deletions umm-spec-lib/test/cmr/umm_spec/test/migration/version/granule.clj
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,14 @@

(deftest test-version-steps
(with-bindings {#'cmr.umm-spec.versioning/versions
{:granule ["1.4" "1.5" "1.6" "1.6.1" "1.6.2" "1.6.3" "1.6.4"]}}
{:granule ["1.4" "1.5" "1.6" "1.6.1" "1.6.2" "1.6.3" "1.6.4" "1.6.5"]}}
(is (= [] (#'vm/version-steps :granule "1.5" "1.5")))
(is (= [["1.4" "1.5"]] (#'vm/version-steps :granule "1.4" "1.5")))
(is (= [["1.5" "1.4"]] (#'vm/version-steps :granule "1.5" "1.4")))
(is (= [["1.4" "1.5"] ["1.5" "1.6"]] (#'vm/version-steps :granule "1.4" "1.6")))
(is (= [["1.4" "1.5"] ["1.5" "1.6"] ["1.6" "1.6.1"]] (#'vm/version-steps :granule "1.4" "1.6.1")))
(is (= [["1.6.1" "1.6.2"] ["1.6.2" "1.6.3"] ["1.6.3" "1.6.4"]] (#'vm/version-steps :granule "1.6.1" "1.6.4")))))
(is (= [["1.6.1" "1.6.2"] ["1.6.2" "1.6.3"] ["1.6.3" "1.6.4"]] (#'vm/version-steps :granule "1.6.1" "1.6.4")))
(is (= [["1.6.4" "1.6.5"]] (#'vm/version-steps :granule "1.6.4" "1.6.5")))))

(defspec all-migrations-produce-valid-umm-spec 100
(for-all [umm-record (gen/no-shrink umm-gen/umm-g-generator)
Expand Down Expand Up @@ -708,6 +709,26 @@
:Name "UMM-G"
:Version "1.6.4"}
:DataGranule {:ArchiveAndDistributionInformation [{:Format "ASCII"} {:Format "ComicSans"}]}
:PGEVersionClass {:PGEName "A PGE Name"
:PGEVersion "1234567890"}
:RelatedUrls [{:URL "https://acdisc.gesdisc.eosdis.nasa.gov/opendap/Aqua_AIRS_Level3/AIRX3STD.006/"
:Type "GET SERVICE"
:Subtype "ALGORITHM THEORETICAL BASIS DOCUMENT (ATBD)"
:MimeType "APPEARS"
:Format "Future-Type"} ;; this was a bad format
{:URL "s3://amazon.something.com/get-data"
:Type "GET DATA VIA DIRECT ACCESS"
:Format "NETCDF-4" ;; this has always been a valid format
:MimeType "application/x-netcdf"}]})

(def granule-1-6-5
{:MetadataSpecification
{:URL "https://cdn.earthdata.nasa.gov/umm/granule/v1.6.5"
:Name "UMM-G"
:Version "1.6.5"}
:DataGranule {:ArchiveAndDistributionInformation [{:Format "ASCII"} {:Format "ComicSans"}]}
:PGEVersionClass {:PGEName "A PGE Name"
:PGEVersion "12345678901234567890123456789012345678901234567890"}
:RelatedUrls [{:URL "https://acdisc.gesdisc.eosdis.nasa.gov/opendap/Aqua_AIRS_Level3/AIRX3STD.006/"
:Type "GET SERVICE"
:Subtype "ALGORITHM THEORETICAL BASIS DOCUMENT (ATBD)"
Expand Down Expand Up @@ -743,3 +764,23 @@
(is (= "Not provided" (:Format (second arch-info))) "New keyword not mappable")
(is (= "Not provided" (:Format (first urls))) "New URL format is not mappable")
(is (= "NETCDF-4" (:Format (second urls))) "Old URL format should be mappable")))

(deftest migrate-1-6-4-up-to-1-6-5
(let [converted (vm/migrate-umm {} :granule "1.6.4" "1.6.5" granule-1-6-4)]
(is (= {:URL "https://cdn.earthdata.nasa.gov/umm/granule/v1.6.5"
:Name "UMM-G"
:Version "1.6.5"}
(:MetadataSpecification converted))
"Specification must be 1.6.5")
(is (= {:PGEName "A PGE Name" :PGEVersion "1234567890"}
(:PGEVersionClass converted)))))

(deftest migrate-1-6-5-down-to-1-6-4
(let [converted (vm/migrate-umm {} :granule "1.6.5" "1.6.4" granule-1-6-5)]
(is (= {:URL "https://cdn.earthdata.nasa.gov/umm/granule/v1.6.4"
:Name "UMM-G"
:Version "1.6.4"}
(:MetadataSpecification converted))
"Specification must be 1.6.4")
(is (= {:PGEName "A PGE Name" :PGEVersion "1234567890"}
(:PGEVersionClass converted)))))

0 comments on commit 963e3b4

Please sign in to comment.