CMR-9383: Remove from elastic search, as well as database when deleting drafts

indexer-app/src/cmr/indexer/services/index_service.clj

@@ -679,7 +679,10 @@
               (dg/index-deleted-granule context concept concept-id revision-id elastic-version elastic-options))
             ;; propagate collection deletion to granules
             (when (= :collection concept-type)
-              (cascade-collection-delete context concept-mapping-types concept-id revision-id))))))))
+              (cascade-collection-delete context concept-mapping-types concept-id revision-id)))))
+      ;; For draft concept, after the index is deleted, remove it from database.
+      (when (cs/is-draft-concept? concept-type)
+        (meta-db/delete-draft context concept)))))
 
 (defn- index-association-concept
   "Index the association concept identified by the given concept-id and revision-id."

ingest-app/src/cmr/ingest/api/core.clj

@@ -330,23 +330,3 @@
                                (ingest/delete-concept
                                 request-context
                                 concept-attribs)))))
-
-(defn delete-draft
-  "Delete the given draft by its concept type, provider id and native id."
-  [concept-type provider-id native-id request]
-  (let [{:keys [request-context params headers]} request
-        concept-attribs (-> {:provider-id provider-id
-                             :native-id native-id
-                             :concept-type concept-type}
-                            (set-revision-id headers)
-                            (set-user-id request-context headers))]
-    (lt-validation/validate-launchpad-token request-context)
-    (common-enabled/validate-write-enabled request-context "ingest")
-    (verify-provider-exists request-context provider-id)
-    (info (format "Deleting %s %s from client %s"
-                  (name concept-type) (pr-str concept-attribs) (:client-id request-context)))
-    (generate-ingest-response headers
-                              (format-and-contextualize-warnings-existing-errors
-                               (ingest/delete-draft
-                                request-context
-                                concept-attribs)))))

ingest-app/src/cmr/ingest/api/generic_documents.clj

@@ -217,10 +217,10 @@
 
 (defn delete-generic-document
   "Deletes a generic document in elasticsearch. Creates a tombstone in the database for non-draft
-   document. Delete all revisions from database for draft document. Returns a 201
-   if successful with the concept id and revision number for non-draft document. Returns a 200
-   if successful with the concept id for draft document. A 404 status is returned if the concept has
-   already been deleted."
+   document. Delete all revisions from database for draft document.
+   Returns a 201 if successful with the concept id and revision number for non-draft document.
+   Returns a 200 if successful with the concept id and revision number for draft document.
+   A 404 status is returned if the concept has already been deleted or removed from database."
   [request]
   (let [{:keys [route-params request-context params]} request
         provider-id (or (:provider params)
@@ -232,9 +232,7 @@
              request-context :update :provider-object provider-id)
             (acl/verify-provider-context-permission
              request-context :read :provider-object provider-id))]
-    (if (is-draft-concept? request)
-      (api-core/delete-draft concept-type provider-id native-id request)
-      (api-core/delete-concept concept-type provider-id native-id request))))
+    (api-core/delete-concept concept-type provider-id native-id request)))
 
 (defn crud-generic-document
   "This function checks for required parameters. If they don't exist then throw an error, otherwise send the request

ingest-app/src/cmr/ingest/services/ingest_service.clj

@@ -14,7 +14,6 @@
   [cmr.ingest.services.ingest-service.util
    ;; Public utility functions
    delete-concept
-   delete-draft
    fix-ingest-concept-format
    health
    reset]

ingest-app/src/cmr/ingest/services/ingest_service/util.clj

@@ -74,26 +74,6 @@
           {:keys [revision-id]} (mdb/save-concept context concept)]
       {:concept-id concept-id, :revision-id revision-id})))
 
-(defn-timed delete-draft
-  "Delete a draft from mdb and indexer. Throws a 404 error if the draft does not exist."
-  [context concept-attribs]
-  (let [{:keys [concept-type provider-id native-id]} concept-attribs
-        existing-concept (first (mdb/find-concepts context
-                                                   {:provider-id provider-id
-                                                    :native-id native-id
-                                                    :exclude-metadata true
-                                                    :latest true}
-                                                   concept-type))
-        concept-id (:concept-id existing-concept)]
-    (when-not concept-id
-      (errors/throw-service-error
-        :not-found (cmsg/invalid-native-id-msg concept-type provider-id native-id)))
-    (let [concept (-> concept-attribs
-                      (dissoc :provider-id :native-id)
-                      (assoc :concept-id concept-id :deleted true))
-          {:keys [concept-id]} (mdb/delete-draft context concept)]
-      {:concept-id concept-id})))
-
 (defn concept-json->concept
   "Returns the concept for the given concept JSON string.
   This is a temporary function and will be replaced by the UMM parse-metadata function once

metadata-db-app/src/cmr/metadata_db/api/concepts.clj

@@ -122,7 +122,7 @@
   "Permanently remove a draft concept from the database."
   [context params concept-id revision-id]
   (let [{:keys [concept-id]} (concept-service/force-delete-draft
-                                context concept-id revision-id)]
+                              context concept-id revision-id)]
     {:status 200
      :body (json/generate-string {:concept-id concept-id})
      :headers rh/json-header}))
@@ -161,11 +161,11 @@
       ;; saves a concept
       (POST "/" {:keys [request-context params body]}
         (save-concept-revision request-context params body))
-      ;; mark a concept as deleted (add a tombstone) specifying the revision the tombstone should have
       ;; remove a draft from the database
       (DELETE "/force-delete-draft/:concept-id" {{:keys [concept-id] :as params} :params
                                                  request-context :request-context}
         (force-delete-draft request-context params concept-id nil))
+      ;; mark a concept as deleted (add a tombstone) specifying the revision the tombstone should have
       (DELETE "/:concept-id/:revision-id" {{:keys [concept-id revision-id] :as params} :params
                                            request-context :request-context}
         (delete-concept request-context params concept-id revision-id))

system-int-test/test/cmr/system_int_test/ingest/generics_test.clj

@@ -8,8 +8,10 @@
    [cmr.common.config :as config]
    [cmr.common.generics :as gcfg]
    [cmr.ingest.api.generic-documents :as gdocs]
+   [cmr.system-int-test.search.generics-test :as search-generics-test]
    [cmr.system-int-test.system :as system]
    [cmr.system-int-test.utils.generic-util :as gen-util]
+   [cmr.system-int-test.utils.index-util :as index]
    [cmr.system-int-test.utils.ingest-util :as ingest]))
 
 (defn grant-all-generic-permission-fixture
@@ -222,17 +224,37 @@
     (is (= (:errors coll-draft) nil))
     (is (= (:errors coll-draft-np) ["You do not have PROVIDER_CONTEXT permission to perform that action."]))))
 
-;; Test that collection-draft is removed from database when deleted.
-(deftest test-collection-draft-removed-from-database-on-delete
+;; Test that collection-draft is removed from database and elastic search when deleted.
+(deftest test-collection-draft-removed-from-database-and-elastic-on-delete
   ;; Drafts have permissions to ingest on PROV1, but not on PROV2.
   (let [native-id "NativeId"
         coll-draft (gen-util/ingest-generic-document
                     nil "PROV1" native-id :collection-draft gen-util/collection-draft :post)
+        concept-id (:concept-id coll-draft)
+        _ (index/wait-until-indexed)
+        ;; search for the draft from elastic search should return result
+        search-result (search-generics-test/search-request "collection-drafts" (str "concept_id=" concept-id))
+        search-status (:status search-result)
+        search-body (:body search-result)
+        ;; deleting draft should delete from database completely.
         result1 (gen-util/ingest-generic-document
-                    nil "PROV1" native-id :collection-draft gen-util/collection-draft :delete) 
+                    nil "PROV1" native-id :collection-draft gen-util/collection-draft :delete)
+        ;; Removal from database happens after the index is deleted.
+        _ (index/wait-until-indexed)
         result2 (gen-util/ingest-generic-document
                     nil "PROV1" native-id :collection-draft gen-util/collection-draft :delete)]
+    ;;Verify that searching for the concept from elastic search does return result.
+    (is (= search-status 200))
+    (is (string/includes? search-body "<hits>1</hits>"))
+
     ;;The first delete removes the draft from database completely and returns concept-id.
     ;;The second delete will not say the draft is already deleted because it no longer exists in the database
-    (is (= result1 {:concept-id "CD1200000011-PROV1", :warnings nil, :existing-errors nil}))
-    (is (= result2 {:errors ["CollectionDraft with native id [NativeId] in provider [PROV1] does not exist."]})))) 
+    (is (= result1 {:concept-id concept-id, :revision-id 2,:warnings nil, :existing-errors nil}))
+    (is (= result2 {:errors ["CollectionDraft with native id [NativeId] in provider [PROV1] does not exist."]}))
+
+    ;;Verify that searching for the concept from elastic search won't return anything after the delete.
+    (let [result (search-generics-test/search-request "collection-drafts" (str "concept_id=" concept-id))
+          status (:status result)
+          body (:body result)]
+      (is (= status 200))
+      (is (string/includes? body "<hits>0</hits>")))))

transmit-lib/src/cmr/transmit/metadata_db.clj

@@ -469,7 +469,7 @@
        (format "Save concept failed. MetadataDb app response status code: %s response: %s"
                status response)))))
 
-(defn-timed delete-draft 
+(defn-timed delete-draft
   "delete a draft in metadata db"
   [context concept]
   (let [conn (config/context->app-connection context :metadata-db)