From a518cfb747e318c76c486c5d041224e616a06568 Mon Sep 17 00:00:00 2001
From: TylerHeald1 <97175638+TylerHeald1@users.noreply.github.com>
Date: Fri, 30 Aug 2024 08:56:12 -0400
Subject: [PATCH] CMR-9443: Reduce size of Elasticsearch query return on some
 calls with unlimited page size (#2168)

* Removing source from some cwic and opensearch ES searchs

* Added a unit test

* add newline

* kondo changes
---
 .../src/cmr/elastic_utils/search/es_index.clj |  5 ++--
 .../test/cmr/elastic_utils/test/es_index.clj  | 29 +++++++++++++++++++
 .../has_granules_or_cwic_results_feature.clj  | 14 +++++----
 3 files changed, 40 insertions(+), 8 deletions(-)
 create mode 100644 elastic-utils-lib/test/cmr/elastic_utils/test/es_index.clj

diff --git a/elastic-utils-lib/src/cmr/elastic_utils/search/es_index.clj b/elastic-utils-lib/src/cmr/elastic_utils/search/es_index.clj
index cc1b864680..eb1e77217c 100644
--- a/elastic-utils-lib/src/cmr/elastic_utils/search/es_index.clj
+++ b/elastic-utils-lib/src/cmr/elastic_utils/search/es_index.clj
@@ -85,7 +85,8 @@
                 highlights
                 scroll
                 scroll-id
-                search-after]} query
+                search-after
+                remove-source]} query
         scroll-timeout (when scroll (es-config/elastic-scroll-timeout))
         search-type (if scroll
                       (es-config/elastic-scroll-search-type)
@@ -100,7 +101,7 @@
      :sort sort-params
      :size page-size
      :from offset
-     :_source fields
+     :_source (if (nil? remove-source) fields false)
      :aggs aggregations
      :scroll scroll-timeout
      :scroll-id scroll-id
diff --git a/elastic-utils-lib/test/cmr/elastic_utils/test/es_index.clj b/elastic-utils-lib/test/cmr/elastic_utils/test/es_index.clj
new file mode 100644
index 0000000000..932b0b8e56
--- /dev/null
+++ b/elastic-utils-lib/test/cmr/elastic_utils/test/es_index.clj
@@ -0,0 +1,29 @@
+(ns cmr.elastic-utils.test.es-index
+  "Tests for the cmr.elastic-utils.search.es-index namespace"
+  (:require 
+   [clojure.test :refer [deftest is testing]]
+   [cmr.elastic-utils.search.es-index :as es-index]
+   [cmr.common.services.search.query-model :as qm]
+   [cmr.elastic-utils.search.es-group-query-conditions :as gc]))
+
+(deftest test-query->execution-params
+  (let [query->execution-params #'es-index/query->execution-params
+        condition (gc/or-conds (map #(qm/string-conditions :consortiums [%])
+                                    ["CWIC" "FEDEO" "GEOSS" "CEOS" "EOSDIS"]))]
+    (testing "query include remove-source"
+      (let [query (qm/query {:concept-type :collection
+                             :result-format :xml
+                             :condition condition
+                             :page-size :unlimited
+                             :remove-source true})
+            execution-params (query->execution-params query)]
+        (is (= false
+               (:_source execution-params)))))
+    (testing "query doesn't include remove-source"
+      (let [query (qm/query {:concept-type :collection
+                             :result-format :xml
+                             :condition condition
+                             :page-size :unlimited})
+            execution-params (query->execution-params query)]
+        (is (not (= false
+                    (:_source execution-params))))))))
diff --git a/search-app/src/cmr/search/services/query_execution/has_granules_or_cwic_results_feature.clj b/search-app/src/cmr/search/services/query_execution/has_granules_or_cwic_results_feature.clj
index aa6baf48ec..8fd29de10e 100644
--- a/search-app/src/cmr/search/services/query_execution/has_granules_or_cwic_results_feature.clj
+++ b/search-app/src/cmr/search/services/query_execution/has_granules_or_cwic_results_feature.clj
@@ -53,11 +53,12 @@
 
 (defn get-cwic-collections
   "Returns the collection granule count by searching elasticsearch by aggregation"
-  [context provider-ids]
+  [context _provider-ids]
   (let [condition (qm/string-conditions :consortiums ["CWIC"])
         query (qm/query {:concept-type :collection
                          :condition condition
-                         :page-size :unlimited})
+                         :page-size :unlimited
+                         :remove-source true})
         results (common-esi/execute-query context query)]
     (into {}
           (for [coll-id (map :_id (get-in results [:hits :hits]))]
@@ -65,12 +66,13 @@
 
 (defn get-opensearch-collections
   "Returns the collection granule count by searching elasticsearch by aggregation"
-  [context provider-ids]
+  [context _provider-ids]
   (let [condition (gc/or-conds (map #(qm/string-conditions :consortiums [%])
                                     (common-config/opensearch-consortiums)))
         query (qm/query {:concept-type :collection
                          :condition condition
-                         :page-size :unlimited})
+                         :page-size :unlimited
+                         :remove-source true})
         results (common-esi/execute-query context query)]
     (into {}
           (for [coll-id (map :_id (get-in results [:hits :hits]))]
@@ -141,11 +143,11 @@
                          (get-opensearch-collections context nil)))))))
 
 (defjob RefreshHasGranulesOrCwicMapJob
-  [ctx system]
+  [_ctx system]
   (refresh-has-granules-or-cwic-map {:system system}))
 
 (defjob RefreshHasGranulesOrOpenSearchMapJob
-  [ctx system]
+  [_ctx system]
   (refresh-has-granules-or-opensearch-map {:system system}))
 
 (defn refresh-has-granules-or-cwic-map-job