From 69c88f4c847c901f54de0511f82f69e8661f23ca Mon Sep 17 00:00:00 2001
From: Paul Pilone <paul@element84.com>
Date: Thu, 22 Aug 2024 17:43:28 -0400
Subject: [PATCH] CUMULUS-3841: Increase fetchRules page size to improve
 performance. (#3779)

* Increase limit of records per page to be 100 from fetchRules function

* Pass pageSize as a parameter from fetchRules function

* minor code cleaning

* Adds change log entry

* Adds unit tests for fetchRules page size argument

* Improves fetchRules pageSize unit test

* Fixes linter errors

---------

Co-authored-by: Hailiang Zhang <hailiang.zhang@nasa.gov>
---
 CHANGELOG.md                                  |  3 +++
 packages/api/lib/rulesHelpers.js              |  6 +++--
 .../api/tests/lib/rules/test-rulesHelpers.js  | 23 ++++++++++++++++---
 3 files changed, 27 insertions(+), 5 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 213bbf452ec..a547fa67f91 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -68,6 +68,9 @@ degraded execution table operations.
 - **CUMULUS-3449**
   - Updated `@cumulus/db` package and configure knex hook postProcessResponse to convert the return string
     from columns ending with "cumulus_id" to number.
+- **CUMULUS-3841**
+  - Increased `fetchRules` page size to default to 100 instead of 10. This improves overall query time when
+    fetching all rules such as in `sqsMessageConsumer`.
 
 ### Fixed
 
diff --git a/packages/api/lib/rulesHelpers.js b/packages/api/lib/rulesHelpers.js
index dfe8ea41a36..19a2ebb692d 100644
--- a/packages/api/lib/rulesHelpers.js
+++ b/packages/api/lib/rulesHelpers.js
@@ -56,12 +56,13 @@ const log = new Logger({ sender: '@cumulus/rulesHelpers' });
  *
  * @param {Object} params - function params
  * @param {number} [params.pageNumber] - current page of API results
+ * @param {number} [params.pageSize] - database query page size used when calling listRules
  * @param {Array<Object>} [params.rules] - partial rules Array
  * @param {Object} [params.queryParams] - API query params, empty query returns all rules
  * @returns {Promise<Array<Object>>} all matching rules
  */
-async function fetchRules({ pageNumber = 1, rules = [], queryParams = {} }) {
-  const query = { ...queryParams, page: pageNumber };
+async function fetchRules({ pageNumber = 1, pageSize = 100, rules = [], queryParams = {} }) {
+  const query = { ...queryParams, page: pageNumber, limit: pageSize };
   const apiResponse = await listRules({
     prefix: process.env.stackName,
     query,
@@ -70,6 +71,7 @@ async function fetchRules({ pageNumber = 1, rules = [], queryParams = {} }) {
   if (responseBody.results.length > 0) {
     return fetchRules({
       pageNumber: (pageNumber + 1),
+      pageSize,
       rules: rules.concat(responseBody.results),
       queryParams,
     });
diff --git a/packages/api/tests/lib/rules/test-rulesHelpers.js b/packages/api/tests/lib/rules/test-rulesHelpers.js
index 6aecb38dc47..ace6fd81977 100644
--- a/packages/api/tests/lib/rules/test-rulesHelpers.js
+++ b/packages/api/tests/lib/rules/test-rulesHelpers.js
@@ -200,20 +200,37 @@ test.serial('fetchRules invokes API to fetch rules', async (t) => {
   t.true(listRulesStub.calledOnce);
 });
 
+test.serial('fetchRules allows for configurable page size', async (t) => {
+  listRulesStub.onFirstCall().callsFake(({ query }) => {
+    t.is(query.limit, 100);
+    return { body: JSON.stringify({ results: [] }) };
+  });
+
+  listRulesStub.onSecondCall().callsFake(({ query }) => {
+    t.is(query.limit, 50);
+    return { body: JSON.stringify({ results: [] }) };
+  });
+
+  await rulesHelpers.fetchRules({});
+  await rulesHelpers.fetchRules({ pageSize: 50 });
+
+  t.true(listRulesStub.calledTwice);
+});
+
 test.serial('fetchRules pages through results until reaching an empty list', async (t) => {
   const rule1 = { name: 'rule-1' };
   const rule2 = { name: 'rule-2' };
   const firstCallArgs = {
     prefix: process.env.stackName,
-    query: { page: 1 },
+    query: { limit: 100, page: 1 },
   };
   const secondCallArgs = {
     prefix: process.env.stackName,
-    query: { page: 2 },
+    query: { limit: 100, page: 2 },
   };
   const thirdCallArgs = {
     prefix: process.env.stackName,
-    query: { page: 3 },
+    query: { limit: 100, page: 3 },
   };
   listRulesStub.onFirstCall().callsFake((params) => {
     t.deepEqual(params, firstCallArgs);