CUMULUS-3841: Increase fetchRules page size to improve performance. (#…

…3779) * Increase limit of records per page to be 100 from fetchRules function * Pass pageSize as a parameter from fetchRules function * minor code cleaning * Adds change log entry * Adds unit tests for fetchRules page size argument * Improves fetchRules pageSize unit test * Fixes linter errors --------- Co-authored-by: Hailiang Zhang <[email protected]>
nasa · Aug 22, 2024 · 69c88f4 · 69c88f4
1 parent 807bb20
commit 69c88f4
Show file tree

Hide file tree

Showing 3 changed files with 27 additions and 5 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -68,6 +68,9 @@ degraded execution table operations.
 - **CUMULUS-3449**
   - Updated `@cumulus/db` package and configure knex hook postProcessResponse to convert the return string
     from columns ending with "cumulus_id" to number.
+- **CUMULUS-3841**
+  - Increased `fetchRules` page size to default to 100 instead of 10. This improves overall query time when
+    fetching all rules such as in `sqsMessageConsumer`.
 
 ### Fixed
 

diff --git a/packages/api/lib/rulesHelpers.js b/packages/api/lib/rulesHelpers.js
@@ -56,12 +56,13 @@ const log = new Logger({ sender: '@cumulus/rulesHelpers' });
  *
  * @param {Object} params - function params
  * @param {number} [params.pageNumber] - current page of API results
+ * @param {number} [params.pageSize] - database query page size used when calling listRules
  * @param {Array<Object>} [params.rules] - partial rules Array
  * @param {Object} [params.queryParams] - API query params, empty query returns all rules
  * @returns {Promise<Array<Object>>} all matching rules
  */
-async function fetchRules({ pageNumber = 1, rules = [], queryParams = {} }) {
-  const query = { ...queryParams, page: pageNumber };
+async function fetchRules({ pageNumber = 1, pageSize = 100, rules = [], queryParams = {} }) {
+  const query = { ...queryParams, page: pageNumber, limit: pageSize };
   const apiResponse = await listRules({
     prefix: process.env.stackName,
     query,
@@ -70,6 +71,7 @@ async function fetchRules({ pageNumber = 1, rules = [], queryParams = {} }) {
   if (responseBody.results.length > 0) {
     return fetchRules({
       pageNumber: (pageNumber + 1),
+      pageSize,
       rules: rules.concat(responseBody.results),
       queryParams,
     });

diff --git a/packages/api/tests/lib/rules/test-rulesHelpers.js b/packages/api/tests/lib/rules/test-rulesHelpers.js
@@ -200,20 +200,37 @@ test.serial('fetchRules invokes API to fetch rules', async (t) => {
   t.true(listRulesStub.calledOnce);
 });
 
+test.serial('fetchRules allows for configurable page size', async (t) => {
+  listRulesStub.onFirstCall().callsFake(({ query }) => {
+    t.is(query.limit, 100);
+    return { body: JSON.stringify({ results: [] }) };
+  });
+
+  listRulesStub.onSecondCall().callsFake(({ query }) => {
+    t.is(query.limit, 50);
+    return { body: JSON.stringify({ results: [] }) };
+  });
+
+  await rulesHelpers.fetchRules({});
+  await rulesHelpers.fetchRules({ pageSize: 50 });
+
+  t.true(listRulesStub.calledTwice);
+});
+
 test.serial('fetchRules pages through results until reaching an empty list', async (t) => {
   const rule1 = { name: 'rule-1' };
   const rule2 = { name: 'rule-2' };
   const firstCallArgs = {
     prefix: process.env.stackName,
-    query: { page: 1 },
+    query: { limit: 100, page: 1 },
   };
   const secondCallArgs = {
     prefix: process.env.stackName,
-    query: { page: 2 },
+    query: { limit: 100, page: 2 },
   };
   const thirdCallArgs = {
     prefix: process.env.stackName,
-    query: { page: 3 },
+    query: { limit: 100, page: 3 },
   };
   listRulesStub.onFirstCall().callsFake((params) => {
     t.deepEqual(params, firstCallArgs);