Allow for larger forms to create csv exports

Tangerine-Community · Oct 9, 2024 · f84315d · f84315d
1 parent 019f8ac
commit f84315d
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 2 deletions.
diff --git a/server/src/scripts/generate-csv-data-set/generate-csv-data-set.js b/server/src/scripts/generate-csv-data-set/generate-csv-data-set.js
@@ -43,7 +43,8 @@ function generateCsv(dbName, formId, outputPath, year = '*', month = '*', csvTem
     }
     cmd = `${cmd} ${csvTemplate ? `"${csvTemplate.headers.join(',')}"` : ''}`
     log.debug("generate-csv: " + cmd)
-    exec(cmd).then(status => {
+    const maxBuffer = 1024 * 1024 * 100;
+    exec(cmd, { maxBuffer }).then(status => {
       resolve(status)
     }).catch(error => {
       log.error("Error when exec-ing generate-csv: " + error)

diff --git a/server/src/scripts/generate-csv/bin.js b/server/src/scripts/generate-csv/bin.js
@@ -78,7 +78,9 @@ async function go(state) {
     //  Run batches.
     while (state.complete === false) {
       log.debug(`Run batch at skip of ${state.skip} at statePath: ${state.statePath}`)
-      const response = await exec(`./batch.js '${state.statePath}'`)
+      const maxBuffer = 1024 * 1024 * 100; // 100MB
+      const response = await exec(`./batch.js '${state.statePath}'`, { maxBuffer });
+
       if (response.stderr) console.error(response.stderr)
       if (process.env.NODE_ENV === 'development') console.log(response)
       state = JSON.parse(await readFile(state.statePath))