Feat/135772 135785 bulk export page to local fs in pdf

apps/app/.env.development

@@ -30,3 +30,4 @@ QUESTIONNAIRE_SERVER_ORIGIN="http://host.docker.internal:3003"
 # AUDIT_LOG_ACTION_GROUP_SIZE=SMALL
 # AUDIT_LOG_ADDITIONAL_ACTIONS=
 # AUDIT_LOG_EXCLUDE_ACTIONS=
+PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium

apps/app/package.json

@@ -161,6 +161,8 @@
     "passport-ldapauth": "^3.0.1",
     "passport-local": "^1.0.0",
     "passport-saml": "^3.2.0",
+    "puppeteer": "^22.0.0",
+    "puppeteer-cluster": "^0.24.0",
     "qs": "^6.11.1",
     "rate-limiter-flexible": "^2.3.7",
     "react": "^18.2.0",
@@ -186,10 +188,12 @@
     "rehype-sanitize": "^5.0.1",
     "rehype-slug": "^5.0.1",
     "rehype-toc": "^3.0.2",
+    "remark": "^13.0.0",
     "remark-breaks": "^3.0.2",
     "remark-emoji": "^3.0.2",
     "remark-frontmatter": "^4.0.1",
     "remark-gfm": "^3.0.1",
+    "remark-html": "^11.0.0",
     "remark-math": "^5.1.1",
     "remark-toc": "^8.0.1",
     "remark-wiki-link": "^1.0.4",

apps/app/src/features/page-bulk-export/client/components/PageBulkExportSelectModal.tsx

@@ -38,7 +38,7 @@ const PageBulkExportSelectModal = (): JSX.Element => {
               </small>
             </div>
             <div className="d-flex justify-content-center mt-2">
-              <button className="btn btn-primary" type="button" onClick={() => startBulkExport(PageBulkExportFormat.markdown)}>
+              <button className="btn btn-primary" type="button" onClick={() => startBulkExport(PageBulkExportFormat.md)}>
                 {t('page_export.markdown')}
               </button>
               <button className="btn btn-primary ms-2" type="button" onClick={() => startBulkExport(PageBulkExportFormat.pdf)}>PDF</button>

apps/app/src/features/page-bulk-export/interfaces/page-bulk-export.ts

@@ -4,7 +4,7 @@ import type {
 } from '@growi/core';
 
 export const PageBulkExportFormat = {
-  markdown: 'markdown',
+  md: 'md',
   pdf: 'pdf',
 } as const;
 

apps/app/src/features/page-bulk-export/server/routes/apiv3/page-bulk-export.ts

@@ -40,7 +40,7 @@ module.exports = (crowi: Crowi): Router => {
     };
 
     try {
-      await pageBulkExportService?.createAndStartPageBulkExportJob(path, req.user, activityParameters);
+      await pageBulkExportService?.createAndStartPageBulkExportJob(path, format, req.user, activityParameters);
       return res.apiv3({}, 204);
     }
     catch (err) {

apps/app/src/features/page-bulk-export/server/service/page-bulk-export.ts

@@ -4,14 +4,16 @@ import type { Readable } from 'stream';
 import { Writable, pipeline } from 'stream';
 import { pipeline as pipelinePromise } from 'stream/promises';
 
-
 import type { HasObjectId } from '@growi/core';
 import { type IPage, isPopulated, SubscriptionStatusType } from '@growi/core';
 import { getParentPath, normalizePath } from '@growi/core/dist/utils/path-utils';
 import type { Archiver } from 'archiver';
 import archiver from 'archiver';
 import gc from 'expose-gc/function';
 import mongoose from 'mongoose';
+import { Cluster } from 'puppeteer-cluster';
+import remark from 'remark';
+import html from 'remark-html';
 
 import type { SupportedActionType } from '~/interfaces/activity';
 import { SupportedAction, SupportedTargetModel } from '~/interfaces/activity';
@@ -21,6 +23,7 @@ import { Attachment } from '~/server/models';
 import type { ActivityDocument } from '~/server/models/activity';
 import type { PageModel, PageDocument } from '~/server/models/page';
 import Subscription from '~/server/models/subscription';
+import { configManager } from '~/server/service/config-manager';
 import type { FileUploader } from '~/server/service/file-uploader';
 import type { IMultipartUploader } from '~/server/service/file-uploader/multipart-uploader';
 import { preNotifyService } from '~/server/service/pre-notify';
@@ -56,12 +59,16 @@ class PageBulkExportService {
   // TODO: If necessary, change to a proper path in https://redmine.weseek.co.jp/issues/149512
   tmpOutputRootDir = '/tmp';
 
+  puppeteerCluster: Cluster | undefined;
+
   constructor(crowi) {
     this.crowi = crowi;
     this.activityEvent = crowi.event('activity');
   }
 
-  async createAndStartPageBulkExportJob(basePagePath: string, currentUser, activityParameters: ActivityParameters): Promise<void> {
+  async createAndStartPageBulkExportJob(
+      basePagePath: string, format: PageBulkExportFormat, currentUser, activityParameters: ActivityParameters,
+  ): Promise<void> {
     const Page = mongoose.model<IPage, PageModel>('Page');
     const basePage = await Page.findByPathAndViewer(basePagePath, currentUser, null, true);
 
@@ -72,24 +79,28 @@ class PageBulkExportService {
     const pageBulkExportJob: PageBulkExportJobDocument & HasObjectId = await PageBulkExportJob.create({
       user: currentUser,
       page: basePage,
-      format: PageBulkExportFormat.markdown,
+      format,
     });
 
     await Subscription.upsertSubscription(currentUser, SupportedTargetModel.MODEL_PAGE_BULK_EXPORT_JOB, pageBulkExportJob, SubscriptionStatusType.SUBSCRIBE);
 
-    this.bulkExportWithBasePagePath(basePagePath, currentUser, activityParameters, pageBulkExportJob);
+    this.bulkExportWithBasePagePath(basePagePath, format, currentUser, activityParameters, pageBulkExportJob);
   }
 
-  async bulkExportWithBasePagePath(
-      basePagePath: string, currentUser, activityParameters: ActivityParameters, pageBulkExportJob: PageBulkExportJobDocument & HasObjectId,
+  private async bulkExportWithBasePagePath(
+      basePagePath: string,
+      format: PageBulkExportFormat,
+      currentUser,
+      activityParameters: ActivityParameters,
+      pageBulkExportJob: PageBulkExportJobDocument & HasObjectId,
   ): Promise<void> {
     const timeStamp = (new Date()).getTime();
     const exportName = `page-bulk-export-${timeStamp}`;
 
     // export pages to fs temporarily
     const tmpOutputDir = `${this.tmpOutputRootDir}/${exportName}`;
     try {
-      await this.exportPagesToFS(basePagePath, tmpOutputDir, currentUser);
+      await this.exportPagesToFS(basePagePath, tmpOutputDir, currentUser, format);
     }
     catch (err) {
       await this.handleExportError(err, activityParameters, pageBulkExportJob, tmpOutputDir);
@@ -152,9 +163,9 @@ class PageBulkExportService {
     }
   }
 
-  private async exportPagesToFS(basePagePath: string, outputDir: string, currentUser): Promise<void> {
+  private async exportPagesToFS(basePagePath: string, outputDir: string, currentUser, format: PageBulkExportFormat): Promise<void> {
     const pagesReadable = await this.getPageReadable(basePagePath, currentUser);
-    const pagesWritable = this.getPageWritable(outputDir);
+    const pagesWritable = await this.getPageWritable(outputDir, format);
 
     return pipelinePromise(pagesReadable, pagesWritable);
   }
@@ -180,21 +191,22 @@ class PageBulkExportService {
   /**
    * Get a Writable that writes the page body temporarily to fs
    */
-  private getPageWritable(outputDir: string): Writable {
+  private async getPageWritable(outputDir: string, format: PageBulkExportFormat): Promise<Writable> {
     return new Writable({
       objectMode: true,
       write: async(page: PageDocument, encoding, callback) => {
         try {
           const revision = page.revision;
 
           if (revision != null && isPopulated(revision)) {
-            const markdownBody = revision.body;
-            const pathNormalized = `${normalizePath(page.path)}.md`;
+            const pageBody = format === PageBulkExportFormat.pdf ? (await this.convertMdToPdf(revision.body)) : revision.body;
+            gc();
+            const pathNormalized = `${normalizePath(page.path)}.${format}`;
             const fileOutputPath = path.join(outputDir, pathNormalized);
             const fileOutputParentPath = getParentPath(fileOutputPath);
 
             await fs.promises.mkdir(fileOutputParentPath, { recursive: true });
-            await fs.promises.writeFile(fileOutputPath, markdownBody);
+            await fs.promises.writeFile(fileOutputPath, pageBody);
           }
         }
         catch (err) {
@@ -268,6 +280,73 @@ class PageBulkExportService {
     });
   }
 
+  /**
+   * Initialize puppeteer cluster for converting markdown to pdf
+   */
+  async initPuppeteerCluster(): Promise<void> {
+    this.puppeteerCluster = await Cluster.launch({
+      concurrency: Cluster.CONCURRENCY_PAGE,
+      maxConcurrency: configManager.getConfig('crowi', 'app:bulkExportPuppeteerClusterMaxConcurrency'),
+      workerCreationDelay: 10000,
+      monitor: true,
+    });
+
+    await this.puppeteerCluster.task(async({ page, data: htmlString }) => {
+      await page.setContent(htmlString, { waitUntil: 'domcontentloaded' });
+      await page.emulateMediaType('screen');
+      const pdfResult = await page.pdf({
+        margin: {
+          top: '100px', right: '50px', bottom: '100px', left: '50px',
+        },
+        printBackground: true,
+        format: 'A4',
+      });
+      return pdfResult;
+    });
+
+    // close cluster on app termination
+    const handleClose = async() => {
+      logger.info('Closing puppeteer cluster...');
+      await this.puppeteerCluster?.idle();
+      await this.puppeteerCluster?.close();
+      process.exit();
+    };
+    process.on('SIGINT', handleClose);
+    process.on('SIGTERM', handleClose);
+  }
+
+  /**
+   * Convert markdown string to html, then to PDF
+   * When PDF conversion is unstable and error occurs, it will retry up to the specified limit
+   */
+  private async convertMdToPdf(md: string): Promise<Buffer> {
+    const executeConvert = async(htmlString: string, retries: number) => {
+      if (this.puppeteerCluster == null) {
+        throw new Error('Puppeteer cluster is not initialized');
+      }
+
+      try {
+        return await this.puppeteerCluster.execute(htmlString);
+      }
+      catch (err) {
+        if (retries > 0) {
+          logger.error('Failed to convert markdown to pdf. Retrying...', err);
+          return executeConvert(htmlString, retries - 1);
+        }
+        throw err;
+      }
+    };
+
+    const htmlString = (await remark()
+      .use(html)
+      .process(md))
+      .toString();
+
+    const result = await executeConvert(htmlString, configManager.getConfig('crowi', 'app:bulkExportPuppeteerRetryLimit'));
+
+    return result;
+  }
+
   private async notifyExportResult(
       activityParameters: ActivityParameters, pageBulkExportJob: PageBulkExportJobDocument, action: SupportedActionType,
   ) {
@@ -290,6 +369,7 @@ class PageBulkExportService {
 
 // eslint-disable-next-line import/no-mutable-exports
 export let pageBulkExportService: PageBulkExportService | undefined; // singleton instance
-export default function instanciate(crowi): void {
+export default async function instanciate(crowi): Promise<void> {
   pageBulkExportService = new PageBulkExportService(crowi);
+  await pageBulkExportService.initPuppeteerCluster();
 }

apps/app/src/server/crowi/index.js

@@ -694,7 +694,7 @@ Crowi.prototype.setupExport = async function() {
 };
 
 Crowi.prototype.setupPageBulkExportService = async function() {
-  instanciatePageBulkExportService(this);
+  await instanciatePageBulkExportService(this);
 };
 
 Crowi.prototype.setupImport = async function() {

apps/app/src/server/service/config-loader.ts

@@ -735,6 +735,18 @@ const ENV_VAR_NAME_TO_CONFIG_INFO = {
     type: ValueType.NUMBER,
     default: 172800, // 2 days
   },
+  BULK_EXPORT_PUPPETEER_CLUSTER_MAX_CONCURRENCY: {
+    ns: 'crowi',
+    key: 'app:bulkExportPuppeteerClusterMaxConcurrency',
+    type: ValueType.NUMBER,
+    default: 10,
+  },
+  BULK_EXPORT_PUPPETEER_RETRY_LIMIT: {
+    ns: 'crowi',
+    key: 'app:bulkExportPuppeteerRetryLimit',
+    type: ValueType.NUMBER,
+    default: 5,
+  },
 };
 
 

apps/app/src/server/service/export.ts

@@ -7,13 +7,13 @@ import archiver from 'archiver';
 import { toArrayIfNot } from '~/utils/array-utils';
 import loggerFactory from '~/utils/logger';
 
-import CollectionProgress from '../models/vo/collection-progress';
+import type CollectionProgress from '../models/vo/collection-progress';
 import CollectionProgressingStatus from '../models/vo/collection-progressing-status';
 
-import AppService from './app';
+import type AppService from './app';
 import ConfigLoader from './config-loader';
-import GrowiBridgeService from './growi-bridge';
-import { ZipFileStat } from './interfaces/export';
+import type GrowiBridgeService from './growi-bridge';
+import type { ZipFileStat } from './interfaces/export';
 
 
 const logger = loggerFactory('growi:services:ExportService');

apps/app/src/server/service/file-uploader/gcs/index.ts

@@ -14,7 +14,6 @@ import {
 } from '../file-uploader';
 import { ContentHeaders } from '../utils';
 
-import type { IGcsMultipartUploader } from './multipart-uploader';
 import { GcsMultipartUploader } from './multipart-uploader';
 
 const logger = loggerFactory('growi:service:fileUploaderGcs');

apps/app/src/server/service/file-uploader/gcs/multipart-uploader.ts

@@ -1,9 +1,11 @@
 import type { Bucket, File } from '@google-cloud/storage';
 // eslint-disable-next-line no-restricted-imports
 import axios from 'axios';
+import urljoin from 'url-join';
 
 import loggerFactory from '~/utils/logger';
 
+import { configManager } from '../../config-manager';
 import { MultipartUploader, UploadStatus, type IMultipartUploader } from '../multipart-uploader';
 
 const logger = loggerFactory('growi:services:fileUploaderGcs:multipartUploader');
@@ -26,7 +28,8 @@ export class GcsMultipartUploader extends MultipartUploader implements IGcsMulti
   constructor(bucket: Bucket, uploadKey: string, maxPartSize: number) {
     super(uploadKey, maxPartSize);
 
-    this.file = bucket.file(this.uploadKey);
+    const namespace = configManager.getConfig('crowi', 'gcs:uploadNamespace');
+    this.file = bucket.file(urljoin(namespace || '', uploadKey));
   }
 
   async initUpload(): Promise<void> {

apps/app/src/server/service/interfaces/export.ts

@@ -1,4 +1,4 @@
-import { Stats } from 'fs';
+import type { Stats } from 'fs';
 
 export type ZipFileStat = {
   meta: object;