feat: Implement BigqueryExporter.exportCustomReports

__tests__/exporter/bigquery_exporter.test.ts

@@ -1,16 +1,22 @@
+import path from "path"
 import { BigqueryExporter } from '../../src/exporter/bigquery_exporter'
 import { BigqueryExporterConfig } from '../../src/config/config'
+import { CustomReportCollection } from '../../src/custom_report_collection'
 
 const bigqueryMock = {
   dataset: jest.fn().mockReturnThis(),
   table: jest.fn().mockReturnThis(),
   load: jest.fn(async () => [{}]) // return success 'results' stub
 }
+const configDir = path.join(__dirname, '..', '..')
+const fixtureSchemaPath = {
+  custom: './__tests__/fixture/custom_table.json'
+}
 
 describe('BigqueryExporter', () => {
   const workflowTable = 'workflow'
   const testReportTable = 'test_report'
-  const config: BigqueryExporterConfig = {
+  const baseConfig: BigqueryExporterConfig = {
     project: 'project',
     dataset: 'dataset',
     reports: [
@@ -21,35 +27,35 @@ describe('BigqueryExporter', () => {
   describe('new', () => {
     it('should not throw when any params are not undefined', async () => {
       expect(() => {
-        new BigqueryExporter(config)
+        new BigqueryExporter(baseConfig, configDir)
       })
     })
 
     it('should throw when project is undefined', async () => {
-      const _config = { ...config, project: undefined }
+      const config = { ...baseConfig, project: undefined }
       expect(() => {
-        new BigqueryExporter(_config)
+        new BigqueryExporter(config, configDir)
       }).toThrow()
     })
 
     it('should throw when dataset is undefined', async () => {
-      const _config = { ...config, dataset: undefined }
+      const config = { ...baseConfig, dataset: undefined }
       expect(() => {
-        new BigqueryExporter(_config)
+        new BigqueryExporter(config, configDir)
       }).toThrow()
     })
 
     it('should throw when workflow table is undefined', async () => {
-      const _config = { ...config, reports: [{ name: 'workflow', table: 'workflow'}] }
+      const config = { ...baseConfig, reports: [{ name: 'workflow', table: 'workflow'}] }
       expect(() => {
-        new BigqueryExporter(_config as any)
+        new BigqueryExporter(config as any, configDir)
       }).toThrow()
     })
 
     it('should throw when test_report table is undefined', async () => {
-      const _config = { ...config, reports: [{ name: 'test_report', table: 'test_report'}] }
+      const config = { ...baseConfig, reports: [{ name: 'test_report', table: 'test_report'}] }
       expect(() => {
-        new BigqueryExporter(_config as any)
+        new BigqueryExporter(config as any, configDir)
       }).toThrow()
     })
   })
@@ -58,7 +64,7 @@ describe('BigqueryExporter', () => {
     const report = [{}]
     let exporter: BigqueryExporter
     beforeEach(() => {
-      exporter = new BigqueryExporter(config)
+      exporter = new BigqueryExporter(baseConfig, configDir)
       exporter.bigquery = bigqueryMock as any
     })
 
@@ -76,4 +82,37 @@ describe('BigqueryExporter', () => {
       expect(bigqueryMock.table.mock.calls[0][0]).toBe(testReportTable)
     })
   })
+
+  describe('exportCustomReports', () => {
+    const config = {
+      ...baseConfig,
+      custom_reports: [
+        { name: 'custom', table: 'custom_table', schema: fixtureSchemaPath.custom }
+      ]
+    }
+    let exporter: BigqueryExporter
+    let reportCollection: CustomReportCollection
+    beforeEach(() => {
+      exporter = new BigqueryExporter(config, configDir)
+      exporter.bigquery = bigqueryMock as any
+      reportCollection = new CustomReportCollection()
+      reportCollection.set('custom', [])
+    })
+
+    it('load to `custom_reports[].table` table', async () => {
+      await exporter.exportCustomReports(reportCollection)
+
+      expect(bigqueryMock.load.mock.calls.length).toBe(1)
+      expect(bigqueryMock.table.mock.calls[0][0]).toBe('custom_table')
+    })
+
+    it('load multi report to `custom_reports[].table` table', async () => {
+    })
+
+    it('should error when define repo.custom_reports but does not exists correspond config in `bigquery.custom_reports`', async () => {
+    })
+
+    it('should error when does found custom_reports table schema json', async () => {
+    })
+  })
 })

__tests__/fixture/custom_table.json

@@ -0,0 +1,12 @@
+[
+  {
+    "mode": "NULLABLE", 
+    "name": "workflowId", 
+    "type": "STRING"
+  }, 
+  {
+    "mode": "NULLABLE", 
+    "name": "workflowRunId", 
+    "type": "STRING"
+  }
+]

src/config/config.ts

@@ -35,6 +35,11 @@ export type BigqueryExporterConfig = {
     name: 'workflow' | 'test_report'
     table: string
   }[]
+  custom_reports?: {
+    name: string
+    table: string
+    schema: string
+  }[]
   maxBadRecords?: number
 }
 

src/exporter/bigquery_exporter.ts

@@ -20,8 +20,9 @@ export class BigqueryExporter implements Exporter {
     workflow: string
     testReport: string
   }
+  customReportTableInfo: Map<string, { table: string, schemaPath: string }>
   maxBadRecords: number
-  constructor( config: BigqueryExporterConfig ) {
+  constructor( config: BigqueryExporterConfig, configDir: string ) {
     if (!config.project || !config.dataset) {
       throw "Must need 'project', 'dataset' parameter in exporter.bigquery config."
     }
@@ -38,6 +39,18 @@ export class BigqueryExporter implements Exporter {
       testReport: testReportTable,
     }
 
+    this.customReportTableInfo = new Map(
+      config.custom_reports?.map((customReport) => {
+        const schemaPath = (path.isAbsolute(customReport.schema))
+          ? customReport.schema
+          : path.resolve(configDir, customReport.schema)
+        return [
+          customReport.name,
+          { table: customReport.table, schemaPath }
+        ]
+      })
+    )
+
     this.maxBadRecords = config.maxBadRecords ?? 0
   }
 
@@ -87,5 +100,11 @@ export class BigqueryExporter implements Exporter {
   }
 
   async exportCustomReports (customReportCollection: CustomReportCollection) {
+    for (const [reportName, customReports] of customReportCollection.customReports) {
+      const table = this.customReportTableInfo.get(reportName)
+      if (table === undefined) throw new Error(`name: '${reportName}' does not exists in exporter.bigquery.custom_reports config!!`)
+
+      await this.export(customReports, table.table, table.schemaPath)
+    }
   }
 }

src/exporter/exporter.ts

@@ -26,7 +26,7 @@ export class CompositExporter implements Exporter {
           return new LocalExporter(service, configDir, _config)
         case 'bigquery':
           _config = config['bigquery'] ?? {}
-          return new BigqueryExporter(_config)
+          return new BigqueryExporter(_config, configDir)
         default:
           return undefined
       }