feat: Export CustomReport (CircleCI)

__tests__/custom_report_collection.test.ts

@@ -1,4 +1,5 @@
-import { CustomReportCollection } from '../src/custom_report_collection'
+import { CustomReportCollection, aggregateCustomReportArtifacts } from '../src/custom_report_collection'
+import { CustomReportArtifact } from '../src/client/client'
 
 describe('CustomReportCollection', () => {
   let customReportCollection: CustomReportCollection
@@ -51,3 +52,23 @@ describe('CustomReportCollection', () => {
     })
   })
 })
+
+describe('aggregateCustomReportArtifacts', () => {
+  it('aggregate CustomReportArtifact[]', () => {
+    const artifact1 = { path: 'custom1.json', data: new ArrayBuffer(1) }
+    const artifact2 = { path: 'custom2.json', data: new ArrayBuffer(1) }
+    const customReportArtifact1: CustomReportArtifact =  new Map([
+      [ 'custom1', [artifact1] ],
+      [ 'custom2', [artifact2] ],
+    ])
+    const customReportArtifact2: CustomReportArtifact =  new Map([
+      [ 'custom2', [artifact2] ]
+    ])
+
+    const actual = aggregateCustomReportArtifacts([customReportArtifact1, customReportArtifact2])
+    expect(actual).toStrictEqual(new Map([
+      [ 'custom1', [artifact1] ],
+      [ 'custom2', [artifact2, artifact2] ],
+    ]))
+  })
+})

src/client/circleci_client.ts

@@ -1,6 +1,8 @@
+import minimatch from 'minimatch'
 import axios, { AxiosInstance } from 'axios'
 import { groupBy, max, minBy } from 'lodash'
-import { axiosRequestLogger } from './client'
+import { axiosRequestLogger, Artifact, CustomReportArtifact } from './client'
+import { CustomReportConfig } from '../config/config'
 
 const DEBUG_PER_PAGE = 10
 
@@ -97,6 +99,13 @@ export type TestResponse = {
   run_id: number // Add for join workflow
 }
 
+type ArtifactsResponse = {
+  path: string,
+  prettry_path: string,
+  node_index: number,
+  url: string,
+}[]
+
 
 export class CircleciClient {
   private axios: AxiosInstance
@@ -116,8 +125,8 @@ export class CircleciClient {
     }
   }
 
+  // https://circleci.com/api/v1.1/project/:vcs-type/:username/:project?circle-token=:token&limit=20&offset=5&filter=completed
   async fetchWorkflowRuns(owner: string, repo: string, vcsType: string, lastRunId?: number) {
-    // https://circleci.com/api/v1.1/project/:vcs-type/:username/:project?circle-token=:token&limit=20&offset=5&filter=completed
     const res = await this.axios.get( `project/${vcsType}/${owner}/${repo}`, {
       params: {
         // API default is 30 and max is 100
@@ -185,6 +194,7 @@ export class CircleciClient {
     return runs
   }
 
+  // ex: https://circleci.com/api/v1.1/project/github/Kesin11/CIAnalyzer/1021
   async fetchJobs(owner: string, repo: string, vcsType: string, runId: number) {
     const res = await this.axios.get( `project/${vcsType}/${owner}/${repo}/${runId}`, {})
     const build = res.data
@@ -207,11 +217,63 @@ export class CircleciClient {
     }
   }
 
+  // ex: https://circleci.com/api/v1.1/project/github/Kesin11/CIAnalyzer/1021/tests
   async fetchTests(owner: string, repo: string, vcsType: string, runId: number) {
     const res = await this.axios.get( `project/${vcsType}/${owner}/${repo}/${runId}/tests`)
     return {
       ...res.data,
       run_id: runId
     } as TestResponse
   }
+
+  // ex: https://circleci.com/api/v1.1/project/github/Kesin11/CIAnalyzer/1021/artifacts
+  async fetchArtifactsList(owner: string, repo: string, vcsType: string, runId: number): Promise<ArtifactsResponse> {
+    const res = await this.axios.get(
+      `project/${vcsType}/${owner}/${repo}/${runId}/artifacts`
+    )
+    return res.data
+  }
+
+  async fetchArtifacts(artifactsResponse: ArtifactsResponse): Promise<Artifact[]> {
+    const pathResponses = artifactsResponse.map((artifact) => {
+      const response = this.axios.get(
+        artifact.url,
+        { responseType: 'arraybuffer'}
+      )
+      return { path: artifact.path, response }
+    })
+
+    const artifacts = []
+    for (const { path, response } of pathResponses) {
+      artifacts.push({
+        path,
+        data: (await response).data as ArrayBuffer
+      })
+    }
+    return artifacts
+  }
+
+  async fetchCustomReports(owner: string, repo: string, vcsType: string, runId: number, customReportsConfigs: CustomReportConfig[]): Promise<CustomReportArtifact> {
+    // Skip if custom report config are not provided
+    if (customReportsConfigs?.length < 1) return new Map()
+
+    const artifactsResponse = await this.fetchArtifactsList(owner, repo, vcsType, runId)
+
+    // Fetch artifacts in parallel
+    const customReports: CustomReportArtifact = new Map<string, Artifact[]>()
+    const nameArtifacts = customReportsConfigs.map((customReportConfig) => {
+      const reportArtifacts = artifactsResponse.filter((artifact) => {
+        return customReportConfig.paths.some((glob) => minimatch(artifact.path, glob))
+      })
+      return {
+        name: customReportConfig.name,
+        artifacts: this.fetchArtifacts(reportArtifacts)
+      }
+    })
+    for (const { name, artifacts } of nameArtifacts) {
+      customReports.set(name, await artifacts)
+    }
+
+    return customReports
+  }
 }

src/custom_report_collection.ts

@@ -8,6 +8,21 @@ export type CustomReport = {
   [key: string]: unknown
 }
 
+export const aggregateCustomReportArtifacts = (customReportArtifactsList: CustomReportArtifact[]) => {
+  const aggregated: CustomReportArtifact = new Map()
+  for (const customReportArtifacts of customReportArtifactsList) {
+    for (const [name, artifacts] of customReportArtifacts) {
+      const aggregatedArtifacts = aggregated.get(name)
+      if (aggregatedArtifacts) {
+        aggregated.set(name, aggregatedArtifacts.concat(artifacts))
+      } else {
+        aggregated.set(name, artifacts)
+      }
+    }
+  }
+  return aggregated
+}
+
 export const createCustomReportCollection = async (workflowReport: WorkflowReport, customReportArtifacts: CustomReportArtifact): Promise<CustomReportCollection> => {
   const reportCollection = new CustomReportCollection()
   for (const [reportName, artifacts] of customReportArtifacts) {

src/runner/circleci_runner.ts

@@ -8,6 +8,7 @@ import { WorkflowReport, TestReport } from "../analyzer/analyzer"
 import { CompositExporter } from "../exporter/exporter"
 import { LastRunStore } from "../last_run_store"
 import { GithubRepositoryClient } from "../client/github_repository_client"
+import { CustomReportCollection, createCustomReportCollection, aggregateCustomReportArtifacts } from "../custom_report_collection"
 
 export class CircleciRunner implements Runner {
   service: string = 'circleci'
@@ -41,6 +42,7 @@ export class CircleciRunner implements Runner {
 
     let workflowReports: WorkflowReport[] = []
     let testReports: TestReport[] = []
+    const customReportCollection = new CustomReportCollection()
     for (const repo of this.config.repos) {
       console.info(`Fetching ${this.service} - ${repo.fullname} ...`)
       const repoWorkflowReports: WorkflowReport[] = []
@@ -52,6 +54,7 @@ export class CircleciRunner implements Runner {
         const tagMap = await this.repoClient.fetchRepositoryTagMap(repo.owner, repo.repo)
 
         for (const workflowRun of workflowRuns) {
+          // Fetch data
           const jobs = await Promise.all(workflowRun.build_nums.map((buildNum) => {
             return this.client.fetchJobs(
               workflowRun.username,
@@ -68,11 +71,26 @@ export class CircleciRunner implements Runner {
               buildNum
             )
           }))
+          const customReportArtifactsList = await Promise.all(workflowRun.build_nums.map((buildNum) => {
+            return this.client.fetchCustomReports(
+              workflowRun.username,
+              workflowRun.reponame,
+              workflowRun.vcs_type,
+              buildNum,
+              repo.customReports,
+            )
+          }))
+          const customReportArtifacts = aggregateCustomReportArtifacts(customReportArtifactsList)
+
+          // Create report
           const workflowReport = this.analyzer.createWorkflowReport(workflowRun, jobs, tagMap)
           const testReports = await this.analyzer.createTestReports(workflowReport, jobs, tests)
+          const runCustomReportCollection = await createCustomReportCollection(workflowReport, customReportArtifacts)
 
+          // Aggregate
           repoWorkflowReports.push(workflowReport)
           repoTestReports = repoTestReports.concat(testReports)
+          customReportCollection.aggregate(runCustomReportCollection)
         }
 
         this.setRepoLastRun(repo.fullname, repoWorkflowReports)
@@ -90,6 +108,7 @@ export class CircleciRunner implements Runner {
     const exporter = new CompositExporter(this.service, this.configDir, this.config.exporter)
     await exporter.exportWorkflowReports(workflowReports)
     await exporter.exportTestReports(testReports)
+    await exporter.exportCustomReports(customReportCollection)
 
     this.store.save()
     console.info(`Success: done execute '${this.service}'`)