[7.17] [ci-stats-reporter] use v2 test group APIs (#131001)

packages/kbn-dev-utils/src/ci_stats_reporter/ci_stats_reporter.ts

@@ -23,6 +23,8 @@ import type { CiStatsTestGroupInfo, CiStatsTestRun } from './ci_stats_test_group
 import { CiStatsMetadata } from './ci_stats_metadata';
 
 const BASE_URL = 'https://ci-stats.kibana.dev';
+const SECOND = 1000;
+const MINUTE = 60 * SECOND;
 
 /** A ci-stats metric record */
 export interface CiStatsMetric {
@@ -85,10 +87,8 @@ export interface CiStatsReportTestsOptions {
 }
 
 /* @internal */
-interface ReportTestsResponse {
-  buildId: string;
+interface ReportTestGroupResponse {
   groupId: string;
-  testRunCount: number;
 }
 
 /* @internal */
@@ -98,6 +98,7 @@ interface ReqOptions {
   body: any;
   bodyDesc: string;
   query?: AxiosRequestConfig['params'];
+  timeout?: number;
 }
 
 /** Object that helps report data to the ci-stats service */
@@ -239,18 +240,52 @@ export class CiStatsReporter {
       );
     }
 
-    return await this.req<ReportTestsResponse>({
+    const groupResp = await this.req<ReportTestGroupResponse>({
       auth: true,
-      path: '/v1/test_group',
+      path: '/v2/test_group',
       query: {
         buildId: this.config?.buildId,
       },
-      bodyDesc: `[${group.name}/${group.type}] test groups with ${testRuns.length} tests`,
-      body: [
-        JSON.stringify({ group }),
-        ...testRuns.map((testRun) => JSON.stringify({ testRun })),
-      ].join('\n'),
+      bodyDesc: `[${group.name}/${group.type}] test group`,
+      body: group,
     });
+
+    if (!groupResp) {
+      return;
+    }
+
+    let bufferBytes = 0;
+    const buffer: string[] = [];
+    const flushBuffer = async () => {
+      await this.req<{ testRunCount: number }>({
+        auth: true,
+        path: '/v2/test_runs',
+        query: {
+          buildId: this.config?.buildId,
+          groupId: groupResp.groupId,
+          groupType: group.type,
+        },
+        bodyDesc: `[${group.name}/${group.type}] Chunk of ${bufferBytes} bytes`,
+        body: buffer.join('\n'),
+        timeout: 5 * MINUTE,
+      });
+      buffer.length = 0;
+      bufferBytes = 0;
+    };
+
+    // send test runs in chunks of ~500kb
+    for (const testRun of testRuns) {
+      const json = JSON.stringify(testRun);
+      bufferBytes += json.length;
+      buffer.push(json);
+      if (bufferBytes >= 450000) {
+        await flushBuffer();
+      }
+    }
+
+    if (bufferBytes) {
+      await flushBuffer();
+    }
   }
 
   /**
@@ -287,7 +322,7 @@ export class CiStatsReporter {
     }
   }
 
-  private async req<T>({ auth, body, bodyDesc, path, query }: ReqOptions) {
+  private async req<T>({ auth, body, bodyDesc, path, query, timeout = 60 * SECOND }: ReqOptions) {
     let attempt = 0;
     const maxAttempts = 5;
 
@@ -312,6 +347,11 @@ export class CiStatsReporter {
           data: body,
           params: query,
           adapter: httpAdapter,
+
+          // if it can be serialized into a string, send it
+          maxBodyLength: Infinity,
+          maxContentLength: Infinity,
+          timeout,
         });
 
         return resp.data;

packages/kbn-pm/dist/index.js

@@ -9050,6 +9050,8 @@ var _ci_stats_config = __webpack_require__(218);
  */
 // @ts-expect-error not "public", but necessary to prevent Jest shimming from breaking things
 const BASE_URL = 'https://ci-stats.kibana.dev';
+const SECOND = 1000;
+const MINUTE = 60 * SECOND;
 /** A ci-stats metric record */
 
 /** Object that helps report data to the ci-stats service */
@@ -9210,19 +9212,56 @@ class CiStatsReporter {
       throw new Error('unable to report tests unless buildId is configured and auth config available');
     }
 
-    return await this.req({
+    const groupResp = await this.req({
       auth: true,
-      path: '/v1/test_group',
+      path: '/v2/test_group',
       query: {
         buildId: (_this$config7 = this.config) === null || _this$config7 === void 0 ? void 0 : _this$config7.buildId
       },
-      bodyDesc: `[${group.name}/${group.type}] test groups with ${testRuns.length} tests`,
-      body: [JSON.stringify({
-        group
-      }), ...testRuns.map(testRun => JSON.stringify({
-        testRun
-      }))].join('\n')
+      bodyDesc: `[${group.name}/${group.type}] test group`,
+      body: group
     });
+
+    if (!groupResp) {
+      return;
+    }
+
+    let bufferBytes = 0;
+    const buffer = [];
+
+    const flushBuffer = async () => {
+      var _this$config8;
+
+      await this.req({
+        auth: true,
+        path: '/v2/test_runs',
+        query: {
+          buildId: (_this$config8 = this.config) === null || _this$config8 === void 0 ? void 0 : _this$config8.buildId,
+          groupId: groupResp.groupId,
+          groupType: group.type
+        },
+        bodyDesc: `[${group.name}/${group.type}] Chunk of ${bufferBytes} bytes`,
+        body: buffer.join('\n'),
+        timeout: 5 * MINUTE
+      });
+      buffer.length = 0;
+      bufferBytes = 0;
+    }; // send test runs in chunks of ~500kb
+
+
+    for (const testRun of testRuns) {
+      const json = JSON.stringify(testRun);
+      bufferBytes += json.length;
+      buffer.push(json);
+
+      if (bufferBytes >= 450000) {
+        await flushBuffer();
+      }
+    }
+
+    if (bufferBytes) {
+      await flushBuffer();
+    }
   }
   /**
    * In order to allow this code to run before @kbn/utils is built, @kbn/pm will pass
@@ -9272,7 +9311,8 @@ class CiStatsReporter {
     body,
     bodyDesc,
     path,
-    query
+    query,
+    timeout = 60 * SECOND
   }) {
     let attempt = 0;
     const maxAttempts = 5;
@@ -9297,7 +9337,11 @@ class CiStatsReporter {
           headers,
           data: body,
           params: query,
-          adapter: _http.default
+          adapter: _http.default,
+          // if it can be serialized into a string, send it
+          maxBodyLength: Infinity,
+          maxContentLength: Infinity,
+          timeout
         });
         return resp.data;
       } catch (error) {