From 7b195afb7b6638b0ee34edf7f5eecd0a47efb20a Mon Sep 17 00:00:00 2001 From: Spencer Date: Wed, 27 Apr 2022 11:16:55 -0700 Subject: [PATCH 1/2] [ci-stats-reporter] use v2 test group APIs (#131001) * [ci-stats-reporter] use v2 test group APIs * fix bazel deps * [CI] Auto-commit changed files from 'yarn kbn run build -i @kbn/pm' * avoid importing kbn/std in kbn-pm * removed kbn/std dependency Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> (cherry picked from commit 7fd166176d21e044374f2cb1335c20b11a2896bd) # Conflicts: # packages/kbn-pm/dist/index.js --- .../ci_stats_reporter/ci_stats_reporter.ts | 51 ++++++++++++++---- packages/kbn-pm/dist/index.js | 52 ++++++++++++++++--- 2 files changed, 85 insertions(+), 18 deletions(-) diff --git a/packages/kbn-dev-utils/src/ci_stats_reporter/ci_stats_reporter.ts b/packages/kbn-dev-utils/src/ci_stats_reporter/ci_stats_reporter.ts index f710f7ec70843..edc5ae5ae9e68 100644 --- a/packages/kbn-dev-utils/src/ci_stats_reporter/ci_stats_reporter.ts +++ b/packages/kbn-dev-utils/src/ci_stats_reporter/ci_stats_reporter.ts @@ -85,10 +85,8 @@ export interface CiStatsReportTestsOptions { } /* @internal */ -interface ReportTestsResponse { - buildId: string; +interface ReportTestGroupResponse { groupId: string; - testRunCount: number; } /* @internal */ @@ -239,18 +237,51 @@ export class CiStatsReporter { ); } - return await this.req({ + const groupResp = await this.req({ auth: true, - path: '/v1/test_group', + path: '/v2/test_group', query: { buildId: this.config?.buildId, }, - bodyDesc: `[${group.name}/${group.type}] test groups with ${testRuns.length} tests`, - body: [ - JSON.stringify({ group }), - ...testRuns.map((testRun) => JSON.stringify({ testRun })), - ].join('\n'), + bodyDesc: `[${group.name}/${group.type}] test group`, + body: group, }); + + if (!groupResp) { + return; + } + + let bufferBytes = 0; + const buffer: string[] = []; + const flushBuffer = async () => { + await this.req<{ testRunCount: number }>({ + auth: true, + path: '/v2/test_runs', + query: { + buildId: this.config?.buildId, + groupId: groupResp.groupId, + groupType: group.type, + }, + bodyDesc: `[${group.name}/${group.type}] Chunk of ${bufferBytes} bytes`, + body: buffer.join('\n'), + }); + buffer.length = 0; + bufferBytes = 0; + }; + + // send test runs in chunks of ~500kb + for (const testRun of testRuns) { + const json = JSON.stringify(testRun); + bufferBytes += json.length; + buffer.push(json); + if (bufferBytes >= 450000) { + await flushBuffer(); + } + } + + if (bufferBytes) { + await flushBuffer(); + } } /** diff --git a/packages/kbn-pm/dist/index.js b/packages/kbn-pm/dist/index.js index 5169623261e67..1d127757393d7 100644 --- a/packages/kbn-pm/dist/index.js +++ b/packages/kbn-pm/dist/index.js @@ -9210,19 +9210,55 @@ class CiStatsReporter { throw new Error('unable to report tests unless buildId is configured and auth config available'); } - return await this.req({ + const groupResp = await this.req({ auth: true, - path: '/v1/test_group', + path: '/v2/test_group', query: { buildId: (_this$config7 = this.config) === null || _this$config7 === void 0 ? void 0 : _this$config7.buildId }, - bodyDesc: `[${group.name}/${group.type}] test groups with ${testRuns.length} tests`, - body: [JSON.stringify({ - group - }), ...testRuns.map(testRun => JSON.stringify({ - testRun - }))].join('\n') + bodyDesc: `[${group.name}/${group.type}] test group`, + body: group }); + + if (!groupResp) { + return; + } + + let bufferBytes = 0; + const buffer = []; + + const flushBuffer = async () => { + var _this$config8; + + await this.req({ + auth: true, + path: '/v2/test_runs', + query: { + buildId: (_this$config8 = this.config) === null || _this$config8 === void 0 ? void 0 : _this$config8.buildId, + groupId: groupResp.groupId, + groupType: group.type + }, + bodyDesc: `[${group.name}/${group.type}] Chunk of ${bufferBytes} bytes`, + body: buffer.join('\n') + }); + buffer.length = 0; + bufferBytes = 0; + }; // send test runs in chunks of ~500kb + + + for (const testRun of testRuns) { + const json = JSON.stringify(testRun); + bufferBytes += json.length; + buffer.push(json); + + if (bufferBytes >= 450000) { + await flushBuffer(); + } + } + + if (bufferBytes) { + await flushBuffer(); + } } /** * In order to allow this code to run before @kbn/utils is built, @kbn/pm will pass From f4631bf279e36d48323f9e9837476048d1e7a2a2 Mon Sep 17 00:00:00 2001 From: Spencer Date: Tue, 3 May 2022 11:00:55 -0700 Subject: [PATCH 2/2] [ci-stats-reporter] use a default timeout of 60 seconds (#131428) Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> (cherry picked from commit 06b958d2a297fb40fd18dbc1aeacd6865ddb9d32) --- .../src/ci_stats_reporter/ci_stats_reporter.ts | 11 ++++++++++- packages/kbn-pm/dist/index.js | 14 +++++++++++--- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/packages/kbn-dev-utils/src/ci_stats_reporter/ci_stats_reporter.ts b/packages/kbn-dev-utils/src/ci_stats_reporter/ci_stats_reporter.ts index edc5ae5ae9e68..ef866bde61cf9 100644 --- a/packages/kbn-dev-utils/src/ci_stats_reporter/ci_stats_reporter.ts +++ b/packages/kbn-dev-utils/src/ci_stats_reporter/ci_stats_reporter.ts @@ -23,6 +23,8 @@ import type { CiStatsTestGroupInfo, CiStatsTestRun } from './ci_stats_test_group import { CiStatsMetadata } from './ci_stats_metadata'; const BASE_URL = 'https://ci-stats.kibana.dev'; +const SECOND = 1000; +const MINUTE = 60 * SECOND; /** A ci-stats metric record */ export interface CiStatsMetric { @@ -96,6 +98,7 @@ interface ReqOptions { body: any; bodyDesc: string; query?: AxiosRequestConfig['params']; + timeout?: number; } /** Object that helps report data to the ci-stats service */ @@ -264,6 +267,7 @@ export class CiStatsReporter { }, bodyDesc: `[${group.name}/${group.type}] Chunk of ${bufferBytes} bytes`, body: buffer.join('\n'), + timeout: 5 * MINUTE, }); buffer.length = 0; bufferBytes = 0; @@ -318,7 +322,7 @@ export class CiStatsReporter { } } - private async req({ auth, body, bodyDesc, path, query }: ReqOptions) { + private async req({ auth, body, bodyDesc, path, query, timeout = 60 * SECOND }: ReqOptions) { let attempt = 0; const maxAttempts = 5; @@ -343,6 +347,11 @@ export class CiStatsReporter { data: body, params: query, adapter: httpAdapter, + + // if it can be serialized into a string, send it + maxBodyLength: Infinity, + maxContentLength: Infinity, + timeout, }); return resp.data; diff --git a/packages/kbn-pm/dist/index.js b/packages/kbn-pm/dist/index.js index 1d127757393d7..fec527b67328e 100644 --- a/packages/kbn-pm/dist/index.js +++ b/packages/kbn-pm/dist/index.js @@ -9050,6 +9050,8 @@ var _ci_stats_config = __webpack_require__(218); */ // @ts-expect-error not "public", but necessary to prevent Jest shimming from breaking things const BASE_URL = 'https://ci-stats.kibana.dev'; +const SECOND = 1000; +const MINUTE = 60 * SECOND; /** A ci-stats metric record */ /** Object that helps report data to the ci-stats service */ @@ -9239,7 +9241,8 @@ class CiStatsReporter { groupType: group.type }, bodyDesc: `[${group.name}/${group.type}] Chunk of ${bufferBytes} bytes`, - body: buffer.join('\n') + body: buffer.join('\n'), + timeout: 5 * MINUTE }); buffer.length = 0; bufferBytes = 0; @@ -9308,7 +9311,8 @@ class CiStatsReporter { body, bodyDesc, path, - query + query, + timeout = 60 * SECOND }) { let attempt = 0; const maxAttempts = 5; @@ -9333,7 +9337,11 @@ class CiStatsReporter { headers, data: body, params: query, - adapter: _http.default + adapter: _http.default, + // if it can be serialized into a string, send it + maxBodyLength: Infinity, + maxContentLength: Infinity, + timeout }); return resp.data; } catch (error) {