From ef6a38f79b9ff6abd542317ccf0d231999da38ec Mon Sep 17 00:00:00 2001 From: Shahzad Date: Tue, 5 May 2026 12:14:21 +0500 Subject: [PATCH] [CI] Speed up telemetry_check by pre-filtering to collector files (#265978) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary The `telemetry.sh` quick check takes **~8m 45s** in CI — the single slowest check in the `quick_checks` step. It also runs in the serial `mayChangeFiles` queue, blocking all other serial checks behind it. ### The problem `extractCollectors()` passes **all ~36K TS files** from each of the 10 telemetry roots into `ts.createProgram()`, even though only **~69 files** across the entire repo actually contain `makeUsageCollector` or `makeStatsCollector` calls. TypeScript parses and type-checks thousands of unrelated source files that will never yield a collector. ### The fix Pre-filter files with a simple regex scan on file contents before feeding them to the TS compiler: ```ts const collectorPaths = fullPaths.filter((p) => COLLECTOR_RE.test(readFileSync(p, 'utf-8'))); ``` TypeScript still resolves all transitive imports needed for type-checking the collector files, but avoids parsing thousands of unrelated files. ### Benchmark results (local) **`src/platform/` root** (largest OSS root): | Approach | Root files | Source files TS loads | Time | |---|---|---|---| | Current (all files) | 8,852 | 17,279 | 48s | | New (collector files only) | 30 | 4,814 | 15s | | **Speedup** | | **3.6x fewer** | **3.1x faster** | **`x-pack/platform/` root** (largest x-pack root): | Approach | Root files | Source files TS loads | Time | |---|---|---|---| | Current (all files) | 15,344 | 25,002 | 71s | | New (collector files only) | 27 | 13,835 | 49s | | **Speedup** | | **1.8x fewer** | **1.5x faster** | **CI estimate**: `telemetry.sh` should drop from **~8m 45s to ~3-4 min**. ### How it was tested - Profiled `createKibanaProgram` locally with the actual Kibana telemetry roots - Verified the same collector files are found and processed - The pre-filter is conservative — any file mentioning the collector function names is included Made with [Cursor](https://cursor.com) Co-authored-by: Ahmad Bamieh (cherry picked from commit b3cb3c89a91ae1ee18b0d7d10c98a1ceb903e82a) --- .../src/tools/extract_collectors.ts | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/platform/packages/private/kbn-telemetry-tools/src/tools/extract_collectors.ts b/src/platform/packages/private/kbn-telemetry-tools/src/tools/extract_collectors.ts index 19560795114a1..1146570171979 100644 --- a/src/platform/packages/private/kbn-telemetry-tools/src/tools/extract_collectors.ts +++ b/src/platform/packages/private/kbn-telemetry-tools/src/tools/extract_collectors.ts @@ -7,12 +7,15 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ +import { readFileSync } from 'fs'; import globby from 'globby'; import * as path from 'path'; import { parseUsageCollection } from './ts_parser'; import { TelemetryRC } from './config'; import { createKibanaProgram, getAllSourceFiles } from './ts_program'; +const COLLECTOR_RE = /makeUsageCollector|makeStatsCollector/; + export async function getProgramPaths({ root, exclude, @@ -53,8 +56,17 @@ export async function getProgramPaths({ } export function* extractCollectors(fullPaths: string[], tsConfig: any) { - const program = createKibanaProgram(fullPaths, tsConfig); - const sourceFiles = getAllSourceFiles(fullPaths, program); + // Pre-filter to only files that reference collector APIs so TS doesn't + // parse thousands of unrelated source files (36K → ~70 root files). + // TS still resolves transitive imports needed for type-checking. + const collectorPaths = fullPaths.filter((p) => COLLECTOR_RE.test(readFileSync(p, 'utf-8'))); + + if (collectorPaths.length === 0) { + return; + } + + const program = createKibanaProgram(collectorPaths, tsConfig); + const sourceFiles = getAllSourceFiles(collectorPaths, program); for (const sourceFile of sourceFiles) { yield* parseUsageCollection(sourceFile, program);