[CI] Jest through moon

.buildkite/pipeline-utils/affected-packages/README.md

@@ -69,7 +69,6 @@ const affectedPackages = await getAffectedPackages(
   {
     strategy: 'git',       // default, can also be 'moon'
     includeDownstream: true,
-    logging: false,
     ignorePatterns: ['**/*.md', 'docs/**'],
     ignoreUncategorizedChanges: false,
   }
@@ -109,7 +108,6 @@ const filteredFiles = filterFilesByPackages(
 |------------------------------------|---------------------------------|--------------|----------------------|
 | `AFFECTED_STRATEGY`                | `git`, `moon`                   | `git`        | `git`                |
 | `AFFECTED_DOWNSTREAM`              | `true`, `false`                 | `false`      | `true`               |
-| `AFFECTED_LOGGING`                 | `true`, `false`                 | `false`      | `true`               |
 | `AFFECTED_IGNORE`                  | comma-separated globs           | —            | —                    |
 | `AFFECTED_IGNORE_UNCATEGORIZED_CHANGES` | `true`, `false`            | `false`      | `false`              |
 | `GITHUB_PR_MERGE_BASE`             | any git ref                     | `origin/main`| —                    |

.buildkite/pipeline-utils/affected-packages/const.ts

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+export const UNCATEGORIZED_MODULE_ID = '[uncategorized]';
+
+export const NO_SELECTIVE_TESTS_LABEL = 'ci:no-selective-tests';
+
+// TODO: find reasonable set of critical files for unit tests
+export const CRITICAL_FILES_JEST_UNIT_TESTS = [
+  'scripts/jest.js',
+  'scripts/jest_all.js',
+  'package.json',
+  'yarn.lock',
+  'tsconfig.json',
+  'src/platform/packages/shared/kbn-test/**/*',
+];
+
+// TODO: find reasonable set of critical files for integration tests
+export const CRITICAL_FILES_JEST_INTEGRATION_TESTS = [
+  'scripts/jest_integration.js',
+  'scripts/jest_all.js',
+  'package.json',
+  'yarn.lock',
+  'tsconfig.json',
+  'src/platform/packages/shared/kbn-test/**/*',
+];

.buildkite/pipeline-utils/affected-packages/index.ts

@@ -11,13 +11,15 @@ import { findModuleForPath } from './module_lookup';
 import { getAffectedModulesGit } from './strategy_git';
 import { getAffectedProjectsMoon } from './strategy_moon';
 
+export * from './const';
+export * from './utils';
+
 export interface AffectedPackagesConfig {
-  strategy: 'git' | 'moon';
-  includeDownstream: boolean;
-  logging: boolean;
+  strategy?: 'git' | 'moon';
+  includeDownstream?: boolean;
   /** Glob patterns for changed files to exclude before module resolution (git strategy only). */
-  ignorePatterns: string[];
-  ignoreUncategorizedChanges: boolean;
+  ignorePatterns?: string[];
+  ignoreUncategorizedChanges?: boolean;
 }
 
 /**
@@ -26,12 +28,19 @@ export interface AffectedPackagesConfig {
  */
 export async function getAffectedPackages(
   mergeBase: string | undefined,
-  config: AffectedPackagesConfig = getConfigFromEnv()
+  configArgs: AffectedPackagesConfig = getConfigFromEnv()
 ): Promise<Set<string>> {
   if (!mergeBase) {
     throw new Error('No merge base found');
   }
 
+  const config = {
+    strategy: configArgs.strategy ?? 'git',
+    includeDownstream: configArgs.includeDownstream ?? false,
+    ignorePatterns: configArgs.ignorePatterns ?? [],
+    ignoreUncategorizedChanges: configArgs.ignoreUncategorizedChanges ?? false,
+  };
+
   try {
     const affectedPackages =
       config.strategy === 'git'
@@ -82,13 +91,12 @@ function getConfigFromEnv(): AffectedPackagesConfig {
   }
   const strategy = rawStrategy;
   const includeDownstream = process.env.AFFECTED_DOWNSTREAM !== 'false';
-  const logging = process.env.AFFECTED_LOGGING !== 'false';
   const ignorePatterns = (process.env.AFFECTED_IGNORE || '')
     .split(',')
     .map((p) => p.trim())
     .filter(Boolean);
 
   const ignoreUncategorizedChanges = process.env.AFFECTED_IGNORE_UNCATEGORIZED_CHANGES !== 'false';
 
-  return { strategy, includeDownstream, logging, ignorePatterns, ignoreUncategorizedChanges };
+  return { strategy, includeDownstream, ignorePatterns, ignoreUncategorizedChanges };
 }

.buildkite/pipeline-utils/affected-packages/list_affected

@@ -85,7 +85,6 @@ async function main() {
     const affectedPackages = await getAffectedPackages(mergeBase, {
       strategy,
       includeDownstream,
-      logging: false,
       ignorePatterns,
       ignoreUncategorizedChanges,
     });

.buildkite/pipeline-utils/affected-packages/module_lookup.ts

@@ -12,8 +12,7 @@ import * as fs from 'fs';
 import { execSync } from 'child_process';
 import * as JSON5 from 'json5';
 import { getKibanaDir } from '../utils';
-
-export const UNCATEGORIZED_MODULE_ID = '[uncategorized]';
+import { UNCATEGORIZED_MODULE_ID } from './const';
 
 export interface ModuleLookup {
   /**

.buildkite/pipeline-utils/affected-packages/strategy_git.ts

@@ -9,11 +9,8 @@
 
 import { execSync } from 'child_process';
 import { getKibanaDir } from '../utils';
-import {
-  findModuleForPath,
-  buildModuleDownstreamGraph,
-  UNCATEGORIZED_MODULE_ID,
-} from './module_lookup';
+import { findModuleForPath, buildModuleDownstreamGraph } from './module_lookup';
+import { UNCATEGORIZED_MODULE_ID } from './const';
 import { filterIgnoredFiles } from './utils';
 
 const isCI = !!process.env.CI?.match(/^(1|true)$/i);

.buildkite/pipeline-utils/affected-packages/utils.ts

@@ -19,3 +19,14 @@ export function filterIgnoredFiles(files: string[], patterns: string[]): string[
   }
   return files.filter((file) => !patterns.some((p) => minimatch(file, p, { dot: true })));
 }
+
+/**
+ * Returns true when any pattern matches any file in the list
+ */
+export function touchedCriticalFiles(files: string[], criticalFiles: string[]): boolean {
+  return (
+    files.some((file) =>
+      criticalFiles.some((criticalFile) => minimatch(file, criticalFile, { dot: true }))
+    ) && false // temporary disable
+  );
+}

.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts

@@ -24,10 +24,18 @@ import DISABLED_JEST_CONFIGS from '../../disabled_jest_configs.json';
 import SHARDED_JEST_CONFIGS from '../../sharded_jest_configs.json';
 import { serverless, stateful } from '../../ftr_configs_manifests.json';
 import { filterEmptyJestConfigs } from './get_tests_from_config';
+import {
+  getAffectedPackages,
+  filterFilesByPackages,
+  NO_SELECTIVE_TESTS_LABEL,
+  CRITICAL_FILES_JEST_UNIT_TESTS,
+  touchedCriticalFiles,
+} from '../affected-packages';
 import { collectEnvFromLabels, expandAgentQueue, getRequiredEnv } from '#pipeline-utils';
 
-const SHARD_ANNOTATION_SEP = '||shard=';
+const NO_SELECTIVE_TESTS = process.env.GITHUB_LABELS?.includes(NO_SELECTIVE_TESTS_LABEL);
 
+const SHARD_ANNOTATION_SEP = '||shard=';
 /**
  * Expands configs that appear in the shard map into N shard-annotated entries.
  * For example, if `fleet/jest.integration.config.js` has 2 shards, it becomes:
@@ -221,7 +229,39 @@ export async function pickTestGroupRunOrder() {
   // Expand sharded integration configs into shard-annotated entries
   const jestIntegrationConfigs = expandShardedJestConfigs(jestIntegrationConfigsRaw);
 
-  if (!ftrConfigsByQueue.size && !jestUnitConfigs.length && !jestIntegrationConfigs.length) {
+  // Apply affected package filtering
+  const affectedPackages = await getAffectedPackages(process.env.GITHUB_PR_MERGE_BASE, {
+    strategy: 'git',
+    includeDownstream: true,
+    ignorePatterns: [], // might want to exclude metadata/text changes in the future
+    ignoreUncategorizedChanges: true,
+  }).catch((error) => {
+    console.error('Error getting affected packages', error);
+    return null;
+  });
+
+  const shouldFilterByAffected =
+    affectedPackages && affectedPackages.size > 0 && !NO_SELECTIVE_TESTS;
+
+  const filteredJestUnitConfigs =
+    shouldFilterByAffected && !touchedCriticalFiles(jestUnitConfigs, CRITICAL_FILES_JEST_UNIT_TESTS)
+      ? filterFilesByPackages(jestUnitConfigs, affectedPackages)
+      : jestUnitConfigs;
+  console.warn(
+    `Filtering Jest unit tests for affected packages: ${jestUnitConfigs.length} -> ${filteredJestUnitConfigs.length}`
+  );
+  const filteredJestIntegrationConfigs = shouldFilterByAffected
+    ? filterFilesByPackages(jestIntegrationConfigs, affectedPackages)
+    : jestIntegrationConfigs;
+  console.warn(
+    `Filtering Jest integration tests for affected packages: ${jestIntegrationConfigs.length} -> ${filteredJestIntegrationConfigs.length}`
+  );
+
+  if (
+    !ftrConfigsByQueue.size &&
+    !filteredJestUnitConfigs.length &&
+    !filteredJestIntegrationConfigs.length
+  ) {
     throw new Error('unable to find any unit, integration, or FTR configs');
   }
 
@@ -288,7 +328,7 @@ export async function pickTestGroupRunOrder() {
         overheadMin: 0.2,
         warmupMin: 4,
         concurrency: 3,
-        names: jestUnitConfigs,
+        names: filteredJestUnitConfigs,
       },
       {
         type: INTEGRATION_TYPE,
@@ -297,7 +337,7 @@ export async function pickTestGroupRunOrder() {
         overheadMin: 0.2,
         warmupMin: 2,
         concurrency: 1,
-        names: jestIntegrationConfigs,
+        names: filteredJestIntegrationConfigs,
       },
       ...Array.from(ftrConfigsByQueue).map(([queue, names]) => ({
         type: FUNCTIONAL_TYPE,

.buildkite/scripts/list-affected-packages

@@ -0,0 +1,105 @@
+#!/usr/bin/env node
+
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+/**
+ * CLI script to list affected packages
+ * 
+ * Usage:
+ *   .buildkite/scripts/list-affected-packages [options]
+ * 
+ * Options:
+ *   --deep         Include downstream dependencies (default: false)
+ *   --json         Output as JSON array
+ *   --merge-base   Git commit to compare against (default: GITHUB_PR_MERGE_BASE env var or 'main')
+ *   --strategy     Strategy to use: git or moon (default: git)
+ * 
+ * Examples:
+ *   .buildkite/scripts/list-affected-packages
+ *   .buildkite/scripts/list-affected-packages --deep
+ *   .buildkite/scripts/list-affected-packages --deep --json
+ *   .buildkite/scripts/list-affected-packages --merge-base HEAD~5
+ *   .buildkite/scripts/list-affected-packages --strategy moon --deep
+ */
+
+// Must be before imports to suppress logging
+process.env.AFFECTED_LOGGING = 'false';
+
+const path = require('path');
+
+// Add the parent directories to the require path so we can load TypeScript via ts-node
+require('ts-node').register({
+  transpileOnly: true,
+  compilerOptions: {
+    module: 'commonjs',
+  },
+  project: path.join(__dirname, '../tsconfig.json'),
+});
+
+const affectedPackages = require(path.join(__dirname, '../pipeline-utils/affected-packages'));
+const getAffectedPackages = affectedPackages.getAffectedPackages;
+
+async function main() {
+  const args = process.argv.slice(2);
+
+  // Parse flags
+  const flags = {
+    deep: args.includes('--deep'),
+    json: args.includes('--json'),
+    mergeBase: undefined,
+    strategy: 'git',
+  };
+
+  // Parse --merge-base value
+  const mergeBaseIndex = args.indexOf('--merge-base');
+  if (mergeBaseIndex !== -1 && args[mergeBaseIndex + 1]) {
+    flags.mergeBase = args[mergeBaseIndex + 1];
+  } else {
+    // Default to GITHUB_PR_MERGE_BASE env var or 'main'
+    flags.mergeBase = process.env.GITHUB_PR_MERGE_BASE || 'main';
+  }
+
+  // Parse --strategy value
+  const strategyIndex = args.indexOf('--strategy');
+  if (strategyIndex !== -1 && args[strategyIndex + 1]) {
+    const strategy = args[strategyIndex + 1];
+    if (strategy !== 'git' && strategy !== 'moon') {
+      console.error(`Invalid strategy: ${strategy}. Must be 'git' or 'moon'.`);
+      process.exit(1);
+    }
+    flags.strategy = strategy;
+  }
+
+  try {
+    // Get affected packages with logging disabled
+    const affectedPackages = await getAffectedPackages(flags.mergeBase, {
+      strategy: flags.strategy,
+      includeDownstream: flags.deep,
+      logging: false,
+    });
+
+    const packageList = Array.from(affectedPackages).sort();
+
+    if (flags.json) {
+      // Output as JSON array
+      console.log(JSON.stringify(packageList, null, 2));
+    } else {
+      // Output one package per line
+      packageList.forEach(pkg => console.log(pkg));
+    }
+
+    process.exit(0);
+  } catch (error) {
+    console.error('Error getting affected packages:', error.message);
+    process.exit(1);
+  }
+}
+
+main();

.buildkite/scripts/pipelines/pull_request/pipeline.sh

@@ -7,5 +7,15 @@ set -euo pipefail
 (buildkite-agent pipeline upload .buildkite/pipelines/pull_request/store_moon_cache.yml > /dev/null \
  && echo "Uploaded cache-warmup step" >&2) || echo "Failed to upload cache-warmup step" >&2
 
-
+set +e
 ts-node .buildkite/scripts/pipelines/pull_request/pipeline.ts
+pipeline_status=$?
+
+if [[ $pipeline_status -ne 0 ]]; then
+  echo "⚠️ Pipeline generation failed - emitting bogus pipeline to ensure build fails" >&2
+  echo "boom"
+  exit 1
+fi
+
+echo "Pipeline generation successful" >&2
+exit 0

.github/workflows/auto-approve-bundled-package-updates.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     if: |
       startsWith(github.event.pull_request.head.ref, 'update-bundled-packages') &&
-      github.event.pull_request.user.login == 'kibanamachine'
+      github.event.pull_request.user.login == 'elasticmachine'
     permissions:
       pull-requests: write
     steps:

.moon/tasks/node.yml

@@ -8,3 +8,24 @@ fileGroups:
   default:
     - '**/*'
   production: [ ]
+
+tasks:
+  # This task can be used to make a task depend on the source files in the workspace
+  source:
+    command: 'true'
+    inputs:
+      - '@group(src)'
+    options:
+      internal: true
+      runFromWorkspaceRoot: true
+
+  # This task can be used to make a task depend on the full upstream source files of a select project
+  source-upstream:
+    command: 'true'
+    inputs:
+      - '@group(src)'
+    deps:
+      - '^:source-upstream'
+    options:
+      internal: true
+      runFromWorkspaceRoot: true