diff --git a/.buildkite/pull_requests.json b/.buildkite/pull_requests.json index 49d29fa99861f..872dfc30d36a3 100644 --- a/.buildkite/pull_requests.json +++ b/.buildkite/pull_requests.json @@ -34,7 +34,7 @@ "always_require_ci_on_changed": [ "^docs/developer/plugin-list.asciidoc$", "^\\.github/CODEOWNERS$", - "/plugins/[^/]+/readme\\.(md|asciidoc)$" + "/plugins/([^/]+/)+readme\\.(md|asciidoc)$" ], "kibana_versions_check": true, "kibana_build_reuse": true, diff --git a/.buildkite/scripts/common/util.sh b/.buildkite/scripts/common/util.sh index d44e2dd052db2..9cd42e68efc14 100755 --- a/.buildkite/scripts/common/util.sh +++ b/.buildkite/scripts/common/util.sh @@ -32,39 +32,27 @@ check_for_changed_files() { C_RESET='\033[0m' # Reset color SHOULD_AUTO_COMMIT_CHANGES="${2:-}" - CUSTOM_FIX_MESSAGE="${3:-}" + CUSTOM_FIX_MESSAGE="${3:-Changes from $1}" GIT_CHANGES="$(git status --porcelain -- . ':!:config/node.options' ':!config/kibana.yml')" if [ "$GIT_CHANGES" ]; then if ! is_auto_commit_disabled && [[ "$SHOULD_AUTO_COMMIT_CHANGES" == "true" && "${BUILDKITE_PULL_REQUEST:-false}" != "false" ]]; then - NEW_COMMIT_MESSAGE="[CI] Auto-commit changed files from '$1'" - PREVIOUS_COMMIT_MESSAGE="$(git log -1 --pretty=%B)" - - if [[ "$NEW_COMMIT_MESSAGE" == "$PREVIOUS_COMMIT_MESSAGE" ]]; then - echo -e "\n${RED}ERROR: '$1' caused changes to the following files:${C_RESET}\n" - echo -e "$GIT_CHANGES\n" - echo -e "CI already attempted to commit these changes, but the file(s) seem to have changed again." - echo -e "Please review and fix manually." - exit 1 - fi - echo "'$1' caused changes to the following files:" echo "$GIT_CHANGES" echo "" - echo "Auto-committing these changes now. A new build should start soon if successful." + echo "Auto-committing & pushing these changes now." git config --global user.name kibanamachine git config --global user.email '42973632+kibanamachine@users.noreply.github.com' gh pr checkout "${BUILDKITE_PULL_REQUEST}" git add -A -- . ':!config/node.options' ':!config/kibana.yml' - git commit -m "$NEW_COMMIT_MESSAGE" + git commit -m "$CUSTOM_FIX_MESSAGE" git push - # After the git push, the new commit will trigger a new build within a few seconds and this build should get cancelled - # So, let's just sleep to give the build time to cancel itself without an error - # If it doesn't get cancelled for some reason, then exit with an error, because we don't want this build to be green (we just don't want it to generate an error either) + # Wait to ensure all commits arrive before we terminate the build sleep 300 + # Still exit with error to fail the current build, a new build should be started after the push exit 1 else echo -e "\n${RED}ERROR: '$1' caused changes to the following files:${C_RESET}\n" diff --git a/.buildkite/scripts/steps/checks/quick_checks.json b/.buildkite/scripts/steps/checks/quick_checks.json new file mode 100644 index 0000000000000..a9dc55e356cfb --- /dev/null +++ b/.buildkite/scripts/steps/checks/quick_checks.json @@ -0,0 +1,80 @@ +[ + { + "script": ".buildkite/scripts/steps/checks/ts_projects.sh", + "mayChangeFiles": true + }, + { + "script": ".buildkite/scripts/steps/checks/packages.sh", + "mayChangeFiles": true + }, + { + "script": ".buildkite/scripts/steps/checks/verify_notice.sh", + "mayChangeFiles": true + }, + { + "script": ".buildkite/scripts/steps/checks/plugin_list_docs.sh", + "mayChangeFiles": true + }, + { + "script": ".buildkite/scripts/steps/checks/event_log.sh", + "mayChangeFiles": true + }, + { + "script": ".buildkite/scripts/steps/checks/telemetry.sh", + "mayChangeFiles": true + }, + { + "script": ".buildkite/scripts/steps/checks/jest_configs.sh" + }, + { + "script": ".buildkite/scripts/steps/checks/bundle_limits.sh" + }, + { + "script": ".buildkite/scripts/steps/checks/i18n.sh" + }, + { + "script": ".buildkite/scripts/steps/checks/file_casing.sh" + }, + { + "script": ".buildkite/scripts/steps/checks/licenses.sh" + }, + { + "script": ".buildkite/scripts/steps/checks/test_projects.sh" + }, + { + "script": ".buildkite/scripts/steps/checks/test_hardening.sh" + }, + { + "script": ".buildkite/scripts/steps/checks/ftr_configs.sh" + }, + { + "script": ".buildkite/scripts/steps/checks/yarn_deduplicate.sh", + "mayChangeFiles": true + }, + { + "script": ".buildkite/scripts/steps/checks/prettier_topology.sh" + }, + { + "script": ".buildkite/scripts/steps/checks/renovate.sh" + }, + { + "script": ".buildkite/scripts/steps/checks/native_modules.sh" + }, + { + "script": ".buildkite/scripts/steps/checks/test_files_missing_owner.sh" + }, + { + "script": ".buildkite/scripts/steps/checks/styled_components_mapping.sh", + "mayChangeFiles": true + }, + { + "script": ".buildkite/scripts/steps/checks/dependencies_missing_owner.sh" + }, + { + "script": ".buildkite/scripts/steps/checks/validate_pipelines.sh", + "mayChangeFiles": true + }, + { + "script": ".buildkite/scripts/steps/checks/check_scout_config.sh" + } +] diff --git a/.buildkite/scripts/steps/checks/yarn_deduplicate.sh b/.buildkite/scripts/steps/checks/yarn_deduplicate.sh index 26c1e1636aa49..afc1f9b3a49df 100755 --- a/.buildkite/scripts/steps/checks/yarn_deduplicate.sh +++ b/.buildkite/scripts/steps/checks/yarn_deduplicate.sh @@ -5,6 +5,6 @@ set -euo pipefail source .buildkite/scripts/common/util.sh echo "--- Check yarn.lock for duplicated modules" -node scripts/yarn_deduplicate && yarn kbn bootstrap +node scripts/yarn_deduplicate && yarn kbn bootstrap --force-install check_for_changed_files 'node scripts/yarn_deduplicate' true 'TO FIX: Run node '"'"'scripts/yarn_deduplicate && yarn kbn bootstrap'"'"' locally, or add an exception to src/dev/yarn_deduplicate/index.ts and then commit the changes and push to your branch' diff --git a/.buildkite/scripts/steps/quick_checks.sh b/.buildkite/scripts/steps/quick_checks.sh index 1b1613d42dc8d..becb7159190b3 100755 --- a/.buildkite/scripts/steps/quick_checks.sh +++ b/.buildkite/scripts/steps/quick_checks.sh @@ -2,9 +2,11 @@ set -euo pipefail +source .buildkite/scripts/common/util.sh + if [[ "${CI:-}" =~ ^(1|true)$ ]]; then export DISABLE_BOOTSTRAP_VALIDATION=false .buildkite/scripts/bootstrap.sh fi -node scripts/quick_checks --file .buildkite/scripts/steps/checks/quick_checks.txt +node scripts/quick_checks --file .buildkite/scripts/steps/checks/quick_checks.json diff --git a/kbn_pm/src/commands/bootstrap/bootstrap_command.mjs b/kbn_pm/src/commands/bootstrap/bootstrap_command.mjs index 94532fa99673b..bf1f7c4a7333c 100644 --- a/kbn_pm/src/commands/bootstrap/bootstrap_command.mjs +++ b/kbn_pm/src/commands/bootstrap/bootstrap_command.mjs @@ -22,6 +22,7 @@ import { regenerateTsconfigPaths } from './regenerate_tsconfig_paths.mjs'; import { regenerateBaseTsconfig } from './regenerate_base_tsconfig.mjs'; import { discovery } from './discovery.mjs'; import { updatePackageJson } from './update_package_json.mjs'; +import { bootstrapBuildkite } from './buildkite.mjs'; const IS_CI = process.env.CI?.match(/(1|true)/i); @@ -86,6 +87,9 @@ export const command = { time('regenerate tsconfig.base.json', async () => { await regenerateBaseTsconfig(packages, log); }), + time('bootstrap .buildkite folder', async () => { + await bootstrapBuildkite(); + }), ]); /** diff --git a/kbn_pm/src/commands/bootstrap/buildkite.mjs b/kbn_pm/src/commands/bootstrap/buildkite.mjs new file mode 100644 index 0000000000000..47754b6e29422 --- /dev/null +++ b/kbn_pm/src/commands/bootstrap/buildkite.mjs @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { run } from '../../lib/spawn.mjs'; +import { REPO_ROOT } from '../../lib/paths.mjs'; +import { resolve } from 'path'; + +/** + * Bootstraps the the .buildkite directory + * @return {Promise<*>} + */ +export function bootstrapBuildkite() { + return run('npm', ['i'], { cwd: resolve(REPO_ROOT, '.buildkite') }); +} diff --git a/packages/kbn-eslint-plugin-telemetry/tsconfig.json b/packages/kbn-eslint-plugin-telemetry/tsconfig.json index 75014884aa312..c1ac0fec6e48a 100644 --- a/packages/kbn-eslint-plugin-telemetry/tsconfig.json +++ b/packages/kbn-eslint-plugin-telemetry/tsconfig.json @@ -7,5 +7,8 @@ }, "include": ["**/*.ts"], "exclude": ["target/**/*"], - "kbn_references": ["@kbn/repo-packages", "@kbn/repo-info"] + "kbn_references": [ + "@kbn/repo-packages", + "@kbn/repo-info", + ] } diff --git a/src/dev/run_quick_checks.ts b/src/dev/run_quick_checks.ts index b856246d40e24..eb4938a671aae 100644 --- a/src/dev/run_quick_checks.ts +++ b/src/dev/run_quick_checks.ts @@ -18,9 +18,15 @@ import { ToolingLog } from '@kbn/tooling-log'; const MAX_PARALLELISM = availableParallelism(); const buildkiteQuickchecksFolder = join('.buildkite', 'scripts', 'steps', 'checks'); -const quickChecksList = join(buildkiteQuickchecksFolder, 'quick_checks.txt'); +const quickChecksList = join(buildkiteQuickchecksFolder, 'quick_checks.json'); const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); +interface QuickCheck { + script: string; + mayChangeFiles?: boolean; + // Additional properties can be added here in the future +} + interface CheckResult { success: boolean; script: string; @@ -51,18 +57,26 @@ let logger: ToolingLog; void run(async ({ log, flagsReader }) => { logger = log; - const scriptsToRun = collectScriptsToRun({ + const checksToRun = collectScriptsToRun({ targetFile: flagsReader.string('file'), targetDir: flagsReader.string('dir'), checks: flagsReader.string('checks'), - }).map((script) => (isAbsolute(script) ? script : join(REPO_ROOT, script))); + }); + + // Partition checks based on mayChangeFiles flag + const fileChangingChecks = checksToRun + .filter((check) => check.mayChangeFiles) + .map((check) => (isAbsolute(check.script) ? check.script : join(REPO_ROOT, check.script))); + + const regularChecks = checksToRun + .filter((check) => !check.mayChangeFiles) + .map((check) => (isAbsolute(check.script) ? check.script : join(REPO_ROOT, check.script))); logger.write( - `--- Running ${scriptsToRun.length} checks, with parallelism ${MAX_PARALLELISM}...`, - scriptsToRun + `--- Running ${checksToRun.length} checks (${fileChangingChecks.length} file-changing with parallelism=1, ${regularChecks.length} regular with parallelism=${MAX_PARALLELISM})...` ); const startTime = Date.now(); - const results = await runAllChecks(scriptsToRun); + const results = await runPartitionedChecks(fileChangingChecks, regularChecks); logger.write('--- All checks finished.'); printResults(startTime, results); @@ -82,7 +96,7 @@ function collectScriptsToRun(inputOptions: { targetFile: string | undefined; targetDir: string | undefined; checks: string | undefined; -}) { +}): QuickCheck[] { const { targetFile, targetDir, checks } = inputOptions; if ([targetFile, targetDir, checks].filter(Boolean).length > 1) { throw new Error('Only one of --file, --dir, or --checks can be used at a time.'); @@ -90,31 +104,55 @@ function collectScriptsToRun(inputOptions: { if (targetDir) { const targetDirAbsolute = isAbsolute(targetDir) ? targetDir : join(REPO_ROOT, targetDir); - return readdirSync(targetDirAbsolute).map((file) => join(targetDir, file)); + return readdirSync(targetDirAbsolute).map((file) => ({ script: join(targetDir, file) })); } else if (checks) { return checks .trim() .split(/[,\n]/) - .map((script) => script.trim()); + .map((script) => ({ script: script.trim() })); } else { const targetFileWithDefault = targetFile || quickChecksList; const targetFileAbsolute = isAbsolute(targetFileWithDefault) ? targetFileWithDefault : join(REPO_ROOT, targetFileWithDefault); - return readFileSync(targetFileAbsolute, 'utf-8') - .trim() - .split('\n') - .map((line) => line.trim()); + const fileContent = readFileSync(targetFileAbsolute, 'utf-8'); + + // Support both JSON and legacy plain text formats for backward compatibility + if (targetFileAbsolute.endsWith('.json')) { + return JSON.parse(fileContent) as QuickCheck[]; + } else { + // Legacy plain text format + return fileContent + .trim() + .split('\n') + .map((line) => ({ script: line.trim() })); + } } } -async function runAllChecks(scriptsToRun: string[]): Promise { +async function runPartitionedChecks( + fileChangingChecks: string[], + regularChecks: string[] +): Promise { + // Run both partitions concurrently, but with different parallelism + const [fileChangingResults, regularResults] = await Promise.all([ + runAllChecks(fileChangingChecks, 1), // File-changing checks run one at a time + runAllChecks(regularChecks, MAX_PARALLELISM), // Regular checks run with full parallelism + ]); + + return [...fileChangingResults, ...regularResults]; +} + +async function runAllChecks( + scriptsToRun: string[], + parallelism = MAX_PARALLELISM +): Promise { const checksRunning: Array> = []; const checksFinished: CheckResult[] = []; while (scriptsToRun.length > 0 || checksRunning.length > 0) { - while (scriptsToRun.length > 0 && checksRunning.length < MAX_PARALLELISM) { + while (scriptsToRun.length > 0 && checksRunning.length < parallelism) { const script = scriptsToRun.shift(); if (!script) { continue;