diff --git a/.buildkite/ftr_configs.yml b/.buildkite/ftr_configs.yml index 3fd459ffc64c1..ebaeb6c3692b7 100644 --- a/.buildkite/ftr_configs.yml +++ b/.buildkite/ftr_configs.yml @@ -56,7 +56,7 @@ disabled: - x-pack/test/fleet_packages/config.ts # Scalability testing config that we run in its own pipeline - - x-pack/test/performance/scalability/config.ts + - x-pack/test/scalability/config.ts defaultQueue: 'n2-4-spot' enabled: @@ -267,10 +267,10 @@ enabled: - x-pack/test/ui_capabilities/spaces_only/config.ts - x-pack/test/upgrade_assistant_integration/config.js - x-pack/test/usage_collection/config.ts - - x-pack/test/performance/journeys/ecommerce_dashboard/config.ts - - x-pack/test/performance/journeys/flight_dashboard/config.ts - - x-pack/test/performance/journeys/login/config.ts - - x-pack/test/performance/journeys/many_fields_discover/config.ts - - x-pack/test/performance/journeys/promotion_tracking_dashboard/config.ts - - x-pack/test/performance/journeys/web_logs_dashboard/config.ts - - x-pack/test/performance/journeys/data_stress_test_lens/config.ts + - x-pack/performance/journeys/ecommerce_dashboard.ts + - x-pack/performance/journeys/flight_dashboard.ts + - x-pack/performance/journeys/login.ts + - x-pack/performance/journeys/many_fields_discover.ts + - x-pack/performance/journeys/promotion_tracking_dashboard.ts + - x-pack/performance/journeys/web_logs_dashboard.ts + - x-pack/performance/journeys/data_stress_test_lens.ts diff --git a/.buildkite/pipelines/performance/daily.yml b/.buildkite/pipelines/performance/daily.yml index 3e0a813adce49..10f137a5c5088 100644 --- a/.buildkite/pipelines/performance/daily.yml +++ b/.buildkite/pipelines/performance/daily.yml @@ -1,19 +1,19 @@ steps: - - label: ':male-mechanic::skin-tone-2: Pre-Build' + - label: '๐Ÿ‘จโ€๐Ÿ”ง Pre-Build' command: .buildkite/scripts/lifecycle/pre_build.sh agents: queue: kibana-default - wait - - label: ':factory_worker: Build Kibana Distribution and Plugins' + - label: '๐Ÿง‘โ€๐Ÿญ Build Kibana Distribution and Plugins' command: .buildkite/scripts/steps/build_kibana.sh agents: queue: c2-16 key: build if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''" - - label: ':muscle: Performance Tests with Playwright config' + - label: '๐Ÿ’ช Performance Tests with Playwright config' command: .buildkite/scripts/steps/functional/performance_playwright.sh agents: queue: kb-static-ubuntu @@ -21,13 +21,13 @@ steps: key: tests timeout_in_minutes: 60 - - label: ':shipit: Performance Tests dataset extraction for scalability benchmarking' + - label: '๐Ÿšข Performance Tests dataset extraction for scalability benchmarking' command: .buildkite/scripts/steps/functional/scalability_dataset_extraction.sh agents: queue: n2-2 depends_on: tests - - label: ':chart_with_upwards_trend: Report performance metrics to ci-stats' + - label: '๐Ÿ“ˆ Report performance metrics to ci-stats' command: .buildkite/scripts/steps/functional/report_performance_metrics.sh agents: queue: n2-2 @@ -36,7 +36,7 @@ steps: - wait: ~ continue_on_failure: true - - label: ':male_superhero::skin-tone-2: Post-Build' + - label: '๐Ÿฆธ Post-Build' command: .buildkite/scripts/lifecycle/post_build.sh agents: queue: kibana-default diff --git a/.buildkite/pull_requests.json b/.buildkite/pull_requests.json index 1c6e1ae3ce7bc..027c2de8bf915 100644 --- a/.buildkite/pull_requests.json +++ b/.buildkite/pull_requests.json @@ -42,7 +42,12 @@ "kibana_versions_check": true, "kibana_build_reuse": true, "kibana_build_reuse_pipeline_slugs": ["kibana-pull-request", "kibana-on-merge"], - "kibana_build_reuse_regexes": ["^test/", "^x-pack/test/"] + "kibana_build_reuse_regexes": [ + "^test/", + "^x-pack/test/", + "/__snapshots__/", + "\\.test\\.(ts|tsx|js|jsx)" + ] } ] } diff --git a/.buildkite/scripts/steps/functional/performance_playwright.sh b/.buildkite/scripts/steps/functional/performance_playwright.sh index adab313e4c382..cdf2e449f7a6b 100644 --- a/.buildkite/scripts/steps/functional/performance_playwright.sh +++ b/.buildkite/scripts/steps/functional/performance_playwright.sh @@ -4,16 +4,33 @@ set -euo pipefail source .buildkite/scripts/common/util.sh +is_test_execution_step + .buildkite/scripts/bootstrap.sh # These tests are running on static workers so we have to make sure we delete previous build of Kibana rm -rf "$KIBANA_BUILD_LOCATION" .buildkite/scripts/download_build_artifacts.sh -echo --- Run Performance Tests with Playwright config +echo "--- ๐Ÿฆบ Starting Elasticsearch" node scripts/es snapshot& +export esPid=$! +trap 'kill ${esPid}' EXIT -esPid=$! +export TEST_ES_URL=http://elastic:changeme@localhost:9200 +export TEST_ES_DISABLE_STARTUP=true + +# Pings the es server every second for up to 2 minutes until it is green +curl \ + --fail \ + --silent \ + --retry 120 \ + --retry-delay 1 \ + --retry-connrefused \ + -XGET "${TEST_ES_URL}/_cluster/health?wait_for_nodes=>=1&wait_for_status=yellow" \ + > /dev/null + +echo "โœ… ES is ready and will continue to run in the background" # unset env vars defined in other parts of CI for automatic APM collection of # Kibana. We manage APM config in our FTR config and performance service, and @@ -29,39 +46,27 @@ unset ELASTIC_APM_SERVER_URL unset ELASTIC_APM_SECRET_TOKEN unset ELASTIC_APM_GLOBAL_LABELS - -export TEST_ES_URL=http://elastic:changeme@localhost:9200 -export TEST_ES_DISABLE_STARTUP=true - -# Pings the es server every seconds 2 mins until it is status is green -curl --retry 120 \ - --retry-delay 1 \ - --retry-all-errors \ - -I -XGET "${TEST_ES_URL}/_cluster/health?wait_for_nodes=>=1&wait_for_status=yellow" - -journeys=("login" "ecommerce_dashboard" "flight_dashboard" "web_logs_dashboard" "promotion_tracking_dashboard" "many_fields_discover" "data_stress_test_lens") - -for i in "${journeys[@]}"; do - echo "JOURNEY[${i}] is running" - - export TEST_PERFORMANCE_PHASE=WARMUP - export JOURNEY_NAME="${i}" - - checks-reporter-with-killswitch "Run Performance Tests with Playwright Config (Journey:${i},Phase: WARMUP)" \ - node scripts/functional_tests \ - --config "x-pack/test/performance/journeys/${i}/config.ts" \ - --kibana-install-dir "$KIBANA_BUILD_LOCATION" \ - --debug \ - --bail - - export TEST_PERFORMANCE_PHASE=TEST - - checks-reporter-with-killswitch "Run Performance Tests with Playwright Config (Journey:${i},Phase: TEST)" \ - node scripts/functional_tests \ - --config "x-pack/test/performance/journeys/${i}/config.ts" \ - --kibana-install-dir "$KIBANA_BUILD_LOCATION" \ - --debug \ - --bail +for journey in x-pack/performance/journeys/*; do + set +e + + phases=("WARMUP" "TEST") + for phase in "${phases[@]}"; do + echo "--- $journey - $phase" + + export TEST_PERFORMANCE_PHASE="$phase" + node scripts/functional_tests \ + --config "$journey" \ + --kibana-install-dir "$KIBANA_BUILD_LOCATION" \ + --debug \ + --bail + + status=$? + if [ $status -ne 0 ]; then + echo "^^^ +++" + echo "โŒ FTR failed with status code: $status" + exit 1 + fi + done + + set -e done - -kill "$esPid" diff --git a/.buildkite/scripts/steps/functional/scalability_dataset_extraction.sh b/.buildkite/scripts/steps/functional/scalability_dataset_extraction.sh index a8711c8d2f58a..a2b81f538b92b 100755 --- a/.buildkite/scripts/steps/functional/scalability_dataset_extraction.sh +++ b/.buildkite/scripts/steps/functional/scalability_dataset_extraction.sh @@ -15,19 +15,16 @@ OUTPUT_DIR="${KIBANA_DIR}/${OUTPUT_REL}" .buildkite/scripts/bootstrap.sh echo "--- Extract APM metrics" -scalabilityJourneys=("login" "ecommerce_dashboard" "flight_dashboard" "web_logs_dashboard" "promotion_tracking_dashboard" "many_fields_discover") - -for i in "${scalabilityJourneys[@]}"; do - JOURNEY_NAME="${i}" - echo "Looking for JOURNEY=${JOURNEY_NAME} and BUILD_ID=${BUILD_ID} in APM traces" - - node scripts/extract_performance_testing_dataset \ - --config "x-pack/test/performance/journeys/${i}/config.ts" \ - --buildId "${BUILD_ID}" \ - --es-url "${ES_SERVER_URL}" \ - --es-username "${USER_FROM_VAULT}" \ - --es-password "${PASS_FROM_VAULT}" \ - --without-static-resources +for journey in x-pack/performance/journeys/*; do + echo "Looking for journey=${journey} and BUILD_ID=${BUILD_ID} in APM traces" + + node scripts/extract_performance_testing_dataset \ + --config "${journey}" \ + --buildId "${BUILD_ID}" \ + --es-url "${ES_SERVER_URL}" \ + --es-username "${USER_FROM_VAULT}" \ + --es-password "${PASS_FROM_VAULT}" \ + --without-static-resources done echo "--- Creating scalability dataset in ${OUTPUT_REL}" diff --git a/.eslintrc.js b/.eslintrc.js index c36e8b5e7e668..df107348cfafc 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -178,6 +178,7 @@ const DEV_PATTERNS = [ 'x-pack/{dev-tools,tasks,scripts,test,build_chromium}/**/*', 'x-pack/plugins/*/server/scripts/**/*', 'x-pack/plugins/fleet/cypress', + 'x-pack/performance/**/*', ]; /** Restricted imports with suggested alternatives */ diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 93cf5cc23dde2..81d4dd81f0cfa 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -860,9 +860,12 @@ packages/kbn-eslint-plugin-disable @elastic/kibana-operations packages/kbn-eslint-plugin-eslint @elastic/kibana-operations packages/kbn-eslint-plugin-imports @elastic/kibana-operations packages/kbn-expect @elastic/kibana-operations +packages/kbn-failed-test-reporter-cli @elastic/kibana-operations packages/kbn-field-types @elastic/kibana-app-services packages/kbn-find-used-node-modules @elastic/kibana-operations packages/kbn-flot-charts @elastic/kibana-operations +packages/kbn-ftr-common-functional-services @elastic/kibana-operations +packages/kbn-ftr-screenshot-filename @elastic/kibana-operations packages/kbn-generate @elastic/kibana-operations packages/kbn-get-repo-files @elastic/kibana-operations packages/kbn-handlebars @elastic/kibana-security @@ -873,6 +876,7 @@ packages/kbn-import-resolver @elastic/kibana-operations packages/kbn-interpreter @elastic/kibana-app-services packages/kbn-io-ts-utils @elastic/apm-ui packages/kbn-jest-serializers @elastic/kibana-operations +packages/kbn-journeys @elastic/kibana-operations packages/kbn-kibana-manifest-schema @elastic/kibana-operations packages/kbn-logging @elastic/kibana-core packages/kbn-logging-mocks @elastic/kibana-core diff --git a/dev_docs/operations/operations_landing.mdx b/dev_docs/operations/operations_landing.mdx index 88e4be9eb93a3..9004141255f58 100644 --- a/dev_docs/operations/operations_landing.mdx +++ b/dev_docs/operations/operations_landing.mdx @@ -26,6 +26,7 @@ layout: landing { pageId: "kibDevDocsOpsWritingStableFunctionalTests" }, { pageId: "kibDevDocsOpsFlakyTestRunner" }, { pageId: "kibDevDocsOpsCiStats" }, + { pageId: "kibDevDocsOpsJourneys" }, ]} /> diff --git a/package.json b/package.json index c257e1a46fc9a..7c8a27447d4c0 100644 --- a/package.json +++ b/package.json @@ -702,11 +702,15 @@ "@kbn/eslint-plugin-eslint": "link:bazel-bin/packages/kbn-eslint-plugin-eslint", "@kbn/eslint-plugin-imports": "link:bazel-bin/packages/kbn-eslint-plugin-imports", "@kbn/expect": "link:bazel-bin/packages/kbn-expect", + "@kbn/failed-test-reporter-cli": "link:bazel-bin/packages/kbn-failed-test-reporter-cli", "@kbn/find-used-node-modules": "link:bazel-bin/packages/kbn-find-used-node-modules", + "@kbn/ftr-common-functional-services": "link:bazel-bin/packages/kbn-ftr-common-functional-services", + "@kbn/ftr-screenshot-filename": "link:bazel-bin/packages/kbn-ftr-screenshot-filename", "@kbn/generate": "link:bazel-bin/packages/kbn-generate", "@kbn/get-repo-files": "link:bazel-bin/packages/kbn-get-repo-files", "@kbn/import-resolver": "link:bazel-bin/packages/kbn-import-resolver", "@kbn/jest-serializers": "link:bazel-bin/packages/kbn-jest-serializers", + "@kbn/journeys": "link:bazel-bin/packages/kbn-journeys", "@kbn/kibana-manifest-schema": "link:bazel-bin/packages/kbn-kibana-manifest-schema", "@kbn/managed-vscode-config": "link:bazel-bin/packages/kbn-managed-vscode-config", "@kbn/managed-vscode-config-cli": "link:bazel-bin/packages/kbn-managed-vscode-config-cli", @@ -1017,8 +1021,11 @@ "@types/kbn__es-types": "link:bazel-bin/packages/kbn-es-types/npm_module_types", "@types/kbn__eslint-plugin-disable": "link:bazel-bin/packages/kbn-eslint-plugin-disable/npm_module_types", "@types/kbn__eslint-plugin-imports": "link:bazel-bin/packages/kbn-eslint-plugin-imports/npm_module_types", + "@types/kbn__failed-test-reporter-cli": "link:bazel-bin/packages/kbn-failed-test-reporter-cli/npm_module_types", "@types/kbn__field-types": "link:bazel-bin/packages/kbn-field-types/npm_module_types", "@types/kbn__find-used-node-modules": "link:bazel-bin/packages/kbn-find-used-node-modules/npm_module_types", + "@types/kbn__ftr-common-functional-services": "link:bazel-bin/packages/kbn-ftr-common-functional-services/npm_module_types", + "@types/kbn__ftr-screenshot-filename": "link:bazel-bin/packages/kbn-ftr-screenshot-filename/npm_module_types", "@types/kbn__generate": "link:bazel-bin/packages/kbn-generate/npm_module_types", "@types/kbn__get-repo-files": "link:bazel-bin/packages/kbn-get-repo-files/npm_module_types", "@types/kbn__handlebars": "link:bazel-bin/packages/kbn-handlebars/npm_module_types", @@ -1032,6 +1039,7 @@ "@types/kbn__interpreter": "link:bazel-bin/packages/kbn-interpreter/npm_module_types", "@types/kbn__io-ts-utils": "link:bazel-bin/packages/kbn-io-ts-utils/npm_module_types", "@types/kbn__jest-serializers": "link:bazel-bin/packages/kbn-jest-serializers/npm_module_types", + "@types/kbn__journeys": "link:bazel-bin/packages/kbn-journeys/npm_module_types", "@types/kbn__kbn-ci-stats-performance-metrics": "link:bazel-bin/packages/kbn-kbn-ci-stats-performance-metrics/npm_module_types", "@types/kbn__kibana-manifest-schema": "link:bazel-bin/packages/kbn-kibana-manifest-schema/npm_module_types", "@types/kbn__logging": "link:bazel-bin/packages/kbn-logging/npm_module_types", diff --git a/packages/BUILD.bazel b/packages/BUILD.bazel index 56ef73801d5a9..d6994772c9ef2 100644 --- a/packages/BUILD.bazel +++ b/packages/BUILD.bazel @@ -204,9 +204,12 @@ filegroup( "//packages/kbn-eslint-plugin-eslint:build", "//packages/kbn-eslint-plugin-imports:build", "//packages/kbn-expect:build", + "//packages/kbn-failed-test-reporter-cli:build", "//packages/kbn-field-types:build", "//packages/kbn-find-used-node-modules:build", "//packages/kbn-flot-charts:build", + "//packages/kbn-ftr-common-functional-services:build", + "//packages/kbn-ftr-screenshot-filename:build", "//packages/kbn-generate:build", "//packages/kbn-get-repo-files:build", "//packages/kbn-handlebars:build", @@ -217,6 +220,7 @@ filegroup( "//packages/kbn-interpreter:build", "//packages/kbn-io-ts-utils:build", "//packages/kbn-jest-serializers:build", + "//packages/kbn-journeys:build", "//packages/kbn-kibana-manifest-schema:build", "//packages/kbn-logging:build", "//packages/kbn-logging-mocks:build", @@ -514,8 +518,11 @@ filegroup( "//packages/kbn-es-types:build_types", "//packages/kbn-eslint-plugin-disable:build_types", "//packages/kbn-eslint-plugin-imports:build_types", + "//packages/kbn-failed-test-reporter-cli:build_types", "//packages/kbn-field-types:build_types", "//packages/kbn-find-used-node-modules:build_types", + "//packages/kbn-ftr-common-functional-services:build_types", + "//packages/kbn-ftr-screenshot-filename:build_types", "//packages/kbn-generate:build_types", "//packages/kbn-get-repo-files:build_types", "//packages/kbn-handlebars:build_types", @@ -526,6 +533,7 @@ filegroup( "//packages/kbn-interpreter:build_types", "//packages/kbn-io-ts-utils:build_types", "//packages/kbn-jest-serializers:build_types", + "//packages/kbn-journeys:build_types", "//packages/kbn-kibana-manifest-schema:build_types", "//packages/kbn-logging:build_types", "//packages/kbn-logging-mocks:build_types", diff --git a/packages/kbn-dev-cli-runner/index.ts b/packages/kbn-dev-cli-runner/index.ts index aa56ab0e976ce..0bc2b64c64d7c 100644 --- a/packages/kbn-dev-cli-runner/index.ts +++ b/packages/kbn-dev-cli-runner/index.ts @@ -9,4 +9,5 @@ export * from './src/run'; export * from './src/run_with_commands'; export * from './src/flags'; +export * from './src/flags_reader'; export type { CleanupTask } from './src/cleanup'; diff --git a/packages/kbn-dev-cli-runner/src/flags.ts b/packages/kbn-dev-cli-runner/src/flags.ts index 919da586f7ba6..595205c3e0333 100644 --- a/packages/kbn-dev-cli-runner/src/flags.ts +++ b/packages/kbn-dev-cli-runner/src/flags.ts @@ -53,6 +53,10 @@ export function mergeFlagOptions(global: FlagOptions = {}, local: FlagOptions = }; } +export const DEFAULT_FLAG_ALIASES = { + v: 'verbose', +}; + export function getFlags( argv: string[], flagOptions: RunOptions['flags'] = {}, @@ -67,7 +71,7 @@ export function getFlags( boolean: [...(flagOptions.boolean || []), ...logLevelFlags, 'help'], alias: { ...flagOptions.alias, - v: 'verbose', + ...DEFAULT_FLAG_ALIASES, }, default: flagOptions.default, unknown: (name: string) => { diff --git a/packages/kbn-dev-cli-runner/src/flags_reader.test.ts b/packages/kbn-dev-cli-runner/src/flags_reader.test.ts new file mode 100644 index 0000000000000..bef3339c5b27a --- /dev/null +++ b/packages/kbn-dev-cli-runner/src/flags_reader.test.ts @@ -0,0 +1,344 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { createAbsolutePathSerializer } from '@kbn/jest-serializers'; + +import { getFlags } from './flags'; +import { FlagsReader } from './flags_reader'; + +const FLAGS = { + string: 'string', + astring: ['foo', 'bar'], + num: '1234', + bool: true, + missing: undefined, +}; + +const basic = new FlagsReader(FLAGS); + +expect.addSnapshotSerializer(createAbsolutePathSerializer()); + +describe('#string()', () => { + it('returns a single string, regardless of flag count', () => { + expect(basic.string('string')).toMatchInlineSnapshot(`"string"`); + expect(basic.string('astring')).toBe(FLAGS.astring.at(-1)); + }); + + it('returns undefined when flag is missing', () => { + expect(basic.string('missing')).toMatchInlineSnapshot(`undefined`); + }); + + it('throws for non-string flags', () => { + expect(() => basic.string('bool')).toThrowErrorMatchingInlineSnapshot( + `"expected --bool to be a string"` + ); + }); + + describe('required version', () => { + it('throws when flag is missing', () => { + expect(() => basic.requiredString('missing')).toThrowErrorMatchingInlineSnapshot( + `"missing required flag --missing"` + ); + }); + }); +}); + +describe('#arrayOfStrings()', () => { + it('returns an array of strings for string flags, regardless of count', () => { + expect(basic.arrayOfStrings('string')).toMatchInlineSnapshot(` + Array [ + "string", + ] + `); + expect(basic.arrayOfStrings('astring')).toMatchInlineSnapshot(` + Array [ + "foo", + "bar", + ] + `); + }); + + it('returns undefined when flag is missing', () => { + expect(basic.arrayOfStrings('missing')).toMatchInlineSnapshot(`undefined`); + }); + + it('throws for non-string flags', () => { + expect(() => basic.arrayOfStrings('bool')).toThrowErrorMatchingInlineSnapshot( + `"expected --bool to be a string"` + ); + }); + + describe('required version', () => { + it('throws when flag is missing', () => { + expect(() => basic.requiredArrayOfStrings('missing')).toThrowErrorMatchingInlineSnapshot( + `"missing required flag --missing"` + ); + }); + }); +}); + +describe('#enum()', () => { + it('validates that values match options', () => { + expect(basic.enum('string', ['a', 'string', 'b'])).toMatchInlineSnapshot(`"string"`); + expect(basic.enum('missing', ['a', 'b'])).toMatchInlineSnapshot(`undefined`); + expect(() => basic.enum('string', ['a', 'b'])).toThrowErrorMatchingInlineSnapshot( + `"invalid --string, expected one of \\"a\\", \\"b\\""` + ); + }); +}); + +describe('#path()', () => { + it('parses the string to an absolute path based on CWD', () => { + expect(basic.path('string')).toMatchInlineSnapshot(`/string`); + expect(basic.path('missing')).toMatchInlineSnapshot(`undefined`); + }); + + describe('required version', () => { + it('throws if the flag is missing', () => { + expect(() => basic.requiredPath('missing')).toThrowErrorMatchingInlineSnapshot( + `"missing required flag --missing"` + ); + }); + }); + + describe('array version', () => { + it('parses a list of paths', () => { + expect(basic.arrayOfPaths('astring')).toMatchInlineSnapshot(` + Array [ + /foo, + /bar, + ] + `); + }); + + describe('required version', () => { + it('throws if the flag is missing', () => { + expect(() => basic.requiredArrayOfPaths('missing')).toThrowErrorMatchingInlineSnapshot( + `"missing required flag --missing"` + ); + }); + }); + }); +}); + +describe('#number()', () => { + it('parses strings as numbers', () => { + expect(basic.number('num')).toMatchInlineSnapshot(`1234`); + expect(basic.number('missing')).toMatchInlineSnapshot(`undefined`); + expect(() => basic.number('bool')).toThrowErrorMatchingInlineSnapshot( + `"expected --bool to be a string"` + ); + expect(() => basic.number('string')).toThrowErrorMatchingInlineSnapshot( + `"unable to parse --string value [string] as a number"` + ); + expect(() => basic.number('astring')).toThrowErrorMatchingInlineSnapshot( + `"unable to parse --astring value [bar] as a number"` + ); + }); + + describe('required version', () => { + it('throws if the flag is missing', () => { + expect(() => basic.requiredNumber('missing')).toThrowErrorMatchingInlineSnapshot( + `"missing required flag --missing"` + ); + }); + }); +}); + +describe('#boolean()', () => { + it('ensures flag is boolean, requires value', () => { + expect(basic.boolean('bool')).toMatchInlineSnapshot(`true`); + expect(() => basic.boolean('missing')).toThrowErrorMatchingInlineSnapshot( + `"expected --missing to be a boolean"` + ); + expect(() => basic.boolean('string')).toThrowErrorMatchingInlineSnapshot( + `"expected --string to be a boolean"` + ); + expect(() => basic.boolean('astring')).toThrowErrorMatchingInlineSnapshot( + `"expected --astring to be a boolean"` + ); + }); +}); + +describe('#getPositionals()', () => { + it('returns all positional arguments in flags', () => { + const flags = new FlagsReader({ + ...FLAGS, + _: ['a', 'b', 'c'], + }); + + expect(flags.getPositionals()).toMatchInlineSnapshot(` + Array [ + "a", + "b", + "c", + ] + `); + }); + + it('handles missing _ flag', () => { + const flags = new FlagsReader({}); + expect(flags.getPositionals()).toMatchInlineSnapshot(`Array []`); + }); +}); + +describe('#getUnused()', () => { + it('returns a map of all unused flags', () => { + const flags = new FlagsReader({ + a: '1', + b: '2', + c: '3', + }); + + expect(flags.getUnused()).toMatchInlineSnapshot(` + Map { + "a" => "1", + "b" => "2", + "c" => "3", + } + `); + + flags.number('a'); + flags.number('b'); + + expect(flags.getUnused()).toMatchInlineSnapshot(` + Map { + "c" => "3", + } + `); + }); + + it('ignores the default flags which are forced on commands', () => { + const rawFlags = getFlags(['--a=1'], { + string: ['a'], + }); + + const flags = new FlagsReader(rawFlags, { + aliases: { + v: 'verbose', + }, + }); + + expect(flags.getUnused()).toMatchInlineSnapshot(` + Map { + "a" => "1", + } + `); + flags.number('a'); + expect(flags.getUnused()).toMatchInlineSnapshot(`Map {}`); + }); + + it('treats aliased flags as used', () => { + const flags = new FlagsReader( + { + f: true, + force: true, + v: true, + verbose: true, + }, + { + aliases: { + f: 'force', + v: 'verbose', + }, + } + ); + + expect(flags.getUnused()).toMatchInlineSnapshot(` + Map { + "f" => true, + "force" => true, + } + `); + flags.boolean('force'); + expect(flags.getUnused()).toMatchInlineSnapshot(`Map {}`); + flags.boolean('v'); + expect(flags.getUnused()).toMatchInlineSnapshot(`Map {}`); + }); + + it('treats failed reads as "uses"', () => { + const flags = new FlagsReader({ a: 'b' }); + + expect(flags.getUnused()).toMatchInlineSnapshot(` + Map { + "a" => "b", + } + `); + expect(() => flags.number('a')).toThrowError(); + expect(flags.getUnused()).toMatchInlineSnapshot(`Map {}`); + }); +}); + +describe('#getUsed()', () => { + it('returns a map of all used flags', () => { + const flags = new FlagsReader({ + a: '1', + b: '2', + c: '3', + }); + + expect(flags.getUsed()).toMatchInlineSnapshot(`Map {}`); + + flags.number('a'); + flags.number('b'); + + expect(flags.getUsed()).toMatchInlineSnapshot(` + Map { + "a" => "1", + "b" => "2", + } + `); + }); + + it('treats aliases flags as used', () => { + const flags = new FlagsReader( + { + f: true, + force: true, + v: true, + verbose: true, + }, + { + aliases: { + f: 'force', + v: 'verbose', + }, + } + ); + + expect(flags.getUsed()).toMatchInlineSnapshot(`Map {}`); + flags.boolean('force'); + expect(flags.getUsed()).toMatchInlineSnapshot(` + Map { + "force" => true, + "f" => true, + } + `); + flags.boolean('v'); + expect(flags.getUsed()).toMatchInlineSnapshot(` + Map { + "force" => true, + "f" => true, + "v" => true, + "verbose" => true, + } + `); + }); + + it('treats failed reads as "uses"', () => { + const flags = new FlagsReader({ a: 'b' }); + + expect(flags.getUsed()).toMatchInlineSnapshot(`Map {}`); + expect(() => flags.number('a')).toThrowError(); + expect(flags.getUsed()).toMatchInlineSnapshot(` + Map { + "a" => "b", + } + `); + }); +}); diff --git a/packages/kbn-dev-cli-runner/src/flags_reader.ts b/packages/kbn-dev-cli-runner/src/flags_reader.ts new file mode 100644 index 0000000000000..156f1a4fba32b --- /dev/null +++ b/packages/kbn-dev-cli-runner/src/flags_reader.ts @@ -0,0 +1,267 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; + +import { createFlagError } from '@kbn/dev-cli-errors'; +import { LOG_LEVEL_FLAGS } from '@kbn/tooling-log'; + +type FlagValue = string | string[] | boolean; +const FORCED_FLAGS = new Set([...LOG_LEVEL_FLAGS.map((l) => l.name), 'help']); + +const makeAbsolute = (rel: string) => Path.resolve(process.cwd(), rel); + +const nonUndefinedValues = (e: [string, FlagValue | undefined]): e is [string, FlagValue] => + e[1] !== undefined; + +export class FlagsReader { + private readonly used: Map; + private readonly unused: Map; + private readonly _: string[]; + private readonly aliasMap: Map; + + constructor( + flags: Record, + private readonly opts?: { aliases?: Record } + ) { + this.used = new Map(); + this.unused = new Map( + Object.entries(flags) + .filter(nonUndefinedValues) + .filter((e) => e[0] !== 'unexpected') + ); + this.aliasMap = new Map( + Object.entries(this.opts?.aliases ?? []).flatMap(([a, b]) => [ + [a, b], + [b, a], + ]) + ); + + this._ = this.arrayOfStrings('_') ?? []; + } + + private use(key: string) { + const alias = this.aliasMap.get(key); + + const used = this.used.get(key); + if (used !== undefined) { + return used; + } + + const unused = this.unused.get(key); + if (unused !== undefined) { + this.used.set(key, unused); + this.unused.delete(key); + + if (alias !== undefined) { + this.used.set(alias, unused); + this.unused.delete(alias); + } + } + + return unused; + } + + /** + * Read a string flag that supports multiple instances into an array of strings. If the + * flag is only passed once an array with a single item will be returned. If the flag is not + * passed then undefined will be returned. + */ + arrayOfStrings(key: string) { + const value = this.use(key); + + switch (typeof value) { + case 'boolean': + throw createFlagError(`expected --${key} to be a string`); + case 'string': + return value ? [value] : []; + default: + return value; + } + } + + /** + * Same as #arrayOfStrings() except when the flag is not passed a "flag error" is thrown telling + * the user that the flag is required and shows them the help text. + */ + requiredArrayOfStrings(key: string) { + const value = this.arrayOfStrings(key); + if (value === undefined) { + throw createFlagError(`missing required flag --${key}`); + } + return value; + } + + /** + * Read the value of a string flag. If the flag is passed multiple times the last value is returned. If + * the flag is not passed then undefined is returned. + */ + string(key: string) { + const value = this.use(key); + + switch (typeof value) { + case 'undefined': + return undefined; + case 'string': + return value || undefined; // convert "" to undefined + case 'object': + const last = value.at(-1); + if (last === undefined) { + throw createFlagError(`expected --${key} to be a string`); + } + return last || undefined; // convert "" to undefined + default: + throw createFlagError(`expected --${key} to be a string`); + } + } + + /** + * Same as #string() except when the flag is passed it is validated against a list + * of valid values + */ + enum(key: string, values: readonly T[]) { + const value = this.string(key); + if (value === undefined) { + return; + } + + if (values.includes(value as T)) { + return value as T; + } + + throw createFlagError(`invalid --${key}, expected one of "${values.join('", "')}"`); + } + + /** + * Same as #string() except when a flag is not passed a "flag error" is thrown telling the user + * that the flag is required and shows them the help text. + */ + requiredString(key: string) { + const value = this.string(key); + if (value === undefined) { + throw createFlagError(`missing required flag --${key}`); + } + return value; + } + + /** + * Same as #string(), except that when there is a value for the string it is resolved to an + * absolute path based on the current working directory + */ + path(key: string) { + const value = this.string(key); + if (value !== undefined) { + return makeAbsolute(value); + } + } + + /** + * Same as #requiredString() except that values are converted to absolute paths based on the + * current working directory + */ + requiredPath(key: string) { + return makeAbsolute(this.requiredString(key)); + } + + /** + * Same as #arrayOfStrings(), except that when there are values they are resolved to + * absolute paths based on the current working directory + */ + arrayOfPaths(key: string) { + const value = this.arrayOfStrings(key); + if (value !== undefined) { + return value.map(makeAbsolute); + } + } + + /** + * Same as #requiredArrayOfStrings(), except that values are resolved to absolute paths + * based on the current working directory + */ + requiredArrayOfPaths(key: string) { + return this.requiredArrayOfStrings(key).map(makeAbsolute); + } + + /** + * Parsed the provided flag as a number, if the value does not parse to a valid number + * using Number.parseFloat() then a "flag error" is thrown. If the flag is not passed + * undefined is returned. + */ + number(key: string) { + const value = this.string(key); + if (value === undefined) { + return; + } + + const num = Number.parseFloat(value); + if (Number.isNaN(num)) { + throw createFlagError(`unable to parse --${key} value [${value}] as a number`); + } + + return num; + } + + /** + * Same as #number() except that when the flag is missing a "flag error" is thrown + */ + requiredNumber(key: string) { + const value = this.number(key); + if (value === undefined) { + throw createFlagError(`missing required flag --${key}`); + } + return value; + } + + /** + * Read a boolean flag value, if the flag is properly defined as a "boolean" in the run options + * then the value will always be a boolean, defaulting to `false`, so there is no need for an + * optional/requiredBoolean() method. + */ + boolean(key: string) { + const value = this.use(key); + if (typeof value !== 'boolean') { + throw createFlagError(`expected --${key} to be a boolean`); + } + return value; + } + + /** + * Get the positional arguments passed, includes any values that are not associated with + * a specific --flag + */ + getPositionals() { + return this._.slice(0); + } + + /** + * Returns all of the unused flags. When a flag is read via any of the key-specific methods + * the key is marked as "used" and this method will return a map of just the flags which + * have not been used yet (excluding the default flags like --debug, --verbose, and --help) + */ + getUnused() { + return new Map( + [...this.unused.entries()].filter(([key]) => { + const alias = this.aliasMap.get(key); + if (alias !== undefined && FORCED_FLAGS.has(alias)) { + return false; + } + + return !FORCED_FLAGS.has(key); + }) + ); + } + + /** + * Returns all of the used flags. When a flag is read via any of the key-specific methods + * the key is marked as "used" and from then on this method will return a map including that + * and any other key used by these methods. + */ + getUsed() { + return new Map(this.used); + } +} diff --git a/packages/kbn-dev-cli-runner/src/run.ts b/packages/kbn-dev-cli-runner/src/run.ts index bbccfdde564f8..08457caaebfd4 100644 --- a/packages/kbn-dev-cli-runner/src/run.ts +++ b/packages/kbn-dev-cli-runner/src/run.ts @@ -10,7 +10,8 @@ import { pickLevelFromFlags, ToolingLog, LogLevel } from '@kbn/tooling-log'; import { ProcRunner, withProcRunner } from '@kbn/dev-proc-runner'; import { createFlagError } from '@kbn/dev-cli-errors'; -import { Flags, getFlags, FlagOptions } from './flags'; +import { Flags, getFlags, FlagOptions, DEFAULT_FLAG_ALIASES } from './flags'; +import { FlagsReader } from './flags_reader'; import { getHelp } from './help'; import { CleanupTask, Cleanup } from './cleanup'; import { Metrics, MetricsMeta } from './metrics'; @@ -21,6 +22,7 @@ export interface RunContext { procRunner: ProcRunner; statsMeta: MetricsMeta; addCleanupTask: (task: CleanupTask) => void; + flagsReader: FlagsReader; } export type RunFn = (context: RunContext) => Promise | void; @@ -71,6 +73,12 @@ export async function run(fn: RunFn, options: RunOptions = {}) { procRunner, statsMeta: metrics.meta, addCleanupTask: cleanup.add.bind(cleanup), + flagsReader: new FlagsReader(flags, { + aliases: { + ...options.flags?.alias, + ...DEFAULT_FLAG_ALIASES, + }, + }), }); }); } catch (error) { diff --git a/packages/kbn-dev-cli-runner/src/run_with_commands.test.ts b/packages/kbn-dev-cli-runner/src/run_with_commands.test.ts index c740087b40c30..329e858b08f5e 100644 --- a/packages/kbn-dev-cli-runner/src/run_with_commands.test.ts +++ b/packages/kbn-dev-cli-runner/src/run_with_commands.test.ts @@ -9,6 +9,7 @@ import { ToolingLog, ToolingLogCollectingWriter } from '@kbn/tooling-log'; import { ProcRunner } from '@kbn/dev-proc-runner'; +import { FlagsReader } from './flags_reader'; import { RunWithCommands } from './run_with_commands'; const testLog = new ToolingLog(); @@ -44,6 +45,7 @@ it('extends the context using extendContext()', async () => { expect(context).toEqual({ log: expect.any(ToolingLog), flags: expect.any(Object), + flagsReader: expect.any(FlagsReader), addCleanupTask: expect.any(Function), procRunner: expect.any(ProcRunner), statsMeta: expect.any(Map), diff --git a/packages/kbn-dev-cli-runner/src/run_with_commands.ts b/packages/kbn-dev-cli-runner/src/run_with_commands.ts index 94b167671d21b..ff93f29f4c631 100644 --- a/packages/kbn-dev-cli-runner/src/run_with_commands.ts +++ b/packages/kbn-dev-cli-runner/src/run_with_commands.ts @@ -11,7 +11,8 @@ import { withProcRunner } from '@kbn/dev-proc-runner'; import { createFlagError } from '@kbn/dev-cli-errors'; import { RunContext, RunOptions } from './run'; -import { getFlags, FlagOptions, mergeFlagOptions } from './flags'; +import { getFlags, FlagOptions, mergeFlagOptions, DEFAULT_FLAG_ALIASES } from './flags'; +import { FlagsReader } from './flags_reader'; import { Cleanup } from './cleanup'; import { getHelpForAllCommands, getCommandLevelHelp } from './help'; import { Metrics } from './metrics'; @@ -116,6 +117,12 @@ export class RunWithCommands { procRunner, statsMeta: metrics.meta, addCleanupTask: cleanup.add.bind(cleanup), + flagsReader: new FlagsReader(commandFlags, { + aliases: { + ...commandFlagOptions.alias, + ...DEFAULT_FLAG_ALIASES, + }, + }), }; const extendedContext = { diff --git a/packages/kbn-failed-test-reporter-cli/BUILD.bazel b/packages/kbn-failed-test-reporter-cli/BUILD.bazel new file mode 100644 index 0000000000000..a3ae8903169a3 --- /dev/null +++ b/packages/kbn-failed-test-reporter-cli/BUILD.bazel @@ -0,0 +1,145 @@ +load("@npm//@bazel/typescript:index.bzl", "ts_config") +load("@build_bazel_rules_nodejs//:index.bzl", "js_library") +load("//src/dev/bazel:index.bzl", "jsts_transpiler", "pkg_npm", "pkg_npm_types", "ts_project") + +PKG_DIRNAME = "kbn-failed-test-reporter-cli" +PKG_REQUIRE_NAME = "@kbn/failed-test-reporter-cli" + +SOURCE_FILES = glob( + [ + "**/*.ts", + "**/*.html", + ], + exclude = [ + "**/*.config.js", + "**/*.mock.*", + "**/*.test.*", + "**/*.stories.*", + "**/__snapshots__/**", + "**/integration_tests/**", + "**/mocks/**", + "**/scripts/**", + "**/storybook/**", + "**/test_fixtures/**", + "**/test_helpers/**", + ], +) + +SRCS = SOURCE_FILES + +filegroup( + name = "srcs", + srcs = SRCS, +) + +NPM_MODULE_EXTRA_FILES = [ + "package.json", +] + +# In this array place runtime dependencies, including other packages and NPM packages +# which must be available for this code to run. +# +# To reference other packages use: +# "//repo/relative/path/to/package" +# eg. "//packages/kbn-utils" +# +# To reference a NPM package use: +# "@npm//name-of-package" +# eg. "@npm//lodash" +RUNTIME_DEPS = [ +] + +# In this array place dependencies necessary to build the types, which will include the +# :npm_module_types target of other packages and packages from NPM, including @types/* +# packages. +# +# To reference the types for another package use: +# "//repo/relative/path/to/package:npm_module_types" +# eg. "//packages/kbn-utils:npm_module_types" +# +# References to NPM packages work the same as RUNTIME_DEPS +TYPES_DEPS = [ + "//packages/kbn-utils:npm_module_types", + "//packages/kbn-ci-stats-reporter:npm_module_types", + "//packages/kbn-dev-cli-runner:npm_module_types", + "//packages/kbn-dev-cli-errors:npm_module_types", + "//packages/kbn-dev-utils:npm_module_types", + "//packages/kbn-tooling-log:npm_module_types", + "//packages/kbn-ftr-screenshot-filename:npm_module_types", + "//packages/kbn-jest-serializers:npm_module_types", + "//packages/kbn-journeys:npm_module_types", + "@npm//@elastic/elasticsearch", + "@npm//@types/node", + "@npm//@types/he", + "@npm//@types/jest", + "@npm//@types/strip-ansi", + "@npm//@types/normalize-path", + "@npm//@types/xml2js", + "@npm//axios", + "@npm//dedent", + "@npm//globby", +] + +jsts_transpiler( + name = "target_node", + srcs = SRCS, + build_pkg_name = package_name(), + additional_args = [ + "--copy-files" + ], +) + +ts_config( + name = "tsconfig", + src = "tsconfig.json", + deps = [ + "//:tsconfig.base.json", + "//:tsconfig.bazel.json", + ], +) + +ts_project( + name = "tsc_types", + args = ['--pretty'], + srcs = SRCS, + deps = TYPES_DEPS, + declaration = True, + declaration_map = True, + emit_declaration_only = True, + out_dir = "target_types", + tsconfig = ":tsconfig", +) + +js_library( + name = PKG_DIRNAME, + srcs = NPM_MODULE_EXTRA_FILES, + deps = RUNTIME_DEPS + [":target_node"], + package_name = PKG_REQUIRE_NAME, + visibility = ["//visibility:public"], +) + +pkg_npm( + name = "npm_module", + deps = [":" + PKG_DIRNAME], +) + +filegroup( + name = "build", + srcs = [":npm_module"], + visibility = ["//visibility:public"], +) + +pkg_npm_types( + name = "npm_module_types", + srcs = SRCS, + deps = [":tsc_types"], + package_name = PKG_REQUIRE_NAME, + tsconfig = ":tsconfig", + visibility = ["//visibility:public"], +) + +filegroup( + name = "build_types", + srcs = [":npm_module_types"], + visibility = ["//visibility:public"], +) diff --git a/packages/kbn-failed-test-reporter-cli/README.md b/packages/kbn-failed-test-reporter-cli/README.md new file mode 100644 index 0000000000000..d577a58dfb856 --- /dev/null +++ b/packages/kbn-failed-test-reporter-cli/README.md @@ -0,0 +1,3 @@ +# @kbn/failed-test-reporter-cli + +Empty package generated by @kbn/generate diff --git a/packages/kbn-test/src/failed_tests_reporter/README.md b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/README.md similarity index 100% rename from packages/kbn-test/src/failed_tests_reporter/README.md rename to packages/kbn-failed-test-reporter-cli/failed_tests_reporter/README.md diff --git a/packages/kbn-test/src/failed_tests_reporter/__fixtures__/cypress_report.xml b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/__fixtures__/cypress_report.xml similarity index 100% rename from packages/kbn-test/src/failed_tests_reporter/__fixtures__/cypress_report.xml rename to packages/kbn-failed-test-reporter-cli/failed_tests_reporter/__fixtures__/cypress_report.xml diff --git a/packages/kbn-test/src/failed_tests_reporter/__fixtures__/ftr_report.xml b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/__fixtures__/ftr_report.xml similarity index 100% rename from packages/kbn-test/src/failed_tests_reporter/__fixtures__/ftr_report.xml rename to packages/kbn-failed-test-reporter-cli/failed_tests_reporter/__fixtures__/ftr_report.xml diff --git a/packages/kbn-test/src/failed_tests_reporter/__fixtures__/index.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/__fixtures__/index.ts similarity index 100% rename from packages/kbn-test/src/failed_tests_reporter/__fixtures__/index.ts rename to packages/kbn-failed-test-reporter-cli/failed_tests_reporter/__fixtures__/index.ts diff --git a/packages/kbn-test/src/failed_tests_reporter/__fixtures__/jest_report.xml b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/__fixtures__/jest_report.xml similarity index 100% rename from packages/kbn-test/src/failed_tests_reporter/__fixtures__/jest_report.xml rename to packages/kbn-failed-test-reporter-cli/failed_tests_reporter/__fixtures__/jest_report.xml diff --git a/packages/kbn-test/src/failed_tests_reporter/__fixtures__/mocha_report.xml b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/__fixtures__/mocha_report.xml similarity index 100% rename from packages/kbn-test/src/failed_tests_reporter/__fixtures__/mocha_report.xml rename to packages/kbn-failed-test-reporter-cli/failed_tests_reporter/__fixtures__/mocha_report.xml diff --git a/packages/kbn-test/src/failed_tests_reporter/add_messages_to_report.test.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/add_messages_to_report.test.ts similarity index 100% rename from packages/kbn-test/src/failed_tests_reporter/add_messages_to_report.test.ts rename to packages/kbn-failed-test-reporter-cli/failed_tests_reporter/add_messages_to_report.test.ts diff --git a/packages/kbn-test/src/failed_tests_reporter/add_messages_to_report.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/add_messages_to_report.ts similarity index 100% rename from packages/kbn-test/src/failed_tests_reporter/add_messages_to_report.ts rename to packages/kbn-failed-test-reporter-cli/failed_tests_reporter/add_messages_to_report.ts diff --git a/packages/kbn-test/src/failed_tests_reporter/buildkite_metadata.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/buildkite_metadata.ts similarity index 100% rename from packages/kbn-test/src/failed_tests_reporter/buildkite_metadata.ts rename to packages/kbn-failed-test-reporter-cli/failed_tests_reporter/buildkite_metadata.ts diff --git a/packages/kbn-test/src/failed_tests_reporter/es_config b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/es_config similarity index 100% rename from packages/kbn-test/src/failed_tests_reporter/es_config rename to packages/kbn-failed-test-reporter-cli/failed_tests_reporter/es_config diff --git a/packages/kbn-test/src/failed_tests_reporter/existing_failed_test_issues.test.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/existing_failed_test_issues.test.ts similarity index 100% rename from packages/kbn-test/src/failed_tests_reporter/existing_failed_test_issues.test.ts rename to packages/kbn-failed-test-reporter-cli/failed_tests_reporter/existing_failed_test_issues.test.ts diff --git a/packages/kbn-test/src/failed_tests_reporter/existing_failed_test_issues.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/existing_failed_test_issues.ts similarity index 100% rename from packages/kbn-test/src/failed_tests_reporter/existing_failed_test_issues.ts rename to packages/kbn-failed-test-reporter-cli/failed_tests_reporter/existing_failed_test_issues.ts diff --git a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts new file mode 100644 index 0000000000000..b105b6d80ac37 --- /dev/null +++ b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts @@ -0,0 +1,208 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; + +import { REPO_ROOT } from '@kbn/utils'; +import { run } from '@kbn/dev-cli-runner'; +import { createFailError, createFlagError } from '@kbn/dev-cli-errors'; +import { CiStatsReporter } from '@kbn/ci-stats-reporter'; +import globby from 'globby'; +import normalize from 'normalize-path'; + +import { getFailures } from './get_failures'; +import { GithubApi } from './github_api'; +import { updateFailureIssue, createFailureIssue } from './report_failure'; +import { readTestReport, getRootMetadata } from './test_report'; +import { addMessagesToReport } from './add_messages_to_report'; +import { getReportMessageIter } from './report_metadata'; +import { reportFailuresToEs } from './report_failures_to_es'; +import { reportFailuresToFile } from './report_failures_to_file'; +import { getBuildkiteMetadata } from './buildkite_metadata'; +import { ExistingFailedTestIssues } from './existing_failed_test_issues'; + +const DEFAULT_PATTERNS = [Path.resolve(REPO_ROOT, 'target/junit/**/*.xml')]; +const DISABLE_MISSING_TEST_REPORT_ERRORS = + process.env.DISABLE_MISSING_TEST_REPORT_ERRORS === 'true'; + +run( + async ({ log, flags }) => { + const indexInEs = flags['index-errors']; + + let updateGithub = flags['github-update']; + if (updateGithub && !process.env.GITHUB_TOKEN) { + throw createFailError( + 'GITHUB_TOKEN environment variable must be set, otherwise use --no-github-update flag' + ); + } + + let branch: string = ''; + if (updateGithub) { + let isPr = false; + + if (process.env.BUILDKITE === 'true') { + branch = process.env.BUILDKITE_BRANCH || ''; + isPr = process.env.BUILDKITE_PULL_REQUEST === 'true'; + updateGithub = process.env.REPORT_FAILED_TESTS_TO_GITHUB === 'true'; + } else { + // JOB_NAME is formatted as `elastic+kibana+7.x` in some places and `elastic+kibana+7.x/JOB=kibana-intake,node=immutable` in others + const jobNameSplit = (process.env.JOB_NAME || '').split(/\+|\//); + branch = jobNameSplit.length >= 3 ? jobNameSplit[2] : process.env.GIT_BRANCH || ''; + isPr = !!process.env.ghprbPullId; + + const isMainOrVersion = branch === 'main' || branch.match(/^\d+\.(x|\d+)$/); + if (!isMainOrVersion || isPr) { + log.info('Failure issues only created on main/version branch jobs'); + updateGithub = false; + } + } + + if (!branch) { + throw createFailError( + 'Unable to determine originating branch from job name or other environment variables' + ); + } + } + + const githubApi = new GithubApi({ + log, + token: process.env.GITHUB_TOKEN, + dryRun: !updateGithub, + }); + + const bkMeta = getBuildkiteMetadata(); + + try { + const buildUrl = flags['build-url'] || (updateGithub ? '' : 'http://buildUrl'); + if (typeof buildUrl !== 'string' || !buildUrl) { + throw createFlagError('Missing --build-url or process.env.BUILD_URL'); + } + + const patterns = (flags._.length ? flags._ : DEFAULT_PATTERNS).map((p) => + normalize(Path.resolve(p)) + ); + log.info('Searching for reports at', patterns); + const reportPaths = await globby(patterns, { + absolute: true, + }); + + if (!reportPaths.length && DISABLE_MISSING_TEST_REPORT_ERRORS) { + // it is fine for code coverage to not have test results + return; + } + + if (!reportPaths.length) { + throw createFailError(`Unable to find any junit reports with patterns [${patterns}]`); + } + + log.info('found', reportPaths.length, 'junit reports', reportPaths); + + const existingIssues = new ExistingFailedTestIssues(log); + for (const reportPath of reportPaths) { + const report = await readTestReport(reportPath); + const messages = Array.from(getReportMessageIter(report)); + const failures = getFailures(report); + + await existingIssues.loadForFailures(failures); + + if (indexInEs) { + await reportFailuresToEs(log, failures); + } + + for (const failure of failures) { + const pushMessage = (msg: string) => { + messages.push({ + classname: failure.classname, + name: failure.name, + message: msg, + }); + }; + + if (failure.likelyIrrelevant) { + pushMessage( + 'Failure is likely irrelevant' + + (updateGithub ? ', so an issue was not created or updated' : '') + ); + continue; + } + + const existingIssue = existingIssues.getForFailure(failure); + if (existingIssue) { + const { newBody, newCount } = await updateFailureIssue( + buildUrl, + existingIssue, + githubApi, + branch + ); + const url = existingIssue.github.htmlUrl; + existingIssue.github.body = newBody; + failure.githubIssue = url; + failure.failureCount = updateGithub ? newCount : newCount - 1; + pushMessage(`Test has failed ${newCount - 1} times on tracked branches: ${url}`); + if (updateGithub) { + pushMessage(`Updated existing issue: ${url} (fail count: ${newCount})`); + } + continue; + } + + const newIssue = await createFailureIssue(buildUrl, failure, githubApi, branch); + existingIssues.addNewlyCreated(failure, newIssue); + pushMessage('Test has not failed recently on tracked branches'); + if (updateGithub) { + pushMessage(`Created new issue: ${newIssue.html_url}`); + failure.githubIssue = newIssue.html_url; + } + failure.failureCount = updateGithub ? 1 : 0; + } + + // mutates report to include messages and writes updated report to disk + await addMessagesToReport({ + report, + messages, + log, + reportPath, + dryRun: !flags['report-update'], + }); + + await reportFailuresToFile(log, failures, bkMeta, getRootMetadata(report)); + } + } finally { + await CiStatsReporter.fromEnv(log).metrics([ + { + group: 'github api request count', + id: `failed test reporter`, + value: githubApi.getRequestCount(), + meta: Object.fromEntries( + Object.entries(bkMeta).map( + ([k, v]) => [`buildkite${k[0].toUpperCase()}${k.slice(1)}`, v] as const + ) + ), + }, + ]); + } + }, + { + description: `a cli that opens issues or updates existing issues based on junit reports`, + flags: { + boolean: ['github-update', 'report-update'], + string: ['build-url'], + default: { + 'github-update': true, + 'report-update': true, + 'index-errors': true, + 'build-url': process.env.BUILD_URL, + }, + help: ` + --no-github-update Execute the CLI without writing to Github + --no-report-update Execute the CLI without writing to the JUnit reports + --no-index-errors Execute the CLI without indexing failures into Elasticsearch + --build-url URL of the failed build, defaults to process.env.BUILD_URL + `, + }, + } +); diff --git a/packages/kbn-test/src/failed_tests_reporter/get_failures.test.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/get_failures.test.ts similarity index 100% rename from packages/kbn-test/src/failed_tests_reporter/get_failures.test.ts rename to packages/kbn-failed-test-reporter-cli/failed_tests_reporter/get_failures.test.ts diff --git a/packages/kbn-test/src/failed_tests_reporter/get_failures.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/get_failures.ts similarity index 100% rename from packages/kbn-test/src/failed_tests_reporter/get_failures.ts rename to packages/kbn-failed-test-reporter-cli/failed_tests_reporter/get_failures.ts diff --git a/packages/kbn-test/src/failed_tests_reporter/github_api.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/github_api.ts similarity index 100% rename from packages/kbn-test/src/failed_tests_reporter/github_api.ts rename to packages/kbn-failed-test-reporter-cli/failed_tests_reporter/github_api.ts diff --git a/packages/kbn-test/src/failed_tests_reporter/issue_metadata.test.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/issue_metadata.test.ts similarity index 100% rename from packages/kbn-test/src/failed_tests_reporter/issue_metadata.test.ts rename to packages/kbn-failed-test-reporter-cli/failed_tests_reporter/issue_metadata.test.ts diff --git a/packages/kbn-test/src/failed_tests_reporter/issue_metadata.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/issue_metadata.ts similarity index 100% rename from packages/kbn-test/src/failed_tests_reporter/issue_metadata.ts rename to packages/kbn-failed-test-reporter-cli/failed_tests_reporter/issue_metadata.ts diff --git a/packages/kbn-test/src/failed_tests_reporter/report_failure.test.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/report_failure.test.ts similarity index 100% rename from packages/kbn-test/src/failed_tests_reporter/report_failure.test.ts rename to packages/kbn-failed-test-reporter-cli/failed_tests_reporter/report_failure.test.ts diff --git a/packages/kbn-test/src/failed_tests_reporter/report_failure.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/report_failure.ts similarity index 100% rename from packages/kbn-test/src/failed_tests_reporter/report_failure.ts rename to packages/kbn-failed-test-reporter-cli/failed_tests_reporter/report_failure.ts diff --git a/packages/kbn-test/src/failed_tests_reporter/report_failures_to_es.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/report_failures_to_es.ts similarity index 100% rename from packages/kbn-test/src/failed_tests_reporter/report_failures_to_es.ts rename to packages/kbn-failed-test-reporter-cli/failed_tests_reporter/report_failures_to_es.ts diff --git a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/report_failures_to_file.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/report_failures_to_file.ts new file mode 100644 index 0000000000000..d34df80f3d0a8 --- /dev/null +++ b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/report_failures_to_file.ts @@ -0,0 +1,199 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; +import Fs from 'fs'; +import { createHash } from 'crypto'; + +import globby from 'globby'; +import { ToolingLog } from '@kbn/tooling-log'; +import { REPO_ROOT } from '@kbn/utils'; +import { escape } from 'he'; +import { FtrScreenshotFilename } from '@kbn/ftr-screenshot-filename'; +import { JourneyScreenshots } from '@kbn/journeys'; + +import { BuildkiteMetadata } from './buildkite_metadata'; +import { TestFailure } from './get_failures'; + +interface JourneyMeta { + journeyName: string; +} +function getJourneyMetadata(rootMeta: Record): JourneyMeta | undefined { + const { journeyName } = rootMeta; + if (typeof journeyName === 'string') { + return { journeyName }; + } + + return undefined; +} + +async function getJourneySnapshotHtml(log: ToolingLog, journeyMeta: JourneyMeta) { + let screenshots; + try { + screenshots = await JourneyScreenshots.load(journeyMeta.journeyName); + } catch (error) { + log.error(`Failed to load journey screenshots: ${error.message}`); + return ''; + } + + return [ + '
', + '
Steps
', + ...screenshots.get().flatMap(({ title, path }) => { + const base64 = Fs.readFileSync(path, 'base64'); + + return [ + `

${escape(title)}

`, + ``, + ]; + }), + '
', + ].join('\n'); +} + +let _allScreenshotsCache: Array<{ path: string; name: string }> | undefined; +function getAllScreenshots(log: ToolingLog) { + return (_allScreenshotsCache ??= findAllScreenshots(log)); +} +function findAllScreenshots(log: ToolingLog) { + try { + return globby + .sync( + [ + 'test/functional/**/screenshots/failure/*.png', + 'x-pack/test/functional/**/screenshots/failure/*.png', + ], + { + cwd: REPO_ROOT, + absolute: true, + } + ) + .map((path) => ({ + path, + name: Path.basename(path, Path.extname(path)), + })); + } catch (error) { + log.error(`Failed to find screenshots: ${error.message}`); + return []; + } +} + +function getFtrScreenshotHtml(log: ToolingLog, failureName: string) { + return getAllScreenshots(log) + .filter((s) => s.name.startsWith(FtrScreenshotFilename.create(failureName, { ext: false }))) + .map((s) => { + const base64 = Fs.readFileSync(s.path).toString('base64'); + return ``; + }) + .join('\n'); +} + +export async function reportFailuresToFile( + log: ToolingLog, + failures: TestFailure[], + bkMeta: BuildkiteMetadata, + rootMeta: Record +) { + if (!failures?.length) { + return; + } + + const journeyMeta = getJourneyMetadata(rootMeta); + + // Jest could, in theory, fail 1000s of tests and write 1000s of failures + // So let's just write files for the first 20 + for (const failure of failures.slice(0, 20)) { + const hash = createHash('md5').update(failure.name).digest('hex'); + const filenameBase = `${ + process.env.BUILDKITE_JOB_ID ? process.env.BUILDKITE_JOB_ID + '_' : '' + }${hash}`; + const dir = Path.join('target', 'test_failures'); + + const failureLog = [ + ['Test:', '-----', failure.classname, failure.name, ''], + ['Failure:', '--------', failure.failure], + failure['system-out'] ? ['', 'Standard Out:', '-------------', failure['system-out']] : [], + ] + .flat() + .join('\n'); + + const failureJSON = JSON.stringify( + { + ...failure, + hash, + buildId: bkMeta.buildId, + jobId: bkMeta.jobId, + url: bkMeta.url, + jobUrl: bkMeta.jobUrl, + jobName: bkMeta.jobName, + }, + null, + 2 + ); + + const failureHTML = Fs.readFileSync( + require.resolve('./report_failures_to_file_html_template.html') + ) + .toString() + .replace('$TITLE', escape(failure.name)) + .replace( + '$MAIN', + ` + ${failure.classname + .split('.') + .map((part) => `
${escape(part.replace('ยท', '.'))}
`) + .join('')} +
+

${escape(failure.name)}

+

+ + Failures in tracked branches: ${ + failure.failureCount || 0 + } + ${ + failure.githubIssue + ? `
${escape( + failure.githubIssue + )}` + : '' + } + +

+ ${ + bkMeta.jobUrl + ? `

+ + Buildkite Job
+ ${escape(bkMeta.jobUrl)} + +

` + : '' + } + 
${escape(failure.failure)}
+ ${ + journeyMeta + ? await getJourneySnapshotHtml(log, journeyMeta) + : getFtrScreenshotHtml(log, failure.name) + } + ${ + failure['system-out'] + ? ` +
Stdout
+ 
${escape(failure['system-out'] || '')}
+ ` + : '' + } + ` + ); + + Fs.mkdirSync(dir, { recursive: true }); + Fs.writeFileSync(Path.join(dir, `${filenameBase}.log`), failureLog, 'utf8'); + Fs.writeFileSync(Path.join(dir, `${filenameBase}.html`), failureHTML, 'utf8'); + Fs.writeFileSync(Path.join(dir, `${filenameBase}.json`), failureJSON, 'utf8'); + } +} diff --git a/packages/kbn-test/src/failed_tests_reporter/report_failures_to_file_html_template.html b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/report_failures_to_file_html_template.html similarity index 100% rename from packages/kbn-test/src/failed_tests_reporter/report_failures_to_file_html_template.html rename to packages/kbn-failed-test-reporter-cli/failed_tests_reporter/report_failures_to_file_html_template.html diff --git a/packages/kbn-test/src/failed_tests_reporter/report_metadata.test.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/report_metadata.test.ts similarity index 100% rename from packages/kbn-test/src/failed_tests_reporter/report_metadata.test.ts rename to packages/kbn-failed-test-reporter-cli/failed_tests_reporter/report_metadata.test.ts diff --git a/packages/kbn-test/src/failed_tests_reporter/report_metadata.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/report_metadata.ts similarity index 100% rename from packages/kbn-test/src/failed_tests_reporter/report_metadata.ts rename to packages/kbn-failed-test-reporter-cli/failed_tests_reporter/report_metadata.ts diff --git a/packages/kbn-test/src/failed_tests_reporter/test_report.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/test_report.ts similarity index 84% rename from packages/kbn-test/src/failed_tests_reporter/test_report.ts rename to packages/kbn-failed-test-reporter-cli/failed_tests_reporter/test_report.ts index 9c83d77b19a99..e70aa44a2a088 100644 --- a/packages/kbn-test/src/failed_tests_reporter/test_report.ts +++ b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/test_report.ts @@ -35,6 +35,8 @@ export interface TestSuite { failures: string; /* number of skipped tests as a string */ skipped: string; + /* optional JSON encoded metadata */ + 'metadata-json'?: string; }; testcase?: TestCase[]; } @@ -93,3 +95,22 @@ export function* makeFailedTestCaseIter(report: TestReport) { yield testCase as FailedTestCase; } } + +export function getRootMetadata(report: TestReport): Record { + const json = + ('testsuites' in report + ? report.testsuites?.testsuite?.[0]?.$?.['metadata-json'] + : report.testsuite?.$?.['metadata-json']) ?? '{}'; + + try { + const obj = JSON.parse(json); + + if (typeof obj === 'object' && obj !== null) { + return obj; + } + + return {}; + } catch { + return {}; + } +} diff --git a/packages/kbn-test/src/failed_tests_reporter/index.ts b/packages/kbn-failed-test-reporter-cli/index.ts similarity index 82% rename from packages/kbn-test/src/failed_tests_reporter/index.ts rename to packages/kbn-failed-test-reporter-cli/index.ts index b750cf44348e1..999da20da72f5 100644 --- a/packages/kbn-test/src/failed_tests_reporter/index.ts +++ b/packages/kbn-failed-test-reporter-cli/index.ts @@ -6,4 +6,4 @@ * Side Public License, v 1. */ -export { runFailedTestsReporterCli } from './run_failed_tests_reporter_cli'; +import './failed_tests_reporter/failed_tests_reporter_cli'; diff --git a/packages/kbn-test/src/functional_tests/cli/index.js b/packages/kbn-failed-test-reporter-cli/jest.config.js similarity index 56% rename from packages/kbn-test/src/functional_tests/cli/index.js rename to packages/kbn-failed-test-reporter-cli/jest.config.js index 9721d70d12262..eb33f488f9e84 100644 --- a/packages/kbn-test/src/functional_tests/cli/index.js +++ b/packages/kbn-failed-test-reporter-cli/jest.config.js @@ -6,7 +6,8 @@ * Side Public License, v 1. */ -export { runTestsCli } from './run_tests/cli'; -export { processOptions as processRunTestsCliOptions } from './run_tests/args'; -export { startServersCli } from './start_servers/cli'; -export { processOptions as processStartServersCliOptions } from './start_servers/args'; +module.exports = { + preset: '@kbn/test/jest_node', + rootDir: '../..', + roots: ['/packages/kbn-failed-test-reporter-cli'], +}; diff --git a/packages/kbn-failed-test-reporter-cli/kibana.jsonc b/packages/kbn-failed-test-reporter-cli/kibana.jsonc new file mode 100644 index 0000000000000..dfaa875e12735 --- /dev/null +++ b/packages/kbn-failed-test-reporter-cli/kibana.jsonc @@ -0,0 +1,8 @@ +{ + "type": "shared-common", + "id": "@kbn/failed-test-reporter-cli", + "owner": "@elastic/kibana-operations", + "devOnly": true, + "runtimeDeps": [], + "typeDeps": [], +} diff --git a/packages/kbn-failed-test-reporter-cli/package.json b/packages/kbn-failed-test-reporter-cli/package.json new file mode 100644 index 0000000000000..daf9a58cd77d7 --- /dev/null +++ b/packages/kbn-failed-test-reporter-cli/package.json @@ -0,0 +1,7 @@ +{ + "name": "@kbn/failed-test-reporter-cli", + "private": true, + "version": "1.0.0", + "main": "./target_node/index.js", + "license": "SSPL-1.0 OR Elastic License 2.0" +} diff --git a/packages/kbn-failed-test-reporter-cli/tsconfig.json b/packages/kbn-failed-test-reporter-cli/tsconfig.json new file mode 100644 index 0000000000000..81935b1385550 --- /dev/null +++ b/packages/kbn-failed-test-reporter-cli/tsconfig.json @@ -0,0 +1,17 @@ +{ + "extends": "../../tsconfig.bazel.json", + "compilerOptions": { + "declaration": true, + "declarationMap": true, + "emitDeclarationOnly": true, + "outDir": "target_types", + "stripInternal": false, + "types": [ + "jest", + "node" + ] + }, + "include": [ + "**/*.ts", + ] +} diff --git a/packages/kbn-ftr-common-functional-services/BUILD.bazel b/packages/kbn-ftr-common-functional-services/BUILD.bazel new file mode 100644 index 0000000000000..8085c75af4af1 --- /dev/null +++ b/packages/kbn-ftr-common-functional-services/BUILD.bazel @@ -0,0 +1,127 @@ +load("@npm//@bazel/typescript:index.bzl", "ts_config") +load("@build_bazel_rules_nodejs//:index.bzl", "js_library") +load("//src/dev/bazel:index.bzl", "jsts_transpiler", "pkg_npm", "pkg_npm_types", "ts_project") + +PKG_DIRNAME = "kbn-ftr-common-functional-services" +PKG_REQUIRE_NAME = "@kbn/ftr-common-functional-services" + +SOURCE_FILES = glob( + [ + "**/*.ts", + ], + exclude = [ + "**/*.config.js", + "**/*.mock.*", + "**/*.test.*", + "**/*.stories.*", + "**/__snapshots__/**", + "**/integration_tests/**", + "**/mocks/**", + "**/scripts/**", + "**/storybook/**", + "**/test_fixtures/**", + "**/test_helpers/**", + ], +) + +SRCS = SOURCE_FILES + +filegroup( + name = "srcs", + srcs = SRCS, +) + +NPM_MODULE_EXTRA_FILES = [ + "package.json", +] + +# In this array place runtime dependencies, including other packages and NPM packages +# which must be available for this code to run. +# +# To reference other packages use: +# "//repo/relative/path/to/package" +# eg. "//packages/kbn-utils" +# +# To reference a NPM package use: +# "@npm//name-of-package" +# eg. "@npm//lodash" +RUNTIME_DEPS = [ +] + +# In this array place dependencies necessary to build the types, which will include the +# :npm_module_types target of other packages and packages from NPM, including @types/* +# packages. +# +# To reference the types for another package use: +# "//repo/relative/path/to/package:npm_module_types" +# eg. "//packages/kbn-utils:npm_module_types" +# +# References to NPM packages work the same as RUNTIME_DEPS +TYPES_DEPS = [ + "@npm//@types/node", + "@npm//@types/jest", + "//packages/kbn-tooling-log:npm_module_types", + "//packages/kbn-es-archiver:npm_module_types", + "//packages/kbn-test:npm_module_types", +] + +jsts_transpiler( + name = "target_node", + srcs = SRCS, + build_pkg_name = package_name(), +) + +ts_config( + name = "tsconfig", + src = "tsconfig.json", + deps = [ + "//:tsconfig.base.json", + "//:tsconfig.bazel.json", + ], +) + +ts_project( + name = "tsc_types", + args = ['--pretty'], + srcs = SRCS, + deps = TYPES_DEPS, + declaration = True, + declaration_map = True, + emit_declaration_only = True, + out_dir = "target_types", + tsconfig = ":tsconfig", +) + +js_library( + name = PKG_DIRNAME, + srcs = NPM_MODULE_EXTRA_FILES, + deps = RUNTIME_DEPS + [":target_node"], + package_name = PKG_REQUIRE_NAME, + visibility = ["//visibility:public"], +) + +pkg_npm( + name = "npm_module", + deps = [":" + PKG_DIRNAME], +) + +filegroup( + name = "build", + srcs = [":npm_module"], + visibility = ["//visibility:public"], +) + +pkg_npm_types( + name = "npm_module_types", + srcs = SRCS, + deps = [":tsc_types"], + package_name = PKG_REQUIRE_NAME, + tsconfig = ":tsconfig", + visibility = ["//visibility:public"], +) + +filegroup( + name = "build_types", + srcs = [":npm_module_types"], + visibility = ["//visibility:public"], +) diff --git a/packages/kbn-ftr-common-functional-services/README.md b/packages/kbn-ftr-common-functional-services/README.md new file mode 100644 index 0000000000000..a2438327a62f7 --- /dev/null +++ b/packages/kbn-ftr-common-functional-services/README.md @@ -0,0 +1,3 @@ +# @kbn/ftr-common-functional-services + +A collection of very common services used by all functional FTR configs, moved to a package so that we can start putting FTR configs in packages. \ No newline at end of file diff --git a/packages/kbn-ftr-common-functional-services/index.ts b/packages/kbn-ftr-common-functional-services/index.ts new file mode 100644 index 0000000000000..950a860f7553f --- /dev/null +++ b/packages/kbn-ftr-common-functional-services/index.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { ProvidedType } from '@kbn/test'; +export { services as commonFunctionalServices } from './services/all'; + +import { KibanaServerProvider } from './services/kibana_server'; +export type KibanaServer = ProvidedType; + +export { RetryService } from './services/retry'; + +import { EsArchiverProvider } from './services/es_archiver'; +export type EsArchiver = ProvidedType; + +import { EsProvider } from './services/es'; +export type Es = ProvidedType; diff --git a/packages/kbn-ftr-common-functional-services/jest.config.js b/packages/kbn-ftr-common-functional-services/jest.config.js new file mode 100644 index 0000000000000..1831bdb22630d --- /dev/null +++ b/packages/kbn-ftr-common-functional-services/jest.config.js @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +module.exports = { + preset: '@kbn/test/jest_node', + rootDir: '../..', + roots: ['/packages/kbn-ftr-common-functional-services'], +}; diff --git a/packages/kbn-ftr-common-functional-services/kibana.jsonc b/packages/kbn-ftr-common-functional-services/kibana.jsonc new file mode 100644 index 0000000000000..5ceecdcda8610 --- /dev/null +++ b/packages/kbn-ftr-common-functional-services/kibana.jsonc @@ -0,0 +1,8 @@ +{ + "type": "shared-common", + "id": "@kbn/ftr-common-functional-services", + "owner": "@elastic/kibana-operations", + "devOnly": true, + "runtimeDeps": [], + "typeDeps": [], +} diff --git a/packages/kbn-ftr-common-functional-services/package.json b/packages/kbn-ftr-common-functional-services/package.json new file mode 100644 index 0000000000000..642a5a39c7141 --- /dev/null +++ b/packages/kbn-ftr-common-functional-services/package.json @@ -0,0 +1,7 @@ +{ + "name": "@kbn/ftr-common-functional-services", + "private": true, + "version": "1.0.0", + "main": "./target_node/index.js", + "license": "SSPL-1.0 OR Elastic License 2.0" +} diff --git a/packages/kbn-ftr-common-functional-services/services/all.ts b/packages/kbn-ftr-common-functional-services/services/all.ts new file mode 100644 index 0000000000000..14019caaa582c --- /dev/null +++ b/packages/kbn-ftr-common-functional-services/services/all.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { EsArchiverProvider } from './es_archiver'; +import { EsProvider } from './es'; +import { KibanaServerProvider } from './kibana_server'; +import { RetryService } from './retry'; + +export const services = { + es: EsProvider, + kibanaServer: KibanaServerProvider, + esArchiver: EsArchiverProvider, + retry: RetryService, +}; diff --git a/test/common/services/elasticsearch.ts b/packages/kbn-ftr-common-functional-services/services/es.ts similarity index 75% rename from test/common/services/elasticsearch.ts rename to packages/kbn-ftr-common-functional-services/services/es.ts index 2f19bfe9105d0..fe9aafbf10736 100644 --- a/test/common/services/elasticsearch.ts +++ b/packages/kbn-ftr-common-functional-services/services/es.ts @@ -9,12 +9,9 @@ import { Client } from '@elastic/elasticsearch'; import { systemIndicesSuperuser, createEsClientForFtrConfig } from '@kbn/test'; -import { FtrProviderContext } from '../ftr_provider_context'; +import { FtrProviderContext } from './ftr_provider_context'; -/* - registers Kibana-specific @elastic/elasticsearch client instance. - */ -export function ElasticsearchProvider({ getService }: FtrProviderContext): Client { +export function EsProvider({ getService }: FtrProviderContext): Client { const config = getService('config'); return createEsClientForFtrConfig(config, { diff --git a/test/common/services/es_archiver.ts b/packages/kbn-ftr-common-functional-services/services/es_archiver.ts similarity index 61% rename from test/common/services/es_archiver.ts rename to packages/kbn-ftr-common-functional-services/services/es_archiver.ts index 865c2ba4b4434..8a81297bf1784 100644 --- a/test/common/services/es_archiver.ts +++ b/packages/kbn-ftr-common-functional-services/services/es_archiver.ts @@ -7,17 +7,15 @@ */ import { EsArchiver } from '@kbn/es-archiver'; -import { FtrProviderContext } from '../ftr_provider_context'; -import * as KibanaServer from './kibana_server'; +import { FtrProviderContext } from './ftr_provider_context'; +import { extendEsArchiver } from './kibana_server'; export function EsArchiverProvider({ getService }: FtrProviderContext): EsArchiver { const config = getService('config'); const client = getService('es'); - const lifecycle = getService('lifecycle'); const log = getService('log'); const kibanaServer = getService('kibanaServer'); const retry = getService('retry'); - const esArchives: string[] = config.get('testData.esArchives'); const esArchiver = new EsArchiver({ client, @@ -25,26 +23,12 @@ export function EsArchiverProvider({ getService }: FtrProviderContext): EsArchiv kbnClient: kibanaServer, }); - KibanaServer.extendEsArchiver({ + extendEsArchiver({ esArchiver, kibanaServer, retry, defaults: config.get('uiSettings.defaults'), }); - if (esArchives.length) { - lifecycle.beforeTests.add(async () => { - for (const archive of esArchives) { - await esArchiver.load(archive); - } - }); - - lifecycle.cleanup.add(async () => { - for (const archive of esArchives) { - await esArchiver.unload(archive); - } - }); - } - return esArchiver; } diff --git a/packages/kbn-ftr-common-functional-services/services/ftr_provider_context.ts b/packages/kbn-ftr-common-functional-services/services/ftr_provider_context.ts new file mode 100644 index 0000000000000..979658fbd8edd --- /dev/null +++ b/packages/kbn-ftr-common-functional-services/services/ftr_provider_context.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { GenericFtrProviderContext, GenericFtrService } from '@kbn/test'; + +import type { services } from './all'; + +type Services = typeof services; + +export type FtrProviderContext = GenericFtrProviderContext; +export class FtrService extends GenericFtrService {} diff --git a/test/common/services/kibana_server/extend_es_archiver.ts b/packages/kbn-ftr-common-functional-services/services/kibana_server/extend_es_archiver.ts similarity index 100% rename from test/common/services/kibana_server/extend_es_archiver.ts rename to packages/kbn-ftr-common-functional-services/services/kibana_server/extend_es_archiver.ts diff --git a/test/common/services/kibana_server/index.ts b/packages/kbn-ftr-common-functional-services/services/kibana_server/index.ts similarity index 100% rename from test/common/services/kibana_server/index.ts rename to packages/kbn-ftr-common-functional-services/services/kibana_server/index.ts diff --git a/test/common/services/kibana_server/kibana_server.ts b/packages/kbn-ftr-common-functional-services/services/kibana_server/kibana_server.ts similarity index 69% rename from test/common/services/kibana_server/kibana_server.ts rename to packages/kbn-ftr-common-functional-services/services/kibana_server/kibana_server.ts index 182b289ed1d8e..bdfc42670f18b 100644 --- a/test/common/services/kibana_server/kibana_server.ts +++ b/packages/kbn-ftr-common-functional-services/services/kibana_server/kibana_server.ts @@ -9,7 +9,7 @@ import Url from 'url'; import { KbnClient } from '@kbn/test'; -import { FtrProviderContext } from '../../ftr_provider_context'; +import { FtrProviderContext } from '../ftr_provider_context'; export function KibanaServerProvider({ getService }: FtrProviderContext): KbnClient { const log = getService('log'); @@ -17,7 +17,6 @@ export function KibanaServerProvider({ getService }: FtrProviderContext): KbnCli const lifecycle = getService('lifecycle'); const url = Url.format(config.get('servers.kibana')); const defaults = config.get('uiSettings.defaults'); - const kbnArchives: string[] = config.get('testData.kbnArchives'); const kbn = new KbnClient({ log, @@ -32,18 +31,5 @@ export function KibanaServerProvider({ getService }: FtrProviderContext): KbnCli }); } - if (kbnArchives.length) { - lifecycle.beforeTests.add(async () => { - for (const archive of kbnArchives) { - await kbn.importExport.load(archive); - } - }); - lifecycle.cleanup.add(async () => { - for (const archive of kbnArchives) { - await kbn.importExport.unload(archive); - } - }); - } - return kbn; } diff --git a/test/common/services/retry/index.ts b/packages/kbn-ftr-common-functional-services/services/retry/index.ts similarity index 100% rename from test/common/services/retry/index.ts rename to packages/kbn-ftr-common-functional-services/services/retry/index.ts diff --git a/test/common/services/retry/retry.ts b/packages/kbn-ftr-common-functional-services/services/retry/retry.ts similarity index 96% rename from test/common/services/retry/retry.ts rename to packages/kbn-ftr-common-functional-services/services/retry/retry.ts index 5c823e256ddc8..231a829225dbc 100644 --- a/test/common/services/retry/retry.ts +++ b/packages/kbn-ftr-common-functional-services/services/retry/retry.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -import { FtrService } from '../../ftr_provider_context'; +import { FtrService } from '../ftr_provider_context'; import { retryForSuccess } from './retry_for_success'; import { retryForTruthy } from './retry_for_truthy'; diff --git a/test/common/services/retry/retry_for_success.ts b/packages/kbn-ftr-common-functional-services/services/retry/retry_for_success.ts similarity index 100% rename from test/common/services/retry/retry_for_success.ts rename to packages/kbn-ftr-common-functional-services/services/retry/retry_for_success.ts diff --git a/test/common/services/retry/retry_for_truthy.ts b/packages/kbn-ftr-common-functional-services/services/retry/retry_for_truthy.ts similarity index 100% rename from test/common/services/retry/retry_for_truthy.ts rename to packages/kbn-ftr-common-functional-services/services/retry/retry_for_truthy.ts diff --git a/packages/kbn-ftr-common-functional-services/tsconfig.json b/packages/kbn-ftr-common-functional-services/tsconfig.json new file mode 100644 index 0000000000000..81935b1385550 --- /dev/null +++ b/packages/kbn-ftr-common-functional-services/tsconfig.json @@ -0,0 +1,17 @@ +{ + "extends": "../../tsconfig.bazel.json", + "compilerOptions": { + "declaration": true, + "declarationMap": true, + "emitDeclarationOnly": true, + "outDir": "target_types", + "stripInternal": false, + "types": [ + "jest", + "node" + ] + }, + "include": [ + "**/*.ts", + ] +} diff --git a/packages/kbn-ftr-screenshot-filename/BUILD.bazel b/packages/kbn-ftr-screenshot-filename/BUILD.bazel new file mode 100644 index 0000000000000..5cbd3e2c87ac7 --- /dev/null +++ b/packages/kbn-ftr-screenshot-filename/BUILD.bazel @@ -0,0 +1,125 @@ +load("@npm//@bazel/typescript:index.bzl", "ts_config") +load("@build_bazel_rules_nodejs//:index.bzl", "js_library") +load("//src/dev/bazel:index.bzl", "jsts_transpiler", "pkg_npm", "pkg_npm_types", "ts_project") + +PKG_DIRNAME = "kbn-ftr-screenshot-filename" +PKG_REQUIRE_NAME = "@kbn/ftr-screenshot-filename" + +SOURCE_FILES = glob( + [ + "**/*.ts", + ], + exclude = [ + "**/*.config.js", + "**/*.mock.*", + "**/*.test.*", + "**/*.stories.*", + "**/__snapshots__/**", + "**/integration_tests/**", + "**/mocks/**", + "**/scripts/**", + "**/storybook/**", + "**/test_fixtures/**", + "**/test_helpers/**", + ], +) + +SRCS = SOURCE_FILES + +filegroup( + name = "srcs", + srcs = SRCS, +) + +NPM_MODULE_EXTRA_FILES = [ + "package.json", +] + +# In this array place runtime dependencies, including other packages and NPM packages +# which must be available for this code to run. +# +# To reference other packages use: +# "//repo/relative/path/to/package" +# eg. "//packages/kbn-utils" +# +# To reference a NPM package use: +# "@npm//name-of-package" +# eg. "@npm//lodash" +RUNTIME_DEPS = [ +] + +# In this array place dependencies necessary to build the types, which will include the +# :npm_module_types target of other packages and packages from NPM, including @types/* +# packages. +# +# To reference the types for another package use: +# "//repo/relative/path/to/package:npm_module_types" +# eg. "//packages/kbn-utils:npm_module_types" +# +# References to NPM packages work the same as RUNTIME_DEPS +TYPES_DEPS = [ + "@npm//@types/node", + "@npm//@types/jest", + "@npm//tslib", +] + +jsts_transpiler( + name = "target_node", + srcs = SRCS, + build_pkg_name = package_name(), +) + +ts_config( + name = "tsconfig", + src = "tsconfig.json", + deps = [ + "//:tsconfig.base.json", + "//:tsconfig.bazel.json", + ], +) + +ts_project( + name = "tsc_types", + args = ['--pretty'], + srcs = SRCS, + deps = TYPES_DEPS, + declaration = True, + declaration_map = True, + emit_declaration_only = True, + out_dir = "target_types", + tsconfig = ":tsconfig", +) + +js_library( + name = PKG_DIRNAME, + srcs = NPM_MODULE_EXTRA_FILES, + deps = RUNTIME_DEPS + [":target_node"], + package_name = PKG_REQUIRE_NAME, + visibility = ["//visibility:public"], +) + +pkg_npm( + name = "npm_module", + deps = [":" + PKG_DIRNAME], +) + +filegroup( + name = "build", + srcs = [":npm_module"], + visibility = ["//visibility:public"], +) + +pkg_npm_types( + name = "npm_module_types", + srcs = SRCS, + deps = [":tsc_types"], + package_name = PKG_REQUIRE_NAME, + tsconfig = ":tsconfig", + visibility = ["//visibility:public"], +) + +filegroup( + name = "build_types", + srcs = [":npm_module_types"], + visibility = ["//visibility:public"], +) diff --git a/packages/kbn-ftr-screenshot-filename/README.md b/packages/kbn-ftr-screenshot-filename/README.md new file mode 100644 index 0000000000000..faf5d4b994e50 --- /dev/null +++ b/packages/kbn-ftr-screenshot-filename/README.md @@ -0,0 +1,3 @@ +# @kbn/ftr-screenshot-filename + +A simple package that exposes a helper function for generating a unique screenshot filename that can be found by `node scripts/failed_test_reporter`. diff --git a/packages/kbn-ftr-screenshot-filename/ftr_screenshot_filename.ts b/packages/kbn-ftr-screenshot-filename/ftr_screenshot_filename.ts new file mode 100644 index 0000000000000..2cce021d50826 --- /dev/null +++ b/packages/kbn-ftr-screenshot-filename/ftr_screenshot_filename.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { createHash } from 'crypto'; + +export function create(fullTitle: string, opts?: { ext?: boolean }) { + const truncatedName = fullTitle.replaceAll(/[^ a-zA-Z0-9-]+/g, '').slice(0, 80); + const failureNameHash = createHash('sha256').update(fullTitle).digest('hex'); + return `${truncatedName}-${failureNameHash}${opts?.ext === false ? '' : `.png`}`; +} diff --git a/packages/kbn-ftr-screenshot-filename/index.ts b/packages/kbn-ftr-screenshot-filename/index.ts new file mode 100644 index 0000000000000..2911989659805 --- /dev/null +++ b/packages/kbn-ftr-screenshot-filename/index.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as FtrScreenshotFilename from './ftr_screenshot_filename'; + +export { FtrScreenshotFilename }; diff --git a/packages/kbn-ftr-screenshot-filename/jest.config.js b/packages/kbn-ftr-screenshot-filename/jest.config.js new file mode 100644 index 0000000000000..0ab6eb759a1dd --- /dev/null +++ b/packages/kbn-ftr-screenshot-filename/jest.config.js @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +module.exports = { + preset: '@kbn/test/jest_node', + rootDir: '../..', + roots: ['/packages/kbn-ftr-screenshot-filename'], +}; diff --git a/packages/kbn-ftr-screenshot-filename/kibana.jsonc b/packages/kbn-ftr-screenshot-filename/kibana.jsonc new file mode 100644 index 0000000000000..61ce39de5a622 --- /dev/null +++ b/packages/kbn-ftr-screenshot-filename/kibana.jsonc @@ -0,0 +1,8 @@ +{ + "type": "shared-common", + "id": "@kbn/ftr-screenshot-filename", + "owner": "@elastic/kibana-operations", + "devOnly": true, + "runtimeDeps": [], + "typeDeps": [], +} diff --git a/packages/kbn-ftr-screenshot-filename/package.json b/packages/kbn-ftr-screenshot-filename/package.json new file mode 100644 index 0000000000000..8e3a9b1e57db4 --- /dev/null +++ b/packages/kbn-ftr-screenshot-filename/package.json @@ -0,0 +1,7 @@ +{ + "name": "@kbn/ftr-screenshot-filename", + "private": true, + "version": "1.0.0", + "main": "./target_node/index.js", + "license": "SSPL-1.0 OR Elastic License 2.0" +} diff --git a/packages/kbn-ftr-screenshot-filename/tsconfig.json b/packages/kbn-ftr-screenshot-filename/tsconfig.json new file mode 100644 index 0000000000000..81935b1385550 --- /dev/null +++ b/packages/kbn-ftr-screenshot-filename/tsconfig.json @@ -0,0 +1,17 @@ +{ + "extends": "../../tsconfig.bazel.json", + "compilerOptions": { + "declaration": true, + "declarationMap": true, + "emitDeclarationOnly": true, + "outDir": "target_types", + "stripInternal": false, + "types": [ + "jest", + "node" + ] + }, + "include": [ + "**/*.ts", + ] +} diff --git a/packages/kbn-journeys/BUILD.bazel b/packages/kbn-journeys/BUILD.bazel new file mode 100644 index 0000000000000..cfadfb4b8b4b7 --- /dev/null +++ b/packages/kbn-journeys/BUILD.bazel @@ -0,0 +1,134 @@ +load("@npm//@bazel/typescript:index.bzl", "ts_config") +load("@build_bazel_rules_nodejs//:index.bzl", "js_library") +load("//src/dev/bazel:index.bzl", "jsts_transpiler", "pkg_npm", "pkg_npm_types", "ts_project") + +PKG_DIRNAME = "kbn-journeys" +PKG_REQUIRE_NAME = "@kbn/journeys" + +SOURCE_FILES = glob( + [ + "**/*.ts", + ], + exclude = [ + "**/*.config.js", + "**/*.mock.*", + "**/*.test.*", + "**/*.stories.*", + "**/__snapshots__/**", + "**/integration_tests/**", + "**/mocks/**", + "**/scripts/**", + "**/storybook/**", + "**/test_fixtures/**", + "**/test_helpers/**", + ], +) + +SRCS = SOURCE_FILES + +filegroup( + name = "srcs", + srcs = SRCS, +) + +NPM_MODULE_EXTRA_FILES = [ + "package.json", +] + +# In this array place runtime dependencies, including other packages and NPM packages +# which must be available for this code to run. +# +# To reference other packages use: +# "//repo/relative/path/to/package" +# eg. "//packages/kbn-utils" +# +# To reference a NPM package use: +# "@npm//name-of-package" +# eg. "@npm//lodash" +RUNTIME_DEPS = [ +] + +# In this array place dependencies necessary to build the types, which will include the +# :npm_module_types target of other packages and packages from NPM, including @types/* +# packages. +# +# To reference the types for another package use: +# "//repo/relative/path/to/package:npm_module_types" +# eg. "//packages/kbn-utils:npm_module_types" +# +# References to NPM packages work the same as RUNTIME_DEPS +TYPES_DEPS = [ + "@npm//@types/node", + "@npm//@types/mocha", + "@npm//playwright", + "@npm//uuid", + "@npm//axios", + "@npm//callsites", + "@npm//rxjs", + "@npm//elastic-apm-node", + "//packages/kbn-ftr-common-functional-services:npm_module_types", + "//packages/kbn-ftr-screenshot-filename:npm_module_types", + "//packages/kbn-test:npm_module_types", + "//packages/kbn-utils:npm_module_types", +] + +jsts_transpiler( + name = "target_node", + srcs = SRCS, + build_pkg_name = package_name(), +) + +ts_config( + name = "tsconfig", + src = "tsconfig.json", + deps = [ + "//:tsconfig.base.json", + "//:tsconfig.bazel.json", + ], +) + +ts_project( + name = "tsc_types", + args = ['--pretty'], + srcs = SRCS, + deps = TYPES_DEPS, + declaration = True, + declaration_map = True, + emit_declaration_only = True, + out_dir = "target_types", + tsconfig = ":tsconfig", +) + +js_library( + name = PKG_DIRNAME, + srcs = NPM_MODULE_EXTRA_FILES, + deps = RUNTIME_DEPS + [":target_node"], + package_name = PKG_REQUIRE_NAME, + visibility = ["//visibility:public"], +) + +pkg_npm( + name = "npm_module", + deps = [":" + PKG_DIRNAME], +) + +filegroup( + name = "build", + srcs = [":npm_module"], + visibility = ["//visibility:public"], +) + +pkg_npm_types( + name = "npm_module_types", + srcs = SRCS, + deps = [":tsc_types"], + package_name = PKG_REQUIRE_NAME, + tsconfig = ":tsconfig", + visibility = ["//visibility:public"], +) + +filegroup( + name = "build_types", + srcs = [":npm_module_types"], + visibility = ["//visibility:public"], +) diff --git a/packages/kbn-journeys/README.mdx b/packages/kbn-journeys/README.mdx new file mode 100644 index 0000000000000..506b5eb21a9e5 --- /dev/null +++ b/packages/kbn-journeys/README.mdx @@ -0,0 +1,32 @@ +--- +id: kibDevDocsOpsJourneys +slug: /kibana-dev-docs/ops/journeys +title: Journeys +description: A new style of functional test, focused on performance testing for now +tags: ['kibana', 'dev', 'contributor', 'operations', 'performance', 'functional', 'testing'] +--- + +Journeys are a slightly newer take on Functional Tests, currently powered by [playwright](https://playwright.dev/docs). + +A Journey is a single pathway through Kibana and looks something like this: + +```ts +import { Journey } from '@kbn/journeys'; +import { subj } from '@kbn/test-subj-selector'; + +export const journey = new Journey({ + esArchives: [ ... ], + kbnArchives: [ ... ], + scalabilitySetup: { ... }, +}) + .step('Go to Discover Page', async ({ page, kbnUrl }) => { + await page.goto(kbnUrl.get(`/app/discover`)); + await page.waitForSelector(subj('discoverDocTable')); + }) + + .step('Expand the first document', async ({ page }) => { + const expandButtons = page.locator(subj('docTableExpandToggleColumn')); + await expandButtons.first().click(); + await page.locator('text="Expanded document"'); + }); +``` \ No newline at end of file diff --git a/packages/kbn-journeys/index.ts b/packages/kbn-journeys/index.ts new file mode 100644 index 0000000000000..cc4c10c685d1c --- /dev/null +++ b/packages/kbn-journeys/index.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export { JourneyConfig } from './journey/journey_config'; +export type { ScalabilityAction, ScalabilitySetup } from './journey/journey_config'; + +export { Journey } from './journey/journey'; +export type { Step } from './journey/journey'; + +export { JourneyScreenshots } from './journey/journey_screenshots'; diff --git a/packages/kbn-journeys/jest.config.js b/packages/kbn-journeys/jest.config.js new file mode 100644 index 0000000000000..3d4735db3ddf8 --- /dev/null +++ b/packages/kbn-journeys/jest.config.js @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +module.exports = { + preset: '@kbn/test/jest_node', + rootDir: '../..', + roots: ['/packages/kbn-journeys'], +}; diff --git a/packages/kbn-journeys/journey/journey.ts b/packages/kbn-journeys/journey/journey.ts new file mode 100644 index 0000000000000..c399db0ba91c4 --- /dev/null +++ b/packages/kbn-journeys/journey/journey.ts @@ -0,0 +1,125 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { inspect } from 'util'; + +import { Page } from 'playwright'; +import callsites from 'callsites'; +import { ToolingLog } from '@kbn/tooling-log'; +import { FtrConfigProvider } from '@kbn/test'; +import { FtrProviderContext } from '@kbn/ftr-common-functional-services'; + +import { Auth } from '../services/auth'; +import { InputDelays } from '../services/input_delays'; +import { KibanaUrl } from '../services/kibana_url'; + +import { JourneyFtrHarness } from './journey_ftr_harness'; +import { makeFtrConfigProvider } from './journey_ftr_config'; +import { JourneyConfig, JourneyConfigOptions } from './journey_config'; + +export interface BaseStepCtx { + page: Page; + log: ToolingLog; + inputDelays: InputDelays; + kbnUrl: KibanaUrl; +} + +export type AnyStep = Step<{}>; + +export interface Step { + name: string; + index: number; + fn(ctx: BaseStepCtx & CtxExt): Promise; +} + +const CONFIG_PROVIDER_CACHE = new WeakMap, FtrConfigProvider>(); + +export class Journey { + static convertToFtrConfigProvider(journey: Journey) { + const cached = CONFIG_PROVIDER_CACHE.get(journey); + if (cached) { + return cached; + } + + const provider = makeFtrConfigProvider(journey.config, journey.#steps); + CONFIG_PROVIDER_CACHE.set(journey, provider); + return provider; + } + + /** + * Load a journey from a file path + */ + static async load(path: string) { + let m; + try { + m = await import(path); + } catch (error) { + throw new Error(`Unable to load file: ${path}`); + } + + if (!m || !m.journey) { + throw new Error(`[${path}] is not a journey`); + } + + const journey = m.journey; + if (journey instanceof Journey) { + return journey; + } + + const dbg = inspect(journey); + throw new Error(`[${path}] does not export a Journey like it should, received ${dbg}`); + } + + #steps: Array> = []; + + config: JourneyConfig; + + /** + * Create a Journey which should be exported from a file in the + * x-pack/performance/journeys directory. + */ + constructor(opts?: JourneyConfigOptions) { + const path = callsites().at(1)?.getFileName(); + + if (!path) { + throw new Error('unable to determine path of journey config file'); + } + + this.config = new JourneyConfig(path, opts); + } + + /** + * Define a step of this Journey. Steps are only separated from each other + * to aid in reading/debuging the journey and reading it's logging output. + * + * If a journey fails, a failure report will be created with a screenshot + * at the point of failure as well as a screenshot at the end of every + * step. + */ + step(name: string, fn: (ctx: BaseStepCtx & CtxExt) => Promise) { + this.#steps.push({ + name, + index: this.#steps.length, + fn, + }); + + return this; + } + + /** called by FTR to setup tests */ + protected testProvider({ getService }: FtrProviderContext) { + new JourneyFtrHarness( + getService('log'), + getService('config'), + getService('esArchiver'), + getService('kibanaServer'), + new Auth(getService('config'), getService('log'), getService('kibanaServer')), + this.config + ).initMochaSuite(this.#steps); + } +} diff --git a/packages/kbn-journeys/journey/journey_config.ts b/packages/kbn-journeys/journey/journey_config.ts new file mode 100644 index 0000000000000..e23b2a748fbe7 --- /dev/null +++ b/packages/kbn-journeys/journey/journey_config.ts @@ -0,0 +1,155 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; + +import { REPO_ROOT } from '@kbn/utils'; + +import { BaseStepCtx } from './journey'; + +export interface RampConcurrentUsersAction { + action: 'rampConcurrentUsers'; + /** + * Duration strings must be formatted as string that starts with an integer and + * ends with either "m" or "s" for minutes and seconds, respectively + * + * eg: "1m" or "30s" + */ + duration: string; + minUsersCount: number; + maxUsersCount: number; +} + +export interface ConstantConcurrentUsersAction { + action: 'constantConcurrentUsers'; + /** + * Duration strings must be formatted as string that starts with an integer and + * ends with either "m" or "s" for minutes and seconds, respectively + * + * eg: "1m" or "30s" + */ + duration: string; + userCount: number; +} + +export type ScalabilityAction = RampConcurrentUsersAction | ConstantConcurrentUsersAction; + +export interface ScalabilitySetup { + /** + * Duration strings must be formatted as string that starts with an integer and + * ends with either "m" or "s" for minutes and seconds, respectively + * + * eg: "1m" or "30s" + */ + maxDuration: string; + warmup: ScalabilityAction[]; + test: ScalabilityAction[]; +} + +export interface JourneyConfigOptions { + /** + * Set to `true` to skip this journey. should probably be preceded + * by a link to a Github issue where the reasoning for why this was + * skipped and not just deleted is outlined. + */ + skipped?: boolean; + /** + * Scalability configuration used to customize automatically generated + * scalability tests. For now chat with Dima/Operations if you want to + * use this option. + */ + scalabilitySetup?: ScalabilitySetup; + /** + * These labels will be attached to all APM data created when running + * this journey. + */ + extraApmLabels?: Record; + /** + * A list of kbnArchives which will be automatically loaded/unloaded + * for this journey. + */ + kbnArchives?: string[]; + /** + * A list of esArchives which will be automatically loaded/unloaded + * for this journey. + */ + esArchives?: string[]; + /** + * By default the API is used to get a cookie that can be used for all + * navigation requests to Kibana, so that we don't ever see the login + * screen. Set this to `false` to disable this behavior. + */ + skipAutoLogin?: boolean; + /** + * Use this to extend the context provided to each step. This function + * is called with the default context and returns an object that will + * be merged with the default context provided to each step function. + */ + extendContext?: (ctx: BaseStepCtx) => CtxExt; +} + +export class JourneyConfig { + #opts: JourneyConfigOptions; + #path: string; + #name: string; + + constructor(path: string, opts: JourneyConfigOptions = {}) { + this.#path = path; + this.#name = Path.basename(this.#path, Path.extname(this.#path)); + this.#opts = opts; + } + + getEsArchives() { + return this.#opts.esArchives ?? []; + } + + getKbnArchives() { + return this.#opts.kbnArchives ?? []; + } + + isXpack() { + return this.getRepoRelPath().split(Path.sep).at(0) === 'x-pack'; + } + + getExtraApmLabels() { + return this.#opts.extraApmLabels ? { ...this.#opts.extraApmLabels } : {}; + } + + getRepoRelPath() { + return Path.relative(REPO_ROOT, this.getPath()); + } + + getPath() { + return this.#path; + } + + getName() { + return this.#name; + } + + shouldAutoLogin() { + return !this.#opts.skipAutoLogin; + } + + isSkipped() { + return !!this.#opts.skipped; + } + + getScalabilityConfig() { + return this.#opts.scalabilitySetup; + } + + getExtendedStepCtx(ctx: BaseStepCtx): BaseStepCtx & CtxExt { + const ext = this.#opts.extendContext ?? (() => ({} as CtxExt)); + + return { + ...ctx, + ...ext(ctx), + }; + } +} diff --git a/packages/kbn-journeys/journey/journey_ftr_config.ts b/packages/kbn-journeys/journey/journey_ftr_config.ts new file mode 100644 index 0000000000000..392ad69b63ba1 --- /dev/null +++ b/packages/kbn-journeys/journey/journey_ftr_config.ts @@ -0,0 +1,127 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; + +import { v4 as uuidV4 } from 'uuid'; +import { REPO_ROOT } from '@kbn/utils'; +import { FtrConfigProviderContext, FtrConfigProvider } from '@kbn/test'; +import { commonFunctionalServices } from '@kbn/ftr-common-functional-services'; + +import { AnyStep } from './journey'; +import { JourneyConfig } from './journey_config'; + +// These "secret" values are intentionally written in the source. We would make the APM server accept anonymous traffic if we could +const APM_SERVER_URL = 'https://kibana-ops-e2e-perf.apm.us-central1.gcp.cloud.es.io:443'; +const APM_PUBLIC_TOKEN = 'CTs9y3cvcfq13bQqsB'; + +export function makeFtrConfigProvider( + config: JourneyConfig, + steps: AnyStep[] +): FtrConfigProvider { + return async ({ readConfigFile }: FtrConfigProviderContext) => { + const baseConfig = ( + await readConfigFile( + Path.resolve( + REPO_ROOT, + config.isXpack() + ? 'x-pack/test/functional/config.base.js' + : 'test/functional/config.base.js' + ) + ) + ).getAll(); + + const testBuildId = process.env.BUILDKITE_BUILD_ID ?? `local-${uuidV4()}`; + const testJobId = process.env.BUILDKITE_JOB_ID ?? `local-${uuidV4()}`; + const prId = process.env.GITHUB_PR_NUMBER + ? Number.parseInt(process.env.GITHUB_PR_NUMBER, 10) + : undefined; + + if (Number.isNaN(prId)) { + throw new Error('invalid GITHUB_PR_NUMBER environment variable'); + } + + const telemetryLabels: Record = { + branch: process.env.BUILDKITE_BRANCH, + ciBuildId: process.env.BUILDKITE_BUILD_ID, + ciBuildJobId: process.env.BUILDKITE_JOB_ID, + ciBuildNumber: Number(process.env.BUILDKITE_BUILD_NUMBER) || 0, + gitRev: process.env.BUILDKITE_COMMIT, + isPr: prId !== undefined, + ...(prId !== undefined ? { prId } : {}), + ciBuildName: process.env.BUILDKITE_PIPELINE_SLUG, + journeyName: config.getName(), + }; + + return { + ...baseConfig, + + mochaOpts: { + ...baseConfig.mochaOpts, + bail: true, + }, + + services: commonFunctionalServices, + pageObjects: {}, + + servicesRequiredForTestAnalysis: ['performance', 'journeyConfig'], + + junit: { + reportName: `Journey: ${config.getName()}`, + metadata: { + journeyName: config.getName(), + stepNames: steps.map((s) => s.name), + }, + }, + + kbnTestServer: { + ...baseConfig.kbnTestServer, + // delay shutdown by 15 seconds to ensure that APM can report the data it collects during test execution + delayShutdown: 15_000, + + serverArgs: [ + ...baseConfig.kbnTestServer.serverArgs, + `--telemetry.optIn=${process.env.TEST_PERFORMANCE_PHASE === 'TEST'}`, + `--telemetry.labels=${JSON.stringify(telemetryLabels)}`, + '--csp.strict=false', + '--csp.warnLegacyBrowsers=false', + ], + + env: { + ELASTIC_APM_ACTIVE: process.env.TEST_PERFORMANCE_PHASE ? 'true' : 'false', + ELASTIC_APM_CONTEXT_PROPAGATION_ONLY: 'false', + ELASTIC_APM_ENVIRONMENT: process.env.CI ? 'ci' : 'development', + ELASTIC_APM_TRANSACTION_SAMPLE_RATE: '1.0', + ELASTIC_APM_SERVER_URL: APM_SERVER_URL, + ELASTIC_APM_SECRET_TOKEN: APM_PUBLIC_TOKEN, + // capture request body for both errors and request transactions + // https://www.elastic.co/guide/en/apm/agent/nodejs/current/configuration.html#capture-body + ELASTIC_APM_CAPTURE_BODY: 'all', + // capture request headers + // https://www.elastic.co/guide/en/apm/agent/nodejs/current/configuration.html#capture-headers + ELASTIC_APM_CAPTURE_HEADERS: true, + // request body with bigger size will be trimmed. + // 300_000 is the default of the APM server. + // for a body with larger size, we might need to reconfigure the APM server to increase the limit. + // https://www.elastic.co/guide/en/apm/agent/nodejs/current/configuration.html#long-field-max-length + ELASTIC_APM_LONG_FIELD_MAX_LENGTH: 300_000, + ELASTIC_APM_GLOBAL_LABELS: Object.entries({ + ...config.getExtraApmLabels(), + testJobId, + testBuildId, + journeyName: config.getName(), + ftrConfig: config.getRepoRelPath(), + performancePhase: process.env.TEST_PERFORMANCE_PHASE, + }) + .flatMap(([key, value]) => (value == null ? [] : `${key}=${value}`)) + .join(','), + }, + }, + }; + }; +} diff --git a/packages/kbn-journeys/journey/journey_ftr_harness.ts b/packages/kbn-journeys/journey/journey_ftr_harness.ts new file mode 100644 index 0000000000000..672b14f0e1a85 --- /dev/null +++ b/packages/kbn-journeys/journey/journey_ftr_harness.ts @@ -0,0 +1,410 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Url from 'url'; +import { inspect, format } from 'util'; +import { setTimeout } from 'timers/promises'; + +import * as Rx from 'rxjs'; +import apmNode from 'elastic-apm-node'; +import playwright, { ChromiumBrowser, Page, BrowserContext, CDPSession, Request } from 'playwright'; +import { asyncMap, asyncForEach } from '@kbn/std'; +import { ToolingLog } from '@kbn/tooling-log'; +import { Config } from '@kbn/test'; +import { EsArchiver, KibanaServer } from '@kbn/ftr-common-functional-services'; + +import { Auth } from '../services/auth'; +import { getInputDelays } from '../services/input_delays'; +import { KibanaUrl } from '../services/kibana_url'; + +import type { Step, AnyStep } from './journey'; +import type { JourneyConfig } from './journey_config'; +import { JourneyScreenshots } from './journey_screenshots'; + +export class JourneyFtrHarness { + private readonly screenshots: JourneyScreenshots; + + constructor( + private readonly log: ToolingLog, + private readonly config: Config, + private readonly esArchiver: EsArchiver, + private readonly kibanaServer: KibanaServer, + private readonly auth: Auth, + private readonly journeyConfig: JourneyConfig + ) { + this.screenshots = new JourneyScreenshots(this.journeyConfig.getName()); + } + + private browser: ChromiumBrowser | undefined; + private page: Page | undefined; + private client: CDPSession | undefined; + private context: BrowserContext | undefined; + private currentSpanStack: Array = []; + private currentTransaction: apmNode.Transaction | undefined | null = undefined; + + private pageTeardown$ = new Rx.Subject(); + private telemetryTrackerSubs = new Map(); + + private apm: apmNode.Agent | null = null; + + private async setupApm() { + const kbnTestServerEnv = this.config.get(`kbnTestServer.env`); + + this.apm = apmNode.start({ + serviceName: 'functional test runner', + environment: process.env.CI ? 'ci' : 'development', + active: kbnTestServerEnv.ELASTIC_APM_ACTIVE !== 'false', + serverUrl: kbnTestServerEnv.ELASTIC_APM_SERVER_URL, + secretToken: kbnTestServerEnv.ELASTIC_APM_SECRET_TOKEN, + globalLabels: kbnTestServerEnv.ELASTIC_APM_GLOBAL_LABELS, + transactionSampleRate: kbnTestServerEnv.ELASTIC_APM_TRANSACTION_SAMPLE_RATE, + logger: { + warn: (...args: any[]) => { + this.log.warning('APM WARN', ...args); + }, + info: (...args: any[]) => { + this.log.info('APM INFO', ...args); + }, + fatal: (...args: any[]) => { + this.log.error(format('APM FATAL', ...args)); + }, + error: (...args: any[]) => { + this.log.error(format('APM ERROR', ...args)); + }, + debug: (...args: any[]) => { + this.log.debug('APM DEBUG', ...args); + }, + trace: (...args: any[]) => { + this.log.verbose('APM TRACE', ...args); + }, + }, + }); + + if (this.currentTransaction) { + throw new Error(`Transaction exist, end prev transaction ${this.currentTransaction?.name}`); + } + + this.currentTransaction = this.apm?.startTransaction( + `Journey: ${this.journeyConfig.getName()}`, + 'performance' + ); + } + + private async setupBrowserAndPage() { + const browser = await this.getBrowserInstance(); + this.context = await browser.newContext({ bypassCSP: true }); + + if (this.journeyConfig.shouldAutoLogin()) { + const cookie = await this.auth.login({ username: 'elastic', password: 'changeme' }); + await this.context.addCookies([cookie]); + } + + this.page = await this.context.newPage(); + + if (!process.env.NO_BROWSER_LOG) { + this.page.on('console', this.onConsoleEvent); + } + + await this.sendCDPCommands(this.context, this.page); + + this.trackTelemetryRequests(this.page); + await this.interceptBrowserRequests(this.page); + } + + private async onSetup() { + await Promise.all([ + this.setupApm(), + this.setupBrowserAndPage(), + asyncForEach(this.journeyConfig.getEsArchives(), async (esArchive) => { + await this.esArchiver.load(esArchive); + }), + asyncForEach(this.journeyConfig.getKbnArchives(), async (kbnArchive) => { + await this.kibanaServer.importExport.load(kbnArchive); + }), + ]); + } + + private async tearDownBrowserAndPage() { + if (this.page) { + const telemetryTracker = this.telemetryTrackerSubs.get(this.page); + this.telemetryTrackerSubs.delete(this.page); + + if (telemetryTracker && !telemetryTracker.closed) { + this.log.info(`Waiting for telemetry requests, including starting within next 3 secs`); + this.pageTeardown$.next(this.page); + await new Promise((resolve) => telemetryTracker.add(resolve)); + } + + this.log.info('destroying page'); + await this.client?.detach(); + await this.page.close(); + await this.context?.close(); + } + + if (this.browser) { + this.log.info('closing browser'); + await this.browser.close(); + } + } + + private async teardownApm() { + if (!this.apm) { + return; + } + + if (this.currentTransaction) { + this.currentTransaction.end('Success'); + this.currentTransaction = undefined; + } + + const apmStarted = this.apm.isStarted(); + // @ts-expect-error + const apmActive = apmStarted && this.apm._conf.active; + + if (!apmActive) { + this.log.warning('APM is not active'); + return; + } + + this.log.info('Flushing APM'); + await new Promise((resolve) => this.apm?.flush(() => resolve())); + // wait for the HTTP request that apm.flush() starts, which we + // can't track but hope it is started within 3 seconds, node will stay + // alive for active requests + // https://github.com/elastic/apm-agent-nodejs/issues/2088 + await setTimeout(3000); + } + + private async onTeardown() { + await Promise.all([ + this.tearDownBrowserAndPage(), + this.teardownApm(), + asyncForEach(this.journeyConfig.getEsArchives(), async (esArchive) => { + await this.esArchiver.unload(esArchive); + }), + asyncForEach(this.journeyConfig.getKbnArchives(), async (kbnArchive) => { + await this.kibanaServer.importExport.unload(kbnArchive); + }), + ]); + } + + private async onStepSuccess(step: AnyStep) { + if (!this.page) { + return; + } + + await this.screenshots.addSuccess(step, await this.page.screenshot()); + } + + private async onStepError(step: AnyStep, err: Error) { + if (this.currentTransaction) { + this.currentTransaction.end(`Failure ${err.message}`); + this.currentTransaction = undefined; + } + + if (this.page) { + await this.screenshots.addError(step, await this.page.screenshot()); + } + } + + private async withSpan(name: string, type: string | undefined, block: () => Promise) { + if (!this.currentTransaction) { + return await block(); + } + + const span = this.apm?.startSpan(name, type ?? null, { + childOf: this.currentTransaction, + }); + if (!span) { + return await block(); + } + + try { + this.currentSpanStack.unshift(span); + const result = await block(); + span.setOutcome('success'); + span.end(); + return result; + } catch (error) { + span.setOutcome('failure'); + span.end(); + throw error; + } finally { + if (span !== this.currentSpanStack.shift()) { + // eslint-disable-next-line no-unsafe-finally + throw new Error('span stack mismatch'); + } + } + } + + private getCurrentTraceparent() { + return (this.currentSpanStack.length ? this.currentSpanStack[0] : this.currentTransaction) + ?.traceparent; + } + + private async getBrowserInstance() { + if (this.browser) { + return this.browser; + } + return await this.withSpan('Browser creation', 'setup', async () => { + const headless = !!(process.env.TEST_BROWSER_HEADLESS || process.env.CI); + this.browser = await playwright.chromium.launch({ headless, timeout: 60_000 }); + return this.browser; + }); + } + + private async sendCDPCommands(context: BrowserContext, page: Page) { + const client = await context.newCDPSession(page); + + await client.send('Network.clearBrowserCache'); + await client.send('Network.setCacheDisabled', { cacheDisabled: true }); + await client.send('Network.emulateNetworkConditions', { + latency: 100, + downloadThroughput: 750_000, + uploadThroughput: 750_000, + offline: false, + }); + + return client; + } + + private telemetryTrackerCount = 0; + + private trackTelemetryRequests(page: Page) { + const id = ++this.telemetryTrackerCount; + + const requestFailure$ = Rx.fromEvent(page, 'requestfailed'); + const requestSuccess$ = Rx.fromEvent(page, 'requestfinished'); + const request$ = Rx.fromEvent(page, 'request').pipe( + Rx.takeUntil( + this.pageTeardown$.pipe( + Rx.first((p) => p === page), + Rx.delay(3000) + // If EBT client buffers: + // Rx.mergeMap(async () => { + // await page.waitForFunction(() => { + // // return window.kibana_ebt_client.buffer_size == 0 + // }); + // }) + ) + ), + Rx.mergeMap((request) => { + if (!request.url().includes('telemetry-staging.elastic.co')) { + return Rx.EMPTY; + } + + this.log.debug(`Waiting for telemetry request #${id} to complete`); + return Rx.merge(requestFailure$, requestSuccess$).pipe( + Rx.first((r) => r === request), + Rx.tap({ + complete: () => this.log.debug(`Telemetry request #${id} complete`), + }) + ); + }) + ); + + this.telemetryTrackerSubs.set(page, request$.subscribe()); + } + + private async interceptBrowserRequests(page: Page) { + await page.route('**', async (route, request) => { + const headers = await request.allHeaders(); + const traceparent = this.getCurrentTraceparent(); + if (traceparent && request.isNavigationRequest()) { + await route.continue({ headers: { traceparent, ...headers } }); + } else { + await route.continue(); + } + }); + } + + #_ctx?: Record; + private getCtx() { + if (this.#_ctx) { + return this.#_ctx; + } + + const page = this.page; + + if (!page) { + throw new Error('performance service is not properly initialized'); + } + + this.#_ctx = this.journeyConfig.getExtendedStepCtx({ + page, + log: this.log, + inputDelays: getInputDelays(), + kbnUrl: new KibanaUrl( + new URL( + Url.format({ + protocol: this.config.get('servers.kibana.protocol'), + hostname: this.config.get('servers.kibana.hostname'), + port: this.config.get('servers.kibana.port'), + }) + ) + ), + }); + + return this.#_ctx; + } + + public initMochaSuite(steps: Array>) { + const journeyName = this.journeyConfig.getName(); + + (this.journeyConfig.isSkipped() ? describe.skip : describe)(`Journey[${journeyName}]`, () => { + before(async () => await this.onSetup()); + after(async () => await this.onTeardown()); + + for (const step of steps) { + it(step.name, async () => { + await this.withSpan(`step: ${step.name}`, 'step', async () => { + try { + await step.fn(this.getCtx()); + await this.onStepSuccess(step); + } catch (e) { + const error = new Error(`Step [${step.name}] failed: ${e.message}`); + error.stack = e.stack; + await this.onStepError(step, error); + throw error; // Rethrow error if step fails otherwise it is silently passing + } + }); + }); + } + }); + } + + private onConsoleEvent = async (message: playwright.ConsoleMessage) => { + try { + const { url, lineNumber, columnNumber } = message.location(); + const location = `${url}:${lineNumber}:${columnNumber}`; + + const args = await asyncMap(message.args(), (handle) => handle.jsonValue()); + const text = args.length + ? args.map((arg) => (typeof arg === 'string' ? arg : inspect(arg, false, null))).join(' ') + : message.text(); + + if ( + url.includes('kbn-ui-shared-deps-npm.dll.js') && + text.includes('moment construction falls') + ) { + // ignore errors from moment about constructing dates with invalid formats + return; + } + + const type = message.type(); + const method = type === 'debug' ? type : type === 'warning' ? 'error' : 'info'; + const name = type === 'warning' ? 'error' : 'log'; + this.log[method](`[console.${name}] @ ${location}:\n${text}`); + } catch (error) { + const dbg = inspect(message); + this.log.error( + `Error interpreting browser console.log:\nerror:${error.message}\nmessage:\n${dbg}` + ); + } + }; +} diff --git a/packages/kbn-journeys/journey/journey_screenshots.ts b/packages/kbn-journeys/journey/journey_screenshots.ts new file mode 100644 index 0000000000000..8cd36444ef7ee --- /dev/null +++ b/packages/kbn-journeys/journey/journey_screenshots.ts @@ -0,0 +1,128 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; +import Fsp from 'fs/promises'; + +import * as Rx from 'rxjs'; +import { REPO_ROOT } from '@kbn/utils'; +import { FtrScreenshotFilename } from '@kbn/ftr-screenshot-filename'; + +import type { AnyStep } from './journey'; + +interface StepShot { + type: 'success' | 'failure'; + title: string; + filename: string; +} + +interface Manifest { + steps: StepShot[]; +} + +const isObj = (v: unknown): v is Record => typeof v === 'object' && v !== null; +const isString = (v: unknown): v is string => typeof v === 'string'; +const isStepShot = (v: unknown): v is StepShot => + isObj(v) && + (v.type === 'success' || v.type === 'failure') && + isString(v.title) && + isString(v.filename); + +const write = async (path: string, content: string | Buffer) => { + await Fsp.mkdir(Path.dirname(path), { recursive: true }); + await Fsp.writeFile(path, content); +}; + +export class JourneyScreenshots { + static async load(journeyName: string) { + const screenshots = new JourneyScreenshots(journeyName); + + const json = await Fsp.readFile(screenshots.#manifestPath, 'utf8'); + const manifest = JSON.parse(json); + + if (!isObj(manifest)) { + throw new Error('invalid manifest, json parsed but not to an object'); + } + + const { steps } = manifest; + + if (!Array.isArray(steps) || !steps.every(isStepShot)) { + throw new Error('invalid manifest, steps must be an array of StepShot objects'); + } + + screenshots.#manifest = { steps }; + return screenshots; + } + + readonly #dir: string; + readonly #manifestPath: string; + + #manifest: Manifest = { + steps: [], + }; + + constructor(journeyName: string) { + this.#dir = Path.resolve(REPO_ROOT, 'data/journey_screenshots', journeyName); + this.#manifestPath = Path.resolve(this.#dir, 'manifest.json'); + } + + readonly #isLocked = new Rx.BehaviorSubject(false); + async lock(fn: () => Promise) { + if (this.#isLocked.getValue()) { + do { + await Rx.firstValueFrom(this.#isLocked.pipe(Rx.skip(1))); + } while (this.#isLocked.getValue()); + } + + try { + this.#isLocked.next(true); + await fn(); + } finally { + this.#isLocked.next(false); + } + } + + async addError(step: AnyStep, screenshot: Buffer) { + await this.lock(async () => { + const filename = FtrScreenshotFilename.create(`${step.index}-${step.name}-failure`); + this.#manifest.steps.push({ + type: 'failure', + title: `Step #${step.index + 1}: ${step.name} - FAILED`, + filename, + }); + + await Promise.all([ + write(Path.resolve(this.#dir, 'manifest.json'), JSON.stringify(this.#manifest)), + write(Path.resolve(this.#dir, filename), screenshot), + ]); + }); + } + + async addSuccess(step: AnyStep, screenshot: Buffer) { + await this.lock(async () => { + const filename = FtrScreenshotFilename.create(`${step.index}-${step.name}`); + this.#manifest.steps.push({ + type: 'success', + title: `Step #${step.index + 1}: ${step.name} - DONE`, + filename, + }); + + await Promise.all([ + write(Path.resolve(this.#dir, 'manifest.json'), JSON.stringify(this.#manifest)), + write(Path.resolve(this.#dir, filename), screenshot), + ]); + }); + } + + get() { + return this.#manifest.steps.map((stepShot) => ({ + ...stepShot, + path: Path.resolve(this.#dir, stepShot.filename), + })); + } +} diff --git a/packages/kbn-journeys/kibana.jsonc b/packages/kbn-journeys/kibana.jsonc new file mode 100644 index 0000000000000..ab8a15547c158 --- /dev/null +++ b/packages/kbn-journeys/kibana.jsonc @@ -0,0 +1,8 @@ +{ + "type": "shared-common", + "id": "@kbn/journeys", + "owner": "@elastic/kibana-operations", + "devOnly": true, + "runtimeDeps": [], + "typeDeps": [], +} diff --git a/packages/kbn-journeys/package.json b/packages/kbn-journeys/package.json new file mode 100644 index 0000000000000..06920a5ebd241 --- /dev/null +++ b/packages/kbn-journeys/package.json @@ -0,0 +1,7 @@ +{ + "name": "@kbn/journeys", + "private": true, + "version": "1.0.0", + "main": "./target_node/index.js", + "license": "SSPL-1.0 OR Elastic License 2.0" +} diff --git a/packages/kbn-journeys/services/auth.ts b/packages/kbn-journeys/services/auth.ts new file mode 100644 index 0000000000000..b8c68a9fbb09c --- /dev/null +++ b/packages/kbn-journeys/services/auth.ts @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Url from 'url'; +import { format } from 'util'; + +import axios, { AxiosResponse } from 'axios'; +import { ToolingLog } from '@kbn/tooling-log'; +import { Config } from '@kbn/test'; +import { KibanaServer } from '@kbn/ftr-common-functional-services'; + +export interface Credentials { + username: string; + password: string; +} + +function extractCookieValue(authResponse: AxiosResponse) { + return authResponse.headers['set-cookie']?.[0].toString().split(';')[0].split('sid=')[1] ?? ''; +} +export class Auth { + constructor( + private readonly config: Config, + private readonly log: ToolingLog, + private readonly kibanaServer: KibanaServer + ) {} + + public async login({ username, password }: Credentials) { + const baseUrl = new URL( + Url.format({ + protocol: this.config.get('servers.kibana.protocol'), + hostname: this.config.get('servers.kibana.hostname'), + port: this.config.get('servers.kibana.port'), + }) + ); + + const loginUrl = new URL('/internal/security/login', baseUrl); + const provider = baseUrl.hostname === 'localhost' ? 'basic' : 'cloud-basic'; + + this.log.info('fetching auth cookie from', loginUrl.href); + const authResponse = await axios.request({ + url: loginUrl.href, + method: 'post', + data: { + providerType: 'basic', + providerName: provider, + currentURL: new URL('/login?next=%2F', baseUrl).href, + params: { username, password }, + }, + headers: { + 'content-type': 'application/json', + 'kbn-version': await this.kibanaServer.version.get(), + 'sec-fetch-mode': 'cors', + 'sec-fetch-site': 'same-origin', + }, + validateStatus: () => true, + maxRedirects: 0, + }); + + const cookie = extractCookieValue(authResponse); + if (cookie) { + this.log.info('captured auth cookie'); + } else { + this.log.error( + format('unable to determine auth cookie from response', { + status: `${authResponse.status} ${authResponse.statusText}`, + body: authResponse.data, + headers: authResponse.headers, + }) + ); + + throw new Error(`failed to determine auth cookie`); + } + + return { + name: 'sid', + value: cookie, + url: baseUrl.href, + }; + } +} diff --git a/x-pack/test/performance/services/input_delays.ts b/packages/kbn-journeys/services/input_delays.ts similarity index 69% rename from x-pack/test/performance/services/input_delays.ts rename to packages/kbn-journeys/services/input_delays.ts index 483974cb9802d..5f66b26f0f36d 100644 --- a/x-pack/test/performance/services/input_delays.ts +++ b/packages/kbn-journeys/services/input_delays.ts @@ -1,10 +1,12 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ -interface InputDelays { + +export interface InputDelays { TYPING: number; MOUSE_CLICK: number; } @@ -20,7 +22,7 @@ const PROFILES: Record = { }, }; -export function InputDelaysProvider(): InputDelays { +export function getInputDelays(): InputDelays { const profile = PROFILES[process.env.INPUT_DELAY_PROFILE ?? 'user']; if (!profile) { diff --git a/packages/kbn-journeys/services/kibana_url.ts b/packages/kbn-journeys/services/kibana_url.ts new file mode 100644 index 0000000000000..d9c54ccfe3c37 --- /dev/null +++ b/packages/kbn-journeys/services/kibana_url.ts @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export interface PathOptions { + /** + * Query string parameters + */ + params?: Record; + /** + * The hash value of the URL + */ + hash?: string; +} + +export class KibanaUrl { + #baseUrl: URL; + + constructor(baseUrl: URL) { + this.#baseUrl = baseUrl; + } + + /** + * Get an absolute URL based on Kibana's URL + * @param rel relative url, resolved relative to Kibana's url + * @param options optional modifications to apply to the URL + */ + get(rel?: string, options?: PathOptions) { + const url = new URL(rel ?? '/', this.#baseUrl); + + if (options?.params) { + for (const [key, value] of Object.entries(options.params)) { + url.searchParams.set(key, value); + } + } + + if (options?.hash !== undefined) { + url.hash = options.hash; + } + + return url.href; + } + + /** + * Get the URL for an app + * @param appName name of the app to get the URL for + * @param options optional modifications to apply to the URL + */ + app(appName: string, options?: PathOptions) { + return this.get(`/app/${appName}`, options); + } + + toString() { + return this.#baseUrl.href; + } +} diff --git a/packages/kbn-journeys/tsconfig.json b/packages/kbn-journeys/tsconfig.json new file mode 100644 index 0000000000000..f4d18db9ffafa --- /dev/null +++ b/packages/kbn-journeys/tsconfig.json @@ -0,0 +1,17 @@ +{ + "extends": "../../tsconfig.bazel.json", + "compilerOptions": { + "declaration": true, + "declarationMap": true, + "emitDeclarationOnly": true, + "outDir": "target_types", + "stripInternal": false, + "types": [ + "mocha", + "node" + ] + }, + "include": [ + "**/*.ts", + ] +} diff --git a/packages/kbn-performance-testing-dataset-extractor/BUILD.bazel b/packages/kbn-performance-testing-dataset-extractor/BUILD.bazel index 479e494bf3b54..53782e9cfbd08 100644 --- a/packages/kbn-performance-testing-dataset-extractor/BUILD.bazel +++ b/packages/kbn-performance-testing-dataset-extractor/BUILD.bazel @@ -66,8 +66,8 @@ RUNTIME_DEPS = [ TYPES_DEPS = [ "//packages/kbn-dev-cli-errors:npm_module_types", "//packages/kbn-dev-cli-runner:npm_module_types", - "//packages/kbn-test:npm_module_types", "//packages/kbn-tooling-log:npm_module_types", + "//packages/kbn-journeys:npm_module_types", "@npm//@elastic/elasticsearch", "@npm//@types/node", "@npm//@types/jest", diff --git a/packages/kbn-performance-testing-dataset-extractor/src/cli.ts b/packages/kbn-performance-testing-dataset-extractor/src/cli.ts index 435f87bcd5818..f2e88addd63fe 100644 --- a/packages/kbn-performance-testing-dataset-extractor/src/cli.ts +++ b/packages/kbn-performance-testing-dataset-extractor/src/cli.ts @@ -12,16 +12,13 @@ * *************************************************************/ +import path from 'path'; + import { run } from '@kbn/dev-cli-runner'; import { createFlagError } from '@kbn/dev-cli-errors'; -import { EsVersion, readConfigFile } from '@kbn/test'; -import path from 'path'; -import { extractor } from './extractor'; -import { ScalabilitySetup, TestData } from './types'; +import { Journey } from '@kbn/journeys'; -interface Vars { - [key: string]: string; -} +import { extractor } from './extractor'; export async function runExtractor() { run( @@ -50,63 +47,46 @@ export async function runExtractor() { throw createFlagError('--es-password must be defined'); } + const withoutStaticResources = !!flags['without-static-resources'] || false; + const buildId = flags.buildId; + if (buildId && typeof buildId !== 'string') { + throw createFlagError('--buildId must be a string'); + } + if (!buildId) { + throw createFlagError('--buildId must be defined'); + } + const configPath = flags.config; if (typeof configPath !== 'string') { throw createFlagError('--config must be a string'); } - const config = await readConfigFile(log, EsVersion.getDefault(), path.resolve(configPath)); - - const scalabilitySetup: ScalabilitySetup = config.get('scalabilitySetup'); + const journey = await Journey.load(path.resolve(configPath)); + const scalabilitySetup = journey.config.getScalabilityConfig(); if (!scalabilitySetup) { log.warning( `'scalabilitySetup' is not defined in config file, output file for Kibana scalability run won't be generated` ); } - const testData: TestData = config.get('testData'); - - const env = config.get(`kbnTestServer.env`); - if ( - typeof env !== 'object' || - typeof env.ELASTIC_APM_GLOBAL_LABELS !== 'string' || - !env.ELASTIC_APM_GLOBAL_LABELS.includes('journeyName=') - ) { - log.error( - `'journeyName' must be defined in config file: - - env: { - ...config.kbnTestServer.env, - ELASTIC_APM_GLOBAL_LABELS: Object.entries({ - journeyName: , - }) - },` - ); - return; - } - - const envVars: Vars = env.ELASTIC_APM_GLOBAL_LABELS.split(',').reduce( - (acc: Vars, pair: string) => { - const [key, value] = pair.split('='); - return { ...acc, [key]: value }; - }, - {} - ); - const journeyName = envVars.journeyName; - - const buildId = flags.buildId; - if (buildId && typeof buildId !== 'string') { - throw createFlagError('--buildId must be a string'); - } - if (!buildId) { - throw createFlagError('--buildId must be defined'); - } - - const withoutStaticResources = !!flags['without-static-resources'] || false; + const testData = { + esArchives: journey.config.getEsArchives(), + kbnArchives: journey.config.getKbnArchives(), + }; return extractor({ - param: { journeyName, scalabilitySetup, testData, buildId, withoutStaticResources }, - client: { baseURL, username, password }, + param: { + journeyName: journey.config.getName(), + scalabilitySetup, + testData, + buildId, + withoutStaticResources, + }, + client: { + baseURL, + username, + password, + }, log, }); }, diff --git a/packages/kbn-performance-testing-dataset-extractor/src/types.ts b/packages/kbn-performance-testing-dataset-extractor/src/types.ts index 69df8a5fd490c..3b7eb1a356adf 100644 --- a/packages/kbn-performance-testing-dataset-extractor/src/types.ts +++ b/packages/kbn-performance-testing-dataset-extractor/src/types.ts @@ -7,6 +7,7 @@ */ import { ToolingLog } from '@kbn/tooling-log'; +import { ScalabilitySetup } from '@kbn/journeys'; export interface Request { transactionId: string; @@ -31,19 +32,6 @@ export interface Stream { requests: T[]; } -export interface InjectionStep { - action: string; - minUsersCount?: number; - maxUsersCount: number; - duration: string; -} - -export interface ScalabilitySetup { - warmup: InjectionStep[]; - test: InjectionStep[]; - maxDuration: string; -} - export interface TestData { kbnArchives?: string[]; esArchives?: string[]; @@ -52,7 +40,7 @@ export interface TestData { export interface CLIParams { param: { journeyName: string; - scalabilitySetup: ScalabilitySetup; + scalabilitySetup?: ScalabilitySetup; testData: TestData; buildId: string; withoutStaticResources: boolean; diff --git a/packages/kbn-test/BUILD.bazel b/packages/kbn-test/BUILD.bazel index a51766a8f6c63..1deca3a0f6d07 100644 --- a/packages/kbn-test/BUILD.bazel +++ b/packages/kbn-test/BUILD.bazel @@ -99,6 +99,7 @@ TYPES_DEPS = [ "//packages/kbn-tooling-log:npm_module_types", "//packages/kbn-bazel-packages:npm_module_types", "//packages/kbn-get-repo-files:npm_module_types", + "//packages/kbn-ftr-screenshot-filename:npm_module_types", "@npm//@elastic/elasticsearch", "@npm//@jest/console", "@npm//@jest/reporters", @@ -116,6 +117,7 @@ TYPES_DEPS = [ "@npm//jest-snapshot", "@npm//redux", "@npm//rxjs", + "@npm//playwright", "@npm//xmlbuilder", "@npm//@types/archiver", "@npm//@types/chance", diff --git a/packages/kbn-test/index.ts b/packages/kbn-test/index.ts index befdf3f3d3a9b..8d1b3cefde463 100644 --- a/packages/kbn-test/index.ts +++ b/packages/kbn-test/index.ts @@ -6,23 +6,13 @@ * Side Public License, v 1. */ -// @internal -import { - runTestsCli, - processRunTestsCliOptions, - startServersCli, - processStartServersCliOptions, - // @ts-ignore not typed yet -} from './src/functional_tests/cli'; - export { KbnClientRequesterError } from './src/kbn_client/kbn_client_requester_error'; // @internal -export { runTestsCli, processRunTestsCliOptions, startServersCli, processStartServersCliOptions }; +export { startServersCli, startServers } from './src/functional_tests/start_servers'; -// @ts-ignore not typed yet // @internal -export { runTests, startServers } from './src/functional_tests/tasks'; +export { runTestsCli, runTests } from './src/functional_tests/run_tests'; export { getKibanaCliArg, getKibanaCliLoggers } from './src/functional_tests/lib/kibana_cli_args'; @@ -48,15 +38,9 @@ export { systemIndicesSuperuser, } from './src/kbn'; -export { readConfigFile } from './src/functional_test_runner/lib/config/read_config_file'; - -export { runFtrCli } from './src/functional_test_runner/cli'; - // @internal export { setupJUnitReportGeneration, escapeCdata } from './src/mocha'; -export { runFailedTestsReporterCli } from './src/failed_tests_reporter'; - export { CI_PARALLEL_PROCESS_PREFIX } from './src/ci_parallel_process_prefix'; export * from './src/functional_test_runner'; diff --git a/packages/kbn-test/src/failed_tests_reporter/report_failures_to_file.ts b/packages/kbn-test/src/failed_tests_reporter/report_failures_to_file.ts deleted file mode 100644 index 9336c40d35bdb..0000000000000 --- a/packages/kbn-test/src/failed_tests_reporter/report_failures_to_file.ts +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { createHash } from 'crypto'; -import { mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from 'fs'; -import { join, basename, resolve } from 'path'; - -import { ToolingLog } from '@kbn/tooling-log'; -import { REPO_ROOT } from '@kbn/utils'; -import { escape } from 'he'; - -import { BuildkiteMetadata } from './buildkite_metadata'; -import { TestFailure } from './get_failures'; - -const findScreenshots = (dirPath: string, allScreenshots: string[] = []) => { - const files = readdirSync(dirPath); - - for (const file of files) { - if (statSync(join(dirPath, file)).isDirectory()) { - if (file.match(/node_modules/)) { - continue; - } - - allScreenshots = findScreenshots(join(dirPath, file), allScreenshots); - } else { - const fullPath = join(dirPath, file); - if (fullPath.match(/screenshots\/failure\/.+\.png$/)) { - allScreenshots.push(fullPath); - } - } - } - - return allScreenshots; -}; - -export function reportFailuresToFile( - log: ToolingLog, - failures: TestFailure[], - bkMeta: BuildkiteMetadata -) { - if (!failures?.length) { - return; - } - - let screenshots: string[]; - try { - screenshots = [ - ...findScreenshots(join(REPO_ROOT, 'test', 'functional')), - ...findScreenshots(join(REPO_ROOT, 'x-pack', 'test', 'functional')), - ]; - } catch (e) { - log.error(e as Error); - screenshots = []; - } - - const screenshotsByName: Record = {}; - for (const screenshot of screenshots) { - const [name] = basename(screenshot).split('.'); - screenshotsByName[name] = screenshot; - } - - // Jest could, in theory, fail 1000s of tests and write 1000s of failures - // So let's just write files for the first 20 - for (const failure of failures.slice(0, 20)) { - const hash = createHash('md5').update(failure.name).digest('hex'); - const filenameBase = `${ - process.env.BUILDKITE_JOB_ID ? process.env.BUILDKITE_JOB_ID + '_' : '' - }${hash}`; - const dir = join('target', 'test_failures'); - - const failureLog = [ - ['Test:', '-----', failure.classname, failure.name, ''], - ['Failure:', '--------', failure.failure], - failure['system-out'] ? ['', 'Standard Out:', '-------------', failure['system-out']] : [], - ] - .flat() - .join('\n'); - - const failureJSON = JSON.stringify( - { - ...failure, - hash, - buildId: bkMeta.buildId, - jobId: bkMeta.jobId, - url: bkMeta.url, - jobUrl: bkMeta.jobUrl, - jobName: bkMeta.jobName, - }, - null, - 2 - ); - - let screenshot = ''; - const truncatedName = failure.name.replace(/([^ a-zA-Z0-9-]+)/g, '_').slice(0, 80); - const failureNameHash = createHash('sha256').update(failure.name).digest('hex'); - const screenshotName = `${truncatedName}-${failureNameHash}`; - - if (screenshotsByName[screenshotName]) { - try { - screenshot = readFileSync(screenshotsByName[screenshotName]).toString('base64'); - } catch (e) { - log.error(e as Error); - } - } - - const screenshotHtml = screenshot - ? `` - : ''; - - const failureHTML = readFileSync( - resolve( - REPO_ROOT, - 'packages/kbn-test/src/failed_tests_reporter/report_failures_to_file_html_template.html' - ) - ) - .toString() - .replace('$TITLE', escape(failure.name)) - .replace( - '$MAIN', - ` - ${failure.classname - .split('.') - .map((part) => `
${escape(part.replace('ยท', '.'))}
`) - .join('')} -
-

${escape(failure.name)}

-

- - Failures in tracked branches: ${ - failure.failureCount || 0 - } - ${ - failure.githubIssue - ? `
${escape( - failure.githubIssue - )}` - : '' - } - -

- ${ - bkMeta.jobUrl - ? `

- - Buildkite Job
- ${escape(bkMeta.jobUrl)} - -

` - : '' - } - 
${escape(failure.failure)}
- ${screenshotHtml} - 
${escape(failure['system-out'] || '')}
- ` - ); - - mkdirSync(dir, { recursive: true }); - writeFileSync(join(dir, `${filenameBase}.log`), failureLog, 'utf8'); - writeFileSync(join(dir, `${filenameBase}.html`), failureHTML, 'utf8'); - writeFileSync(join(dir, `${filenameBase}.json`), failureJSON, 'utf8'); - } -} diff --git a/packages/kbn-test/src/failed_tests_reporter/run_failed_tests_reporter_cli.ts b/packages/kbn-test/src/failed_tests_reporter/run_failed_tests_reporter_cli.ts deleted file mode 100644 index 5702372aab7be..0000000000000 --- a/packages/kbn-test/src/failed_tests_reporter/run_failed_tests_reporter_cli.ts +++ /dev/null @@ -1,210 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import Path from 'path'; - -import { REPO_ROOT } from '@kbn/utils'; -import { run } from '@kbn/dev-cli-runner'; -import { createFailError, createFlagError } from '@kbn/dev-cli-errors'; -import { CiStatsReporter } from '@kbn/ci-stats-reporter'; -import globby from 'globby'; -import normalize from 'normalize-path'; - -import { getFailures } from './get_failures'; -import { GithubApi } from './github_api'; -import { updateFailureIssue, createFailureIssue } from './report_failure'; -import { readTestReport } from './test_report'; -import { addMessagesToReport } from './add_messages_to_report'; -import { getReportMessageIter } from './report_metadata'; -import { reportFailuresToEs } from './report_failures_to_es'; -import { reportFailuresToFile } from './report_failures_to_file'; -import { getBuildkiteMetadata } from './buildkite_metadata'; -import { ExistingFailedTestIssues } from './existing_failed_test_issues'; - -const DEFAULT_PATTERNS = [Path.resolve(REPO_ROOT, 'target/junit/**/*.xml')]; -const DISABLE_MISSING_TEST_REPORT_ERRORS = - process.env.DISABLE_MISSING_TEST_REPORT_ERRORS === 'true'; - -export function runFailedTestsReporterCli() { - run( - async ({ log, flags }) => { - const indexInEs = flags['index-errors']; - - let updateGithub = flags['github-update']; - if (updateGithub && !process.env.GITHUB_TOKEN) { - throw createFailError( - 'GITHUB_TOKEN environment variable must be set, otherwise use --no-github-update flag' - ); - } - - let branch: string = ''; - if (updateGithub) { - let isPr = false; - - if (process.env.BUILDKITE === 'true') { - branch = process.env.BUILDKITE_BRANCH || ''; - isPr = process.env.BUILDKITE_PULL_REQUEST === 'true'; - updateGithub = process.env.REPORT_FAILED_TESTS_TO_GITHUB === 'true'; - } else { - // JOB_NAME is formatted as `elastic+kibana+7.x` in some places and `elastic+kibana+7.x/JOB=kibana-intake,node=immutable` in others - const jobNameSplit = (process.env.JOB_NAME || '').split(/\+|\//); - branch = jobNameSplit.length >= 3 ? jobNameSplit[2] : process.env.GIT_BRANCH || ''; - isPr = !!process.env.ghprbPullId; - - const isMainOrVersion = branch === 'main' || branch.match(/^\d+\.(x|\d+)$/); - if (!isMainOrVersion || isPr) { - log.info('Failure issues only created on main/version branch jobs'); - updateGithub = false; - } - } - - if (!branch) { - throw createFailError( - 'Unable to determine originating branch from job name or other environment variables' - ); - } - } - - const githubApi = new GithubApi({ - log, - token: process.env.GITHUB_TOKEN, - dryRun: !updateGithub, - }); - - const bkMeta = getBuildkiteMetadata(); - - try { - const buildUrl = flags['build-url'] || (updateGithub ? '' : 'http://buildUrl'); - if (typeof buildUrl !== 'string' || !buildUrl) { - throw createFlagError('Missing --build-url or process.env.BUILD_URL'); - } - - const patterns = (flags._.length ? flags._ : DEFAULT_PATTERNS).map((p) => - normalize(Path.resolve(p)) - ); - log.info('Searching for reports at', patterns); - const reportPaths = await globby(patterns, { - absolute: true, - }); - - if (!reportPaths.length && DISABLE_MISSING_TEST_REPORT_ERRORS) { - // it is fine for code coverage to not have test results - return; - } - - if (!reportPaths.length) { - throw createFailError(`Unable to find any junit reports with patterns [${patterns}]`); - } - - log.info('found', reportPaths.length, 'junit reports', reportPaths); - - const existingIssues = new ExistingFailedTestIssues(log); - for (const reportPath of reportPaths) { - const report = await readTestReport(reportPath); - const messages = Array.from(getReportMessageIter(report)); - const failures = getFailures(report); - - await existingIssues.loadForFailures(failures); - - if (indexInEs) { - await reportFailuresToEs(log, failures); - } - - for (const failure of failures) { - const pushMessage = (msg: string) => { - messages.push({ - classname: failure.classname, - name: failure.name, - message: msg, - }); - }; - - if (failure.likelyIrrelevant) { - pushMessage( - 'Failure is likely irrelevant' + - (updateGithub ? ', so an issue was not created or updated' : '') - ); - continue; - } - - const existingIssue = existingIssues.getForFailure(failure); - if (existingIssue) { - const { newBody, newCount } = await updateFailureIssue( - buildUrl, - existingIssue, - githubApi, - branch - ); - const url = existingIssue.github.htmlUrl; - existingIssue.github.body = newBody; - failure.githubIssue = url; - failure.failureCount = updateGithub ? newCount : newCount - 1; - pushMessage(`Test has failed ${newCount - 1} times on tracked branches: ${url}`); - if (updateGithub) { - pushMessage(`Updated existing issue: ${url} (fail count: ${newCount})`); - } - continue; - } - - const newIssue = await createFailureIssue(buildUrl, failure, githubApi, branch); - existingIssues.addNewlyCreated(failure, newIssue); - pushMessage('Test has not failed recently on tracked branches'); - if (updateGithub) { - pushMessage(`Created new issue: ${newIssue.html_url}`); - failure.githubIssue = newIssue.html_url; - } - failure.failureCount = updateGithub ? 1 : 0; - } - - // mutates report to include messages and writes updated report to disk - await addMessagesToReport({ - report, - messages, - log, - reportPath, - dryRun: !flags['report-update'], - }); - - reportFailuresToFile(log, failures, bkMeta); - } - } finally { - await CiStatsReporter.fromEnv(log).metrics([ - { - group: 'github api request count', - id: `failed test reporter`, - value: githubApi.getRequestCount(), - meta: Object.fromEntries( - Object.entries(bkMeta).map( - ([k, v]) => [`buildkite${k[0].toUpperCase()}${k.slice(1)}`, v] as const - ) - ), - }, - ]); - } - }, - { - description: `a cli that opens issues or updates existing issues based on junit reports`, - flags: { - boolean: ['github-update', 'report-update'], - string: ['build-url'], - default: { - 'github-update': true, - 'report-update': true, - 'index-errors': true, - 'build-url': process.env.BUILD_URL, - }, - help: ` - --no-github-update Execute the CLI without writing to Github - --no-report-update Execute the CLI without writing to the JUnit reports - --no-index-errors Execute the CLI without indexing failures into Elasticsearch - --build-url URL of the failed build, defaults to process.env.BUILD_URL - `, - }, - } - ); -} diff --git a/packages/kbn-test/src/functional_test_runner/cli.ts b/packages/kbn-test/src/functional_test_runner/cli.ts index de20c93c39995..dfd1edc1d8fc4 100644 --- a/packages/kbn-test/src/functional_test_runner/cli.ts +++ b/packages/kbn-test/src/functional_test_runner/cli.ts @@ -9,26 +9,15 @@ import Path from 'path'; import { inspect } from 'util'; -import { run, Flags } from '@kbn/dev-cli-runner'; +import { run } from '@kbn/dev-cli-runner'; import { createFlagError } from '@kbn/dev-cli-errors'; import { ToolingLog } from '@kbn/tooling-log'; import { getTimeReporter } from '@kbn/ci-stats-reporter'; import exitHook from 'exit-hook'; +import { readConfigFile, EsVersion } from './lib'; import { FunctionalTestRunner } from './functional_test_runner'; -const makeAbsolutePath = (v: string) => Path.resolve(process.cwd(), v); -const toArray = (v: string | string[]) => ([] as string[]).concat(v || []); -const parseInstallDir = (flags: Flags) => { - const flag = flags['kibana-install-dir']; - - if (typeof flag !== 'string' && flag !== undefined) { - throw createFlagError('--kibana-install-dir must be a string or not defined'); - } - - return flag ? makeAbsolutePath(flag) : undefined; -}; - export function runFtrCli() { const runStartTime = Date.now(); const toolingLog = new ToolingLog({ @@ -37,52 +26,49 @@ export function runFtrCli() { }); const reportTime = getTimeReporter(toolingLog, 'scripts/functional_test_runner'); run( - async ({ flags, log }) => { - const esVersion = flags['es-version'] || undefined; // convert "" to undefined - if (esVersion !== undefined && typeof esVersion !== 'string') { - throw createFlagError('expected --es-version to be a string'); + async ({ flagsReader, log }) => { + const esVersionInput = flagsReader.string('es-version'); + + const configPaths = [ + ...(flagsReader.arrayOfStrings('config') ?? []), + ...(flagsReader.arrayOfStrings('journey') ?? []), + ].map((rel) => Path.resolve(rel)); + if (configPaths.length !== 1) { + throw createFlagError(`Expected there to be exactly one --config/--journey flag`); } - const configRel = flags.config; - if (typeof configRel !== 'string' || !configRel) { - throw createFlagError('--config is required'); - } - const configPath = makeAbsolutePath(configRel); - - const functionalTestRunner = new FunctionalTestRunner( - log, - configPath, - { - mochaOpts: { - bail: flags.bail, - dryRun: flags['dry-run'], - grep: flags.grep || undefined, - invert: flags.invert, - }, - kbnTestServer: { - installDir: parseInstallDir(flags), - }, - suiteFiles: { - include: toArray(flags.include as string | string[]).map(makeAbsolutePath), - exclude: toArray(flags.exclude as string | string[]).map(makeAbsolutePath), - }, - suiteTags: { - include: toArray(flags['include-tag'] as string | string[]), - exclude: toArray(flags['exclude-tag'] as string | string[]), - }, - updateBaselines: flags.updateBaselines || flags.u, - updateSnapshots: flags.updateSnapshots || flags.u, + const esVersion = esVersionInput ? new EsVersion(esVersionInput) : EsVersion.getDefault(); + const settingOverrides = { + mochaOpts: { + bail: flagsReader.boolean('bail'), + dryRun: flagsReader.boolean('dry-run'), + grep: flagsReader.string('grep'), + invert: flagsReader.boolean('invert'), }, - esVersion - ); + kbnTestServer: { + installDir: flagsReader.path('kibana-install-dir'), + }, + suiteFiles: { + include: flagsReader.arrayOfPaths('include') ?? [], + exclude: flagsReader.arrayOfPaths('exclude') ?? [], + }, + suiteTags: { + include: flagsReader.arrayOfStrings('include-tag') ?? [], + exclude: flagsReader.arrayOfStrings('exclude-tag') ?? [], + }, + updateBaselines: flagsReader.boolean('updateBaselines') || flagsReader.boolean('u'), + updateSnapshots: flagsReader.boolean('updateSnapshots') || flagsReader.boolean('u'), + }; + + const config = await readConfigFile(log, esVersion, configPaths[0], settingOverrides); - await functionalTestRunner.readConfigFile(); + const functionalTestRunner = new FunctionalTestRunner(log, config, esVersion); - if (flags.throttle) { + if (flagsReader.boolean('throttle')) { process.env.TEST_THROTTLE_NETWORK = '1'; } - if (flags.headless) { + if (flagsReader.boolean('headless')) { process.env.TEST_BROWSER_HEADLESS = '1'; } @@ -95,7 +81,7 @@ export function runFtrCli() { await reportTime(runStartTime, 'total', { success: false, err: err.message, - ...flags, + ...Object.fromEntries(flagsReader.getUsed().entries()), }); log.indent(-log.getIndent()); log.error(err); @@ -103,7 +89,7 @@ export function runFtrCli() { } else { await reportTime(runStartTime, 'total', { success: true, - ...flags, + ...Object.fromEntries(flagsReader.getUsed().entries()), }); } @@ -118,7 +104,7 @@ export function runFtrCli() { exitHook(teardown); try { - if (flags['test-stats']) { + if (flagsReader.boolean('test-stats')) { process.stderr.write( JSON.stringify(await functionalTestRunner.getTestStats(), null, 2) + '\n' ); @@ -139,6 +125,7 @@ export function runFtrCli() { flags: { string: [ 'config', + 'journey', 'grep', 'include', 'exclude', @@ -159,7 +146,8 @@ export function runFtrCli() { 'dry-run', ], help: ` - --config=path path to a config file + --config=path path to a config file (either this or --journey is required) + --journey=path path to a journey file (either this or --config is required) --bail stop tests after the first failure --grep pattern used to select which tests to run --invert invert grep to exclude tests diff --git a/packages/kbn-test/src/functional_test_runner/fake_mocha_types.ts b/packages/kbn-test/src/functional_test_runner/fake_mocha_types.ts index 506b6f139f736..17e9663e33883 100644 --- a/packages/kbn-test/src/functional_test_runner/fake_mocha_types.ts +++ b/packages/kbn-test/src/functional_test_runner/fake_mocha_types.ts @@ -15,6 +15,7 @@ import { EventEmitter } from 'events'; export interface Suite { + currentTest?: Test; suites: Suite[]; tests: Test[]; title: string; diff --git a/packages/kbn-test/src/functional_test_runner/functional_test_runner.ts b/packages/kbn-test/src/functional_test_runner/functional_test_runner.ts index 4e549e960dc26..11f99abfa6fbf 100644 --- a/packages/kbn-test/src/functional_test_runner/functional_test_runner.ts +++ b/packages/kbn-test/src/functional_test_runner/functional_test_runner.ts @@ -14,7 +14,6 @@ import { REPO_ROOT } from '@kbn/utils'; import { Suite, Test } from './fake_mocha_types'; import { Lifecycle, - readConfigFile, ProviderCollection, Providers, readProviderSpec, @@ -32,8 +31,7 @@ export class FunctionalTestRunner { private readonly esVersion: EsVersion; constructor( private readonly log: ToolingLog, - private readonly configFile: string, - private readonly configOverrides: any, + private readonly config: Config, esVersion?: string | EsVersion ) { this.esVersion = @@ -47,8 +45,8 @@ export class FunctionalTestRunner { async run(abortSignal?: AbortSignal) { const testStats = await this.getTestStats(); - return await this.runHarness(async (config, lifecycle, coreProviders) => { - SuiteTracker.startTracking(lifecycle, this.configFile); + return await this.runHarness(async (lifecycle, coreProviders) => { + SuiteTracker.startTracking(lifecycle, this.config.path); const realServices = !testStats || (testStats.testCount > 0 && testStats.nonSkippedTestCount > 0); @@ -56,19 +54,19 @@ export class FunctionalTestRunner { const providers = realServices ? new ProviderCollection(this.log, [ ...coreProviders, - ...readProviderSpec('Service', config.get('services')), - ...readProviderSpec('PageObject', config.get('pageObjects')), + ...readProviderSpec('Service', this.config.get('services')), + ...readProviderSpec('PageObject', this.config.get('pageObjects')), ]) - : this.getStubProviderCollection(config, coreProviders); + : this.getStubProviderCollection(coreProviders); if (realServices) { if (providers.hasService('es')) { - await this.validateEsVersion(config); + await this.validateEsVersion(); } await providers.loadAll(); } - const customTestRunner = config.get('testRunner'); + const customTestRunner = this.config.get('testRunner'); if (customTestRunner) { this.log.warning( 'custom test runner defined, ignoring all mocha/suite/filtering related options' @@ -78,7 +76,7 @@ export class FunctionalTestRunner { let reporter; let reporterOptions; - if (config.get('mochaOpts.dryRun')) { + if (this.config.get('mochaOpts.dryRun')) { // override default reporter for dryRun results const targetFile = Path.resolve(REPO_ROOT, 'target/functional-tests/dryRunOutput.json'); reporter = 'json'; @@ -88,22 +86,22 @@ export class FunctionalTestRunner { this.log.info(`Dry run results will be stored in ${targetFile}`); } - const mocha = await setupMocha( + const mocha = await setupMocha({ lifecycle, - this.log, - config, + log: this.log, + config: this.config, providers, - this.esVersion, + esVersion: this.esVersion, reporter, - reporterOptions - ); + reporterOptions, + }); // there's a bug in mocha's dry run, see https://github.com/mochajs/mocha/issues/4838 // until we can update to a mocha version where this is fixed, we won't actually // execute the mocha dry run but simulate it by reading the suites and tests of // the mocha object and writing a report file with similar structure to the json report // (just leave out some execution details like timing, retry and erros) - if (config.get('mochaOpts.dryRun')) { + if (this.config.get('mochaOpts.dryRun')) { return this.simulateMochaDryRun(mocha); } @@ -123,8 +121,8 @@ export class FunctionalTestRunner { }); } - private async validateEsVersion(config: Config) { - const es = createEsClientForFtrConfig(config); + private async validateEsVersion() { + const es = createEsClientForFtrConfig(this.config); let esInfo; try { @@ -151,13 +149,19 @@ export class FunctionalTestRunner { } async getTestStats() { - return await this.runHarness(async (config, lifecycle, coreProviders) => { - if (config.get('testRunner')) { + return await this.runHarness(async (lifecycle, coreProviders) => { + if (this.config.get('testRunner')) { return; } - const providers = this.getStubProviderCollection(config, coreProviders); - const mocha = await setupMocha(lifecycle, this.log, config, providers, this.esVersion); + const providers = this.getStubProviderCollection(coreProviders); + const mocha = await setupMocha({ + lifecycle, + log: this.log, + config: this.config, + providers, + esVersion: this.esVersion, + }); const queue = new Set([mocha.suite]); const allTests: Test[] = []; @@ -178,7 +182,7 @@ export class FunctionalTestRunner { }); } - private getStubProviderCollection(config: Config, coreProviders: Providers) { + private getStubProviderCollection(coreProviders: Providers) { // when we want to load the tests but not actually run anything we can // use stubbed providers which allow mocha to do it's thing without taking // too much time @@ -206,32 +210,30 @@ export class FunctionalTestRunner { ...coreProviders, ...readStubbedProviderSpec( 'Service', - config.get('services'), - config.get('servicesRequiredForTestAnalysis') + this.config.get('services'), + this.config.get('servicesRequiredForTestAnalysis') ), - ...readStubbedProviderSpec('PageObject', config.get('pageObjects'), []), + ...readStubbedProviderSpec('PageObject', this.config.get('pageObjects'), []), ]); } private async runHarness( - handler: (config: Config, lifecycle: Lifecycle, coreProviders: Providers) => Promise + handler: (lifecycle: Lifecycle, coreProviders: Providers) => Promise ): Promise { let runErrorOccurred = false; const lifecycle = new Lifecycle(this.log); try { - const config = await this.readConfigFile(); - this.log.debug('Config loaded'); - if ( - (!config.get('testFiles') || config.get('testFiles').length === 0) && - !config.get('testRunner') + this.config.module.type !== 'journey' && + (!this.config.get('testFiles') || this.config.get('testFiles').length === 0) && + !this.config.get('testRunner') ) { throw new Error('No tests defined.'); } const dockerServers = new DockerServersService( - config.get('dockerServers'), + this.config.get('dockerServers'), this.log, lifecycle ); @@ -240,13 +242,13 @@ export class FunctionalTestRunner { const coreProviders = readProviderSpec('Service', { lifecycle: () => lifecycle, log: () => this.log, - config: () => config, + config: () => this.config, dockerServers: () => dockerServers, esVersion: () => this.esVersion, - dedicatedTaskRunner: () => new DedicatedTaskRunner(config, this.log), + dedicatedTaskRunner: () => new DedicatedTaskRunner(this.config, this.log), }); - return await handler(config, lifecycle, coreProviders); + return await handler(lifecycle, coreProviders); } catch (runError) { runErrorOccurred = true; throw runError; @@ -265,10 +267,6 @@ export class FunctionalTestRunner { } } - public async readConfigFile() { - return await readConfigFile(this.log, this.esVersion, this.configFile, this.configOverrides); - } - simulateMochaDryRun(mocha: any) { interface TestEntry { file: string; diff --git a/packages/kbn-test/src/functional_test_runner/lib/config/config.test.ts b/packages/kbn-test/src/functional_test_runner/lib/config/config.test.ts index d551e7a884b41..f55a68d3e025c 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/config/config.test.ts +++ b/packages/kbn-test/src/functional_test_runner/lib/config/config.test.ts @@ -23,6 +23,7 @@ describe('Config', () => { }, primary: true, path: process.cwd(), + module: {} as any, }); expect(config.has('services.foo')).toEqual(true); diff --git a/packages/kbn-test/src/functional_test_runner/lib/config/config.ts b/packages/kbn-test/src/functional_test_runner/lib/config/config.ts index d6248b9628e73..6fa2f2acc9046 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/config/config.ts +++ b/packages/kbn-test/src/functional_test_runner/lib/config/config.ts @@ -10,6 +10,7 @@ import { Schema } from 'joi'; import { cloneDeepWith, get, has, toPath } from 'lodash'; import { schema } from './schema'; +import { ConfigModule } from './config_loading'; const $values = Symbol('values'); @@ -17,25 +18,27 @@ interface Options { settings?: Record; primary?: boolean; path: string; + module: ConfigModule; } export class Config { public readonly path: string; + public readonly module: ConfigModule; private [$values]: Record; constructor(options: Options) { - const { settings = {}, primary = false, path = null } = options || {}; - - if (!path) { + if (!options.path) { throw new TypeError('path is a required option'); } - this.path = path; - const { error, value } = schema.validate(settings, { + this.path = options.path; + this.module = options.module; + + const { error, value } = schema.validate(options.settings, { abortEarly: false, context: { - primary: !!primary, - path, + primary: !!options?.primary, + path: options.path, }, }); diff --git a/packages/kbn-test/src/functional_test_runner/lib/config/read_config_file.test.ts b/packages/kbn-test/src/functional_test_runner/lib/config/config_loading.test.ts similarity index 97% rename from packages/kbn-test/src/functional_test_runner/lib/config/read_config_file.test.ts rename to packages/kbn-test/src/functional_test_runner/lib/config/config_loading.test.ts index 29b723dae7195..a5ccac5edea81 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/config/read_config_file.test.ts +++ b/packages/kbn-test/src/functional_test_runner/lib/config/config_loading.test.ts @@ -7,7 +7,7 @@ */ import { ToolingLog } from '@kbn/tooling-log'; -import { readConfigFile } from './read_config_file'; +import { readConfigFile } from './config_loading'; import { Config } from './config'; import { EsVersion } from '../es_version'; diff --git a/packages/kbn-test/src/functional_test_runner/lib/config/config_loading.ts b/packages/kbn-test/src/functional_test_runner/lib/config/config_loading.ts new file mode 100644 index 0000000000000..cfa2cabec4dfc --- /dev/null +++ b/packages/kbn-test/src/functional_test_runner/lib/config/config_loading.ts @@ -0,0 +1,182 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; +import { ToolingLog } from '@kbn/tooling-log'; +import { defaultsDeep } from 'lodash'; +import { createFlagError, createFailError } from '@kbn/dev-cli-errors'; +import { REPO_ROOT } from '@kbn/utils'; + +import { FtrConfigProvider, GenericFtrProviderContext } from '../../public_types'; +import { Config } from './config'; +import { EsVersion } from '../es_version'; +import { FTR_CONFIGS_MANIFEST_REL, FTR_CONFIGS_MANIFEST_PATHS } from './ftr_configs_manifest'; + +interface LoadSettingsOptions { + path: string; + settingOverrides: any; + primary: boolean; +} + +interface Journey { + config: { + isSkipped(): boolean; + }; + testProvider(ctx: GenericFtrProviderContext): void; +} + +export type ConfigModule = + | { + type: 'config'; + path: string; + provider: FtrConfigProvider; + } + | { + type: 'journey'; + path: string; + provider: FtrConfigProvider; + journey: Journey; + }; + +async function getConfigModule({ + path, + primary, +}: { + path: string; + primary: boolean; +}): Promise { + let resolvedPath; + try { + resolvedPath = require.resolve(path); + } catch (error) { + if (error.code === 'MODULE_NOT_FOUND') { + throw createFlagError(`Unable to find config file [${path}]`); + } + throw error; + } + + if ( + primary && + !FTR_CONFIGS_MANIFEST_PATHS.includes(resolvedPath) && + !resolvedPath.includes(`${Path.sep}__fixtures__${Path.sep}`) + ) { + const rel = Path.relative(REPO_ROOT, resolvedPath); + throw createFlagError( + `Refusing to load FTR Config at [${rel}] which is not listed in [${FTR_CONFIGS_MANIFEST_REL}]. All FTR Config files must be listed there, use the "enabled" key if the FTR Config should be run on automatically on PR CI, or the "disabled" key if it is run manually or by a special job.` + ); + } + + // eslint-disable-next-line @typescript-eslint/no-var-requires + const exports = require(resolvedPath); + const defaultExport = exports.__esModule ? exports.default : exports; + if (typeof defaultExport === 'function') { + return { + type: 'config', + path: resolvedPath, + provider: defaultExport, + }; + } + + const { journey } = exports; + if ( + !journey.constructor || + typeof journey.constructor !== 'function' || + journey.constructor.name !== 'Journey' + ) { + const rel = Path.relative(process.cwd(), resolvedPath); + throw createFailError( + `"journey" export in journey at [${rel}] is not a valid instance of Journey` + ); + } + + return { + type: 'journey', + path: resolvedPath, + provider: journey.constructor.convertToFtrConfigProvider(journey), + journey, + }; +} + +const cache = new WeakMap>(); +async function executeConfigModule( + log: ToolingLog, + esVersion: EsVersion, + options: LoadSettingsOptions, + module: ConfigModule +): Promise { + const cached = cache.get(module.provider); + + if (cached) { + return defaultsDeep({}, options.settingOverrides, await cached); + } + + log.debug(`Loading config file from ${Path.relative(process.cwd(), options.path)}`); + const settings: Promise = module.provider({ + log, + esVersion, + async readConfigFile(p: string) { + const childModule = await getConfigModule({ + primary: false, + path: p, + }); + + return new Config({ + settings: await executeConfigModule( + log, + esVersion, + { + path: childModule.path, + settingOverrides: {}, + primary: false, + }, + childModule + ), + primary: false, + path: p, + module: childModule, + }); + }, + }); + + cache.set(module.provider, Promise.resolve(settings)); + + return defaultsDeep({}, options.settingOverrides, await settings); +} + +const ident = (vars: T) => vars; + +export async function readConfigFile( + log: ToolingLog, + esVersion: EsVersion, + path: string, + settingOverrides: any = {}, + extendSettings: (vars: any) => any = ident +) { + const module = await getConfigModule({ + primary: true, + path, + }); + + return new Config({ + settings: extendSettings( + await executeConfigModule( + log, + esVersion, + { + path, + settingOverrides, + primary: true, + }, + module + ) + ), + primary: true, + path, + module, + }); +} diff --git a/packages/kbn-test/src/functional_test_runner/lib/config/index.ts b/packages/kbn-test/src/functional_test_runner/lib/config/index.ts index a1f22e215307d..f0fa7e9989c86 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/config/index.ts +++ b/packages/kbn-test/src/functional_test_runner/lib/config/index.ts @@ -7,5 +7,6 @@ */ export { Config } from './config'; -export { readConfigFile } from './read_config_file'; +export { readConfigFile } from './config_loading'; +export type { ConfigModule } from './config_loading'; export { runCheckFtrConfigsCli } from './run_check_ftr_configs_cli'; diff --git a/packages/kbn-test/src/functional_test_runner/lib/config/read_config_file.ts b/packages/kbn-test/src/functional_test_runner/lib/config/read_config_file.ts deleted file mode 100644 index 142e5c9da9b3b..0000000000000 --- a/packages/kbn-test/src/functional_test_runner/lib/config/read_config_file.ts +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import Path from 'path'; -import { ToolingLog } from '@kbn/tooling-log'; -import { defaultsDeep } from 'lodash'; -import { createFlagError } from '@kbn/dev-cli-errors'; -import { REPO_ROOT } from '@kbn/utils'; - -import { Config } from './config'; -import { EsVersion } from '../es_version'; -import { FTR_CONFIGS_MANIFEST_REL, FTR_CONFIGS_MANIFEST_PATHS } from './ftr_configs_manifest'; - -const cache = new WeakMap(); - -async function getSettingsFromFile( - log: ToolingLog, - esVersion: EsVersion, - options: { - path: string; - settingOverrides: any; - primary: boolean; - } -) { - let resolvedPath; - try { - resolvedPath = require.resolve(options.path); - } catch (error) { - if (error.code === 'MODULE_NOT_FOUND') { - throw createFlagError(`Unable to find config file [${options.path}]`); - } - - throw error; - } - - if ( - options.primary && - !FTR_CONFIGS_MANIFEST_PATHS.includes(resolvedPath) && - !resolvedPath.includes(`${Path.sep}__fixtures__${Path.sep}`) - ) { - const rel = Path.relative(REPO_ROOT, resolvedPath); - throw createFlagError( - `Refusing to load FTR Config at [${rel}] which is not listed in [${FTR_CONFIGS_MANIFEST_REL}]. All FTR Config files must be listed there, use the "enabled" key if the FTR Config should be run on automatically on PR CI, or the "disabled" key if it is run manually or by a special job.` - ); - } - - const configModule = require(resolvedPath); // eslint-disable-line @typescript-eslint/no-var-requires - const configProvider = configModule.__esModule ? configModule.default : configModule; - - if (!cache.has(configProvider)) { - log.debug('Loading config file from %j', resolvedPath); - cache.set( - configProvider, - configProvider({ - log, - esVersion, - async readConfigFile(p: string, o: any) { - return new Config({ - settings: await getSettingsFromFile(log, esVersion, { - path: p, - settingOverrides: o, - primary: false, - }), - primary: false, - path: p, - }); - }, - }) - ); - } - - const settingsWithDefaults: any = defaultsDeep( - {}, - options.settingOverrides, - await cache.get(configProvider)! - ); - - return settingsWithDefaults; -} - -export async function readConfigFile( - log: ToolingLog, - esVersion: EsVersion, - path: string, - settingOverrides: any = {} -) { - return new Config({ - settings: await getSettingsFromFile(log, esVersion, { - path, - settingOverrides, - primary: true, - }), - primary: true, - path, - }); -} diff --git a/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts b/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts index 7e7ba9e26eb48..ce44dd3cc0496 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts +++ b/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts @@ -14,7 +14,6 @@ import type { CustomHelpers } from 'joi'; // valid pattern for ID // enforced camel-case identifiers for consistency const ID_PATTERN = /^[a-zA-Z0-9_]+$/; -const SCALABILITY_DURATION_PATTERN = /^[1-9]\d{0,}[m|s]$/; // it will search both --inspect and --inspect-brk const INSPECTING = !!process.execArgv.find((arg) => arg.includes('--inspect')); @@ -164,6 +163,7 @@ export const schema = Joi.object() .keys({ enabled: Joi.boolean().default(!!process.env.CI && !process.env.DISABLE_JUNIT_REPORTER), reportName: Joi.string(), + metadata: Joi.object().unknown(true).default(), }) .default(), @@ -267,63 +267,6 @@ export const schema = Joi.object() }) .default(), - /** - * Optional settings to list test data archives, that will be loaded during the 'beforeTests' - * lifecycle phase and unloaded during the 'cleanup' lifecycle phase. - */ - testData: Joi.object() - .keys({ - kbnArchives: Joi.array().items(Joi.string()).default([]), - esArchives: Joi.array().items(Joi.string()).default([]), - }) - .default(), - - /** - * Optional settings to enable scalability testing for single user performance journey. - * If defined, 'scalabilitySetup' must include 'warmup' and 'test' stage array, - * 'maxDuration', e.g. '10m' to limit execution time to 10 minutes. - * Each stage must include 'action', 'duration' and 'maxUsersCount'. - * In addition, 'rampConcurrentUsers' requires 'minUsersCount' to ramp users from - * min to max within provided time duration. - */ - scalabilitySetup: Joi.object() - .keys({ - warmup: Joi.array() - .items( - Joi.object().keys({ - action: Joi.string() - .valid('constantConcurrentUsers', 'rampConcurrentUsers') - .required(), - duration: Joi.string().pattern(SCALABILITY_DURATION_PATTERN).required(), - minUsersCount: Joi.number().when('action', { - is: 'rampConcurrentUsers', - then: Joi.number().required().less(Joi.ref('maxUsersCount')), - otherwise: Joi.forbidden(), - }), - maxUsersCount: Joi.number().required().greater(0), - }) - ) - .required(), - test: Joi.array() - .items( - Joi.object().keys({ - action: Joi.string() - .valid('constantConcurrentUsers', 'rampConcurrentUsers') - .required(), - duration: Joi.string().pattern(SCALABILITY_DURATION_PATTERN).required(), - minUsersCount: Joi.number().when('action', { - is: 'rampConcurrentUsers', - then: Joi.number().required().less(Joi.ref('maxUsersCount')), - otherwise: Joi.forbidden(), - }), - maxUsersCount: Joi.number().required().greater(0), - }) - ) - .required(), - maxDuration: Joi.string().pattern(SCALABILITY_DURATION_PATTERN).required(), - }) - .optional(), - // settings for the kibanaServer.uiSettings module uiSettings: Joi.object() .keys({ diff --git a/packages/kbn-test/src/functional_test_runner/lib/es_version.ts b/packages/kbn-test/src/functional_test_runner/lib/es_version.ts index 8b3acde47a4dc..976a2c417c747 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/es_version.ts +++ b/packages/kbn-test/src/functional_test_runner/lib/es_version.ts @@ -11,6 +11,10 @@ import { kibanaPackageJson } from '@kbn/utils'; export class EsVersion { static getDefault() { + if (typeof jest === 'object' && jest) { + return new EsVersion('9.9.9'); + } + // example: https://storage.googleapis.com/kibana-ci-es-snapshots-daily/8.0.0/manifest-latest-verified.json const manifestUrl = process.env.ES_SNAPSHOT_MANIFEST; if (manifestUrl) { diff --git a/packages/kbn-test/src/functional_test_runner/lib/index.ts b/packages/kbn-test/src/functional_test_runner/lib/index.ts index 983a185dee682..3b12849a9ec13 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/index.ts +++ b/packages/kbn-test/src/functional_test_runner/lib/index.ts @@ -8,7 +8,7 @@ export { Lifecycle } from './lifecycle'; export { LifecyclePhase } from './lifecycle_phase'; -export { readConfigFile, Config, runCheckFtrConfigsCli } from './config'; +export * from './config'; export * from './providers'; // @internal export { runTests, setupMocha } from './mocha'; diff --git a/packages/kbn-test/src/functional_test_runner/lib/mocha/load_test_files.js b/packages/kbn-test/src/functional_test_runner/lib/mocha/load_test_files.js deleted file mode 100644 index 7c4a6ddcd1430..0000000000000 --- a/packages/kbn-test/src/functional_test_runner/lib/mocha/load_test_files.js +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { isAbsolute } from 'path'; - -import { loadTracer } from '../load_tracer'; -import { decorateMochaUi } from './decorate_mocha_ui'; -import { decorateSnapshotUi } from '../snapshots/decorate_snapshot_ui'; - -/** - * Load an array of test files into a mocha instance - * - * @param {Mocha} mocha - * @param {ToolingLog} log - * @param {Config} config - * @param {ProviderCollection} providers - * @param {String} path - * @return {undefined} - mutates mocha, no return value - */ -export const loadTestFiles = ({ - mocha, - log, - config, - lifecycle, - providers, - paths, - updateBaselines, - updateSnapshots, -}) => { - const dockerServers = config.get('dockerServers'); - const isDockerGroup = dockerServers && Object.keys(dockerServers).length; - - decorateSnapshotUi({ lifecycle, updateSnapshots, isCi: !!process.env.CI }); - - const innerLoadTestFile = (path) => { - if (typeof path !== 'string' || !isAbsolute(path)) { - throw new TypeError('loadTestFile() only accepts absolute paths'); - } - - loadTracer(path, `testFile[${path}]`, () => { - log.verbose('Loading test file %s', path); - - const testModule = require(path); // eslint-disable-line import/no-dynamic-require - const testProvider = testModule.__esModule ? testModule.default : testModule; - - runTestProvider(testProvider, path); // eslint-disable-line no-use-before-define - }); - }; - - const runTestProvider = (provider, path) => { - if (typeof provider !== 'function') { - throw new Error(`Default export of test files must be a function, got ${provider}`); - } - - loadTracer(provider, `testProvider[${path}]`, () => { - // mocha.suite hocus-pocus comes from: https://git.io/vDnXO - - const context = decorateMochaUi(log, lifecycle, global, { - isDockerGroup, - rootTags: config.get('rootTags'), - }); - mocha.suite.emit('pre-require', context, path, mocha); - - const returnVal = provider({ - loadTestFile: innerLoadTestFile, - getService: providers.getService, - getPageObject: providers.getPageObject, - getPageObjects: providers.getPageObjects, - updateBaselines, - }); - - if (returnVal && typeof returnVal.then === 'function') { - throw new TypeError('Default export of test files must not be an async function'); - } - - mocha.suite.emit('require', returnVal, path, mocha); - mocha.suite.emit('post-require', global, path, mocha); - - context.revertProxiedAssignments(); - }); - }; - - paths.forEach(innerLoadTestFile); -}; diff --git a/packages/kbn-test/src/functional_test_runner/lib/mocha/load_tests.ts b/packages/kbn-test/src/functional_test_runner/lib/mocha/load_tests.ts new file mode 100644 index 0000000000000..32f61caf1b3c7 --- /dev/null +++ b/packages/kbn-test/src/functional_test_runner/lib/mocha/load_tests.ts @@ -0,0 +1,120 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { isAbsolute } from 'path'; + +import type { ToolingLog } from '@kbn/tooling-log'; + +import type { Config } from '../config'; +import type { GenericFtrProviderContext } from '../../public_types'; +import type { Lifecycle } from '../lifecycle'; +import type { ProviderCollection } from '../providers'; +import { loadTracer } from '../load_tracer'; +import { decorateSnapshotUi } from '../snapshots/decorate_snapshot_ui'; + +// @ts-expect-error not js yet +import { decorateMochaUi } from './decorate_mocha_ui'; + +type TestProvider = (ctx: GenericFtrProviderContext) => void; + +interface Options { + mocha: any; + log: ToolingLog; + config: Config; + lifecycle: Lifecycle; + providers: ProviderCollection; + paths: string[]; + updateBaselines: boolean; + updateSnapshots: boolean; +} + +const isObj = (v: unknown): v is Record => typeof v === 'object' && v !== null; + +/** + * Load an array of test files or a test provider into a mocha instance + */ +export const loadTests = ({ + mocha, + log, + config, + lifecycle, + providers, + paths, + updateBaselines, + updateSnapshots, +}: Options) => { + const dockerServers = config.get('dockerServers'); + const isDockerGroup = dockerServers && Object.keys(dockerServers).length; + + const ctx: GenericFtrProviderContext = { + loadTestFile, + getService: providers.getService as any, + hasService: providers.hasService as any, + getPageObject: providers.getPageObject as any, + getPageObjects: providers.getPageObjects as any, + updateBaselines, + }; + + decorateSnapshotUi({ lifecycle, updateSnapshots, isCi: !!process.env.CI }); + + function loadTestFile(path: string) { + if (typeof path !== 'string' || !isAbsolute(path)) { + throw new TypeError('loadTestFile() only accepts absolute paths'); + } + + loadTracer(path, `testFile[${path}]`, () => { + log.verbose('Loading test file %s', path); + + // eslint-disable-next-line @typescript-eslint/no-var-requires + const testModule = require(path); + const testProvider = testModule.__esModule ? testModule.default : testModule; + + runTestProvider(testProvider, path); + }); + } + + function withMocha(debugPath: string, fn: () => void) { + // mocha.suite hocus-pocus comes from: https://git.io/vDnXO + const context = decorateMochaUi(log, lifecycle, global, { + isDockerGroup, + rootTags: config.get('rootTags'), + }); + mocha.suite.emit('pre-require', context, debugPath, mocha); + + fn(); + + mocha.suite.emit('require', undefined, debugPath, mocha); + mocha.suite.emit('post-require', global, debugPath, mocha); + + context.revertProxiedAssignments(); + } + + function runTestProvider(provider: TestProvider, path: string) { + if (typeof provider !== 'function') { + throw new Error(`Default export of test files must be a function, got ${provider}`); + } + + loadTracer(provider, `testProvider[${path}]`, () => { + withMocha(path, () => { + const returnVal = provider(ctx); + if (isObj(returnVal) && typeof returnVal.then === 'function') { + throw new TypeError('Test file providers must not be async or return promises'); + } + }); + }); + } + + const cm = config.module; + if (cm.type === 'journey') { + withMocha(cm.path, () => { + cm.journey.testProvider(ctx); + }); + } else { + paths.forEach(loadTestFile); + } +}; diff --git a/packages/kbn-test/src/functional_test_runner/lib/mocha/reporter/reporter.js b/packages/kbn-test/src/functional_test_runner/lib/mocha/reporter/reporter.js index 66a4c9ce4fd04..b2bea7b079c9b 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/mocha/reporter/reporter.js +++ b/packages/kbn-test/src/functional_test_runner/lib/mocha/reporter/reporter.js @@ -47,6 +47,7 @@ export function MochaReporterProvider({ getService }) { if (config.get('junit.enabled') && config.get('junit.reportName')) { setupJUnitReportGeneration(runner, { reportName: config.get('junit.reportName'), + metadata: config.get('junit.metadata'), }); } diff --git a/packages/kbn-test/src/functional_test_runner/lib/mocha/setup_mocha.js b/packages/kbn-test/src/functional_test_runner/lib/mocha/setup_mocha.ts similarity index 59% rename from packages/kbn-test/src/functional_test_runner/lib/mocha/setup_mocha.js rename to packages/kbn-test/src/functional_test_runner/lib/mocha/setup_mocha.ts index 9261391c5bf6a..10c51517aec94 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/mocha/setup_mocha.js +++ b/packages/kbn-test/src/functional_test_runner/lib/mocha/setup_mocha.ts @@ -6,34 +6,50 @@ * Side Public License, v 1. */ -import Mocha from 'mocha'; import { relative } from 'path'; + import { REPO_ROOT } from '@kbn/utils'; +import { ToolingLog } from '@kbn/tooling-log'; -import { loadTestFiles } from './load_test_files'; +import { Suite } from '../../fake_mocha_types'; +import { loadTests } from './load_tests'; import { filterSuites } from './filter_suites'; +import { Lifecycle } from '../lifecycle'; +import { Config } from '../config'; +import { ProviderCollection } from '../providers'; +import { EsVersion } from '../es_version'; + +// @ts-expect-error not ts yet import { MochaReporterProvider } from './reporter'; +// @ts-expect-error not ts yet import { validateCiGroupTags } from './validate_ci_group_tags'; +interface Options { + lifecycle: Lifecycle; + log: ToolingLog; + config: Config; + providers: ProviderCollection; + esVersion: EsVersion; + reporter?: any; + reporterOptions?: any; +} + +// we use require so that @types/mocha isn't loaded +const Mocha = require('mocha'); // eslint-disable-line @typescript-eslint/no-var-requires + /** * Instantiate mocha and load testfiles into it - * - * @param {Lifecycle} lifecycle - * @param {ToolingLog} log - * @param {Config} config - * @param {ProviderCollection} providers - * @param {EsVersion} esVersion * @return {Promise} */ -export async function setupMocha( +export async function setupMocha({ lifecycle, log, config, providers, esVersion, reporter, - reporterOptions -) { + reporterOptions, +}: Options) { // configure mocha const mocha = new Mocha({ ...config.get('mochaOpts'), @@ -43,19 +59,19 @@ export async function setupMocha( }); // global beforeEach hook in root suite triggers before all others - mocha.suite.beforeEach('global before each', async function () { - await lifecycle.beforeEachTest.trigger(this.currentTest); + mocha.suite.beforeEach('global before each', async function (this: Suite) { + await lifecycle.beforeEachTest.trigger(this.currentTest!); }); - loadTestFiles({ + loadTests({ mocha, log, config, lifecycle, providers, - paths: config.get('testFiles'), updateBaselines: config.get('updateBaselines'), updateSnapshots: config.get('updateSnapshots'), + paths: config.get('testFiles'), }); // valiate that there aren't any tests in multiple ciGroups @@ -76,15 +92,15 @@ export async function setupMocha( filterSuites({ log, mocha, - include: config.get('suiteFiles.include').map((file) => relative(REPO_ROOT, file)), - exclude: config.get('suiteFiles.exclude').map((file) => relative(REPO_ROOT, file)), + include: config.get('suiteFiles.include').map((file: string) => relative(REPO_ROOT, file)), + exclude: config.get('suiteFiles.exclude').map((file: string) => relative(REPO_ROOT, file)), }); filterSuites({ log, mocha, - include: config.get('suiteTags.include').map((tag) => tag.replace(/-\d+$/, '')), - exclude: config.get('suiteTags.exclude').map((tag) => tag.replace(/-\d+$/, '')), + include: config.get('suiteTags.include').map((tag: string) => tag.replace(/-\d+$/, '')), + exclude: config.get('suiteTags.exclude').map((tag: string) => tag.replace(/-\d+$/, '')), }); return mocha; diff --git a/packages/kbn-test/src/functional_test_runner/public_types.ts b/packages/kbn-test/src/functional_test_runner/public_types.ts index 3faa19de73ce1..248b914b2ebe7 100644 --- a/packages/kbn-test/src/functional_test_runner/public_types.ts +++ b/packages/kbn-test/src/functional_test_runner/public_types.ts @@ -105,6 +105,11 @@ export interface GenericFtrProviderContext< * @param path */ loadTestFile(path: string): void; + + /** + * Did the user request that baselines get updated? + */ + updateBaselines: boolean; } export class GenericFtrService> { @@ -117,4 +122,6 @@ export interface FtrConfigProviderContext { readConfigFile(path: string): Promise; } +export type FtrConfigProvider = (ctx: FtrConfigProviderContext) => T | Promise; + export type { Test, Suite }; diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/args.test.js.snap b/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/args.test.js.snap deleted file mode 100644 index ff8961e263f17..0000000000000 --- a/packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/args.test.js.snap +++ /dev/null @@ -1,332 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`display help for run tests CLI displays as expected 1`] = ` -"Run Functional Tests - -Usage: - node scripts/functional_tests --help - node scripts/functional_tests [--config [--config ...]] - node scripts/functional_tests [options] [-- --] - -Options: - --help Display this menu and exit. - --config Pass in a config. Can pass in multiple configs. - --esFrom Build Elasticsearch from source or run from snapshot. Default: $TEST_ES_FROM or snapshot - --kibana-install-dir Run Kibana from existing install directory instead of from source. - --bail Stop the test run at the first failure. - --grep Pattern to select which tests to run. - --updateBaselines Replace baseline screenshots with whatever is generated from the test. - --updateSnapshots Replace inline and file snapshots with whatever is generated from the test. - --u Replace both baseline screenshots and snapshots - --include Files that must included to be run, can be included multiple times. - --exclude Files that must NOT be included to be run, can be included multiple times. - --include-tag Tags that suites must include to be run, can be included multiple times. - --exclude-tag Tags that suites must NOT include to be run, can be included multiple times. - --assert-none-excluded Exit with 1/0 based on if any test is excluded with the current set of tags. - --logToFile Write the log output from Kibana/Elasticsearch to files instead of to stdout - --verbose Log everything. - --debug Run in debug mode. - --quiet Only log errors. - --silent Log nothing. - --dry-run Report tests without executing them." -`; - -exports[`process options for run tests CLI accepts boolean value for updateBaselines 1`] = ` -Object { - "assertNoneExcluded": false, - "configs": Array [ - /foo, - ], - "createLogger": [Function], - "esFrom": "snapshot", - "esVersion": "999.999.999", - "extraKbnOpts": undefined, - "logsDir": undefined, - "suiteFiles": Object { - "exclude": Array [], - "include": Array [], - }, - "suiteTags": Object { - "exclude": Array [], - "include": Array [], - }, - "updateBaselines": true, -} -`; - -exports[`process options for run tests CLI accepts boolean value for updateSnapshots 1`] = ` -Object { - "assertNoneExcluded": false, - "configs": Array [ - /foo, - ], - "createLogger": [Function], - "esFrom": "snapshot", - "esVersion": "999.999.999", - "extraKbnOpts": undefined, - "logsDir": undefined, - "suiteFiles": Object { - "exclude": Array [], - "include": Array [], - }, - "suiteTags": Object { - "exclude": Array [], - "include": Array [], - }, - "updateSnapshots": true, -} -`; - -exports[`process options for run tests CLI accepts debug option 1`] = ` -Object { - "assertNoneExcluded": false, - "configs": Array [ - /foo, - ], - "createLogger": [Function], - "debug": true, - "esFrom": "snapshot", - "esVersion": "999.999.999", - "extraKbnOpts": undefined, - "logsDir": undefined, - "suiteFiles": Object { - "exclude": Array [], - "include": Array [], - }, - "suiteTags": Object { - "exclude": Array [], - "include": Array [], - }, -} -`; - -exports[`process options for run tests CLI accepts empty config value if default passed 1`] = ` -Object { - "assertNoneExcluded": false, - "config": "", - "configs": Array [ - /foo, - ], - "createLogger": [Function], - "esFrom": "snapshot", - "esVersion": "999.999.999", - "extraKbnOpts": undefined, - "logsDir": undefined, - "suiteFiles": Object { - "exclude": Array [], - "include": Array [], - }, - "suiteTags": Object { - "exclude": Array [], - "include": Array [], - }, -} -`; - -exports[`process options for run tests CLI accepts extra server options 1`] = ` -Object { - "_": Object { - "server.foo": "bar", - }, - "assertNoneExcluded": false, - "configs": Array [ - /foo, - ], - "createLogger": [Function], - "esFrom": "snapshot", - "esVersion": "999.999.999", - "extraKbnOpts": Object { - "server.foo": "bar", - }, - "logsDir": undefined, - "suiteFiles": Object { - "exclude": Array [], - "include": Array [], - }, - "suiteTags": Object { - "exclude": Array [], - "include": Array [], - }, -} -`; - -exports[`process options for run tests CLI accepts quiet option 1`] = ` -Object { - "assertNoneExcluded": false, - "configs": Array [ - /foo, - ], - "createLogger": [Function], - "esFrom": "snapshot", - "esVersion": "999.999.999", - "extraKbnOpts": undefined, - "logsDir": undefined, - "quiet": true, - "suiteFiles": Object { - "exclude": Array [], - "include": Array [], - }, - "suiteTags": Object { - "exclude": Array [], - "include": Array [], - }, -} -`; - -exports[`process options for run tests CLI accepts silent option 1`] = ` -Object { - "assertNoneExcluded": false, - "configs": Array [ - /foo, - ], - "createLogger": [Function], - "esFrom": "snapshot", - "esVersion": "999.999.999", - "extraKbnOpts": undefined, - "logsDir": undefined, - "silent": true, - "suiteFiles": Object { - "exclude": Array [], - "include": Array [], - }, - "suiteTags": Object { - "exclude": Array [], - "include": Array [], - }, -} -`; - -exports[`process options for run tests CLI accepts source value for $TEST_ES_FROM 1`] = ` -Object { - "assertNoneExcluded": false, - "configs": Array [ - /foo, - ], - "createLogger": [Function], - "esFrom": "source", - "esVersion": "999.999.999", - "extraKbnOpts": undefined, - "logsDir": undefined, - "suiteFiles": Object { - "exclude": Array [], - "include": Array [], - }, - "suiteTags": Object { - "exclude": Array [], - "include": Array [], - }, -} -`; - -exports[`process options for run tests CLI accepts source value for esFrom 1`] = ` -Object { - "assertNoneExcluded": false, - "configs": Array [ - /foo, - ], - "createLogger": [Function], - "esFrom": "source", - "esVersion": "999.999.999", - "extraKbnOpts": undefined, - "logsDir": undefined, - "suiteFiles": Object { - "exclude": Array [], - "include": Array [], - }, - "suiteTags": Object { - "exclude": Array [], - "include": Array [], - }, -} -`; - -exports[`process options for run tests CLI accepts string value for kibana-install-dir 1`] = ` -Object { - "assertNoneExcluded": false, - "configs": Array [ - /foo, - ], - "createLogger": [Function], - "esFrom": "snapshot", - "esVersion": "999.999.999", - "extraKbnOpts": undefined, - "installDir": "foo", - "logsDir": undefined, - "suiteFiles": Object { - "exclude": Array [], - "include": Array [], - }, - "suiteTags": Object { - "exclude": Array [], - "include": Array [], - }, -} -`; - -exports[`process options for run tests CLI accepts value for grep 1`] = ` -Object { - "assertNoneExcluded": false, - "configs": Array [ - /foo, - ], - "createLogger": [Function], - "esFrom": "snapshot", - "esVersion": "999.999.999", - "extraKbnOpts": undefined, - "grep": "management", - "logsDir": undefined, - "suiteFiles": Object { - "exclude": Array [], - "include": Array [], - }, - "suiteTags": Object { - "exclude": Array [], - "include": Array [], - }, -} -`; - -exports[`process options for run tests CLI accepts verbose option 1`] = ` -Object { - "assertNoneExcluded": false, - "configs": Array [ - /foo, - ], - "createLogger": [Function], - "esFrom": "snapshot", - "esVersion": "999.999.999", - "extraKbnOpts": undefined, - "logsDir": undefined, - "suiteFiles": Object { - "exclude": Array [], - "include": Array [], - }, - "suiteTags": Object { - "exclude": Array [], - "include": Array [], - }, - "verbose": true, -} -`; - -exports[`process options for run tests CLI prioritizes source flag over $TEST_ES_FROM 1`] = ` -Object { - "assertNoneExcluded": false, - "configs": Array [ - /foo, - ], - "createLogger": [Function], - "esFrom": "snapshot", - "esVersion": "999.999.999", - "extraKbnOpts": undefined, - "logsDir": undefined, - "suiteFiles": Object { - "exclude": Array [], - "include": Array [], - }, - "suiteTags": Object { - "exclude": Array [], - "include": Array [], - }, -} -`; diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/args.js b/packages/kbn-test/src/functional_tests/cli/run_tests/args.js deleted file mode 100644 index 8b1bf471f4e98..0000000000000 --- a/packages/kbn-test/src/functional_tests/cli/run_tests/args.js +++ /dev/null @@ -1,193 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import Path from 'path'; - -import { v4 as uuid } from 'uuid'; -import dedent from 'dedent'; -import { REPO_ROOT } from '@kbn/utils'; -import { ToolingLog, pickLevelFromFlags } from '@kbn/tooling-log'; -import { EsVersion } from '../../../functional_test_runner'; - -const options = { - help: { desc: 'Display this menu and exit.' }, - config: { - arg: '', - desc: 'Pass in a config. Can pass in multiple configs.', - }, - esFrom: { - arg: '', - choices: ['snapshot', 'source'], - desc: 'Build Elasticsearch from source or run from snapshot.', - defaultHelp: 'Default: $TEST_ES_FROM or snapshot', - }, - 'kibana-install-dir': { - arg: '', - desc: 'Run Kibana from existing install directory instead of from source.', - }, - bail: { desc: 'Stop the test run at the first failure.' }, - grep: { - arg: '', - desc: 'Pattern to select which tests to run.', - }, - updateBaselines: { - desc: 'Replace baseline screenshots with whatever is generated from the test.', - }, - updateSnapshots: { - desc: 'Replace inline and file snapshots with whatever is generated from the test.', - }, - u: { - desc: 'Replace both baseline screenshots and snapshots', - }, - include: { - arg: '', - desc: 'Files that must included to be run, can be included multiple times.', - }, - exclude: { - arg: '', - desc: 'Files that must NOT be included to be run, can be included multiple times.', - }, - 'include-tag': { - arg: '', - desc: 'Tags that suites must include to be run, can be included multiple times.', - }, - 'exclude-tag': { - arg: '', - desc: 'Tags that suites must NOT include to be run, can be included multiple times.', - }, - 'assert-none-excluded': { - desc: 'Exit with 1/0 based on if any test is excluded with the current set of tags.', - }, - logToFile: { - desc: 'Write the log output from Kibana/Elasticsearch to files instead of to stdout', - }, - verbose: { desc: 'Log everything.' }, - debug: { desc: 'Run in debug mode.' }, - quiet: { desc: 'Only log errors.' }, - silent: { desc: 'Log nothing.' }, - 'dry-run': { desc: 'Report tests without executing them.' }, -}; - -export function displayHelp() { - const helpOptions = Object.keys(options) - .filter((name) => name !== '_') - .map((name) => { - const option = options[name]; - return { - ...option, - usage: `${name} ${option.arg || ''}`, - default: option.defaultHelp || '', - }; - }) - .map((option) => { - return `--${option.usage.padEnd(28)} ${option.desc} ${option.default}`; - }) - .join(`\n `); - - return dedent(` - Run Functional Tests - - Usage: - node scripts/functional_tests --help - node scripts/functional_tests [--config [--config ...]] - node scripts/functional_tests [options] [-- --] - - Options: - ${helpOptions} - `); -} - -export function processOptions(userOptions, defaultConfigPaths) { - validateOptions(userOptions); - - let configs; - if (userOptions.config) { - configs = [].concat(userOptions.config); - } else { - if (!defaultConfigPaths || defaultConfigPaths.length === 0) { - throw new Error(`functional_tests: config is required`); - } else { - configs = defaultConfigPaths; - } - } - - if (!userOptions.esFrom) { - userOptions.esFrom = process.env.TEST_ES_FROM || 'snapshot'; - } - - if (userOptions['kibana-install-dir']) { - userOptions.installDir = userOptions['kibana-install-dir']; - delete userOptions['kibana-install-dir']; - } - - userOptions.suiteFiles = { - include: [].concat(userOptions.include || []), - exclude: [].concat(userOptions.exclude || []), - }; - delete userOptions.include; - delete userOptions.exclude; - - userOptions.suiteTags = { - include: [].concat(userOptions['include-tag'] || []), - exclude: [].concat(userOptions['exclude-tag'] || []), - }; - delete userOptions['include-tag']; - delete userOptions['exclude-tag']; - - userOptions.assertNoneExcluded = !!userOptions['assert-none-excluded']; - delete userOptions['assert-none-excluded']; - - if (userOptions['dry-run']) { - userOptions.dryRun = userOptions['dry-run']; - delete userOptions['dry-run']; - } - - const log = new ToolingLog({ - level: pickLevelFromFlags(userOptions), - writeTo: process.stdout, - }); - function createLogger() { - return log; - } - - const logToFile = !!userOptions.logToFile; - const logsDir = logToFile ? Path.resolve(REPO_ROOT, 'data/ftr_servers_logs', uuid()) : undefined; - - return { - ...userOptions, - configs: configs.map((c) => Path.resolve(c)), - createLogger, - extraKbnOpts: userOptions._, - esVersion: EsVersion.getDefault(), - logsDir, - }; -} - -function validateOptions(userOptions) { - Object.entries(userOptions).forEach(([key, val]) => { - if (key === '_' || key === 'suiteTags') { - return; - } - - // Validate flags passed - if (options[key] === undefined) { - throw new Error(`functional_tests: invalid option [${key}]`); - } - - if ( - // Validate boolean flags - (!options[key].arg && typeof val !== 'boolean') || - // Validate string/array flags - (options[key].arg && typeof val !== 'string' && !Array.isArray(val)) || - // Validate enum flags - (options[key].choices && !options[key].choices.includes(val)) - ) { - throw new Error(`functional_tests: invalid argument [${val}] to option [${key}]`); - } - }); -} diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/args.test.js b/packages/kbn-test/src/functional_tests/cli/run_tests/args.test.js deleted file mode 100644 index 888708a2b9d69..0000000000000 --- a/packages/kbn-test/src/functional_tests/cli/run_tests/args.test.js +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { createAbsolutePathSerializer } from '@kbn/jest-serializers'; - -import { displayHelp, processOptions } from './args'; - -jest.mock('../../../functional_test_runner/lib/es_version', () => { - return { - EsVersion: class { - static getDefault() { - return '999.999.999'; - } - }, - }; -}); - -expect.addSnapshotSerializer(createAbsolutePathSerializer(process.cwd())); - -const INITIAL_TEST_ES_FROM = process.env.TEST_ES_FROM; -beforeEach(() => { - process.env.TEST_ES_FROM = 'snapshot'; -}); -afterEach(() => { - process.env.TEST_ES_FROM = INITIAL_TEST_ES_FROM; -}); - -describe('display help for run tests CLI', () => { - it('displays as expected', () => { - expect(displayHelp()).toMatchSnapshot(); - }); -}); - -describe('process options for run tests CLI', () => { - it('rejects boolean config value', () => { - expect(() => { - processOptions({ config: true }); - }).toThrow('functional_tests: invalid argument [true] to option [config]'); - }); - - it('rejects empty config value if no default passed', () => { - expect(() => { - processOptions({}); - }).toThrow('functional_tests: config is required'); - }); - - it('accepts empty config value if default passed', () => { - const options = processOptions({ config: '' }, ['foo']); - expect(options).toMatchSnapshot(); - }); - - it('rejects non-boolean value for bail', () => { - expect(() => { - processOptions({ bail: 'peanut' }, ['foo']); - }).toThrow('functional_tests: invalid argument [peanut] to option [bail]'); - }); - - it('accepts string value for kibana-install-dir', () => { - const options = processOptions({ 'kibana-install-dir': 'foo' }, ['foo']); - expect(options).toMatchSnapshot(); - }); - - it('rejects boolean value for kibana-install-dir', () => { - expect(() => { - processOptions({ 'kibana-install-dir': true }, ['foo']); - }).toThrow('functional_tests: invalid argument [true] to option [kibana-install-dir]'); - }); - - it('accepts boolean value for updateBaselines', () => { - const options = processOptions({ updateBaselines: true }, ['foo']); - expect(options).toMatchSnapshot(); - }); - - it('accepts boolean value for updateSnapshots', () => { - const options = processOptions({ updateSnapshots: true }, ['foo']); - expect(options).toMatchSnapshot(); - }); - - it('accepts source value for esFrom', () => { - const options = processOptions({ esFrom: 'source' }, ['foo']); - expect(options).toMatchSnapshot(); - }); - - it('accepts source value for $TEST_ES_FROM', () => { - process.env.TEST_ES_FROM = 'source'; - const options = processOptions({}, ['foo']); - expect(options).toMatchSnapshot(); - }); - - it('prioritizes source flag over $TEST_ES_FROM', () => { - process.env.TEST_ES_FROM = 'source'; - const options = processOptions({ esFrom: 'snapshot' }, ['foo']); - expect(options).toMatchSnapshot(); - }); - - it('rejects non-enum value for esFrom', () => { - expect(() => { - processOptions({ esFrom: 'butter' }, ['foo']); - }).toThrow('functional_tests: invalid argument [butter] to option [esFrom]'); - }); - - it('accepts value for grep', () => { - const options = processOptions({ grep: 'management' }, ['foo']); - expect(options).toMatchSnapshot(); - }); - - it('accepts debug option', () => { - const options = processOptions({ debug: true }, ['foo']); - expect(options).toMatchSnapshot(); - }); - - it('accepts silent option', () => { - const options = processOptions({ silent: true }, ['foo']); - expect(options).toMatchSnapshot(); - }); - - it('accepts quiet option', () => { - const options = processOptions({ quiet: true }, ['foo']); - expect(options).toMatchSnapshot(); - }); - - it('accepts verbose option', () => { - const options = processOptions({ verbose: true }, ['foo']); - expect(options).toMatchSnapshot(); - }); - - it('accepts extra server options', () => { - const options = processOptions({ _: { 'server.foo': 'bar' } }, ['foo']); - expect(options).toMatchSnapshot(); - }); - - it('rejects invalid options even if valid options exist', () => { - expect(() => { - processOptions({ debug: true, aintnothang: true, bail: true }, ['foo']); - }).toThrow('functional_tests: invalid option [aintnothang]'); - }); -}); diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/cli.js b/packages/kbn-test/src/functional_tests/cli/run_tests/cli.js deleted file mode 100644 index 3958c1503cd30..0000000000000 --- a/packages/kbn-test/src/functional_tests/cli/run_tests/cli.js +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { runTests, initLogsDir } from '../../tasks'; -import { runCli } from '../../lib'; -import { processOptions, displayHelp } from './args'; - -/** - * Run servers and tests for each config - * Only cares about --config option. Other options - * are passed directly to functional_test_runner, such as - * --bail, --verbose, etc. - * @param {string[]} defaultConfigPaths Optional paths to configs - * if no config option is passed - */ -export async function runTestsCli(defaultConfigPaths) { - await runCli(displayHelp, async (userOptions) => { - const options = processOptions(userOptions, defaultConfigPaths); - initLogsDir(options); - await runTests(options); - }); -} diff --git a/packages/kbn-test/src/functional_tests/cli/start_servers/__snapshots__/args.test.js.snap b/packages/kbn-test/src/functional_tests/cli/start_servers/__snapshots__/args.test.js.snap deleted file mode 100644 index 1f572578119f7..0000000000000 --- a/packages/kbn-test/src/functional_tests/cli/start_servers/__snapshots__/args.test.js.snap +++ /dev/null @@ -1,141 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`display help for start servers CLI displays as expected 1`] = ` -"Start Functional Test Servers - -Usage: - node scripts/functional_tests_server --help - node scripts/functional_tests_server [--config ] - node scripts/functional_tests_server [options] [-- --] - -Options: - --help Display this menu and exit. - --config Pass in a config - --esFrom Build Elasticsearch from source, snapshot or path to existing install dir. Default: $TEST_ES_FROM or snapshot - --kibana-install-dir Run Kibana from existing install directory instead of from source. - --logToFile Write the log output from Kibana/Elasticsearch to files instead of to stdout - --verbose Log everything. - --debug Run in debug mode. - --quiet Only log errors. - --silent Log nothing." -`; - -exports[`process options for start servers CLI accepts debug option 1`] = ` -Object { - "config": /foo, - "createLogger": [Function], - "debug": true, - "esFrom": "snapshot", - "extraKbnOpts": undefined, - "logsDir": undefined, - "useDefaultConfig": true, -} -`; - -exports[`process options for start servers CLI accepts empty config value if default passed 1`] = ` -Object { - "config": /foo, - "createLogger": [Function], - "esFrom": "snapshot", - "extraKbnOpts": undefined, - "logsDir": undefined, - "useDefaultConfig": true, -} -`; - -exports[`process options for start servers CLI accepts extra server options 1`] = ` -Object { - "_": Object { - "server.foo": "bar", - }, - "config": /foo, - "createLogger": [Function], - "esFrom": "snapshot", - "extraKbnOpts": Object { - "server.foo": "bar", - }, - "logsDir": undefined, - "useDefaultConfig": true, -} -`; - -exports[`process options for start servers CLI accepts quiet option 1`] = ` -Object { - "config": /foo, - "createLogger": [Function], - "esFrom": "snapshot", - "extraKbnOpts": undefined, - "logsDir": undefined, - "quiet": true, - "useDefaultConfig": true, -} -`; - -exports[`process options for start servers CLI accepts silent option 1`] = ` -Object { - "config": /foo, - "createLogger": [Function], - "esFrom": "snapshot", - "extraKbnOpts": undefined, - "logsDir": undefined, - "silent": true, - "useDefaultConfig": true, -} -`; - -exports[`process options for start servers CLI accepts source value for $TEST_ES_FROM 1`] = ` -Object { - "config": /foo, - "createLogger": [Function], - "esFrom": "source", - "extraKbnOpts": undefined, - "logsDir": undefined, - "useDefaultConfig": true, -} -`; - -exports[`process options for start servers CLI accepts source value for esFrom 1`] = ` -Object { - "config": /foo, - "createLogger": [Function], - "esFrom": "source", - "extraKbnOpts": undefined, - "logsDir": undefined, - "useDefaultConfig": true, -} -`; - -exports[`process options for start servers CLI accepts string value for kibana-install-dir 1`] = ` -Object { - "config": /foo, - "createLogger": [Function], - "esFrom": "snapshot", - "extraKbnOpts": undefined, - "installDir": "foo", - "logsDir": undefined, - "useDefaultConfig": true, -} -`; - -exports[`process options for start servers CLI accepts verbose option 1`] = ` -Object { - "config": /foo, - "createLogger": [Function], - "esFrom": "snapshot", - "extraKbnOpts": undefined, - "logsDir": undefined, - "useDefaultConfig": true, - "verbose": true, -} -`; - -exports[`process options for start servers CLI prioritizes source flag over $TEST_ES_FROM 1`] = ` -Object { - "config": /foo, - "createLogger": [Function], - "esFrom": "snapshot", - "extraKbnOpts": undefined, - "logsDir": undefined, - "useDefaultConfig": true, -} -`; diff --git a/packages/kbn-test/src/functional_tests/cli/start_servers/args.js b/packages/kbn-test/src/functional_tests/cli/start_servers/args.js deleted file mode 100644 index e025bdc339331..0000000000000 --- a/packages/kbn-test/src/functional_tests/cli/start_servers/args.js +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import Path from 'path'; - -import { v4 as uuid } from 'uuid'; -import dedent from 'dedent'; -import { REPO_ROOT } from '@kbn/utils'; -import { ToolingLog, pickLevelFromFlags } from '@kbn/tooling-log'; - -const options = { - help: { desc: 'Display this menu and exit.' }, - config: { - arg: '', - desc: 'Pass in a config', - }, - esFrom: { - arg: '', - desc: 'Build Elasticsearch from source, snapshot or path to existing install dir.', - defaultHelp: 'Default: $TEST_ES_FROM or snapshot', - }, - 'kibana-install-dir': { - arg: '', - desc: 'Run Kibana from existing install directory instead of from source.', - }, - logToFile: { - desc: 'Write the log output from Kibana/Elasticsearch to files instead of to stdout', - }, - verbose: { desc: 'Log everything.' }, - debug: { desc: 'Run in debug mode.' }, - quiet: { desc: 'Only log errors.' }, - silent: { desc: 'Log nothing.' }, -}; - -export function displayHelp() { - const helpOptions = Object.keys(options) - .filter((name) => name !== '_') - .map((name) => { - const option = options[name]; - return { - ...option, - usage: `${name} ${option.arg || ''}`, - default: option.defaultHelp || '', - }; - }) - .map((option) => { - return `--${option.usage.padEnd(30)} ${option.desc} ${option.default}`; - }) - .join(`\n `); - - return dedent(` - Start Functional Test Servers - - Usage: - node scripts/functional_tests_server --help - node scripts/functional_tests_server [--config ] - node scripts/functional_tests_server [options] [-- --] - - Options: - ${helpOptions} - `); -} - -export function processOptions(userOptions, defaultConfigPath) { - validateOptions(userOptions); - - const useDefaultConfig = !userOptions.config; - const config = useDefaultConfig ? defaultConfigPath : userOptions.config; - - if (!config) { - throw new Error(`functional_tests_server: config is required`); - } - - if (!userOptions.esFrom) { - userOptions.esFrom = process.env.TEST_ES_FROM || 'snapshot'; - } - - if (userOptions['kibana-install-dir']) { - userOptions.installDir = userOptions['kibana-install-dir']; - delete userOptions['kibana-install-dir']; - } - - const log = new ToolingLog({ - level: pickLevelFromFlags(userOptions), - writeTo: process.stdout, - }); - - function createLogger() { - return log; - } - - const logToFile = !!userOptions.logToFile; - const logsDir = logToFile ? Path.resolve(REPO_ROOT, 'data/ftr_servers_logs', uuid()) : undefined; - - return { - ...userOptions, - logsDir, - config: Path.resolve(config), - useDefaultConfig, - createLogger, - extraKbnOpts: userOptions._, - }; -} - -function validateOptions(userOptions) { - Object.entries(userOptions).forEach(([key, val]) => { - if (key === '_') return; - - // Validate flags passed - if (options[key] === undefined) { - throw new Error(`functional_tests_server: invalid option [${key}]`); - } - - if ( - // Validate boolean flags - (!options[key].arg && typeof val !== 'boolean') || - // Validate string/array flags - (options[key].arg && typeof val !== 'string' && !Array.isArray(val)) || - // Validate enum flags - (options[key].choices && !options[key].choices.includes(val)) - ) { - throw new Error(`functional_tests_server: invalid argument [${val}] to option [${key}]`); - } - }); -} diff --git a/packages/kbn-test/src/functional_tests/cli/start_servers/args.test.js b/packages/kbn-test/src/functional_tests/cli/start_servers/args.test.js deleted file mode 100644 index 7d6c77be2539e..0000000000000 --- a/packages/kbn-test/src/functional_tests/cli/start_servers/args.test.js +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { displayHelp, processOptions } from './args'; -import { createAbsolutePathSerializer } from '@kbn/jest-serializers'; - -expect.addSnapshotSerializer(createAbsolutePathSerializer(process.cwd())); - -const INITIAL_TEST_ES_FROM = process.env.TEST_ES_FROM; -beforeEach(() => { - process.env.TEST_ES_FROM = 'snapshot'; -}); -afterEach(() => { - process.env.TEST_ES_FROM = INITIAL_TEST_ES_FROM; -}); - -describe('display help for start servers CLI', () => { - it('displays as expected', () => { - expect(displayHelp()).toMatchSnapshot(); - }); -}); - -describe('process options for start servers CLI', () => { - it('rejects boolean config value', () => { - expect(() => { - processOptions({ config: true }); - }).toThrow('functional_tests_server: invalid argument [true] to option [config]'); - }); - - it('rejects empty config value if no default passed', () => { - expect(() => { - processOptions({}); - }).toThrow('functional_tests_server: config is required'); - }); - - it('accepts empty config value if default passed', () => { - const options = processOptions({ config: '' }, 'foo'); - expect(options).toMatchSnapshot(); - }); - - it('rejects invalid option', () => { - expect(() => { - processOptions({ bail: true }, 'foo'); - }).toThrow('functional_tests_server: invalid option [bail]'); - }); - - it('accepts string value for kibana-install-dir', () => { - const options = processOptions({ 'kibana-install-dir': 'foo' }, 'foo'); - expect(options).toMatchSnapshot(); - }); - - it('rejects boolean value for kibana-install-dir', () => { - expect(() => { - processOptions({ 'kibana-install-dir': true }, 'foo'); - }).toThrow('functional_tests_server: invalid argument [true] to option [kibana-install-dir]'); - }); - - it('accepts source value for esFrom', () => { - const options = processOptions({ esFrom: 'source' }, 'foo'); - expect(options).toMatchSnapshot(); - }); - - it('accepts source value for $TEST_ES_FROM', () => { - process.env.TEST_ES_FROM = 'source'; - const options = processOptions({}, 'foo'); - expect(options).toMatchSnapshot(); - }); - - it('prioritizes source flag over $TEST_ES_FROM', () => { - process.env.TEST_ES_FROM = 'source'; - const options = processOptions({ esFrom: 'snapshot' }, 'foo'); - expect(options).toMatchSnapshot(); - }); - - it('accepts debug option', () => { - const options = processOptions({ debug: true }, 'foo'); - expect(options).toMatchSnapshot(); - }); - - it('accepts silent option', () => { - const options = processOptions({ silent: true }, 'foo'); - expect(options).toMatchSnapshot(); - }); - - it('accepts quiet option', () => { - const options = processOptions({ quiet: true }, 'foo'); - expect(options).toMatchSnapshot(); - }); - - it('accepts verbose option', () => { - const options = processOptions({ verbose: true }, 'foo'); - expect(options).toMatchSnapshot(); - }); - - it('accepts extra server options', () => { - const options = processOptions({ _: { 'server.foo': 'bar' } }, 'foo'); - expect(options).toMatchSnapshot(); - }); - - it('rejects invalid options even if valid options exist', () => { - expect(() => { - processOptions({ debug: true, aintnothang: true, bail: true }, 'foo'); - }).toThrow('functional_tests_server: invalid option [aintnothang]'); - }); -}); diff --git a/packages/kbn-test/src/functional_tests/cli/start_servers/cli.js b/packages/kbn-test/src/functional_tests/cli/start_servers/cli.js deleted file mode 100644 index d57d5c4761f6e..0000000000000 --- a/packages/kbn-test/src/functional_tests/cli/start_servers/cli.js +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { startServers, initLogsDir } from '../../tasks'; -import { runCli } from '../../lib'; -import { processOptions, displayHelp } from './args'; - -/** - * Start servers - * @param {string} defaultConfigPath Optional path to config - * if no config option is passed - */ -export async function startServersCli(defaultConfigPath) { - await runCli(displayHelp, async (userOptions) => { - const options = processOptions(userOptions, defaultConfigPath); - initLogsDir(options); - await startServers({ - ...options, - }); - }); -} diff --git a/packages/kbn-test/src/functional_tests/lib/__snapshots__/run_cli.test.js.snap b/packages/kbn-test/src/functional_tests/lib/__snapshots__/run_cli.test.js.snap deleted file mode 100644 index 6506675cea9bc..0000000000000 --- a/packages/kbn-test/src/functional_tests/lib/__snapshots__/run_cli.test.js.snap +++ /dev/null @@ -1,26 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`does right thing when non-error is thrown 1`] = ` -" -'foo bar' thrown! - ...stack trace... -" -`; - -exports[`logs no stack trace then exits when stack missing 1`] = ` -" -foo error - (no stack trace) - -" -`; - -exports[`logs the stack then exits when run function throws an error 1`] = ` -" -foo error - stack 1 - stack 2 - stack 3 - -" -`; diff --git a/packages/kbn-test/src/functional_tests/lib/index.ts b/packages/kbn-test/src/functional_tests/lib/index.ts index 8844a2ee59a19..7adf9c9d6420a 100644 --- a/packages/kbn-test/src/functional_tests/lib/index.ts +++ b/packages/kbn-test/src/functional_tests/lib/index.ts @@ -8,6 +8,4 @@ export { runKibanaServer } from './run_kibana_server'; export { runElasticsearch } from './run_elasticsearch'; -export type { CreateFtrOptions, CreateFtrParams } from './run_ftr'; -export { runFtr, hasTests, assertNoneExcluded } from './run_ftr'; -export { runCli } from './run_cli'; +export * from './run_ftr'; diff --git a/packages/kbn-test/src/functional_tests/test_helpers.ts b/packages/kbn-test/src/functional_tests/lib/logs_dir.ts similarity index 52% rename from packages/kbn-test/src/functional_tests/test_helpers.ts rename to packages/kbn-test/src/functional_tests/lib/logs_dir.ts index 4131f23770a05..0671a1ffd01e3 100644 --- a/packages/kbn-test/src/functional_tests/test_helpers.ts +++ b/packages/kbn-test/src/functional_tests/lib/logs_dir.ts @@ -6,14 +6,12 @@ * Side Public License, v 1. */ -/* eslint-env jest */ +import Path from 'path'; +import Fs from 'fs'; -import { format } from 'util'; +import { ToolingLog } from '@kbn/tooling-log'; -export function checkMockConsoleLogSnapshot(logMock: jest.Mock) { - const output = logMock.mock.calls - .reduce((acc, args) => `${acc}${format(...args)}\n`, '') - .replace(/(^ at.+[>)\d]$\n?)+/m, ' ...stack trace...'); - - expect(output).toMatchSnapshot(); +export async function initLogsDir(log: ToolingLog, logsDir: string) { + log.info(`Kibana/ES logs will be written to ${Path.relative(process.cwd(), logsDir)}/`); + Fs.mkdirSync(logsDir, { recursive: true }); } diff --git a/packages/kbn-test/src/functional_tests/lib/run_cli.test.js b/packages/kbn-test/src/functional_tests/lib/run_cli.test.js deleted file mode 100644 index eccb1405d51dc..0000000000000 --- a/packages/kbn-test/src/functional_tests/lib/run_cli.test.js +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { runCli } from './run_cli'; -import { checkMockConsoleLogSnapshot } from '../test_helpers'; - -const mockProcessExit = jest.spyOn(process, 'exit').mockImplementation(() => {}); -const mockConsoleLog = jest.spyOn(console, 'log').mockImplementation(() => {}); - -const actualProcessArgv = process.argv; - -const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); - -beforeEach(() => { - process.argv = actualProcessArgv.slice(0, 2); - jest.clearAllMocks(); -}); - -afterAll(() => { - process.argv = actualProcessArgv; -}); - -it('accepts help option even if invalid options passed', async () => { - process.argv.push('--foo', '--bar', '--help'); - - const mockGetHelpText = jest.fn().mockReturnValue('mock help text'); - const mockRun = jest.fn(); - await runCli(mockGetHelpText, mockRun); - - expect(mockProcessExit).not.toHaveBeenCalled(); - expect(mockGetHelpText).toHaveBeenCalledTimes(1); - expect(mockConsoleLog).toHaveBeenCalledTimes(1); - expect(mockConsoleLog).toHaveBeenCalledWith('mock help text'); - expect(mockRun).not.toHaveBeenCalled(); -}); - -it('passes parsed argv to run function', async () => { - process.argv.push('--foo', 'bar', '--baz=box', '--', 'a', 'b', 'c'); - - const mockGetHelpText = jest.fn(); - const mockRun = jest.fn(); - await runCli(mockGetHelpText, mockRun); - - expect(mockGetHelpText).not.toHaveBeenCalled(); - expect(mockConsoleLog).not.toHaveBeenCalled(); - expect(mockProcessExit).not.toHaveBeenCalled(); - expect(mockRun).toHaveBeenCalledTimes(1); - expect(mockRun).toHaveBeenCalledWith({ - foo: 'bar', - baz: 'box', - _: ['a', 'b', 'c'], - }); -}); - -it('waits for promise returned from run function to resolve before resolving', async () => { - let resolveMockRun; - const mockRun = jest.fn().mockImplementation( - () => - new Promise((resolve) => { - resolveMockRun = resolve; - }) - ); - - const onResolved = jest.fn(); - const promise = runCli(null, mockRun).then(onResolved); - - expect(mockRun).toHaveBeenCalled(); - expect(onResolved).not.toHaveBeenCalled(); - - await sleep(500); - - expect(onResolved).not.toHaveBeenCalled(); - - resolveMockRun(); - await promise; - expect(onResolved).toHaveBeenCalled(); -}); - -it('logs the stack then exits when run function throws an error', async () => { - await runCli(null, () => { - const error = new Error('foo error'); - error.stack = 'foo error\n stack 1\n stack 2\n stack 3'; - throw error; - }); - - expect(mockProcessExit).toHaveBeenCalledTimes(1); - expect(mockProcessExit).toHaveBeenCalledWith(1); - - expect(mockConsoleLog).toHaveBeenCalled(); - checkMockConsoleLogSnapshot(mockConsoleLog); -}); - -it('logs no stack trace then exits when stack missing', async () => { - await runCli(null, () => { - const error = new Error('foo error'); - error.stack = undefined; - throw error; - }); - - expect(mockProcessExit).toHaveBeenCalledTimes(1); - expect(mockProcessExit).toHaveBeenCalledWith(1); - - expect(mockConsoleLog).toHaveBeenCalled(); - checkMockConsoleLogSnapshot(mockConsoleLog); -}); - -it('does right thing when non-error is thrown', async () => { - await runCli(null, () => { - throw 'foo bar'; - }); - - expect(mockProcessExit).toHaveBeenCalledTimes(1); - expect(mockProcessExit).toHaveBeenCalledWith(1); - - expect(mockConsoleLog).toHaveBeenCalled(); - checkMockConsoleLogSnapshot(mockConsoleLog); -}); diff --git a/packages/kbn-test/src/functional_tests/lib/run_cli.ts b/packages/kbn-test/src/functional_tests/lib/run_cli.ts deleted file mode 100644 index 3e2cb50ff2e78..0000000000000 --- a/packages/kbn-test/src/functional_tests/lib/run_cli.ts +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { inspect } from 'util'; - -import chalk from 'chalk'; -import getopts from 'getopts'; -/* eslint-disable no-console */ -export class CliError extends Error { - constructor(message: string) { - super(message); - Error.captureStackTrace(this, CliError); - } -} - -export async function runCli( - getHelpText: () => string, - run: (options: getopts.ParsedOptions) => Promise -) { - try { - const userOptions = getopts(process.argv.slice(2)) || {}; - if (userOptions.help) { - console.log(getHelpText()); - return; - } - - await run(userOptions); - } catch (error) { - if (!(error instanceof Error)) { - error = new Error(`${inspect(error)} thrown!`); - } - - console.log(); - console.log(chalk.red(error.message)); - - // CliError is a special error class that indicates that the error is produced as a part - // of using the CLI, and does not need a stack trace to make sense, so we skip the stack - // trace logging if the error thrown is an instance of this class - if (!(error instanceof CliError)) { - // first line in the stack trace is the message, skip it as we log it directly and color it red - if (error.stack) { - console.log(error.stack.split('\n').slice(1).join('\n')); - } else { - console.log(' (no stack trace)'); - } - } - - console.log(); - - process.exit(error.exitCode || 1); - } -} diff --git a/packages/kbn-test/src/functional_tests/lib/run_ftr.ts b/packages/kbn-test/src/functional_tests/lib/run_ftr.ts index b9945adbdfb56..665d27c00754e 100644 --- a/packages/kbn-test/src/functional_tests/lib/run_ftr.ts +++ b/packages/kbn-test/src/functional_tests/lib/run_ftr.ts @@ -6,127 +6,45 @@ * Side Public License, v 1. */ import type { ToolingLog } from '@kbn/tooling-log'; -import { FunctionalTestRunner, readConfigFile, EsVersion } from '../../functional_test_runner'; -import { CliError } from './run_cli'; +import { createFailError } from '@kbn/dev-cli-errors'; -export interface CreateFtrOptions { - /** installation dir from which to run Kibana */ - installDir: string; +import { EsVersion, Config, FunctionalTestRunner } from '../../functional_test_runner'; + +export async function runFtr(options: { log: ToolingLog; - /** Whether to exit test run at the first failure */ - bail?: boolean; - grep: string; - updateBaselines?: boolean; - suiteFiles?: { - include?: string[]; - exclude?: string[]; - }; - suiteTags?: { - include?: string[]; - exclude?: string[]; - }; - updateSnapshots?: boolean; + config: Config; esVersion: EsVersion; - dryRun?: boolean; -} - -export interface CreateFtrParams { - configPath: string; - options: CreateFtrOptions; -} -async function createFtr({ - configPath, - options: { - installDir, - log, - bail, - grep, - updateBaselines, - suiteFiles, - suiteTags, - updateSnapshots, - esVersion, - dryRun, - }, -}: CreateFtrParams) { - const config = await readConfigFile(log, esVersion, configPath); - - return { - config, - ftr: new FunctionalTestRunner( - log, - configPath, - { - mochaOpts: { - bail: !!bail, - grep, - dryRun: !!dryRun, - }, - kbnTestServer: { - installDir, - }, - updateBaselines, - updateSnapshots, - suiteFiles: { - include: [...(suiteFiles?.include || []), ...config.get('suiteFiles.include')], - exclude: [...(suiteFiles?.exclude || []), ...config.get('suiteFiles.exclude')], - }, - suiteTags: { - include: [...(suiteTags?.include || []), ...config.get('suiteTags.include')], - exclude: [...(suiteTags?.exclude || []), ...config.get('suiteTags.exclude')], - }, - }, - esVersion - ), - }; -} - -export async function assertNoneExcluded(params: CreateFtrParams) { - const { config, ftr } = await createFtr(params); - - if (config.get('testRunner')) { - // tests with custom test runners are not included in this check - return; - } - - const stats = await ftr.getTestStats(); - if (!stats) { - throw new Error('unable to get test stats'); - } - if (stats.testsExcludedByTag.length > 0) { - throw new CliError(` - ${stats.testsExcludedByTag.length} tests in the ${params.configPath} config - are excluded when filtering by the tags run on CI. Make sure that all suites are - tagged with one of the following tags: + signal?: AbortSignal; +}) { + const ftr = new FunctionalTestRunner(options.log, options.config, options.esVersion); - ${JSON.stringify(params.options.suiteTags)} - - - ${stats.testsExcludedByTag.join('\n - ')} - `); - } -} - -export async function runFtr(params: CreateFtrParams, signal?: AbortSignal) { - const { ftr } = await createFtr(params); - - const failureCount = await ftr.run(signal); + const failureCount = await ftr.run(options.signal); if (failureCount > 0) { - throw new CliError( + throw createFailError( `${failureCount} functional test ${failureCount === 1 ? 'failure' : 'failures'}` ); } } -export async function hasTests(params: CreateFtrParams) { - const { ftr, config } = await createFtr(params); - - if (config.get('testRunner')) { +export async function checkForEnabledTestsInFtrConfig(options: { + log: ToolingLog; + config: Config; + esVersion: EsVersion; +}) { + if (options.config.get('testRunner')) { // configs with custom test runners are assumed to always have tests return true; } + + if (options.config.module.type === 'journey') { + return !options.config.module.journey.config.isSkipped(); + } + + const ftr = new FunctionalTestRunner(options.log, options.config, options.esVersion); const stats = await ftr.getTestStats(); if (!stats) { - throw new Error('unable to get test stats'); + throw createFailError('unable to get test stats'); } + return stats.nonSkippedTestCount > 0; } diff --git a/packages/kbn-test/src/functional_tests/lib/run_kibana_server.ts b/packages/kbn-test/src/functional_tests/lib/run_kibana_server.ts index 2ae15ca5f83f8..2ab4af2df2e2d 100644 --- a/packages/kbn-test/src/functional_tests/lib/run_kibana_server.ts +++ b/packages/kbn-test/src/functional_tests/lib/run_kibana_server.ts @@ -34,25 +34,19 @@ function extendNodeOptions(installDir?: string) { }; } -export async function runKibanaServer({ - procs, - config, - options, - onEarlyExit, -}: { +export async function runKibanaServer(options: { procs: ProcRunner; config: Config; - options: { - installDir?: string; - extraKbnOpts?: string[]; - logsDir?: string; - }; + installDir?: string; + extraKbnOpts?: string[]; + logsDir?: string; onEarlyExit?: (msg: string) => void; }) { - const runOptions = config.get('kbnTestServer.runOptions'); + const { config, procs } = options; + const runOptions = options.config.get('kbnTestServer.runOptions'); const installDir = runOptions.alwaysUseSource ? undefined : options.installDir; const devMode = !installDir; - const useTaskRunner = config.get('kbnTestServer.useDedicatedTaskRunner'); + const useTaskRunner = options.config.get('kbnTestServer.useDedicatedTaskRunner'); const procRunnerOpts = { cwd: installDir || REPO_ROOT, @@ -64,11 +58,11 @@ export async function runKibanaServer({ env: { FORCE_COLOR: 1, ...process.env, - ...config.get('kbnTestServer.env'), + ...options.config.get('kbnTestServer.env'), ...extendNodeOptions(installDir), }, wait: runOptions.wait, - onEarlyExit, + onEarlyExit: options.onEarlyExit, }; const prefixArgs = devMode diff --git a/packages/kbn-test/src/functional_tests/run_tests/cli.ts b/packages/kbn-test/src/functional_tests/run_tests/cli.ts new file mode 100644 index 0000000000000..19a003dd973cf --- /dev/null +++ b/packages/kbn-test/src/functional_tests/run_tests/cli.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { run } from '@kbn/dev-cli-runner'; + +import { initLogsDir } from '../lib/logs_dir'; +import { runTests } from './run_tests'; +import { parseFlags, FLAG_OPTIONS } from './flags'; + +export function runTestsCli() { + run( + async ({ flagsReader, log }) => { + const options = parseFlags(flagsReader); + + if (options.logsDir) { + initLogsDir(log, options.logsDir); + } + + await runTests(log, options); + }, + { + description: `Run Functional Tests`, + usage: ` + node scripts/functional_tests --help + node scripts/functional_tests [--config [--config ...]] + node scripts/functional_tests [options] [-- --] + `, + flags: FLAG_OPTIONS, + } + ); +} diff --git a/packages/kbn-test/src/functional_tests/run_tests/flags.test.ts b/packages/kbn-test/src/functional_tests/run_tests/flags.test.ts new file mode 100644 index 0000000000000..dbb8b1e9762e5 --- /dev/null +++ b/packages/kbn-test/src/functional_tests/run_tests/flags.test.ts @@ -0,0 +1,151 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { createAbsolutePathSerializer, createAnyInstanceSerializer } from '@kbn/jest-serializers'; +import { FlagsReader, getFlags } from '@kbn/dev-cli-runner'; + +import { EsVersion } from '../../functional_test_runner'; +import { parseFlags, FLAG_OPTIONS } from './flags'; + +jest.mock('uuid', () => ({ v4: () => 'some-uuid' })); + +expect.addSnapshotSerializer(createAbsolutePathSerializer()); +expect.addSnapshotSerializer( + createAnyInstanceSerializer(EsVersion, (v: EsVersion) => `EsVersion ${v.toString()}`) +); + +const INITIAL_TEST_ES_FROM = process.env.TEST_ES_FROM; +beforeEach(() => { + process.env.TEST_ES_FROM = 'snapshot'; +}); +afterEach(() => { + process.env.TEST_ES_FROM = INITIAL_TEST_ES_FROM; +}); + +const defaults = getFlags(['--config=foo'], FLAG_OPTIONS); + +const test = (opts: Record) => + parseFlags(new FlagsReader({ ...defaults, ...opts })); + +describe('parse runTest flags', () => { + it('validates defaults', () => { + expect(test({})).toMatchInlineSnapshot(` + Object { + "bail": false, + "configs": Array [ + /foo, + ], + "dryRun": false, + "esFrom": "snapshot", + "esVersion": , + "grep": undefined, + "installDir": undefined, + "logsDir": undefined, + "suiteFilters": Object { + "exclude": Array [], + "include": Array [], + }, + "suiteTags": Object { + "exclude": Array [], + "include": Array [], + }, + "updateBaselines": false, + "updateSnapshots": false, + } + `); + }); + + it('allows combinations of config and journey', () => { + expect(() => test({ config: undefined })).toThrowErrorMatchingInlineSnapshot( + `"At least one --config or --journey flag is required"` + ); + + expect(test({ config: ['configFoo'], journey: 'journeyFoo' }).configs).toMatchInlineSnapshot(` + Array [ + /configFoo, + /journeyFoo, + ] + `); + + expect(test({ config: undefined, journey: 'foo' }).configs).toMatchInlineSnapshot(` + Array [ + /foo, + ] + `); + + expect(test({ config: undefined, journey: ['foo', 'bar', 'baz'] }).configs) + .toMatchInlineSnapshot(` + Array [ + /foo, + /bar, + /baz, + ] + `); + + expect(test({ config: ['bar'], journey: ['foo', 'baz'] }).configs).toMatchInlineSnapshot(` + Array [ + /bar, + /foo, + /baz, + ] + `); + }); + + it('updates all with updateAll', () => { + const { updateBaselines, updateSnapshots } = test({ updateAll: true }); + expect({ updateBaselines, updateSnapshots }).toMatchInlineSnapshot(` + Object { + "updateBaselines": true, + "updateSnapshots": true, + } + `); + }); + + it('validates esFrom', () => { + expect(() => test({ esFrom: 'foo' })).toThrowErrorMatchingInlineSnapshot( + `"invalid --esFrom, expected one of \\"snapshot\\", \\"source\\""` + ); + }); + + it('accepts multiple tags', () => { + const { suiteFilters, suiteTags } = test({ + 'include-tag': ['foo', 'bar'], + include: 'path', + exclude: ['foo'], + 'exclude-tag': ['foo'], + }); + + expect({ suiteFilters, suiteTags }).toMatchInlineSnapshot(` + Object { + "suiteFilters": Object { + "exclude": Array [ + /foo, + ], + "include": Array [ + /path, + ], + }, + "suiteTags": Object { + "exclude": Array [ + "foo", + ], + "include": Array [ + "foo", + "bar", + ], + }, + } + `); + }); +}); + +it('supports logsDir', () => { + expect(test({ logToFile: true }).logsDir).toMatchInlineSnapshot( + `/data/ftr_servers_logs/some-uuid` + ); +}); diff --git a/packages/kbn-test/src/functional_tests/run_tests/flags.ts b/packages/kbn-test/src/functional_tests/run_tests/flags.ts new file mode 100644 index 0000000000000..7639ae341f071 --- /dev/null +++ b/packages/kbn-test/src/functional_tests/run_tests/flags.ts @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; + +import { v4 as uuidV4 } from 'uuid'; +import { REPO_ROOT } from '@kbn/utils'; +import { FlagsReader, FlagOptions } from '@kbn/dev-cli-runner'; +import { createFlagError } from '@kbn/dev-cli-errors'; + +import { EsVersion } from '../../functional_test_runner'; + +export type RunTestsOptions = ReturnType; + +export const FLAG_OPTIONS: FlagOptions = { + boolean: ['bail', 'logToFile', 'dry-run', 'updateBaselines', 'updateSnapshots', 'updateAll'], + string: [ + 'config', + 'journey', + 'esFrom', + 'kibana-install-dir', + 'grep', + 'include-tag', + 'exclude-tag', + 'include', + 'exclude', + ], + alias: { + updateAll: 'u', + }, + help: ` + --config Define a FTR config that should be executed. Can be specified multiple times + --journey Define a Journey that should be executed. Can be specified multiple times + --esFrom Build Elasticsearch from source or run from snapshot. Default: $TEST_ES_FROM or "snapshot" + --include-tag Tags that suites must include to be run, can be included multiple times + --exclude-tag Tags that suites must NOT include to be run, can be included multiple times + --include Files that must included to be run, can be included multiple times + --exclude Files that must NOT be included to be run, can be included multiple times + --grep Pattern to select which tests to run + --kibana-install-dir Run Kibana from existing install directory instead of from source + --bail Stop the test run at the first failure + --logToFile Write the log output from Kibana/ES to files instead of to stdout + --dry-run Report tests without executing them + --updateBaselines Replace baseline screenshots with whatever is generated from the test + --updateSnapshots Replace inline and file snapshots with whatever is generated from the test + --updateAll, -u Replace both baseline screenshots and snapshots + `, +}; + +export function parseFlags(flags: FlagsReader) { + const configs = [ + ...(flags.arrayOfPaths('config') ?? []), + ...(flags.arrayOfPaths('journey') ?? []), + ]; + + if (!configs.length) { + throw createFlagError('At least one --config or --journey flag is required'); + } + + const esVersionString = flags.string('es-version'); + + return { + configs, + esVersion: esVersionString ? new EsVersion(esVersionString) : EsVersion.getDefault(), + bail: flags.boolean('bail'), + dryRun: flags.boolean('dry-run'), + updateBaselines: flags.boolean('updateBaselines') || flags.boolean('updateAll'), + updateSnapshots: flags.boolean('updateSnapshots') || flags.boolean('updateAll'), + logsDir: flags.boolean('logToFile') + ? Path.resolve(REPO_ROOT, 'data/ftr_servers_logs', uuidV4()) + : undefined, + esFrom: flags.enum('esFrom', ['snapshot', 'source']) ?? 'snapshot', + installDir: flags.path('kibana-install-dir'), + grep: flags.string('grep'), + suiteTags: { + include: flags.arrayOfStrings('include-tag'), + exclude: flags.arrayOfStrings('exclude-tag'), + }, + suiteFilters: { + include: flags.arrayOfPaths('include'), + exclude: flags.arrayOfPaths('exclude'), + }, + }; +} diff --git a/packages/kbn-test/src/functional_tests/run_tests/index.ts b/packages/kbn-test/src/functional_tests/run_tests/index.ts new file mode 100644 index 0000000000000..6b0ed25db6849 --- /dev/null +++ b/packages/kbn-test/src/functional_tests/run_tests/index.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export { runTestsCli } from './cli'; +export { runTests } from './run_tests'; diff --git a/packages/kbn-test/src/functional_tests/run_tests/run_tests.ts b/packages/kbn-test/src/functional_tests/run_tests/run_tests.ts new file mode 100644 index 0000000000000..3eb8348691a1b --- /dev/null +++ b/packages/kbn-test/src/functional_tests/run_tests/run_tests.ts @@ -0,0 +1,116 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; +import { setTimeout } from 'timers/promises'; + +import { REPO_ROOT } from '@kbn/utils'; +import { ToolingLog } from '@kbn/tooling-log'; +import { withProcRunner } from '@kbn/dev-proc-runner'; + +import { readConfigFile } from '../../functional_test_runner'; + +import { checkForEnabledTestsInFtrConfig, runFtr } from '../lib/run_ftr'; +import { runElasticsearch } from '../lib/run_elasticsearch'; +import { runKibanaServer } from '../lib/run_kibana_server'; +import { RunTestsOptions } from './flags'; + +/** + * Run servers and tests for each config + */ +export async function runTests(log: ToolingLog, options: RunTestsOptions) { + if (!process.env.CI) { + log.warning('โ—๏ธโ—๏ธโ—๏ธ'); + log.warning('โ—๏ธโ—๏ธโ—๏ธ'); + log.warning('โ—๏ธโ—๏ธโ—๏ธ'); + log.warning( + " Don't forget to use `node scripts/build_kibana_platform_plugins` to build plugins you plan on testing" + ); + log.warning('โ—๏ธโ—๏ธโ—๏ธ'); + log.warning('โ—๏ธโ—๏ธโ—๏ธ'); + log.warning('โ—๏ธโ—๏ธโ—๏ธ'); + } + + for (const [i, path] of options.configs.entries()) { + await log.indent(0, async () => { + if (options.configs.length > 1) { + const progress = `${i + 1}/${options.configs.length}`; + log.write(`--- [${progress}] Running ${Path.relative(REPO_ROOT, path)}`); + } + + const config = await readConfigFile(log, options.esVersion, path); + + const hasTests = await checkForEnabledTestsInFtrConfig({ + config, + esVersion: options.esVersion, + log, + }); + if (!hasTests) { + // just run the FTR, no Kibana or ES, which will quickly report a skipped test group to ci-stats and continue + await runFtr({ + log, + config, + esVersion: options.esVersion, + }); + return; + } + + await withProcRunner(log, async (procs) => { + const abortCtrl = new AbortController(); + + const onEarlyExit = (msg: string) => { + log.error(msg); + abortCtrl.abort(); + }; + + let shutdownEs; + try { + if (process.env.TEST_ES_DISABLE_STARTUP !== 'true') { + shutdownEs = await runElasticsearch({ ...options, log, config, onEarlyExit }); + if (abortCtrl.signal.aborted) { + return; + } + } + + await runKibanaServer({ + procs, + config, + logsDir: options.logsDir, + installDir: options.installDir, + onEarlyExit, + }); + + if (abortCtrl.signal.aborted) { + return; + } + + await runFtr({ + log, + config, + esVersion: options.esVersion, + signal: abortCtrl.signal, + }); + } finally { + try { + const delay = config.get('kbnTestServer.delayShutdown'); + if (typeof delay === 'number') { + log.info('Delaying shutdown of Kibana for', delay, 'ms'); + await setTimeout(delay); + } + + await procs.stop('kibana'); + } finally { + if (shutdownEs) { + await shutdownEs(); + } + } + } + }); + }); + } +} diff --git a/packages/kbn-test/src/functional_tests/start_servers/cli.ts b/packages/kbn-test/src/functional_tests/start_servers/cli.ts new file mode 100644 index 0000000000000..548e0310d5bcc --- /dev/null +++ b/packages/kbn-test/src/functional_tests/start_servers/cli.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { run } from '@kbn/dev-cli-runner'; + +import { initLogsDir } from '../lib/logs_dir'; + +import { parseFlags, FLAG_OPTIONS } from './flags'; +import { startServers } from './start_servers'; + +/** + * Start servers + */ +export function startServersCli() { + run( + async ({ flagsReader: flags, log }) => { + const options = parseFlags(flags); + + if (options.logsDir) { + initLogsDir(log, options.logsDir); + } + + await startServers(log, options); + }, + { + flags: FLAG_OPTIONS, + } + ); +} diff --git a/packages/kbn-test/src/functional_tests/start_servers/flags.test.ts b/packages/kbn-test/src/functional_tests/start_servers/flags.test.ts new file mode 100644 index 0000000000000..5f40b2ae66828 --- /dev/null +++ b/packages/kbn-test/src/functional_tests/start_servers/flags.test.ts @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { getFlags, FlagsReader } from '@kbn/dev-cli-runner'; +import { createAnyInstanceSerializer, createAbsolutePathSerializer } from '@kbn/jest-serializers'; +import { EsVersion } from '../../functional_test_runner'; +import { parseFlags, FLAG_OPTIONS } from './flags'; + +jest.mock('uuid', () => ({ v4: () => 'some-uuid' })); + +expect.addSnapshotSerializer( + createAnyInstanceSerializer(EsVersion, (v: EsVersion) => `EsVersion ${v.toString()}`) +); +expect.addSnapshotSerializer(createAbsolutePathSerializer()); + +const defaults = getFlags(['--config=foo'], FLAG_OPTIONS); + +const test = (opts: Record) => + parseFlags(new FlagsReader({ ...defaults, ...opts })); + +it('parses a subset of the flags from runTests', () => { + expect(test({ config: 'foo' })).toMatchInlineSnapshot(` + Object { + "config": "foo", + "esFrom": undefined, + "esVersion": , + "installDir": undefined, + "logsDir": undefined, + } + `); +}); + +it('rejects zero configs', () => { + expect(() => test({ config: [] })).toThrowErrorMatchingInlineSnapshot( + `"expected exactly one --config or --journey flag"` + ); +}); + +it('rejects two configs', () => { + expect(() => test({ config: ['foo'], journey: ['bar'] })).toThrowErrorMatchingInlineSnapshot( + `"expected exactly one --config or --journey flag"` + ); +}); + +it('supports logsDir', () => { + expect(test({ logToFile: true }).logsDir).toMatchInlineSnapshot( + `/data/ftr_servers_logs/some-uuid` + ); +}); diff --git a/packages/kbn-test/src/functional_tests/start_servers/flags.ts b/packages/kbn-test/src/functional_tests/start_servers/flags.ts new file mode 100644 index 0000000000000..8ce3af9f5917b --- /dev/null +++ b/packages/kbn-test/src/functional_tests/start_servers/flags.ts @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; + +import { v4 as uuidV4 } from 'uuid'; +import { FlagsReader, FlagOptions } from '@kbn/dev-cli-runner'; +import { createFlagError } from '@kbn/dev-cli-errors'; +import { REPO_ROOT } from '@kbn/utils'; + +import { EsVersion } from '../../functional_test_runner'; + +export type StartServerOptions = ReturnType; + +export const FLAG_OPTIONS: FlagOptions = { + string: ['config', 'journey', 'esFrom', 'kibana-install-dir'], + boolean: ['logToFile'], + help: ` + --config Define a FTR config that should be executed. Can be specified multiple times + --journey Define a Journey that should be executed. Can be specified multiple times + --esFrom Build Elasticsearch from source or run from snapshot. Default: $TEST_ES_FROM or "snapshot" + --kibana-install-dir Run Kibana from existing install directory instead of from source + --logToFile Write the log output from Kibana/ES to files instead of to stdout + `, +}; + +export function parseFlags(flags: FlagsReader) { + const configs = [ + ...(flags.arrayOfStrings('config') ?? []), + ...(flags.arrayOfStrings('journey') ?? []), + ]; + if (configs.length !== 1) { + throw createFlagError(`expected exactly one --config or --journey flag`); + } + + return { + config: configs[0], + esFrom: flags.enum('esFrom', ['source', 'snapshot']), + esVersion: EsVersion.getDefault(), + installDir: flags.string('kibana-install-dir'), + logsDir: flags.boolean('logToFile') + ? Path.resolve(REPO_ROOT, 'data/ftr_servers_logs', uuidV4()) + : undefined, + }; +} diff --git a/packages/kbn-test/src/functional_tests/start_servers/index.ts b/packages/kbn-test/src/functional_tests/start_servers/index.ts new file mode 100644 index 0000000000000..ff88a1f4cb476 --- /dev/null +++ b/packages/kbn-test/src/functional_tests/start_servers/index.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export { startServersCli } from './cli'; +export { startServers } from './start_servers'; diff --git a/packages/kbn-test/src/functional_tests/start_servers/start_servers.ts b/packages/kbn-test/src/functional_tests/start_servers/start_servers.ts new file mode 100644 index 0000000000000..3bb601fabe002 --- /dev/null +++ b/packages/kbn-test/src/functional_tests/start_servers/start_servers.ts @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; + +import * as Rx from 'rxjs'; +import dedent from 'dedent'; +import { REPO_ROOT } from '@kbn/utils'; +import { ToolingLog } from '@kbn/tooling-log'; +import { withProcRunner } from '@kbn/dev-proc-runner'; +import { getTimeReporter } from '@kbn/ci-stats-reporter'; + +import { readConfigFile } from '../../functional_test_runner'; +import { runElasticsearch } from '../lib/run_elasticsearch'; +import { runKibanaServer } from '../lib/run_kibana_server'; +import { StartServerOptions } from './flags'; + +const FTR_SCRIPT_PATH = Path.resolve(REPO_ROOT, 'scripts/functional_test_runner'); + +export async function startServers(log: ToolingLog, options: StartServerOptions) { + const runStartTime = Date.now(); + const reportTime = getTimeReporter(log, 'scripts/functional_tests_server'); + + await withProcRunner(log, async (procs) => { + const config = await readConfigFile(log, options.esVersion, options.config); + + const shutdownEs = await runElasticsearch({ + config, + log, + esFrom: options.esFrom, + logsDir: options.logsDir, + }); + + await runKibanaServer({ + procs, + config, + installDir: options.installDir, + extraKbnOpts: options.installDir ? [] : ['--dev', '--no-dev-config', '--no-dev-credentials'], + }); + + reportTime(runStartTime, 'ready', { + success: true, + ...options, + }); + + // wait for 5 seconds of silence before logging the + // success message so that it doesn't get buried + await silence(log, 5000); + + const installDirFlag = options.installDir ? ` --kibana-install-dir=${options.installDir}` : ''; + const rel = Path.relative(process.cwd(), config.module.path); + const pathsMessage = ` --${config.module.type}=${rel}`; + + log.success( + '\n\n' + + dedent` + Elasticsearch and Kibana are ready for functional testing. Start the functional tests + in another terminal session by running this command from this directory: + + node ${Path.relative(process.cwd(), FTR_SCRIPT_PATH)}${installDirFlag}${pathsMessage} + ` + + '\n\n' + ); + + await procs.waitForAllToStop(); + await shutdownEs(); + }); +} + +async function silence(log: ToolingLog, milliseconds: number) { + await Rx.firstValueFrom( + log.getWritten$().pipe( + Rx.startWith(null), + Rx.switchMap(() => Rx.timer(milliseconds)) + ) + ); +} diff --git a/packages/kbn-test/src/functional_tests/tasks.ts b/packages/kbn-test/src/functional_tests/tasks.ts deleted file mode 100644 index 26504b07544b0..0000000000000 --- a/packages/kbn-test/src/functional_tests/tasks.ts +++ /dev/null @@ -1,230 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import Fs from 'fs'; -import Path from 'path'; -import { setTimeout } from 'timers/promises'; - -import * as Rx from 'rxjs'; -import { startWith, switchMap, take } from 'rxjs/operators'; -import { withProcRunner } from '@kbn/dev-proc-runner'; -import { ToolingLog } from '@kbn/tooling-log'; -import { getTimeReporter } from '@kbn/ci-stats-reporter'; -import { REPO_ROOT } from '@kbn/utils'; -import dedent from 'dedent'; - -import { readConfigFile, EsVersion } from '../functional_test_runner/lib'; -import { - runElasticsearch, - runKibanaServer, - runFtr, - assertNoneExcluded, - hasTests, - CreateFtrOptions, -} from './lib'; - -const FTR_SCRIPT_PATH = Path.resolve(REPO_ROOT, 'scripts/functional_test_runner'); - -const makeSuccessMessage = (options: StartServerOptions) => { - const installDirFlag = options.installDir ? ` --kibana-install-dir=${options.installDir}` : ''; - const configPaths: string[] = Array.isArray(options.config) ? options.config : [options.config]; - const pathsMessage = options.useDefaultConfig - ? '' - : configPaths - .map((path) => Path.relative(process.cwd(), path)) - .map((path) => ` --config ${path}`) - .join(''); - - return ( - '\n\n' + - dedent` - Elasticsearch and Kibana are ready for functional testing. Start the functional tests - in another terminal session by running this command from this directory: - - node ${Path.relative(process.cwd(), FTR_SCRIPT_PATH)}${installDirFlag}${pathsMessage} - ` + - '\n\n' - ); -}; - -export async function initLogsDir(options: { logsDir?: string; createLogger(): ToolingLog }) { - if (options.logsDir) { - options - .createLogger() - .info(`Kibana/ES logs will be written to ${Path.relative(process.cwd(), options.logsDir)}/`); - - Fs.mkdirSync(options.logsDir, { recursive: true }); - } -} - -/** - * Run servers and tests for each config - */ -interface RunTestsParams extends CreateFtrOptions { - /** Array of paths to configs */ - configs: string[]; - /** run from source instead of snapshot */ - esFrom?: string; - esVersion: EsVersion; - createLogger: () => ToolingLog; - extraKbnOpts: string[]; - assertNoneExcluded: boolean; -} -export async function runTests(options: RunTestsParams) { - if (!process.env.CI && !options.assertNoneExcluded) { - const log = options.createLogger(); - log.warning('โ—๏ธโ—๏ธโ—๏ธ'); - log.warning('โ—๏ธโ—๏ธโ—๏ธ'); - log.warning('โ—๏ธโ—๏ธโ—๏ธ'); - log.warning( - " Don't forget to use `node scripts/build_kibana_platform_plugins` to build plugins you plan on testing" - ); - log.warning('โ—๏ธโ—๏ธโ—๏ธ'); - log.warning('โ—๏ธโ—๏ธโ—๏ธ'); - log.warning('โ—๏ธโ—๏ธโ—๏ธ'); - } - - const log = options.createLogger(); - - if (options.assertNoneExcluded) { - log.write('--- asserting that all tests belong to a ciGroup'); - for (const configPath of options.configs) { - log.info('loading', configPath); - await log.indent(4, async () => { - await assertNoneExcluded({ configPath, options: { ...options, log } }); - }); - continue; - } - - return; - } - - for (const [i, configPath] of options.configs.entries()) { - await log.indent(0, async () => { - if (options.configs.length > 1) { - const progress = `${i + 1}/${options.configs.length}`; - log.write(`--- [${progress}] Running ${Path.relative(REPO_ROOT, configPath)}`); - } - - if (!(await hasTests({ configPath, options: { ...options, log } }))) { - // just run the FTR, no Kibana or ES, which will quickly report a skipped test group to ci-stats and continue - await runFtr({ configPath, options: { ...options, log } }); - return; - } - - await withProcRunner(log, async (procs) => { - const config = await readConfigFile(log, options.esVersion, configPath); - const abortCtrl = new AbortController(); - - const onEarlyExit = (msg: string) => { - log.error(msg); - abortCtrl.abort(); - }; - - let shutdownEs; - try { - if (process.env.TEST_ES_DISABLE_STARTUP !== 'true') { - shutdownEs = await runElasticsearch({ ...options, log, config, onEarlyExit }); - if (abortCtrl.signal.aborted) { - return; - } - } - await runKibanaServer({ procs, config, options, onEarlyExit }); - if (abortCtrl.signal.aborted) { - return; - } - await runFtr({ configPath, options: { ...options, log } }, abortCtrl.signal); - } finally { - try { - const delay = config.get('kbnTestServer.delayShutdown'); - if (typeof delay === 'number') { - log.info('Delaying shutdown of Kibana for', delay, 'ms'); - await setTimeout(delay); - } - - await procs.stop('kibana'); - } finally { - if (shutdownEs) { - await shutdownEs(); - } - } - } - }); - }); - } -} - -interface StartServerOptions { - /** Path to a config file */ - config: string; - log: ToolingLog; - /** installation dir from which to run Kibana */ - installDir?: string; - /** run from source instead of snapshot */ - esFrom?: string; - createLogger: () => ToolingLog; - extraKbnOpts: string[]; - useDefaultConfig?: boolean; - esVersion: EsVersion; -} - -export async function startServers({ ...options }: StartServerOptions) { - const runStartTime = Date.now(); - const toolingLog = new ToolingLog({ - level: 'info', - writeTo: process.stdout, - }); - const reportTime = getTimeReporter(toolingLog, 'scripts/functional_tests_server'); - - const log = options.createLogger(); - const opts = { - ...options, - log, - }; - - await withProcRunner(log, async (procs) => { - const config = await readConfigFile(log, options.esVersion, options.config); - - const shutdownEs = await runElasticsearch({ ...opts, config }); - await runKibanaServer({ - procs, - config, - options: { - ...opts, - extraKbnOpts: [ - ...options.extraKbnOpts, - ...(options.installDir ? [] : ['--dev', '--no-dev-config', '--no-dev-credentials']), - ], - }, - }); - - reportTime(runStartTime, 'ready', { - success: true, - ...options, - }); - - // wait for 5 seconds of silence before logging the - // success message so that it doesn't get buried - await silence(log, 5000); - log.success(makeSuccessMessage(options)); - - await procs.waitForAllToStop(); - await shutdownEs(); - }); -} - -async function silence(log: ToolingLog, milliseconds: number) { - await log - .getWritten$() - .pipe( - startWith(null), - switchMap(() => Rx.timer(milliseconds)), - take(1) - ) - .toPromise(); -} diff --git a/packages/kbn-test/src/mocha/junit_report_generation.js b/packages/kbn-test/src/mocha/junit_report_generation.js index 98210aba770b7..599d1f366194f 100644 --- a/packages/kbn-test/src/mocha/junit_report_generation.js +++ b/packages/kbn-test/src/mocha/junit_report_generation.js @@ -24,6 +24,7 @@ export function setupJUnitReportGeneration(runner, options = {}) { reportName = 'Unnamed Mocha Tests', rootDirectory = REPO_ROOT, getTestMetadata = () => ({}), + metadata, } = options; const stats = {}; @@ -104,6 +105,7 @@ export function setupJUnitReportGeneration(runner, options = {}) { tests: allTests.length + failedHooks.length, failures: failures.length, skipped: skippedResults.length, + 'metadata-json': JSON.stringify(metadata ?? {}), }); function addTestcaseEl(node) { diff --git a/packages/kbn-test/src/mocha/junit_report_generation.test.js b/packages/kbn-test/src/mocha/junit_report_generation.test.js index c19550349fd85..ac23d91390ed9 100644 --- a/packages/kbn-test/src/mocha/junit_report_generation.test.js +++ b/packages/kbn-test/src/mocha/junit_report_generation.test.js @@ -60,6 +60,7 @@ describe('dev/mocha/junit report generation', () => { name: 'test', skipped: '1', tests: '4', + 'metadata-json': '{}', time: testsuite.$.time, timestamp: testsuite.$.timestamp, }, diff --git a/scripts/functional_tests_server.js b/scripts/functional_tests_server.js index 836a1ede126e3..1d51041dccd45 100644 --- a/scripts/functional_tests_server.js +++ b/scripts/functional_tests_server.js @@ -7,4 +7,4 @@ */ require('../src/setup_node_env'); -require('@kbn/test').startServersCli(require.resolve('../test/functional/config.base.js')); +require('@kbn/test').startServersCli(); diff --git a/scripts/report_failed_tests.js b/scripts/report_failed_tests.js index 3d69999a2bb1e..a56675523bba3 100644 --- a/scripts/report_failed_tests.js +++ b/scripts/report_failed_tests.js @@ -7,4 +7,4 @@ */ require('../src/setup_node_env'); -require('@kbn/test').runFailedTestsReporterCli(); +require('@kbn/failed-test-reporter-cli'); diff --git a/src/dev/typescript/projects.ts b/src/dev/typescript/projects.ts index d88ba3a327941..f57854b83550d 100644 --- a/src/dev/typescript/projects.ts +++ b/src/dev/typescript/projects.ts @@ -30,6 +30,7 @@ export const PROJECTS = [ createProject('tsconfig.json'), createProject('test/tsconfig.json', { name: 'kibana/test' }), createProject('x-pack/test/tsconfig.json', { name: 'x-pack/test' }), + createProject('x-pack/performance/tsconfig.json', { name: 'x-pack/performance' }), createProject('src/core/tsconfig.json'), createProject('.buildkite/tsconfig.json'), createProject('kbn_pm/tsconfig.json'), diff --git a/test/common/services/bsearch.ts b/test/common/services/bsearch.ts index 3c8a58c46867b..2a0278840a7c8 100644 --- a/test/common/services/bsearch.ts +++ b/test/common/services/bsearch.ts @@ -10,8 +10,7 @@ import expect from '@kbn/expect'; import request from 'superagent'; import type SuperTest from 'supertest'; import { IEsSearchResponse } from '@kbn/data-plugin/common'; -import { FtrProviderContext } from '../ftr_provider_context'; -import { RetryService } from './retry/retry'; +import { FtrService } from '../ftr_provider_context'; /** * Function copied from here: @@ -46,9 +45,8 @@ interface SendOptions { } /** - * Bsearch factory which will return a new bsearch capable service that can reduce flake - * on the CI systems when they are under pressure and bsearch returns an async search - * response or a sync response. + * Bsearch Service that can reduce flake on the CI systems when they are under + * pressure and bsearch returns an async search response or a sync response. * * @example * const supertest = getService('supertest'); @@ -57,21 +55,18 @@ interface SendOptions { * supertest, * options: { * defaultIndex: ['large_volume_dns_data'], - * } - * strategy: 'securitySolutionSearchStrategy', - * }); + * }, + * strategy: 'securitySolutionSearchStrategy', + * }); * expect(response).eql({ ... your value ... }); */ -export const BSearchFactory = (retry: RetryService) => ({ +export class BsearchService extends FtrService { + private readonly retry = this.ctx.getService('retry'); + /** Send method to send in your supertest, url, options, and strategy name */ - send: async ({ - supertest, - options, - strategy, - space, - }: SendOptions): Promise => { + async send({ supertest, options, strategy, space }: SendOptions) { const spaceUrl = getSpaceUrlPrefix(space); - const { body } = await retry.try(async () => { + const { body } = await this.retry.try(async () => { return supertest .post(`${spaceUrl}/internal/search/${strategy}`) .set('kbn-xsrf', 'true') @@ -79,44 +74,32 @@ export const BSearchFactory = (retry: RetryService) => ({ .expect(200); }); - if (body.isRunning) { - const result = await retry.try(async () => { - const resp = await supertest - .post(`${spaceUrl}/internal/bsearch`) - .set('kbn-xsrf', 'true') - .send({ - batch: [ - { - request: { - id: body.id, - ...options, - }, - options: { - strategy, - }, - }, - ], - }) - .expect(200); - const [parsedResponse] = parseBfetchResponse(resp); - expect(parsedResponse.result.isRunning).equal(false); - return parsedResponse.result; - }); - return result; - } else { + if (!body.isRunning) { return body; } - }, -}); -/** - * Bsearch provider which will return a new bsearch capable service that can reduce flake - * on the CI systems when they are under pressure and bsearch returns an async search response - * or a sync response. - */ -export function BSearchProvider({ - getService, -}: FtrProviderContext): ReturnType { - const retry = getService('retry'); - return BSearchFactory(retry); + const result = await this.retry.try(async () => { + const resp = await supertest + .post(`${spaceUrl}/internal/bsearch`) + .set('kbn-xsrf', 'true') + .send({ + batch: [ + { + request: { + id: body.id, + ...options, + }, + options: { + strategy, + }, + }, + ], + }) + .expect(200); + const [parsedResponse] = parseBfetchResponse(resp); + expect(parsedResponse.result.isRunning).equal(false); + return parsedResponse.result as T; + }); + return result; + } } diff --git a/test/common/services/index.ts b/test/common/services/index.ts index 91d17ce1bb3e8..25c8ea0cdd27d 100644 --- a/test/common/services/index.ts +++ b/test/common/services/index.ts @@ -6,28 +6,22 @@ * Side Public License, v 1. */ +import { commonFunctionalServices } from '@kbn/ftr-common-functional-services'; import { DeploymentService } from './deployment'; -import { ElasticsearchProvider } from './elasticsearch'; -import { EsArchiverProvider } from './es_archiver'; -import { KibanaServerProvider } from './kibana_server'; -import { RetryService } from './retry'; import { RandomnessService } from './randomness'; import { SecurityServiceProvider } from './security'; import { EsDeleteAllIndicesProvider } from './es_delete_all_indices'; import { SavedObjectInfoService } from './saved_object_info'; import { IndexPatternsService } from './index_patterns'; -import { BSearchProvider } from './bsearch'; +import { BsearchService } from './bsearch'; export const services = { + ...commonFunctionalServices, deployment: DeploymentService, - es: ElasticsearchProvider, - esArchiver: EsArchiverProvider, - kibanaServer: KibanaServerProvider, - retry: RetryService, randomness: RandomnessService, security: SecurityServiceProvider, esDeleteAllIndices: EsDeleteAllIndicesProvider, savedObjectInfo: SavedObjectInfoService, indexPatterns: IndexPatternsService, - bsearch: BSearchProvider, + bsearch: BsearchService, }; diff --git a/test/functional/services/common/failure_debugging.ts b/test/functional/services/common/failure_debugging.ts index 343036436293d..5555ae78bccf8 100644 --- a/test/functional/services/common/failure_debugging.ts +++ b/test/functional/services/common/failure_debugging.ts @@ -9,9 +9,9 @@ import { resolve } from 'path'; import { writeFile, mkdir } from 'fs'; import { promisify } from 'util'; -import { createHash } from 'crypto'; import del from 'del'; +import { FtrScreenshotFilename } from '@kbn/ftr-screenshot-filename'; import { FtrProviderContext } from '../../ftr_provider_context'; interface Test { @@ -49,15 +49,7 @@ export async function FailureDebuggingProvider({ getService }: FtrProviderContex } async function onFailure(_: any, test: Test) { - const fullName = test.fullTitle(); - - // include a hash of the full title of the test in the filename so that even with truncation filenames are - // always unique and deterministic based on the test title - const hash = createHash('sha256').update(fullName).digest('hex'); - - // Replace characters in test names which can't be used in filenames, like * - const name = `${fullName.replace(/([^ a-zA-Z0-9-]+)/g, '_').slice(0, 80)}-${hash}`; - + const name = FtrScreenshotFilename.create(test.fullTitle(), { ext: false }); await Promise.all([screenshots.takeForFailure(name), logCurrentUrl(), savePageHtml(name)]); } diff --git a/x-pack/test/performance/es_archives/ecommerce_sample_data/data.json.gz b/x-pack/performance/es_archives/ecommerce_sample_data/data.json.gz similarity index 100% rename from x-pack/test/performance/es_archives/ecommerce_sample_data/data.json.gz rename to x-pack/performance/es_archives/ecommerce_sample_data/data.json.gz diff --git a/x-pack/test/performance/es_archives/ecommerce_sample_data/mappings.json b/x-pack/performance/es_archives/ecommerce_sample_data/mappings.json similarity index 100% rename from x-pack/test/performance/es_archives/ecommerce_sample_data/mappings.json rename to x-pack/performance/es_archives/ecommerce_sample_data/mappings.json diff --git a/x-pack/test/performance/es_archives/reporting_dashboard/data.json.gz b/x-pack/performance/es_archives/reporting_dashboard/data.json.gz similarity index 100% rename from x-pack/test/performance/es_archives/reporting_dashboard/data.json.gz rename to x-pack/performance/es_archives/reporting_dashboard/data.json.gz diff --git a/x-pack/test/performance/es_archives/reporting_dashboard/mappings.json b/x-pack/performance/es_archives/reporting_dashboard/mappings.json similarity index 100% rename from x-pack/test/performance/es_archives/reporting_dashboard/mappings.json rename to x-pack/performance/es_archives/reporting_dashboard/mappings.json diff --git a/x-pack/performance/journeys/data_stress_test_lens.ts b/x-pack/performance/journeys/data_stress_test_lens.ts new file mode 100644 index 0000000000000..6d65dc5e4a0d2 --- /dev/null +++ b/x-pack/performance/journeys/data_stress_test_lens.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Journey } from '@kbn/journeys'; +import { waitForVisualizations } from '../utils'; + +export const journey = new Journey({ + kbnArchives: ['test/functional/fixtures/kbn_archiver/stress_test'], + esArchives: ['test/functional/fixtures/es_archiver/stress_test'], +}).step('Go to dashboard', async ({ page, kbnUrl }) => { + await page.goto(kbnUrl.get(`/app/dashboards#/view/92b143a0-2e9c-11ed-b1b6-a504560b392c`)); + + await waitForVisualizations(page, 1); +}); diff --git a/x-pack/performance/journeys/ecommerce_dashboard.ts b/x-pack/performance/journeys/ecommerce_dashboard.ts new file mode 100644 index 0000000000000..89f05902f4153 --- /dev/null +++ b/x-pack/performance/journeys/ecommerce_dashboard.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Journey } from '@kbn/journeys'; +import { subj } from '@kbn/test-subj-selector'; + +import { ToastsService } from '../services/toasts'; +import { waitForVisualizations } from '../utils'; + +export const journey = new Journey({ + extendContext: ({ page, log }) => ({ + toasts: new ToastsService(log, page), + }), +}) + .step('Go to Sample Data Page', async ({ page, kbnUrl }) => { + await page.goto(kbnUrl.get(`/app/home#/tutorial_directory/sampleData`)); + + await page.waitForSelector(subj('showSampleDataButton')); + }) + + .step('Open Sample Data pane', async ({ page }) => { + // open the "other sample data sets" section + await page.click(subj('showSampleDataButton')); + // wait for the logs card to be visible + await page.waitForSelector(subj('sampleDataSetCardecommerce')); + }) + + .step('Remove Ecommerce Sample Data if installed', async ({ page, log, toasts }) => { + if (!(await page.$(subj('removeSampleDataSetecommerce')))) { + log.info('Ecommerce data does not need to be removed'); + return; + } + + // click the "remove" button + await page.click(subj('removeSampleDataSetecommerce')); + // wait for the toast acknowledging uninstallation + await toasts.waitForAndClear('uninstalled'); + }) + + .step('Install Ecommerce Sample Data', async ({ page, toasts }) => { + // click the "add data" button + await page.click(subj('addSampleDataSetecommerce')); + // wait for the toast acknowledging installation + await toasts.waitForAndClear('installed'); + }) + + .step('Go to Ecommerce Dashboard', async ({ page }) => { + await page.click(subj('launchSampleDataSetecommerce')); + await page.click(subj('viewSampleDataSetecommerce-dashboard')); + + await waitForVisualizations(page, 13); + }); diff --git a/x-pack/performance/journeys/flight_dashboard.ts b/x-pack/performance/journeys/flight_dashboard.ts new file mode 100644 index 0000000000000..ac6e589d391a5 --- /dev/null +++ b/x-pack/performance/journeys/flight_dashboard.ts @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Journey } from '@kbn/journeys'; +import { subj } from '@kbn/test-subj-selector'; + +import { ToastsService } from '../services/toasts'; +import { waitForVisualizations } from '../utils'; + +export const journey = new Journey({ + extendContext: ({ page, log }) => ({ + toasts: new ToastsService(log, page), + }), +}) + .step('Go to Sample Data Page', async ({ page, kbnUrl }) => { + await page.goto(kbnUrl.get(`/app/home#/tutorial_directory/sampleData`)); + + await page.waitForSelector(subj('showSampleDataButton')); + }) + + .step('Open Sample Data pane', async ({ page }) => { + // open the "other sample data sets" section + await page.click(subj('showSampleDataButton')); + // wait for the logs card to be visible + await page.waitForSelector(subj('sampleDataSetCardflights')); + }) + + .step('Remove Flights Sample Data if installed', async ({ page, log, toasts }) => { + if (!(await page.$(subj('removeSampleDataSetflights')))) { + log.info('Flights data does not need to be removed'); + return; + } + + // click the "remove" button + await page.click(subj('removeSampleDataSetflights')); + // wait for the toast acknowledging uninstallation + await toasts.waitForAndClear('uninstalled'); + }) + + .step('Install Flights Sample Data', async ({ page, toasts }) => { + // click the "add data" button + await page.click(subj('addSampleDataSetflights')); + // wait for the toast acknowledging installation + await toasts.waitForAndClear('installed'); + }) + + .step('Go to Flights Dashboard', async ({ page }) => { + await page.click(subj('launchSampleDataSetflights')); + await page.click(subj('viewSampleDataSetflights-dashboard')); + + await waitForVisualizations(page, 15); + }) + + .step('Go to Airport Connections Visualizations Edit', async ({ page }) => { + await page.click(subj('dashboardEditMode')); + + const flightsPanelHeadingSelector = `embeddablePanelHeading-[Flights]AirportConnections(HoverOverAirport)`; + const panelToggleMenuIconSelector = `embeddablePanelToggleMenuIcon`; + await page.click(subj(`${flightsPanelHeadingSelector} > ${panelToggleMenuIconSelector}`)); + + await page.click(subj('embeddablePanelAction-editPanel')); + + await waitForVisualizations(page, 1); + }); diff --git a/x-pack/performance/journeys/login.ts b/x-pack/performance/journeys/login.ts new file mode 100644 index 0000000000000..7c5808be83607 --- /dev/null +++ b/x-pack/performance/journeys/login.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Journey } from '@kbn/journeys'; +import { subj } from '@kbn/test-subj-selector'; + +export const journey = new Journey({ + skipAutoLogin: true, + scalabilitySetup: { + warmup: [ + { + action: 'constantConcurrentUsers', + userCount: 10, + duration: '30s', + }, + { + action: 'rampConcurrentUsers', + minUsersCount: 10, + maxUsersCount: 50, + duration: '2m', + }, + ], + test: [ + { + action: 'constantConcurrentUsers', + userCount: 50, + duration: '5m', + }, + ], + maxDuration: '10m', + }, +}).step('Login', async ({ page, kbnUrl, inputDelays }) => { + await page.goto(kbnUrl.get()); + + await page.type(subj('loginUsername'), 'elastic', { delay: inputDelays.TYPING }); + await page.type(subj('loginPassword'), 'changeme', { delay: inputDelays.TYPING }); + await page.click(subj('loginSubmit'), { delay: inputDelays.MOUSE_CLICK }); + + await page.waitForSelector('#headerUserMenu'); +}); diff --git a/x-pack/performance/journeys/many_fields_discover.ts b/x-pack/performance/journeys/many_fields_discover.ts new file mode 100644 index 0000000000000..41ec0373c700c --- /dev/null +++ b/x-pack/performance/journeys/many_fields_discover.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Journey } from '@kbn/journeys'; +import { subj } from '@kbn/test-subj-selector'; + +export const journey = new Journey({ + // FAILING: https://github.com/elastic/kibana/issues/130287 + skipped: true, + kbnArchives: ['test/functional/fixtures/kbn_archiver/many_fields_data_view'], + esArchives: ['test/functional/fixtures/es_archiver/many_fields'], +}) + .step('Go to Discover Page', async ({ page, kbnUrl }) => { + await page.goto(kbnUrl.get(`/app/discover`)); + await page.waitForSelector(subj('discoverDocTable')); + }) + .step('Expand the first document', async ({ page }) => { + const expandButtons = page.locator(subj('docTableExpandToggleColumn')); + await expandButtons.first().click(); + await page.locator('text="Expanded document"'); + }); diff --git a/x-pack/performance/journeys/promotion_tracking_dashboard.ts b/x-pack/performance/journeys/promotion_tracking_dashboard.ts new file mode 100644 index 0000000000000..e6bd67a2819c5 --- /dev/null +++ b/x-pack/performance/journeys/promotion_tracking_dashboard.ts @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Journey } from '@kbn/journeys'; +import { subj } from '@kbn/test-subj-selector'; +import { waitForVisualizations } from '../utils'; + +export const journey = new Journey({ + kbnArchives: ['x-pack/performance/kbn_archives/promotion_tracking_dashboard'], + esArchives: ['x-pack/performance/es_archives/ecommerce_sample_data'], + scalabilitySetup: { + warmup: [ + { + action: 'constantConcurrentUsers', + userCount: 10, + duration: '30s', + }, + { + action: 'rampConcurrentUsers', + minUsersCount: 10, + maxUsersCount: 50, + duration: '2m', + }, + ], + test: [ + { + action: 'constantConcurrentUsers', + userCount: 50, + duration: '5m', + }, + ], + maxDuration: '10m', + }, +}) + .step('Go to Dashboards Page', async ({ page, kbnUrl }) => { + await page.goto(kbnUrl.get(`/app/dashboards`)); + await page.waitForSelector('#dashboardListingHeading'); + }) + + .step('Go to Promotion Tracking Dashboard', async ({ page }) => { + await page.click(subj('dashboardListingTitleLink-Promotion-Dashboard')); + }) + + .step('Change time range', async ({ page }) => { + await page.click(subj('superDatePickerToggleQuickMenuButton')); + await page.click(subj('superDatePickerCommonlyUsed_Last_30 days')); + }) + + .step('Wait for visualization animations to finish', async ({ page }) => { + await waitForVisualizations(page, 1); + }); diff --git a/x-pack/performance/journeys/web_logs_dashboard.ts b/x-pack/performance/journeys/web_logs_dashboard.ts new file mode 100644 index 0000000000000..64ea47d412e0e --- /dev/null +++ b/x-pack/performance/journeys/web_logs_dashboard.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Journey } from '@kbn/journeys'; +import { subj } from '@kbn/test-subj-selector'; + +import { ToastsService } from '../services/toasts'; +import { waitForVisualizations } from '../utils'; + +export const journey = new Journey({ + extendContext: ({ page, log }) => ({ + toasts: new ToastsService(log, page), + }), +}) + .step('Go to Sample Data Page', async ({ page, kbnUrl }) => { + await page.goto(kbnUrl.get(`/app/home#/tutorial_directory/sampleData`)); + + await page.waitForSelector(subj('showSampleDataButton')); + }) + + .step('Open Sample Data pane', async ({ page }) => { + // open the "other sample data sets" section + await page.click(subj('showSampleDataButton')); + // wait for the logs card to be visible + await page.waitForSelector(subj('sampleDataSetCardlogs')); + }) + + .step('Remove Sample Data Logs if installed', async ({ page, log, toasts }) => { + if (!(await page.$(subj('removeSampleDataSetlogs')))) { + log.info('Logs data does not need to be removed'); + return; + } + + // click the "remove" button + await page.click(subj('removeSampleDataSetlogs')); + // wait for the toast acknowledging uninstallation + await toasts.waitForAndClear('uninstalled'); + }) + + .step('Install Logs Sample Data', async ({ page, toasts }) => { + // click the "add data" button + await page.click(subj('addSampleDataSetlogs')); + // wait for the toast acknowledging installation + await toasts.waitForAndClear('installed'); + }) + + .step('Go to Web Logs Dashboard', async ({ page }) => { + await page.click(subj('launchSampleDataSetlogs')); + await page.click(subj('viewSampleDataSetlogs-dashboard')); + + await waitForVisualizations(page, 12); + }); diff --git a/x-pack/test/performance/kbn_archives/promotion_tracking_dashboard.json b/x-pack/performance/kbn_archives/promotion_tracking_dashboard.json similarity index 100% rename from x-pack/test/performance/kbn_archives/promotion_tracking_dashboard.json rename to x-pack/performance/kbn_archives/promotion_tracking_dashboard.json diff --git a/x-pack/test/performance/kbn_archives/reporting_dashboard.json b/x-pack/performance/kbn_archives/reporting_dashboard.json similarity index 100% rename from x-pack/test/performance/kbn_archives/reporting_dashboard.json rename to x-pack/performance/kbn_archives/reporting_dashboard.json diff --git a/x-pack/performance/services/toasts.ts b/x-pack/performance/services/toasts.ts new file mode 100644 index 0000000000000..b3859ddb92ec4 --- /dev/null +++ b/x-pack/performance/services/toasts.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ToolingLog } from '@kbn/tooling-log'; +import { subj } from '@kbn/test-subj-selector'; +import { Page } from 'playwright'; + +export class ToastsService { + constructor(private readonly log: ToolingLog, private readonly page: Page) {} + + /** + * Wait for a toast with some bit of text matching the provided `textSnipped`, then clear + * it and resolve the promise. + */ + async waitForAndClear(textSnippet: string) { + const txt = JSON.stringify(textSnippet); + this.log.info(`waiting for toast that has the text ${txt}`); + const toastSel = `.euiToast:has-text(${txt})`; + + const toast = this.page.locator(toastSel); + await toast.waitFor(); + + this.log.info('toast found, closing'); + + const close = toast.locator(subj('toastCloseButton')); + await close.click(); + + await toast.waitFor({ state: 'hidden' }); + } +} diff --git a/x-pack/performance/tsconfig.json b/x-pack/performance/tsconfig.json new file mode 100644 index 0000000000000..caff6d53f4476 --- /dev/null +++ b/x-pack/performance/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "./target/types", + "emitDeclarationOnly": true, + "declaration": true, + "declarationMap": true, + "types": ["node", "mocha"] + }, + "include": ["**/*.ts"], +} diff --git a/x-pack/test/performance/utils.ts b/x-pack/performance/utils.ts similarity index 67% rename from x-pack/test/performance/utils.ts rename to x-pack/performance/utils.ts index 1e9e754088418..c0a7ba95f7ee1 100644 --- a/x-pack/test/performance/utils.ts +++ b/x-pack/performance/utils.ts @@ -7,14 +7,8 @@ import { Page } from 'playwright'; -export function serializeApmGlobalLabels(obj: any) { - return Object.entries(obj) - .filter(([, v]) => !!v) - .reduce((acc, [k, v]) => (acc ? `${acc},${k}=${v}` : `${k}=${v}`), ''); -} - -export function waitForVisualizations(page: Page, visCount: number) { - return page.waitForFunction(function renderCompleted(cnt) { +export async function waitForVisualizations(page: Page, visCount: number) { + return await page.waitForFunction(function renderCompleted(cnt) { const visualizations = Array.from(document.querySelectorAll('[data-rendering-count]')); const visualizationElementsLoaded = visualizations.length === cnt; const visualizationAnimationsFinished = visualizations.every( diff --git a/x-pack/scripts/functional_tests_server.js b/x-pack/scripts/functional_tests_server.js index 329fea019221b..938d4a465af50 100755 --- a/x-pack/scripts/functional_tests_server.js +++ b/x-pack/scripts/functional_tests_server.js @@ -8,4 +8,4 @@ process.env.ALLOW_PERFORMANCE_HOOKS_IN_TASK_MANAGER = true; require('../../src/setup_node_env'); -require('@kbn/test').startServersCli(require.resolve('../test/functional/config.base.js')); +require('@kbn/test').startServersCli(); diff --git a/x-pack/test/common/ftr_provider_context.ts b/x-pack/test/common/ftr_provider_context.ts index aa56557c09df8..e8c18508a202f 100644 --- a/x-pack/test/common/ftr_provider_context.ts +++ b/x-pack/test/common/ftr_provider_context.ts @@ -5,8 +5,9 @@ * 2.0. */ -import { GenericFtrProviderContext } from '@kbn/test'; +import { GenericFtrProviderContext, GenericFtrService } from '@kbn/test'; import { services } from './services'; export type FtrProviderContext = GenericFtrProviderContext; +export class FtrService extends GenericFtrService {} diff --git a/x-pack/test/common/services/bsearch_secure.ts b/x-pack/test/common/services/bsearch_secure.ts index f60dc6aa4c3cd..94a5abe73c901 100644 --- a/x-pack/test/common/services/bsearch_secure.ts +++ b/x-pack/test/common/services/bsearch_secure.ts @@ -12,8 +12,7 @@ import expect from '@kbn/expect'; import request from 'superagent'; import type SuperTest from 'supertest'; import { IEsSearchResponse } from '@kbn/data-plugin/common'; -import { FtrProviderContext } from '../ftr_provider_context'; -import { RetryService } from '../../../../test/common/services/retry/retry'; +import { FtrService } from '../ftr_provider_context'; const parseBfetchResponse = (resp: request.Response): Array> => { return resp.text @@ -36,8 +35,10 @@ interface SendOptions { space?: string; } -export const BSecureSearchFactory = (retry: RetryService) => ({ - send: async ({ +export class BsearchSecureService extends FtrService { + private readonly retry = this.ctx.getService('retry'); + + async send({ supertestWithoutAuth, auth, referer, @@ -45,9 +46,10 @@ export const BSecureSearchFactory = (retry: RetryService) => ({ options, strategy, space, - }: SendOptions): Promise => { + }: SendOptions) { const spaceUrl = getSpaceUrlPrefix(space); - const { body } = await retry.try(async () => { + + const { body } = await this.retry.try(async () => { let result; const url = `${spaceUrl}/internal/search/${strategy}`; if (referer && kibanaVersion) { @@ -84,40 +86,35 @@ export const BSecureSearchFactory = (retry: RetryService) => ({ } throw new Error('try again'); }); - if (body.isRunning) { - const result = await retry.try(async () => { - const resp = await supertestWithoutAuth - .post(`${spaceUrl}/internal/bsearch`) - .auth(auth.username, auth.password) - .set('kbn-xsrf', 'true') - .send({ - batch: [ - { - request: { - id: body.id, - ...options, - }, - options: { - strategy, - }, - }, - ], - }) - .expect(200); - const [parsedResponse] = parseBfetchResponse(resp); - expect(parsedResponse.result.isRunning).equal(false); - return parsedResponse.result; - }); - return result; - } else { - return body; + + if (!body.isRunning) { + return body as T; } - }, -}); -export function BSecureSearchProvider({ - getService, -}: FtrProviderContext): ReturnType { - const retry = getService('retry'); - return BSecureSearchFactory(retry); + const result = await this.retry.try(async () => { + const resp = await supertestWithoutAuth + .post(`${spaceUrl}/internal/bsearch`) + .auth(auth.username, auth.password) + .set('kbn-xsrf', 'true') + .send({ + batch: [ + { + request: { + id: body.id, + ...options, + }, + options: { + strategy, + }, + }, + ], + }) + .expect(200); + const [parsedResponse] = parseBfetchResponse(resp); + expect(parsedResponse.result.isRunning).equal(false); + return parsedResponse.result; + }); + + return result as T; + } } diff --git a/x-pack/test/common/services/index.ts b/x-pack/test/common/services/index.ts index c51fe7a06e6ac..0f247ad743edf 100644 --- a/x-pack/test/common/services/index.ts +++ b/x-pack/test/common/services/index.ts @@ -9,12 +9,12 @@ import { services as kibanaApiIntegrationServices } from '../../../../test/api_i import { services as kibanaCommonServices } from '../../../../test/common/services'; import { InfraLogViewsServiceProvider } from './infra_log_views'; import { SpacesServiceProvider } from './spaces'; -import { BSecureSearchProvider } from './bsearch_secure'; +import { BsearchSecureService } from './bsearch_secure'; export const services = { ...kibanaCommonServices, infraLogViews: InfraLogViewsServiceProvider, supertest: kibanaApiIntegrationServices.supertest, spaces: SpacesServiceProvider, - secureBsearch: BSecureSearchProvider, + secureBsearch: BsearchSecureService, }; diff --git a/x-pack/test/functional/page_objects/infra_logs_page.ts b/x-pack/test/functional/page_objects/infra_logs_page.ts index fe43dd1767d94..c1d20c2e977ad 100644 --- a/x-pack/test/functional/page_objects/infra_logs_page.ts +++ b/x-pack/test/functional/page_objects/infra_logs_page.ts @@ -5,7 +5,6 @@ * 2.0. */ -// import testSubjSelector from '@kbn/test-subj-selector'; // import moment from 'moment'; import querystring from 'querystring'; import { encode, RisonValue } from 'rison-node'; diff --git a/x-pack/test/functional_execution_context/test_utils.ts b/x-pack/test/functional_execution_context/test_utils.ts index 03299689186a2..6cf1af27b0bb2 100644 --- a/x-pack/test/functional_execution_context/test_utils.ts +++ b/x-pack/test/functional_execution_context/test_utils.ts @@ -8,7 +8,7 @@ import Fs from 'fs/promises'; import Path from 'path'; import { isEqualWith } from 'lodash'; import type { Ecs, KibanaExecutionContext } from '@kbn/core/server'; -import type { RetryService } from '../../../test/common/services/retry'; +import type { RetryService } from '@kbn/ftr-common-functional-services'; export const logFilePath = Path.resolve(__dirname, './kibana.log'); export const ANY = Symbol('any'); diff --git a/x-pack/test/performance/journeys/base.config.ts b/x-pack/test/performance/journeys/base.config.ts deleted file mode 100644 index f2924b3a8a785..0000000000000 --- a/x-pack/test/performance/journeys/base.config.ts +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import uuid from 'uuid'; -import { FtrConfigProviderContext } from '@kbn/test'; - -import { TelemetryConfigLabels } from '@kbn/telemetry-plugin/server/config'; -import { services } from '../services'; -import { pageObjects } from '../page_objects'; - -// These "secret" values are intentionally written in the source. We would make the APM server accept anonymous traffic if we could -const APM_SERVER_URL = 'https://kibana-ops-e2e-perf.apm.us-central1.gcp.cloud.es.io:443'; -const APM_PUBLIC_TOKEN = 'CTs9y3cvcfq13bQqsB'; - -export default async function ({ readConfigFile, log }: FtrConfigProviderContext) { - const functionalConfig = await readConfigFile(require.resolve('../../functional/config.base.js')); - - const testBuildId = process.env.BUILDKITE_BUILD_ID ?? `local-${uuid()}`; - const testJobId = process.env.BUILDKITE_JOB_ID ?? `local-${uuid()}`; - const executionId = uuid(); - - log.info(` ๐Ÿ‘ทโ€โ™€๏ธ BUILD ID ${testBuildId}\n ๐Ÿ‘ท JOB ID ${testJobId}\n ๐Ÿ‘ทโ€โ™‚๏ธ EXECUTION ID:${executionId}`); - - const prId = process.env.GITHUB_PR_NUMBER - ? Number.parseInt(process.env.GITHUB_PR_NUMBER, 10) - : undefined; - - if (Number.isNaN(prId)) { - throw new Error('invalid GITHUB_PR_NUMBER environment variable'); - } - - const telemetryLabels: TelemetryConfigLabels = { - branch: process.env.BUILDKITE_BRANCH, - ciBuildId: process.env.BUILDKITE_BUILD_ID, - ciBuildJobId: process.env.BUILDKITE_JOB_ID, - ciBuildNumber: Number(process.env.BUILDKITE_BUILD_NUMBER) || 0, - gitRev: process.env.BUILDKITE_COMMIT, - isPr: prId !== undefined, - ...(prId !== undefined ? { prId } : {}), - testJobId, - testBuildId, - ciBuildName: process.env.BUILDKITE_PIPELINE_SLUG, - }; - - return { - services, - pageObjects, - servicesRequiredForTestAnalysis: ['performance'], - servers: functionalConfig.get('servers'), - esTestCluster: functionalConfig.get('esTestCluster'), - apps: functionalConfig.get('apps'), - screenshots: functionalConfig.get('screenshots'), - junit: { - reportName: 'Performance Tests', - }, - kbnTestServer: { - ...functionalConfig.get('kbnTestServer'), - serverArgs: [ - ...functionalConfig.get('kbnTestServer.serverArgs'), - `--telemetry.optIn=${process.env.TEST_PERFORMANCE_PHASE === 'TEST'}`, - `--telemetry.labels=${JSON.stringify(telemetryLabels)}`, - '--csp.strict=false', - '--csp.warnLegacyBrowsers=false', - ], - env: { - ELASTIC_APM_ACTIVE: process.env.TEST_PERFORMANCE_PHASE ? 'true' : 'false', - ELASTIC_APM_CONTEXT_PROPAGATION_ONLY: 'false', - ELASTIC_APM_ENVIRONMENT: process.env.CI ? 'ci' : 'development', - ELASTIC_APM_TRANSACTION_SAMPLE_RATE: '1.0', - ELASTIC_APM_SERVER_URL: APM_SERVER_URL, - ELASTIC_APM_SECRET_TOKEN: APM_PUBLIC_TOKEN, - // capture request body for both errors and request transactions - // https://www.elastic.co/guide/en/apm/agent/nodejs/current/configuration.html#capture-body - ELASTIC_APM_CAPTURE_BODY: 'all', - // capture request headers - // https://www.elastic.co/guide/en/apm/agent/nodejs/current/configuration.html#capture-headers - ELASTIC_APM_CAPTURE_HEADERS: true, - // request body with bigger size will be trimmed. - // 300_000 is the default of the APM server. - // for a body with larger size, we might need to reconfigure the APM server to increase the limit. - // https://www.elastic.co/guide/en/apm/agent/nodejs/current/configuration.html#long-field-max-length - ELASTIC_APM_LONG_FIELD_MAX_LENGTH: 300_000, - ELASTIC_APM_GLOBAL_LABELS: { - testJobId, - testBuildId, - }, - }, - // delay shutdown by 15 seconds to ensure that APM can report the data it collects during test execution - delayShutdown: 15_000, - }, - }; -} diff --git a/x-pack/test/performance/journeys/data_stress_test_lens/config.ts b/x-pack/test/performance/journeys/data_stress_test_lens/config.ts deleted file mode 100644 index 0eb5f841692fa..0000000000000 --- a/x-pack/test/performance/journeys/data_stress_test_lens/config.ts +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import { FtrConfigProviderContext } from '@kbn/test'; -import { serializeApmGlobalLabels } from '../../utils'; - -export const JOURNEY_DATA_STRESS_TEST_LENS = 'data_stress_test_lens'; - -export default async function ({ readConfigFile }: FtrConfigProviderContext) { - const performanceConfig = await readConfigFile(require.resolve('../base.config')); - - const testFiles = [require.resolve(`./${JOURNEY_DATA_STRESS_TEST_LENS}`)]; - - const config = { - ...performanceConfig.getAll(), - testFiles, - testData: { - kbnArchives: ['test/functional/fixtures/kbn_archiver/stress_test'], - esArchives: ['test/functional/fixtures/es_archiver/stress_test'], - }, - }; - - const apmGlobalLabels = { - ...performanceConfig.get('kbnTestServer').env.ELASTIC_APM_GLOBAL_LABELS, - ftrConfig: `x-pack/test/performance/tests/journeys/${JOURNEY_DATA_STRESS_TEST_LENS}/config.ts`, - performancePhase: process.env.TEST_PERFORMANCE_PHASE, - journeyName: JOURNEY_DATA_STRESS_TEST_LENS, - }; - - return { - ...config, - kbnTestServer: { - ...config.kbnTestServer, - serverArgs: [ - ...performanceConfig.get('kbnTestServer.serverArgs'), - `--telemetry.labels.journeyName=${JOURNEY_DATA_STRESS_TEST_LENS}`, - ], - env: { - ...config.kbnTestServer.env, - ELASTIC_APM_GLOBAL_LABELS: serializeApmGlobalLabels(apmGlobalLabels), - }, - }, - }; -} diff --git a/x-pack/test/performance/journeys/data_stress_test_lens/data_stress_test_lens.ts b/x-pack/test/performance/journeys/data_stress_test_lens/data_stress_test_lens.ts deleted file mode 100644 index e30b6ee67d897..0000000000000 --- a/x-pack/test/performance/journeys/data_stress_test_lens/data_stress_test_lens.ts +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import { FtrProviderContext } from '../../ftr_provider_context'; -import { StepCtx } from '../../services/performance'; -import { waitForVisualizations } from '../../utils'; -import { JOURNEY_DATA_STRESS_TEST_LENS } from './config'; - -export default function ({ getService }: FtrProviderContext) { - describe(JOURNEY_DATA_STRESS_TEST_LENS, () => { - const performance = getService('performance'); - - it(JOURNEY_DATA_STRESS_TEST_LENS, async () => { - await performance.runUserJourney( - JOURNEY_DATA_STRESS_TEST_LENS, - [ - { - name: 'Go to dashboard', - handler: async ({ page, kibanaUrl }: StepCtx) => { - await page.goto( - `${kibanaUrl}/app/dashboards#/view/92b143a0-2e9c-11ed-b1b6-a504560b392c` - ); - - await waitForVisualizations(page, 1); - }, - }, - ], - { - requireAuth: false, - } - ); - }); - }); -} diff --git a/x-pack/test/performance/journeys/ecommerce_dashboard/config.ts b/x-pack/test/performance/journeys/ecommerce_dashboard/config.ts deleted file mode 100644 index 3b4d19b9a5bfc..0000000000000 --- a/x-pack/test/performance/journeys/ecommerce_dashboard/config.ts +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import { FtrConfigProviderContext } from '@kbn/test'; -import { serializeApmGlobalLabels } from '../../utils'; - -const JOURNEY_ECOMMERCE_DASHBOARD = 'ecommerce_dashboard'; - -export default async function ({ readConfigFile }: FtrConfigProviderContext) { - const performanceConfig = await readConfigFile(require.resolve('../base.config')); - - const testFiles = [require.resolve(`./${JOURNEY_ECOMMERCE_DASHBOARD}`)]; - - const config = { - ...performanceConfig.getAll(), - testFiles, - }; - - const apmGlobalLabels = { - ...performanceConfig.get('kbnTestServer').env.ELASTIC_APM_GLOBAL_LABELS, - ftrConfig: `x-pack/test/performance/tests/journeys/${JOURNEY_ECOMMERCE_DASHBOARD}/config.ts`, - performancePhase: process.env.TEST_PERFORMANCE_PHASE, - journeyName: JOURNEY_ECOMMERCE_DASHBOARD, - }; - - return { - ...config, - kbnTestServer: { - ...config.kbnTestServer, - serverArgs: [ - ...performanceConfig.get('kbnTestServer.serverArgs'), - `--telemetry.labels.journeyName=${JOURNEY_ECOMMERCE_DASHBOARD}`, - ], - env: { - ...config.kbnTestServer.env, - ELASTIC_APM_GLOBAL_LABELS: serializeApmGlobalLabels(apmGlobalLabels), - }, - }, - }; -} diff --git a/x-pack/test/performance/journeys/ecommerce_dashboard/ecommerce_dashboard.ts b/x-pack/test/performance/journeys/ecommerce_dashboard/ecommerce_dashboard.ts deleted file mode 100644 index 5263c19902b93..0000000000000 --- a/x-pack/test/performance/journeys/ecommerce_dashboard/ecommerce_dashboard.ts +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import { Page } from 'playwright'; -import { FtrProviderContext } from '../../ftr_provider_context'; -import { StepCtx } from '../../services/performance'; -import { waitForVisualizations } from '../../utils'; - -export default function ({ getService }: FtrProviderContext) { - describe('ecommerce_dashboard', () => { - it('ecommerce_dashboard', async () => { - const performance = getService('performance'); - const logger = getService('log'); - - await performance.runUserJourney( - 'ecommerce_dashboard', - [ - { - name: 'Go to Sample Data Page', - handler: async ({ page, kibanaUrl }: StepCtx) => { - await page.goto(`${kibanaUrl}/app/home#/tutorial_directory/sampleData`); - await page.waitForSelector('text="More ways to add data"'); - }, - }, - { - name: 'Add Ecommerce Sample Data', - handler: async ({ page }: { page: Page }) => { - const showSampleDataButton = page.locator('[data-test-subj=showSampleDataButton]'); - await showSampleDataButton.click(); - const removeButton = page.locator('[data-test-subj=removeSampleDataSetecommerce]'); - try { - await removeButton.click({ timeout: 1_000 }); - } catch (e) { - logger.info('Ecommerce data does not exist'); - } - const addDataButton = page.locator('[data-test-subj=addSampleDataSetecommerce]'); - if (addDataButton) { - await addDataButton.click(); - } - }, - }, - { - name: 'Go to Ecommerce Dashboard', - handler: async ({ page }: { page: Page }) => { - await page.click('[data-test-subj=launchSampleDataSetecommerce]'); - await page.click('[data-test-subj=viewSampleDataSetecommerce-dashboard]'); - - await waitForVisualizations(page, 13); - }, - }, - ], - { - requireAuth: false, - } - ); - }); - }); -} diff --git a/x-pack/test/performance/journeys/flight_dashboard/config.ts b/x-pack/test/performance/journeys/flight_dashboard/config.ts deleted file mode 100644 index 0b060694a2881..0000000000000 --- a/x-pack/test/performance/journeys/flight_dashboard/config.ts +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import { FtrConfigProviderContext } from '@kbn/test'; -import { serializeApmGlobalLabels } from '../../utils'; - -const JOURNEY_FLIGHT_DASHBOARD = 'flight_dashboard'; - -export default async function ({ readConfigFile }: FtrConfigProviderContext) { - const performanceConfig = await readConfigFile(require.resolve('../base.config')); - - const testFiles = [require.resolve(`./${JOURNEY_FLIGHT_DASHBOARD}`)]; - - const config = { - ...performanceConfig.getAll(), - testFiles, - }; - - const apmGlobalLabels = { - ...performanceConfig.get('kbnTestServer').env.ELASTIC_APM_GLOBAL_LABELS, - ftrConfig: `x-pack/test/performance/tests/journeys/${JOURNEY_FLIGHT_DASHBOARD}/config.ts`, - performancePhase: process.env.TEST_PERFORMANCE_PHASE, - journeyName: JOURNEY_FLIGHT_DASHBOARD, - }; - - return { - ...config, - kbnTestServer: { - ...config.kbnTestServer, - serverArgs: [ - ...performanceConfig.get('kbnTestServer.serverArgs'), - `--telemetry.labels.journeyName=${JOURNEY_FLIGHT_DASHBOARD}`, - ], - env: { - ...config.kbnTestServer.env, - ELASTIC_APM_GLOBAL_LABELS: serializeApmGlobalLabels(apmGlobalLabels), - }, - }, - }; -} diff --git a/x-pack/test/performance/journeys/flight_dashboard/flight_dashboard.ts b/x-pack/test/performance/journeys/flight_dashboard/flight_dashboard.ts deleted file mode 100644 index 5967cd6ca80f3..0000000000000 --- a/x-pack/test/performance/journeys/flight_dashboard/flight_dashboard.ts +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import { FtrProviderContext } from '../../ftr_provider_context'; -import { StepCtx } from '../../services/performance'; -import { waitForVisualizations } from '../../utils'; - -export default function ({ getService }: FtrProviderContext) { - describe('flight_dashboard', () => { - it('flight_dashboard', async () => { - const performance = getService('performance'); - const logger = getService('log'); - - await performance.runUserJourney( - 'flight_dashboard', - [ - { - name: 'Go to Sample Data Page', - handler: async ({ page, kibanaUrl }: StepCtx) => { - await page.goto(`${kibanaUrl}/app/home#/tutorial_directory/sampleData`); - await page.waitForSelector('[data-test-subj=sampleDataSetCardflights]'); - }, - }, - { - name: 'Add Flights Sample Data', - handler: async ({ page }) => { - const showSampleDataButton = page.locator('[data-test-subj=showSampleDataButton]'); - await showSampleDataButton.click(); - const removeButton = page.locator('[data-test-subj=removeSampleDataSetflights]'); - try { - await removeButton.click({ timeout: 1_000 }); - } catch (e) { - logger.info('Flights data does not exist'); - } - - const addDataButton = page.locator('[data-test-subj=addSampleDataSetflights]'); - if (addDataButton) { - await addDataButton.click(); - } - }, - }, - { - name: 'Go to Flights Dashboard', - handler: async ({ page }) => { - await page.click('[data-test-subj=launchSampleDataSetflights]'); - await page.click('[data-test-subj=viewSampleDataSetflights-dashboard]'); - - await waitForVisualizations(page, 15); - }, - }, - { - name: 'Go to Airport Connections Visualizations Edit', - handler: async ({ page }) => { - await page.click('[data-test-subj="dashboardEditMode"]'); - - const flightsPanelHeadingSelector = `[data-test-subj="embeddablePanelHeading-[Flights]AirportConnections(HoverOverAirport)"]`; - const panelToggleMenuIconSelector = `[data-test-subj="embeddablePanelToggleMenuIcon"]`; - - await page.click(`${flightsPanelHeadingSelector} ${panelToggleMenuIconSelector}`); - - await page.click('[data-test-subj="embeddablePanelAction-editPanel"]'); - - await waitForVisualizations(page, 1); - }, - }, - ], - { - requireAuth: false, - } - ); - }); - }); -} diff --git a/x-pack/test/performance/journeys/login/config.ts b/x-pack/test/performance/journeys/login/config.ts deleted file mode 100644 index adf9bc76416a4..0000000000000 --- a/x-pack/test/performance/journeys/login/config.ts +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import { FtrConfigProviderContext } from '@kbn/test'; -import { serializeApmGlobalLabels } from '../../utils'; - -const JOURNEY_LOGIN = 'login'; - -export default async function ({ readConfigFile }: FtrConfigProviderContext) { - const performanceConfig = await readConfigFile(require.resolve('../base.config')); - - const testFiles = [require.resolve(`./${JOURNEY_LOGIN}`)]; - - const config = { - ...performanceConfig.getAll(), - testFiles, - scalabilitySetup: { - warmup: [ - { - action: 'constantConcurrentUsers', - maxUsersCount: 10, - duration: '30s', - }, - { - action: 'rampConcurrentUsers', - minUsersCount: 10, - maxUsersCount: 50, - duration: '2m', - }, - ], - test: [ - { - action: 'constantConcurrentUsers', - maxUsersCount: 50, - duration: '5m', - }, - ], - maxDuration: '10m', - }, - }; - - const apmGlobalLabels = { - ...performanceConfig.get('kbnTestServer').env.ELASTIC_APM_GLOBAL_LABELS, - ftrConfig: `x-pack/test/performance/tests/journeys/${JOURNEY_LOGIN}/config.ts`, - performancePhase: process.env.TEST_PERFORMANCE_PHASE, - journeyName: JOURNEY_LOGIN, - }; - - return { - ...config, - kbnTestServer: { - ...config.kbnTestServer, - serverArgs: [ - ...performanceConfig.get('kbnTestServer.serverArgs'), - `--telemetry.labels.journeyName=${JOURNEY_LOGIN}`, - ], - env: { - ...config.kbnTestServer.env, - ELASTIC_APM_GLOBAL_LABELS: serializeApmGlobalLabels(apmGlobalLabels), - }, - }, - }; -} diff --git a/x-pack/test/performance/journeys/login/login.ts b/x-pack/test/performance/journeys/login/login.ts deleted file mode 100644 index f1fd17174322f..0000000000000 --- a/x-pack/test/performance/journeys/login/login.ts +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import { FtrProviderContext } from '../../ftr_provider_context'; -import { StepCtx } from '../../services/performance'; - -export default function ({ getService }: FtrProviderContext) { - describe('login', () => { - it('login', async () => { - const inputDelays = getService('inputDelays'); - const performance = getService('performance'); - - await performance.runUserJourney( - 'login', - [ - { - name: 'Login', - handler: async ({ page, kibanaUrl }: StepCtx) => { - await page.goto(`${kibanaUrl}`); - - const usernameLocator = page.locator('[data-test-subj=loginUsername]'); - const passwordLocator = page.locator('[data-test-subj=loginPassword]'); - const submitButtonLocator = page.locator('[data-test-subj=loginSubmit]'); - - await usernameLocator?.type('elastic', { delay: inputDelays.TYPING }); - await passwordLocator?.type('changeme', { delay: inputDelays.TYPING }); - await submitButtonLocator?.click({ delay: inputDelays.MOUSE_CLICK }); - - await page.waitForSelector('#headerUserMenu'); - }, - }, - ], - { - requireAuth: true, - } - ); - }); - }); -} diff --git a/x-pack/test/performance/journeys/many_fields_discover/config.ts b/x-pack/test/performance/journeys/many_fields_discover/config.ts deleted file mode 100644 index 11ce52089a730..0000000000000 --- a/x-pack/test/performance/journeys/many_fields_discover/config.ts +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import { FtrConfigProviderContext } from '@kbn/test'; -import { serializeApmGlobalLabels } from '../../utils'; - -const JOURNEY_MANY_FIELDS_DISCOVER = 'many_fields_discover'; - -export default async function ({ readConfigFile }: FtrConfigProviderContext) { - const performanceConfig = await readConfigFile(require.resolve('../base.config')); - - const testFiles = [require.resolve(`./${JOURNEY_MANY_FIELDS_DISCOVER}`)]; - - const config = { - ...performanceConfig.getAll(), - testFiles, - testData: { - kbnArchives: ['test/functional/fixtures/kbn_archiver/many_fields_data_view'], - esArchives: ['test/functional/fixtures/es_archiver/many_fields'], - }, - }; - - const apmGlobalLabels = { - ...performanceConfig.get('kbnTestServer').env.ELASTIC_APM_GLOBAL_LABELS, - ftrConfig: `x-pack/test/performance/tests/journeys/${JOURNEY_MANY_FIELDS_DISCOVER}/config.ts`, - performancePhase: process.env.TEST_PERFORMANCE_PHASE, - journeyName: JOURNEY_MANY_FIELDS_DISCOVER, - }; - - return { - ...config, - kbnTestServer: { - ...config.kbnTestServer, - serverArgs: [ - ...performanceConfig.get('kbnTestServer.serverArgs'), - `--telemetry.labels.journeyName=${JOURNEY_MANY_FIELDS_DISCOVER}`, - ], - env: { - ...config.kbnTestServer.env, - ELASTIC_APM_GLOBAL_LABELS: serializeApmGlobalLabels(apmGlobalLabels), - }, - }, - }; -} diff --git a/x-pack/test/performance/journeys/many_fields_discover/many_fields_discover.ts b/x-pack/test/performance/journeys/many_fields_discover/many_fields_discover.ts deleted file mode 100644 index 648ed9fca37db..0000000000000 --- a/x-pack/test/performance/journeys/many_fields_discover/many_fields_discover.ts +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import { FtrProviderContext } from '../../ftr_provider_context'; -import { StepCtx } from '../../services/performance'; - -export default function ({ getService }: FtrProviderContext) { - // FAILING: https://github.com/elastic/kibana/issues/130287 - describe.skip('many_fields_discover', () => { - const performance = getService('performance'); - - it('many_fields_discover', async () => { - await performance.runUserJourney( - 'many_fields_discover', - [ - { - name: 'Go to Discover Page', - handler: async ({ page, kibanaUrl }: StepCtx) => { - await page.goto(`${kibanaUrl}/app/discover`); - await page.waitForSelector('[data-test-subj="discoverDocTable"]'); - }, - }, - { - name: 'Expand the first document', - handler: async ({ page }) => { - const expandButtons = page.locator('[data-test-subj="docTableExpandToggleColumn"]'); - await expandButtons.first().click(); - await page.locator('text="Expanded document"'); - }, - }, - ], - { - requireAuth: false, - } - ); - }); - }); -} diff --git a/x-pack/test/performance/journeys/promotion_tracking_dashboard/config.ts b/x-pack/test/performance/journeys/promotion_tracking_dashboard/config.ts deleted file mode 100644 index cc0074503576f..0000000000000 --- a/x-pack/test/performance/journeys/promotion_tracking_dashboard/config.ts +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import { FtrConfigProviderContext } from '@kbn/test'; -import { serializeApmGlobalLabels } from '../../utils'; - -const JOURNEY_PROMOTION_TRACKING_DASHBOARD = 'promotion_tracking_dashboard'; - -export default async function ({ readConfigFile }: FtrConfigProviderContext) { - const performanceConfig = await readConfigFile(require.resolve('../base.config')); - - const testFiles = [require.resolve(`./${JOURNEY_PROMOTION_TRACKING_DASHBOARD}`)]; - - const config = { - ...performanceConfig.getAll(), - testFiles, - testData: { - kbnArchives: ['x-pack/test/performance/kbn_archives/promotion_tracking_dashboard'], - esArchives: ['x-pack/test/performance/es_archives/ecommerce_sample_data'], - }, - scalabilitySetup: { - warmup: [ - { - action: 'constantConcurrentUsers', - maxUsersCount: 10, - duration: '30s', - }, - { - action: 'rampConcurrentUsers', - minUsersCount: 10, - maxUsersCount: 50, - duration: '2m', - }, - ], - test: [ - { - action: 'constantConcurrentUsers', - maxUsersCount: 50, - duration: '5m', - }, - ], - maxDuration: '10m', - }, - }; - - const apmGlobalLabels = { - ...performanceConfig.get('kbnTestServer').env.ELASTIC_APM_GLOBAL_LABELS, - ftrConfig: `x-pack/test/performance/tests/journeys/${JOURNEY_PROMOTION_TRACKING_DASHBOARD}/config.ts`, - performancePhase: process.env.TEST_PERFORMANCE_PHASE, - journeyName: JOURNEY_PROMOTION_TRACKING_DASHBOARD, - }; - - return { - ...config, - kbnTestServer: { - ...config.kbnTestServer, - serverArgs: [ - ...performanceConfig.get('kbnTestServer.serverArgs'), - `--telemetry.labels.journeyName=${JOURNEY_PROMOTION_TRACKING_DASHBOARD}`, - ], - env: { - ...config.kbnTestServer.env, - ELASTIC_APM_GLOBAL_LABELS: serializeApmGlobalLabels(apmGlobalLabels), - }, - }, - }; -} diff --git a/x-pack/test/performance/journeys/promotion_tracking_dashboard/promotion_tracking_dashboard.ts b/x-pack/test/performance/journeys/promotion_tracking_dashboard/promotion_tracking_dashboard.ts deleted file mode 100644 index 590bf38b401c0..0000000000000 --- a/x-pack/test/performance/journeys/promotion_tracking_dashboard/promotion_tracking_dashboard.ts +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import { FtrProviderContext } from '../../ftr_provider_context'; -import { StepCtx } from '../../services/performance'; -import { waitForVisualizations } from '../../utils'; - -export default function ({ getService }: FtrProviderContext) { - describe('promotion_tracking_dashboard', () => { - const performance = getService('performance'); - - it('promotion_tracking_dashboard', async () => { - await performance.runUserJourney( - 'promotion_tracking_dashboard', - [ - { - name: 'Go to Dashboards Page', - handler: async ({ page, kibanaUrl }: StepCtx) => { - await page.goto(`${kibanaUrl}/app/dashboards`); - await page.waitForSelector('#dashboardListingHeading'); - }, - }, - { - name: 'Go to Promotion Tracking Dashboard', - handler: async ({ page }) => { - const promotionDashboardButton = page.locator( - '[data-test-subj="dashboardListingTitleLink-Promotion-Dashboard"]' - ); - await promotionDashboardButton.click(); - }, - }, - { - name: 'Change time range', - handler: async ({ page }) => { - const beginningTimeRangeButton = page.locator( - '[data-test-subj="superDatePickerToggleQuickMenuButton"]' - ); - await beginningTimeRangeButton.click(); - - const lastYearButton = page.locator( - '[data-test-subj="superDatePickerCommonlyUsed_Last_30 days"]' - ); - await lastYearButton.click(); - }, - }, - { - name: 'Wait for visualization animations to finish', - handler: async ({ page }) => { - await waitForVisualizations(page, 1); - }, - }, - ], - { - requireAuth: false, - } - ); - }); - }); -} diff --git a/x-pack/test/performance/journeys/web_logs_dashboard/config.ts b/x-pack/test/performance/journeys/web_logs_dashboard/config.ts deleted file mode 100644 index 0c84f7bcc2071..0000000000000 --- a/x-pack/test/performance/journeys/web_logs_dashboard/config.ts +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import { FtrConfigProviderContext } from '@kbn/test'; -import { serializeApmGlobalLabels } from '../../utils'; - -const JOURNEY_WEBLOGS_DASHBOARD = 'web_logs_dashboard'; - -export default async function ({ readConfigFile }: FtrConfigProviderContext) { - const performanceConfig = await readConfigFile(require.resolve('../base.config')); - - const testFiles = [require.resolve(`./${JOURNEY_WEBLOGS_DASHBOARD}`)]; - - const config = { - ...performanceConfig.getAll(), - testFiles, - }; - - const apmGlobalLabels = { - ...performanceConfig.get('kbnTestServer').env.ELASTIC_APM_GLOBAL_LABELS, - ftrConfig: `x-pack/test/performance/tests/journeys/${JOURNEY_WEBLOGS_DASHBOARD}/config.ts`, - performancePhase: process.env.TEST_PERFORMANCE_PHASE, - journeyName: JOURNEY_WEBLOGS_DASHBOARD, - }; - - return { - ...config, - kbnTestServer: { - ...config.kbnTestServer, - serverArgs: [ - ...performanceConfig.get('kbnTestServer.serverArgs'), - `--telemetry.labels.journeyName=${JOURNEY_WEBLOGS_DASHBOARD}`, - ], - env: { - ...config.kbnTestServer.env, - ELASTIC_APM_GLOBAL_LABELS: serializeApmGlobalLabels(apmGlobalLabels), - }, - }, - }; -} diff --git a/x-pack/test/performance/journeys/web_logs_dashboard/web_logs_dashboard.ts b/x-pack/test/performance/journeys/web_logs_dashboard/web_logs_dashboard.ts deleted file mode 100644 index d7df55cac159c..0000000000000 --- a/x-pack/test/performance/journeys/web_logs_dashboard/web_logs_dashboard.ts +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import { FtrProviderContext } from '../../ftr_provider_context'; -import { StepCtx } from '../../services/performance'; -import { waitForVisualizations } from '../../utils'; - -export default function ({ getService }: FtrProviderContext) { - describe('weblogs_dashboard', () => { - it('weblogs_dashboard', async () => { - const performance = getService('performance'); - const logger = getService('log'); - - await performance.runUserJourney( - 'weblogs_dashboard', - [ - { - name: 'Go to Sample Data Page', - handler: async ({ page, kibanaUrl }: StepCtx) => { - await page.goto(`${kibanaUrl}/app/home#/tutorial_directory/sampleData`); - await page.waitForSelector('text="More ways to add data"'); - }, - }, - { - name: 'Add Web Logs Sample Data', - handler: async ({ page }) => { - const showSampleDataButton = page.locator('[data-test-subj=showSampleDataButton]'); - await showSampleDataButton.click(); - const removeButton = page.locator('[data-test-subj=removeSampleDataSetlogs]'); - try { - await removeButton.click({ timeout: 1_000 }); - } catch (e) { - logger.info('Weblogs data does not exist'); - } - - const addDataButton = page.locator('[data-test-subj=addSampleDataSetlogs]'); - if (addDataButton) { - await addDataButton.click(); - } - }, - }, - { - name: 'Go to Web Logs Dashboard', - handler: async ({ page }) => { - await page.click('[data-test-subj=launchSampleDataSetlogs]'); - await page.click('[data-test-subj=viewSampleDataSetlogs-dashboard]'); - - await waitForVisualizations(page, 12); - }, - }, - ], - { - requireAuth: false, - } - ); - }); - }); -} diff --git a/x-pack/test/performance/page_objects.ts b/x-pack/test/performance/page_objects.ts deleted file mode 100644 index 6c273213bf4a1..0000000000000 --- a/x-pack/test/performance/page_objects.ts +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export const pageObjects = {}; diff --git a/x-pack/test/performance/scalability/config.ts b/x-pack/test/performance/scalability/config.ts deleted file mode 100644 index 6a8302f33fe34..0000000000000 --- a/x-pack/test/performance/scalability/config.ts +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { FtrConfigProviderContext } from '@kbn/test'; -import fs from 'fs'; -import path from 'path'; -import { REPO_ROOT } from '@kbn/utils'; -import { createFlagError } from '@kbn/dev-cli-errors'; -import { serializeApmGlobalLabels } from '../utils'; -import { ScalabilityTestRunner } from './runner'; -import { FtrProviderContext } from '../ftr_provider_context'; - -// These "secret" values are intentionally written in the source. -const APM_SERVER_URL = 'https://142fea2d3047486e925eb8b223559cae.apm.europe-west1.gcp.cloud.es.io'; -const APM_PUBLIC_TOKEN = 'pWFFEym07AKBBhUE2i'; -const AGGS_SHARD_DELAY = process.env.LOAD_TESTING_SHARD_DELAY; -const DISABLE_PLUGINS = process.env.LOAD_TESTING_DISABLE_PLUGINS; -const scalabilityJsonPath = process.env.SCALABILITY_JOURNEY_PATH; -const gatlingProjectRootPath: string = - process.env.GATLING_PROJECT_PATH || path.resolve(REPO_ROOT, '../kibana-load-testing'); - -const readScalabilityJourney = (filePath: string): ScalabilityJourney => { - if (path.extname(filePath) !== '.json') { - throw createFlagError(`Path to scalability journey json is non-json file: '${filePath}'`); - } - try { - return JSON.parse(fs.readFileSync(filePath, 'utf-8')); - } catch (error) { - if (error.code === 'ENOENT') { - throw createFlagError(`Path to scalability journey json is invalid: ${filePath}`); - } - throw createFlagError(`Invalid JSON provided: '${filePath}', ${error}`); - } -}; - -interface ScalabilityJourney { - journeyName: string; -} - -export default async function ({ readConfigFile }: FtrConfigProviderContext) { - const performanceConfig = await readConfigFile(require.resolve('../journeys/base.config.ts')); - - if (!fs.existsSync(gatlingProjectRootPath)) { - throw createFlagError( - `Incorrect path to load testing project: '${gatlingProjectRootPath}'\n - Clone 'elastic/kibana-load-testing' and set path using 'GATLING_PROJECT_PATH' env var` - ); - } - - if (!scalabilityJsonPath) { - throw createFlagError( - `Set path to scalability journey json using 'SCALABILITY_JOURNEY_PATH' env var` - ); - } - const scalabilityJourney = readScalabilityJourney(scalabilityJsonPath); - - const apmGlobalLabels = { - ...performanceConfig.get('kbnTestServer').env.ELASTIC_APM_GLOBAL_LABELS, - journeyFilePath: path.basename(scalabilityJsonPath), - journeyName: scalabilityJourney.journeyName, - }; - - return { - ...performanceConfig.getAll(), - - testRunner: (context: FtrProviderContext) => - ScalabilityTestRunner(context, scalabilityJsonPath, gatlingProjectRootPath), - - esTestCluster: { - ...performanceConfig.get('esTestCluster'), - serverArgs: [...performanceConfig.get('esTestCluster.serverArgs')], - esJavaOpts: '-Xms8g -Xmx8g', - }, - - kbnTestServer: { - ...performanceConfig.get('kbnTestServer'), - sourceArgs: [ - ...performanceConfig.get('kbnTestServer.sourceArgs'), - '--no-base-path', - '--env.name=development', - ...(!!AGGS_SHARD_DELAY ? ['--data.search.aggs.shardDelay.enabled=true'] : []), - ...(!!DISABLE_PLUGINS ? ['--plugins.initialize=false'] : []), - ], - serverArgs: [ - ...performanceConfig.get('kbnTestServer.serverArgs'), - `--telemetry.labels.journeyName=${scalabilityJourney.journeyName}`, - ], - env: { - ELASTIC_APM_ACTIVE: process.env.ELASTIC_APM_ACTIVE, - ELASTIC_APM_CONTEXT_PROPAGATION_ONLY: 'false', - ELASTIC_APM_ENVIRONMENT: process.env.CI ? 'ci' : 'development', - ELASTIC_APM_TRANSACTION_SAMPLE_RATE: '1.0', - ELASTIC_APM_SERVER_URL: APM_SERVER_URL, - ELASTIC_APM_SECRET_TOKEN: APM_PUBLIC_TOKEN, - ELASTIC_APM_BREAKDOWN_METRICS: false, - ELASTIC_APM_CAPTURE_SPAN_STACK_TRACES: false, - ELASTIC_APM_METRICS_INTERVAL: '80s', - ELASTIC_APM_MAX_QUEUE_SIZE: 20480, - ELASTIC_APM_GLOBAL_LABELS: serializeApmGlobalLabels(apmGlobalLabels), - }, - // delay shutdown to ensure that APM can report the data it collects during test execution - delayShutdown: 90_000, - }, - }; -} diff --git a/x-pack/test/performance/services/auth.ts b/x-pack/test/performance/services/auth.ts deleted file mode 100644 index 282369751472e..0000000000000 --- a/x-pack/test/performance/services/auth.ts +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import axios, { AxiosResponse } from 'axios'; -import Url from 'url'; -import { FtrService, FtrProviderContext } from '../ftr_provider_context'; - -export interface Credentials { - username: string; - password: string; -} - -function extractCookieValue(authResponse: AxiosResponse) { - return authResponse.headers['set-cookie']?.[0].toString().split(';')[0].split('sid=')[1] ?? ''; -} -export class AuthService extends FtrService { - private readonly kibanaServer = this.ctx.getService('kibanaServer'); - private readonly config = this.ctx.getService('config'); - - constructor(ctx: FtrProviderContext) { - super(ctx); - } - - public async login({ username, password }: Credentials) { - const headers = { - 'content-type': 'application/json', - 'kbn-version': await this.kibanaServer.version.get(), - 'sec-fetch-mode': 'cors', - 'sec-fetch-site': 'same-origin', - }; - - const baseUrl = Url.format({ - protocol: this.config.get('servers.kibana.protocol'), - hostname: this.config.get('servers.kibana.hostname'), - port: this.config.get('servers.kibana.port'), - }); - - const loginUrl = baseUrl + '/internal/security/login'; - const provider = baseUrl.includes('localhost') ? 'basic' : 'cloud-basic'; - - const authBody = { - providerType: 'basic', - providerName: provider, - currentURL: `${baseUrl}/login?next=%2F`, - params: { username, password }, - }; - - const authResponse = await axios.post(loginUrl, authBody, { headers }); - - return { - name: 'sid', - value: extractCookieValue(authResponse), - url: baseUrl, - }; - } -} - -export const AuthProvider = (ctx: FtrProviderContext) => new AuthService(ctx); diff --git a/x-pack/test/performance/services/index.ts b/x-pack/test/performance/services/index.ts deleted file mode 100644 index d8cd6075c91c2..0000000000000 --- a/x-pack/test/performance/services/index.ts +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { services as functionalServices } from '../../functional/services'; -import { PerformanceTestingService } from './performance'; -import { InputDelaysProvider } from './input_delays'; -import { AuthProvider } from './auth'; - -export const services = { - es: functionalServices.es, - kibanaServer: functionalServices.kibanaServer, - esArchiver: functionalServices.esArchiver, - retry: functionalServices.retry, - performance: PerformanceTestingService, - inputDelays: InputDelaysProvider, - auth: AuthProvider, -}; diff --git a/x-pack/test/performance/services/performance.ts b/x-pack/test/performance/services/performance.ts deleted file mode 100644 index ffddc834dc115..0000000000000 --- a/x-pack/test/performance/services/performance.ts +++ /dev/null @@ -1,306 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -/* eslint-disable no-console */ - -import Url from 'url'; -import * as Rx from 'rxjs'; -import { inspect } from 'util'; -import { setTimeout } from 'timers/promises'; -import apm, { Span, Transaction } from 'elastic-apm-node'; -import playwright, { ChromiumBrowser, Page, BrowserContext, CDPSession, Request } from 'playwright'; -import { FtrService, FtrProviderContext } from '../ftr_provider_context'; - -export interface StepCtx { - page: Page; - kibanaUrl: string; -} - -type StepFn = (ctx: StepCtx) => Promise; -export type Steps = Array<{ name: string; handler: StepFn }>; - -export class PerformanceTestingService extends FtrService { - private readonly auth = this.ctx.getService('auth'); - private readonly log = this.ctx.getService('log'); - private readonly config = this.ctx.getService('config'); - - private browser: ChromiumBrowser | undefined; - private currentSpanStack: Array = []; - private currentTransaction: Transaction | undefined | null = undefined; - - private pageTeardown$ = new Rx.Subject(); - private telemetryTrackerSubs = new Map(); - - constructor(ctx: FtrProviderContext) { - super(ctx); - - ctx.getService('lifecycle').beforeTests.add(() => { - apm.start({ - serviceName: 'functional test runner', - environment: process.env.CI ? 'ci' : 'development', - active: this.config.get(`kbnTestServer.env`).ELASTIC_APM_ACTIVE !== 'false', - serverUrl: this.config.get(`kbnTestServer.env`).ELASTIC_APM_SERVER_URL, - secretToken: this.config.get(`kbnTestServer.env`).ELASTIC_APM_SECRET_TOKEN, - globalLabels: this.config.get(`kbnTestServer.env`).ELASTIC_APM_GLOBAL_LABELS, - transactionSampleRate: - this.config.get(`kbnTestServer.env`).ELASTIC_APM_TRANSACTION_SAMPLE_RATE, - logger: process.env.VERBOSE_APM_LOGGING - ? { - warn(...args: any[]) { - console.log('APM WARN', ...args); - }, - info(...args: any[]) { - console.log('APM INFO', ...args); - }, - fatal(...args: any[]) { - console.log('APM FATAL', ...args); - }, - error(...args: any[]) { - console.log('APM ERROR', ...args); - }, - debug(...args: any[]) { - console.log('APM DEBUG', ...args); - }, - trace(...args: any[]) { - console.log('APM TRACE', ...args); - }, - } - : undefined, - }); - }); - - ctx.getService('lifecycle').cleanup.add(async () => { - await this.shutdownBrowser(); - await new Promise((resolve) => apm.flush(() => resolve())); - // wait for the HTTP request that apm.flush() starts, which we - // can't track but hope is complete within 3 seconds - // https://github.com/elastic/apm-agent-nodejs/issues/2088 - await setTimeout(3000); - }); - } - - private getKibanaUrl() { - return Url.format({ - protocol: this.config.get('servers.kibana.protocol'), - hostname: this.config.get('servers.kibana.hostname'), - port: this.config.get('servers.kibana.port'), - }); - } - - private async withTransaction(name: string, block: () => Promise) { - try { - if (this.currentTransaction) { - throw new Error( - `Transaction already started, make sure you end transaction ${this.currentTransaction?.name}` - ); - } - this.currentTransaction = apm.startTransaction(name, 'performance'); - const result = await block(); - if (this.currentTransaction === undefined) { - throw new Error(`No transaction started`); - } - this.currentTransaction?.end('success'); - this.currentTransaction = undefined; - return result; - } catch (e) { - if (this.currentTransaction === undefined) { - throw new Error(`No transaction started`); - } - this.currentTransaction?.end('failure'); - this.currentTransaction = undefined; - throw e; - } - } - - private async withSpan(name: string, type: string | undefined, block: () => Promise) { - try { - this.currentSpanStack.unshift(apm.startSpan(name, type ?? null)); - const result = await block(); - if (this.currentSpanStack.length === 0) { - throw new Error(`No Span started`); - } - const span = this.currentSpanStack.shift(); - span?.setOutcome('success'); - span?.end(); - return result; - } catch (e) { - if (this.currentSpanStack.length === 0) { - throw new Error(`No Span started`); - } - const span = this.currentSpanStack.shift(); - span?.setOutcome('failure'); - span?.end(); - throw e; - } - } - - private getCurrentTraceparent() { - return (this.currentSpanStack.length ? this.currentSpanStack[0] : this.currentTransaction) - ?.traceparent; - } - - private async getBrowserInstance() { - if (this.browser) { - return this.browser; - } - return await this.withSpan('browser creation', 'setup', async () => { - const headless = !!(process.env.TEST_BROWSER_HEADLESS || process.env.CI); - this.browser = await playwright.chromium.launch({ headless, timeout: 60000 }); - return this.browser; - }); - } - - private async sendCDPCommands(context: BrowserContext, page: Page) { - const client = await context.newCDPSession(page); - - await client.send('Network.clearBrowserCache'); - await client.send('Network.setCacheDisabled', { cacheDisabled: true }); - await client.send('Network.emulateNetworkConditions', { - latency: 100, - downloadThroughput: 750_000, - uploadThroughput: 750_000, - offline: false, - }); - - return client; - } - - private telemetryTrackerCount = 0; - - private trackTelemetryRequests(page: Page) { - const id = ++this.telemetryTrackerCount; - - const requestFailure$ = Rx.fromEvent(page, 'requestfailed'); - const requestSuccess$ = Rx.fromEvent(page, 'requestfinished'); - const request$ = Rx.fromEvent(page, 'request').pipe( - Rx.takeUntil( - this.pageTeardown$.pipe( - Rx.first((p) => p === page), - Rx.delay(3000) - // If EBT client buffers: - // Rx.mergeMap(async () => { - // await page.waitForFunction(() => { - // // return window.kibana_ebt_client.buffer_size == 0 - // }); - // }) - ) - ), - Rx.mergeMap((request) => { - if (!request.url().includes('telemetry-staging.elastic.co')) { - return Rx.EMPTY; - } - - this.log.debug(`Waiting for telemetry request #${id} to complete`); - return Rx.merge(requestFailure$, requestSuccess$).pipe( - Rx.first((r) => r === request), - Rx.tap({ - complete: () => this.log.debug(`Telemetry request #${id} complete`), - }) - ); - }) - ); - - this.telemetryTrackerSubs.set(page, request$.subscribe()); - } - - private async interceptBrowserRequests(page: Page) { - await page.route('**', async (route, request) => { - const headers = await request.allHeaders(); - const traceparent = this.getCurrentTraceparent(); - if (traceparent && request.isNavigationRequest()) { - await route.continue({ headers: { traceparent, ...headers } }); - } else { - await route.continue(); - } - }); - } - - public runUserJourney( - journeyName: string, - steps: Steps, - { requireAuth }: { requireAuth: boolean } - ) { - return this.withTransaction(`Journey ${journeyName}`, async () => { - const browser = await this.getBrowserInstance(); - const context = await browser.newContext({ bypassCSP: true }); - - if (!requireAuth) { - const cookie = await this.auth.login({ username: 'elastic', password: 'changeme' }); - await context.addCookies([cookie]); - } - - const page = await context.newPage(); - if (!process.env.NO_BROWSER_LOG) { - page.on('console', this.onConsoleEvent()); - } - const client = await this.sendCDPCommands(context, page); - - this.trackTelemetryRequests(page); - await this.interceptBrowserRequests(page); - await this.handleSteps(steps, page); - await this.tearDown(page, client, context); - }); - } - - private async tearDown(page: Page, client: CDPSession, context: BrowserContext) { - if (page) { - const telemetryTracker = this.telemetryTrackerSubs.get(page); - this.telemetryTrackerSubs.delete(page); - - if (telemetryTracker && !telemetryTracker.closed) { - this.log.info( - `Waiting for telemetry requests to complete, including requests starting within next 3 secs` - ); - this.pageTeardown$.next(page); - await new Promise((resolve) => telemetryTracker.add(resolve)); - } - await client.detach(); - await page.close(); - await context.close(); - } - } - - private async shutdownBrowser() { - if (this.browser) { - await (await this.getBrowserInstance()).close(); - } - } - - private async handleSteps(steps: Array<{ name: string; handler: StepFn }>, page: Page) { - for (const step of steps) { - await this.withSpan(`step: ${step.name}`, 'step', async () => { - try { - await step.handler({ page, kibanaUrl: this.getKibanaUrl() }); - } catch (e) { - const error = new Error(`Step [${step.name}] failed: ${e.message}`); - error.stack = e.stack; - } - }); - } - } - - private onConsoleEvent() { - return async (message: playwright.ConsoleMessage) => { - try { - const args = await Promise.all(message.args().map(async (handle) => handle.jsonValue())); - - const { url, lineNumber, columnNumber } = message.location(); - - const location = `${url},${lineNumber},${columnNumber}`; - - const text = args.length - ? args.map((arg) => (typeof arg === 'string' ? arg : inspect(arg, false, null))).join(' ') - : message.text(); - - console.log(`[console.${message.type()}]`, text); - console.log(' ', location); - } catch (e) { - console.error('Failed to evaluate console.log line', e); - } - }; - } -} diff --git a/x-pack/test/scalability/config.ts b/x-pack/test/scalability/config.ts new file mode 100644 index 0000000000000..49bcfee2cf199 --- /dev/null +++ b/x-pack/test/scalability/config.ts @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrConfigProviderContext } from '@kbn/test'; +import fs from 'fs'; +import path from 'path'; +import { REPO_ROOT } from '@kbn/utils'; +import { createFlagError } from '@kbn/dev-cli-errors'; +import { commonFunctionalServices } from '@kbn/ftr-common-functional-services'; +import { ScalabilityTestRunner } from './runner'; +import { FtrProviderContext } from './ftr_provider_context'; + +// These "secret" values are intentionally written in the source. +const AGGS_SHARD_DELAY = process.env.LOAD_TESTING_SHARD_DELAY; +const DISABLE_PLUGINS = process.env.LOAD_TESTING_DISABLE_PLUGINS; +const scalabilityJsonPath = process.env.SCALABILITY_JOURNEY_PATH; +const gatlingProjectRootPath: string = + process.env.GATLING_PROJECT_PATH || path.resolve(REPO_ROOT, '../kibana-load-testing'); + +export default async function ({ readConfigFile }: FtrConfigProviderContext) { + if (!fs.existsSync(gatlingProjectRootPath)) { + throw createFlagError( + `Incorrect path to load testing project: '${gatlingProjectRootPath}'\n + Clone 'elastic/kibana-load-testing' and set path using 'GATLING_PROJECT_PATH' env var` + ); + } + + if (!scalabilityJsonPath) { + throw createFlagError( + `Set path to scalability journey json using 'SCALABILITY_JOURNEY_PATH' env var` + ); + } + + const baseConfig = ( + await readConfigFile(require.resolve('../../performance/journeys/login.ts')) + ).getAll(); + + return { + ...baseConfig, + + services: commonFunctionalServices, + pageObjects: {}, + + testRunner: (context: FtrProviderContext) => + ScalabilityTestRunner(context, scalabilityJsonPath, gatlingProjectRootPath), + + esTestCluster: { + ...baseConfig.esTestCluster, + esJavaOpts: '-Xms8g -Xmx8g', + }, + + kbnTestServer: { + ...baseConfig.kbnTestServer, + sourceArgs: [ + ...baseConfig.kbnTestServer.sourceArgs, + '--no-base-path', + '--env.name=development', + ...(!!AGGS_SHARD_DELAY ? ['--data.search.aggs.shardDelay.enabled=true'] : []), + ...(!!DISABLE_PLUGINS ? ['--plugins.initialize=false'] : []), + ], + // delay shutdown to ensure that APM can report the data it collects during test execution + delayShutdown: 90_000, + }, + }; +} diff --git a/x-pack/test/performance/ftr_provider_context.ts b/x-pack/test/scalability/ftr_provider_context.ts similarity index 79% rename from x-pack/test/performance/ftr_provider_context.ts rename to x-pack/test/scalability/ftr_provider_context.ts index e757164fa1de9..19c510a9ec8e7 100644 --- a/x-pack/test/performance/ftr_provider_context.ts +++ b/x-pack/test/scalability/ftr_provider_context.ts @@ -5,10 +5,8 @@ * 2.0. */ +import { commonFunctionalServices } from '@kbn/ftr-common-functional-services'; import { GenericFtrProviderContext, GenericFtrService } from '@kbn/test'; -import { pageObjects } from './page_objects'; -import { services } from './services'; - -export type FtrProviderContext = GenericFtrProviderContext; +export type FtrProviderContext = GenericFtrProviderContext; export class FtrService extends GenericFtrService {} diff --git a/x-pack/test/performance/scalability/runner.ts b/x-pack/test/scalability/runner.ts similarity index 95% rename from x-pack/test/performance/scalability/runner.ts rename to x-pack/test/scalability/runner.ts index e768339b0d498..e09a9d438b410 100644 --- a/x-pack/test/performance/scalability/runner.ts +++ b/x-pack/test/scalability/runner.ts @@ -6,7 +6,7 @@ */ import { withProcRunner } from '@kbn/dev-proc-runner'; -import { FtrProviderContext } from '../ftr_provider_context'; +import { FtrProviderContext } from './ftr_provider_context'; /** * ScalabilityTestRunner is used to run load simulation against local Kibana instance diff --git a/x-pack/test/security_api_integration/tests/audit/file_wrapper.ts b/x-pack/test/security_api_integration/tests/audit/file_wrapper.ts index bb7e707eefd8b..6f6aef69ed406 100644 --- a/x-pack/test/security_api_integration/tests/audit/file_wrapper.ts +++ b/x-pack/test/security_api_integration/tests/audit/file_wrapper.ts @@ -6,7 +6,7 @@ */ import Fs from 'fs'; -import { RetryService } from '../../../../../test/common/services/retry'; +import type { RetryService } from '@kbn/ftr-common-functional-services'; export class FileWrapper { constructor(private readonly path: string, private readonly retry: RetryService) {} diff --git a/x-pack/test/stack_functional_integration/services/es_archiver.js b/x-pack/test/stack_functional_integration/services/es_archiver.js index b34be03d896ad..821cf72e2c6bc 100644 --- a/x-pack/test/stack_functional_integration/services/es_archiver.js +++ b/x-pack/test/stack_functional_integration/services/es_archiver.js @@ -10,7 +10,7 @@ import Path from 'path'; import { EsArchiver } from '@kbn/es-archiver'; import { REPO_ROOT } from '@kbn/utils'; -import * as KibanaServer from '../../../../test/common/services/kibana_server'; +import { KibanaServer } from '@kbn/ftr-common-functional-services'; const INTEGRATION_TEST_ROOT = process.env.WORKSPACE || Path.resolve(REPO_ROOT, '../integration-test'); diff --git a/yarn.lock b/yarn.lock index 1d81724e537b8..535bf43b01e18 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3372,6 +3372,10 @@ version "0.0.0" uid "" +"@kbn/failed-test-reporter-cli@link:bazel-bin/packages/kbn-failed-test-reporter-cli": + version "0.0.0" + uid "" + "@kbn/field-types@link:bazel-bin/packages/kbn-field-types": version "0.0.0" uid "" @@ -3384,6 +3388,14 @@ version "0.0.0" uid "" +"@kbn/ftr-common-functional-services@link:bazel-bin/packages/kbn-ftr-common-functional-services": + version "0.0.0" + uid "" + +"@kbn/ftr-screenshot-filename@link:bazel-bin/packages/kbn-ftr-screenshot-filename": + version "0.0.0" + uid "" + "@kbn/generate@link:bazel-bin/packages/kbn-generate": version "0.0.0" uid "" @@ -3436,6 +3448,10 @@ version "0.0.0" uid "" +"@kbn/journeys@link:bazel-bin/packages/kbn-journeys": + version "0.0.0" + uid "" + "@kbn/kibana-manifest-schema@link:bazel-bin/packages/kbn-kibana-manifest-schema": version "0.0.0" uid "" @@ -7401,6 +7417,10 @@ version "0.0.0" uid "" +"@types/kbn__failed-test-reporter-cli@link:bazel-bin/packages/kbn-failed-test-reporter-cli/npm_module_types": + version "0.0.0" + uid "" + "@types/kbn__field-types@link:bazel-bin/packages/kbn-field-types/npm_module_types": version "0.0.0" uid "" @@ -7409,6 +7429,14 @@ version "0.0.0" uid "" +"@types/kbn__ftr-common-functional-services@link:bazel-bin/packages/kbn-ftr-common-functional-services/npm_module_types": + version "0.0.0" + uid "" + +"@types/kbn__ftr-screenshot-filename@link:bazel-bin/packages/kbn-ftr-screenshot-filename/npm_module_types": + version "0.0.0" + uid "" + "@types/kbn__generate@link:bazel-bin/packages/kbn-generate/npm_module_types": version "0.0.0" uid "" @@ -7461,6 +7489,10 @@ version "0.0.0" uid "" +"@types/kbn__journeys@link:bazel-bin/packages/kbn-journeys/npm_module_types": + version "0.0.0" + uid "" + "@types/kbn__kbn-ci-stats-performance-metrics@link:bazel-bin/packages/kbn-kbn-ci-stats-performance-metrics/npm_module_types": version "0.0.0" uid ""