diff --git a/.buildkite/it/run_serverless.sh b/.buildkite/it/run_serverless.sh old mode 100644 new mode 100755 index cb59efdb8..8ab58bba8 --- a/.buildkite/it/run_serverless.sh +++ b/.buildkite/it/run_serverless.sh @@ -14,6 +14,7 @@ echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf PYTHON_VERSION="$1" TEST_NAME="$2" +IFS=',' read -ra RUN_FULL_CI_WHEN_CHANGED <<< "$3" echo "--- System dependencies" @@ -29,7 +30,36 @@ echo "--- Python modules" source .venv/bin/activate python -m pip install .[develop] -echo "--- Run IT serverless test \"$TEST_NAME\" :pytest:" +echo "--- Track filter modification" -hatch -v -e it_serverless run $TEST_NAME +CHANGED_FILES=$(git diff --name-only origin/master...HEAD) +if [[ -z "$CHANGED_FILES" ]]; then + echo "No changed files detected between origin/master and HEAD. Running full CI" + TRACK_FILTER_ARG="" +else + readarray -t changed_files_arr <<< "$CHANGED_FILES" + CHANGED_TOP_LEVEL_DIRS=$(printf '%s\n' "$CHANGED_FILES" | awk -F/ '/\//{print $1}' | sort -u | paste -sd, -) + CHANGED_TOP_LEVEL_DIRS=${CHANGED_TOP_LEVEL_DIRS%,} + IFS=',' read -ra changed_dirs_arr <<< "$CHANGED_TOP_LEVEL_DIRS" + + all_changed_arr=("${changed_files_arr[@]}" "${changed_dirs_arr[@]}") + + TRACK_FILTER_ARG=" --track-filter=${CHANGED_TOP_LEVEL_DIRS}" + + # If any changes match one of the RUN_FULL_CI_WHEN_CHANGED paths, run full CI + for static_path in "${RUN_FULL_CI_WHEN_CHANGED[@]}"; do + for changed in "${all_changed_arr[@]}"; do + if [[ "$static_path" == "$changed" ]]; then + echo "Matched '$static_path' in changed files/dirs. Running full CI." + TRACK_FILTER_ARG="" + break 2 + fi + done + done +fi + + +echo "--- Run IT serverless test \"$TEST_NAME\"$TRACK_FILTER_ARG :pytest:" + +hatch -v -e it_serverless run $TEST_NAME$TRACK_FILTER_ARG diff --git a/.buildkite/it/serverless-pipeline.yml b/.buildkite/it/serverless-pipeline.yml index 4d19a455e..e074d9ecd 100644 --- a/.buildkite/it/serverless-pipeline.yml +++ b/.buildkite/it/serverless-pipeline.yml @@ -1,3 +1,6 @@ +env: + RUN_FULL_CI_WHEN_CHANGED: pyproject.toml,.buildkite,it_serverless + common: plugins: - elastic/vault-secrets#v0.0.2: &vault-base_url @@ -24,11 +27,11 @@ steps: - elastic/vault-secrets#v0.0.2: *vault-base_url - elastic/vault-secrets#v0.0.2: *vault-get_credentials_endpoint - elastic/vault-secrets#v0.0.2: *vault-api_key - command: bash .buildkite/it/run_serverless.sh 3.11 test_user + command: bash .buildkite/it/run_serverless.sh 3.11 test_user $RUN_FULL_CI_WHEN_CHANGED - label: "Run IT Serverless tests with operator privileges" if: build.pull_request.base_branch == "master" || build.source == "schedule" plugins: - elastic/vault-secrets#v0.0.2: *vault-base_url - elastic/vault-secrets#v0.0.2: *vault-get_credentials_endpoint - elastic/vault-secrets#v0.0.2: *vault-api_key - command: bash .buildkite/it/run_serverless.sh 3.11 test_operator + command: bash .buildkite/it/run_serverless.sh 3.11 test_operator $RUN_FULL_CI_WHEN_CHANGED diff --git a/.github/scripts/track-filter.py b/.github/scripts/track-filter.py new file mode 100644 index 000000000..92d548941 --- /dev/null +++ b/.github/scripts/track-filter.py @@ -0,0 +1,22 @@ +import os + +import yaml + +filters = {} + +# static file paths should be a comma-separated list of files or directories (omitting the trailing '/') +static_paths = os.environ.get("RUN_FULL_CI_WHEN_CHANGED", []) + +# Statically include some files that should always trigger a full CI run +if static_paths: + filters["full_ci"] = [f"{path}/**" if os.path.isdir(path.strip()) else path.strip() for path in static_paths.split(",")] + +# Dynamically create filters for each track (top-level subdirectory) in the repo +for entry in os.listdir("."): + if os.path.isdir(entry) and entry not in static_paths: + filters[entry] = [f"{entry}/**"] + + +with open(".github/filters.yml", "w") as f: + yaml.dump(filters, f, default_flow_style=False) +print(f"Created .github/filters.yml with {len(filters)} track(s): {', '.join(filters.keys())}") diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f8ca66f5f..c3f13dba3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,6 +11,8 @@ on: env: DEFAULT_BRANCH: master + # comma-separated list of paths that if changed will trigger a full CI run (Note: don't use trailing '/' at the end) + RUN_FULL_CI_WHEN_CHANGED: 'pyproject.toml,.github,it' permissions: "read-all" @@ -35,7 +37,55 @@ jobs: slack_channel: ${{ secrets.SLACK_CHANNEL }} status: FAILED + determine-revision: + runs-on: ubuntu-22.04 + outputs: + revision: ${{ steps.revision-argument.outputs.revision }} + steps: + - uses: actions/checkout@v4 + - name: "Determine ES version" + id: es-version + run: | + ES_VERSION=$(cat es-version) + echo "Determined es-version: $ES_VERSION" + echo "version=$ES_VERSION" >> $GITHUB_OUTPUT + - name: "Determine --revision argument" + id: revision-argument + run: | + echo "revision= --revision=${{ steps.es-version.outputs.version }}" >> $GITHUB_OUTPUT + + filter-pr-changes: + runs-on: ubuntu-22.04 + outputs: + track_filter: ${{ steps.track-filter.outputs.track_filter }} + steps: + - uses: actions/checkout@v4 + - name: Parse repo and create filters.yml + run: python3 .github/scripts/track-filter.py + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 #v3.0.2 + id: changes + with: + token: ${{ secrets.GITHUB_TOKEN }} + filters: .github/filters.yml + - name: Collect changed tracks and calculate --track-filter argument + id: track-filter + run: | + TRACKS=$(echo '${{ toJSON(steps.changes.outputs) }}' | jq -r ' + to_entries + | map(select(.value == "true")) + | map(.key) + | join(",") + ') + if echo "$TRACKS" | grep -qw "full_ci"; then + echo 'track_filter=' >> $GITHUB_OUTPUT + else + echo "track_filter= --track-filter=$TRACKS" >> $GITHUB_OUTPUT + fi + test: + needs: + - filter-pr-changes + - determine-revision strategy: fail-fast: false matrix: @@ -55,8 +105,8 @@ jobs: cache-dependency-path: pyproject.toml - name: "Install dependencies" run: python -m pip install .[develop] - - name: "Run tests" - run: hatch -v -e unit run test + - name: "Run tests${{ needs.filter-pr-changes.outputs.track_filter }}${{ needs.determine-revision.outputs.revision }}" + run: hatch -v -e unit run test${{ needs.filter-pr-changes.outputs.track_filter }}${{ needs.determine-revision.outputs.revision }} - uses: elastic/es-perf-github-status@v2 if: ${{ failure() && ( github.event_name == 'schedule' || ( github.event_name == 'push' && github.ref_name == env.DEFAULT_BRANCH ) ) }} with: @@ -65,6 +115,10 @@ jobs: status: FAILED rally-tracks-compat: + needs: + - filter-pr-changes + - determine-revision + strategy: fail-fast: false matrix: @@ -88,8 +142,8 @@ jobs: - run: echo "JAVA11_HOME=$JAVA_HOME_11_X64" >> $GITHUB_ENV - name: "Install dependencies" run: python -m pip install .[develop] - - name: "Run tests" - run: hatch -v -e it run test + - name: "Run tests${{ needs.filter-pr-changes.outputs.track_filter }}${{ needs.determine-revision.outputs.revision }}" + run: hatch -v -e it run test${{ needs.filter-pr-changes.outputs.track_filter }}${{ needs.determine-revision.outputs.revision }} timeout-minutes: 120 env: # elastic/endpoint fetches assets from GitHub, authenticate to avoid diff --git a/es-version b/es-version index 964fadb24..1a2c3557b 100644 --- a/es-version +++ b/es-version @@ -1 +1 @@ -9.2 \ No newline at end of file +9.2 diff --git a/it/logs/test_logs.py b/it/logs/test_logs.py index 4e07dc785..bde162416 100644 --- a/it/logs/test_logs.py +++ b/it/logs/test_logs.py @@ -22,6 +22,7 @@ pytest_rally = pytest.importorskip("pytest_rally") +@pytest.mark.track("elastic/logs") class TestLogs: def test_logs_fails_if_assets_not_installed(self, es_cluster, rally, capsys): ret = rally.race(track="elastic/logs", exclude_tasks="tag:setup") diff --git a/it/logs/test_logs_unmapped.py b/it/logs/test_logs_unmapped.py index eb341f560..2cf365db4 100644 --- a/it/logs/test_logs_unmapped.py +++ b/it/logs/test_logs_unmapped.py @@ -22,6 +22,7 @@ pytest_rally = pytest.importorskip("pytest_rally") +@pytest.mark.track("elastic/logs") class TestLogsUnmapped: def test_logs_chicken(self, es_cluster, rally): custom = {"mapping": "unmapped"} diff --git a/it/test_custom_parameters.py b/it/test_custom_parameters.py index b02cc4667..387d85e2d 100644 --- a/it/test_custom_parameters.py +++ b/it/test_custom_parameters.py @@ -21,6 +21,7 @@ class TestCustomParameters: + @pytest.mark.track("tsdb") def test_tsdb_esql(self, es_cluster, rally): ret = rally.race( track="tsdb", @@ -28,6 +29,7 @@ def test_tsdb_esql(self, es_cluster, rally): ) assert ret == 0 + @pytest.mark.track("tsdb") def test_tsdb_data_stream(self, es_cluster, rally): ret = rally.race( track="tsdb", diff --git a/it/test_security.py b/it/test_security.py index 1b403c9ab..61cb9c641 100644 --- a/it/test_security.py +++ b/it/test_security.py @@ -21,6 +21,7 @@ pytest_rally = pytest.importorskip("pytest_rally") +@pytest.mark.track("elastic/security") class TestSecurity: def test_security_indexing(self, es_cluster, rally): ret = rally.race(track="elastic/security", challenge="security-indexing", track_params={"number_of_replicas": "0"}) diff --git a/it/test_synthetic_source.py b/it/test_synthetic_source.py index f0893570c..3e10fc25c 100644 --- a/it/test_synthetic_source.py +++ b/it/test_synthetic_source.py @@ -34,6 +34,7 @@ def params(updates=None): class TestSyntheticSource: + @pytest.mark.track("tsdb") def test_tsdb_default(self, es_cluster, rally): ret = rally.race( track="tsdb", @@ -41,6 +42,7 @@ def test_tsdb_default(self, es_cluster, rally): ) assert ret == 0 + @pytest.mark.track("nyc_taxis") def test_nyc_taxis_default(self, es_cluster, rally): ret = rally.race( track="nyc_taxis", diff --git a/it_serverless/test_logs.py b/it_serverless/test_logs.py index ee17f9086..1761d6672 100644 --- a/it_serverless/test_logs.py +++ b/it_serverless/test_logs.py @@ -42,6 +42,7 @@ def params(updates=None): return {**base, **updates} +@pytest.mark.track("elastic/logs") @pytest.mark.operator_only class TestLogs: def test_logs_fails_if_assets_not_installed(self, operator, rally, capsys, project_config: ServerlessProjectConfig): diff --git a/pyproject.toml b/pyproject.toml index dad778548..901afcd6e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,14 +45,14 @@ extra-dependencies = [ ] [tool.hatch.envs.unit.scripts] -test = "pytest" +test = "pytest {args}" [tool.hatch.envs.it.scripts] -test = "pytest it --log-cli-level=INFO" +test = "pytest it --log-cli-level=INFO {args}" [tool.hatch.envs.it_serverless.scripts] -test_user = "pytest -s it_serverless --log-cli-level=INFO" -test_operator = "pytest -s it_serverless --log-cli-level=INFO --operator" +test_user = "pytest -s it_serverless --log-cli-level=INFO {args}" +test_operator = "pytest -s it_serverless --log-cli-level=INFO --operator {args}" [tool.pytest.ini_options] # set to true for more verbose output of tests @@ -61,6 +61,9 @@ addopts = "--verbose --color=yes --ignore=it --ignore=it_serverless" junit_family = "xunit2" junit_logging = "all" asyncio_mode = "strict" +markers = [ + 'track(name): Optionally associate a test class, function or module with a track. Usage: @pytest.mark.track("track1", "track2"). You can use a comma-separated list of track names. When given the --track-filter option (--track-filter=track1,track3), pytest will only run the tests marked with at least one of the track names. Unmarked objects will run by default.' +] [tool.black] line-length = 140