9.2 manual backports

.buildkite/it/run_serverless.sh

@@ -14,6 +14,7 @@ echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf
 
 PYTHON_VERSION="$1"
 TEST_NAME="$2"
+IFS=',' read -ra RUN_FULL_CI_WHEN_CHANGED <<< "$3"
 
 echo "--- System dependencies"
 
@@ -29,7 +30,36 @@ echo "--- Python modules"
 source .venv/bin/activate
 python -m pip install .[develop]
 
-echo "--- Run IT serverless test \"$TEST_NAME\" :pytest:"
+echo "--- Track filter modification"
 
-hatch -v -e it_serverless run $TEST_NAME
+CHANGED_FILES=$(git diff --name-only origin/master...HEAD)
 
+if [[ -z "$CHANGED_FILES" ]]; then
+    echo "No changed files detected between origin/master and HEAD. Running full CI"
+    TRACK_FILTER_ARG=""
+else
+    readarray -t changed_files_arr <<< "$CHANGED_FILES"
+    CHANGED_TOP_LEVEL_DIRS=$(printf '%s\n' "$CHANGED_FILES" | awk -F/ '/\//{print $1}' | sort -u | paste -sd, -)
+    CHANGED_TOP_LEVEL_DIRS=${CHANGED_TOP_LEVEL_DIRS%,}
+    IFS=',' read -ra changed_dirs_arr <<< "$CHANGED_TOP_LEVEL_DIRS"
+
+    all_changed_arr=("${changed_files_arr[@]}" "${changed_dirs_arr[@]}")
+
+    TRACK_FILTER_ARG=" --track-filter=${CHANGED_TOP_LEVEL_DIRS}"
+
+    # If any changes match one of the RUN_FULL_CI_WHEN_CHANGED paths, run full CI
+    for static_path in "${RUN_FULL_CI_WHEN_CHANGED[@]}"; do
+        for changed in "${all_changed_arr[@]}"; do
+            if [[ "$static_path" == "$changed" ]]; then
+                echo "Matched '$static_path' in changed files/dirs. Running full CI."
+                TRACK_FILTER_ARG=""
+                break 2
+            fi
+        done
+    done
+fi
+
+
+echo "--- Run IT serverless test \"$TEST_NAME\"$TRACK_FILTER_ARG :pytest:"
+
+hatch -v -e it_serverless run $TEST_NAME$TRACK_FILTER_ARG

.buildkite/it/serverless-pipeline.yml

@@ -1,3 +1,6 @@
+env:
+    RUN_FULL_CI_WHEN_CHANGED: pyproject.toml,.buildkite,it_serverless
+
 common:
   plugins:
     - elastic/vault-secrets#v0.0.2: &vault-base_url
@@ -24,11 +27,11 @@ steps:
       - elastic/vault-secrets#v0.0.2: *vault-base_url
       - elastic/vault-secrets#v0.0.2: *vault-get_credentials_endpoint
       - elastic/vault-secrets#v0.0.2: *vault-api_key
-    command: bash .buildkite/it/run_serverless.sh 3.11 test_user
+    command: bash .buildkite/it/run_serverless.sh 3.11 test_user $RUN_FULL_CI_WHEN_CHANGED
   - label: "Run IT Serverless tests with operator privileges"
     if: build.pull_request.base_branch == "master" || build.source == "schedule"
     plugins:
       - elastic/vault-secrets#v0.0.2: *vault-base_url
       - elastic/vault-secrets#v0.0.2: *vault-get_credentials_endpoint
       - elastic/vault-secrets#v0.0.2: *vault-api_key
-    command: bash .buildkite/it/run_serverless.sh 3.11 test_operator
+    command: bash .buildkite/it/run_serverless.sh 3.11 test_operator $RUN_FULL_CI_WHEN_CHANGED

.github/scripts/track-filter.py

@@ -0,0 +1,22 @@
+import os
+
+import yaml
+
+filters = {}
+
+# static file paths should be a comma-separated list of files or directories (omitting the trailing '/')
+static_paths = os.environ.get("RUN_FULL_CI_WHEN_CHANGED", [])
+
+# Statically include some files that should always trigger a full CI run
+if static_paths:
+    filters["full_ci"] = [f"{path}/**" if os.path.isdir(path.strip()) else path.strip() for path in static_paths.split(",")]
+
+# Dynamically create filters for each track (top-level subdirectory) in the repo
+for entry in os.listdir("."):
+    if os.path.isdir(entry) and entry not in static_paths:
+        filters[entry] = [f"{entry}/**"]
+
+
+with open(".github/filters.yml", "w") as f:
+    yaml.dump(filters, f, default_flow_style=False)
+print(f"Created .github/filters.yml with {len(filters)} track(s): {', '.join(filters.keys())}")

.github/workflows/ci.yml

@@ -11,6 +11,8 @@ on:
 
 env:
   DEFAULT_BRANCH: master
+  # comma-separated list of paths that if changed will trigger a full CI run (Note: don't use trailing '/' at the end)
+  RUN_FULL_CI_WHEN_CHANGED: 'pyproject.toml,.github,it'
 
 permissions: "read-all"
 
@@ -35,7 +37,55 @@ jobs:
           slack_channel: ${{ secrets.SLACK_CHANNEL }}
           status: FAILED
 
+  determine-revision:
+    runs-on: ubuntu-22.04
+    outputs:
+      revision: ${{ steps.revision-argument.outputs.revision }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: "Determine ES version"
+        id: es-version
+        run: |
+          ES_VERSION=$(cat es-version)
+          echo "Determined es-version: $ES_VERSION"
+          echo "version=$ES_VERSION" >> $GITHUB_OUTPUT
+      - name: "Determine --revision argument"
+        id: revision-argument
+        run: |
+          echo "revision= --revision=${{ steps.es-version.outputs.version }}" >> $GITHUB_OUTPUT
+
+  filter-pr-changes:
+    runs-on: ubuntu-22.04
+    outputs:
+      track_filter: ${{ steps.track-filter.outputs.track_filter }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Parse repo and create filters.yml
+        run: python3 .github/scripts/track-filter.py
+      - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 #v3.0.2
+        id: changes
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          filters: .github/filters.yml
+      - name: Collect changed tracks and calculate --track-filter argument
+        id: track-filter
+        run: |
+            TRACKS=$(echo '${{ toJSON(steps.changes.outputs) }}' | jq -r '
+            to_entries
+            | map(select(.value == "true"))
+            | map(.key)
+            | join(",")
+            ')
+            if echo "$TRACKS" | grep -qw "full_ci"; then
+            echo 'track_filter=' >> $GITHUB_OUTPUT
+            else
+            echo "track_filter= --track-filter=$TRACKS" >> $GITHUB_OUTPUT
+            fi
+
   test:
+    needs: 
+      - filter-pr-changes
+      - determine-revision
     strategy:
       fail-fast: false
       matrix:
@@ -55,8 +105,8 @@ jobs:
           cache-dependency-path: pyproject.toml
       - name: "Install dependencies"
         run: python -m pip install .[develop]
-      - name: "Run tests"
-        run: hatch -v -e unit run test
+      - name: "Run tests${{ needs.filter-pr-changes.outputs.track_filter }}${{ needs.determine-revision.outputs.revision }}"
+        run: hatch -v -e unit run test${{ needs.filter-pr-changes.outputs.track_filter }}${{ needs.determine-revision.outputs.revision }}
       - uses: elastic/es-perf-github-status@v2
         if: ${{ failure() && ( github.event_name == 'schedule' || ( github.event_name == 'push' && github.ref_name == env.DEFAULT_BRANCH ) ) }}
         with:
@@ -65,6 +115,10 @@ jobs:
           status: FAILED
 
   rally-tracks-compat:
+    needs: 
+      - filter-pr-changes
+      - determine-revision
+
     strategy:
       fail-fast: false
       matrix:
@@ -88,8 +142,8 @@ jobs:
       - run: echo "JAVA11_HOME=$JAVA_HOME_11_X64" >> $GITHUB_ENV
       - name: "Install dependencies"
         run: python -m pip install .[develop]
-      - name: "Run tests"
-        run: hatch -v -e it run test
+      - name: "Run tests${{ needs.filter-pr-changes.outputs.track_filter }}${{ needs.determine-revision.outputs.revision }}"
+        run: hatch -v -e it run test${{ needs.filter-pr-changes.outputs.track_filter }}${{ needs.determine-revision.outputs.revision }}
         timeout-minutes: 120
         env:
           # elastic/endpoint fetches assets from GitHub, authenticate to avoid

es-version

@@ -1 +1 @@
-9.2
+9.2

it/logs/test_logs.py

@@ -22,6 +22,7 @@
 pytest_rally = pytest.importorskip("pytest_rally")
 
 
+@pytest.mark.track("elastic/logs")
 class TestLogs:
     def test_logs_fails_if_assets_not_installed(self, es_cluster, rally, capsys):
         ret = rally.race(track="elastic/logs", exclude_tasks="tag:setup")

it/logs/test_logs_unmapped.py

@@ -22,6 +22,7 @@
 pytest_rally = pytest.importorskip("pytest_rally")
 
 
+@pytest.mark.track("elastic/logs")
 class TestLogsUnmapped:
     def test_logs_chicken(self, es_cluster, rally):
         custom = {"mapping": "unmapped"}

it/test_custom_parameters.py

@@ -21,13 +21,15 @@
 
 
 class TestCustomParameters:
+    @pytest.mark.track("tsdb")
     def test_tsdb_esql(self, es_cluster, rally):
         ret = rally.race(
             track="tsdb",
             track_params={"run_esql_aggs": True, "index_mode": "time_series"},
         )
         assert ret == 0
 
+    @pytest.mark.track("tsdb")
     def test_tsdb_data_stream(self, es_cluster, rally):
         ret = rally.race(
             track="tsdb",

it/test_security.py

@@ -21,6 +21,7 @@
 pytest_rally = pytest.importorskip("pytest_rally")
 
 
+@pytest.mark.track("elastic/security")
 class TestSecurity:
     def test_security_indexing(self, es_cluster, rally):
         ret = rally.race(track="elastic/security", challenge="security-indexing", track_params={"number_of_replicas": "0"})

it/test_synthetic_source.py

@@ -34,13 +34,15 @@ def params(updates=None):
 
 
 class TestSyntheticSource:
+    @pytest.mark.track("tsdb")
     def test_tsdb_default(self, es_cluster, rally):
         ret = rally.race(
             track="tsdb",
             track_params=params(),
         )
         assert ret == 0
 
+    @pytest.mark.track("nyc_taxis")
     def test_nyc_taxis_default(self, es_cluster, rally):
         ret = rally.race(
             track="nyc_taxis",

it_serverless/test_logs.py

@@ -42,6 +42,7 @@ def params(updates=None):
         return {**base, **updates}
 
 
+@pytest.mark.track("elastic/logs")
 @pytest.mark.operator_only
 class TestLogs:
     def test_logs_fails_if_assets_not_installed(self, operator, rally, capsys, project_config: ServerlessProjectConfig):

pyproject.toml

@@ -45,14 +45,14 @@ extra-dependencies = [
 ]
 
 [tool.hatch.envs.unit.scripts]
-test = "pytest"
+test = "pytest {args}"
 
 [tool.hatch.envs.it.scripts]
-test = "pytest it --log-cli-level=INFO"
+test = "pytest it --log-cli-level=INFO {args}"
 
 [tool.hatch.envs.it_serverless.scripts]
-test_user = "pytest -s it_serverless --log-cli-level=INFO"
-test_operator = "pytest -s it_serverless --log-cli-level=INFO --operator"
+test_user = "pytest -s it_serverless --log-cli-level=INFO {args}"
+test_operator = "pytest -s it_serverless --log-cli-level=INFO --operator {args}"
 
 [tool.pytest.ini_options]
 # set to true for more verbose output of tests
@@ -61,6 +61,9 @@ addopts = "--verbose --color=yes --ignore=it --ignore=it_serverless"
 junit_family = "xunit2"
 junit_logging = "all"
 asyncio_mode = "strict"
+markers = [
+    'track(name): Optionally associate a test class, function or module with a track. Usage: @pytest.mark.track("track1", "track2"). You can use a comma-separated list of track names. When given the --track-filter option (--track-filter=track1,track3), pytest will only run the tests marked with at least one of the track names. Unmarked objects will run by default.'
+]
 
 [tool.black]
 line-length = 140