ci: benchmark optimize (#19086)

Author: everpcpc

.github/actions/benchmark_cloud/action.yml

@@ -46,14 +46,8 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - name: Install script dependencies
-      shell: bash
-      run: |
-        sudo apt-get update -yq
-        sudo apt-get install -yq python3
-
     - name: Prepare
-      working-directory: benchmark/clickbench
+      working-directory: benchmark
       shell: bash
       id: prepare
       env:
@@ -64,7 +58,7 @@ runs:
         else
           database="${{ inputs.database }}"
           if [[ -z "$database" ]]; then
-            database="clickbench"
+            database="benchmark"
           fi
           echo "database=$database" >> $GITHUB_OUTPUT
         fi
@@ -79,7 +73,7 @@ runs:
         echo "tries=$tries" >> $GITHUB_OUTPUT
 
     - name: Run Benchmark
-      working-directory: benchmark/clickbench
+      working-directory: benchmark
       env:
         BENCHMARK_ID: ${{ inputs.run_id }}
         BENCHMARK_DATASET: ${{ inputs.dataset }}
@@ -104,8 +98,8 @@ runs:
       with:
         name: benchmark-${{ inputs.dataset }}-${{ inputs.size }}-cache-${{ inputs.cache_size }}
         path: |
-          benchmark/clickbench/result-${{ inputs.dataset }}-cloud-${{ inputs.size }}-cache-${{ inputs.cache_size }}.json
-          benchmark/clickbench/result-${{ inputs.dataset }}-cloud-${{ inputs.size }}-cache-${{ inputs.cache_size }}-*.ndjson
+          benchmark/result-${{ inputs.dataset }}-cloud-${{ inputs.size }}-cache-${{ inputs.cache_size }}.json
+          benchmark/result-${{ inputs.dataset }}-cloud-${{ inputs.size }}-cache-${{ inputs.cache_size }}-*.ndjson
 
     - name: Remove warehouse
       if: always()

.github/actions/benchmark_local/action.yml

@@ -20,14 +20,8 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - name: Install script dependencies
-      shell: bash
-      run: |
-        sudo apt-get update -yq
-        sudo apt-get install -yq python3
-
     - name: Run Benchmark
-      working-directory: benchmark/clickbench
+      working-directory: benchmark
       env:
         BENCHMARK_ID: ${{ inputs.run_id }}
         BENCHMARK_DATASET: ${{ inputs.dataset }}
@@ -42,7 +36,7 @@ runs:
         name: benchmark_local
 
     - name: Prepare Metadata
-      working-directory: benchmark/clickbench
+      working-directory: benchmark
       shell: bash
       run: |
         case ${{ inputs.source }} in
@@ -63,4 +57,4 @@ runs:
       uses: actions/upload-artifact@v4
       with:
         name: benchmark-${{ inputs.dataset }}-local
-        path: benchmark/clickbench/result-${{ inputs.dataset }}-local.json
+        path: benchmark/result-${{ inputs.dataset }}-local.json

.github/workflows/reuse.benchmark.yml

@@ -189,17 +189,14 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - name: Install Dependencies
-        run: |
-          sudo apt-get update -yq
-          sudo apt-get install -yq python3-jinja2
+      - uses: astral-sh/setup-uv@v5
       - uses: actions/download-artifact@v4
         with:
-          path: benchmark/clickbench/results
+          path: benchmark/results
           pattern: benchmark-*
           merge-multiple: true
       - name: Get Report Prefix
-        working-directory: benchmark/clickbench
+        working-directory: benchmark
         run: |
           shopt -s nullglob
           for result in results/*.json; do
@@ -220,16 +217,17 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
           AWS_DEFAULT_REGION: auto
           AWS_ENDPOINT_URL: ${{ secrets.R2_ENDPOINT_URL }}
-        working-directory: benchmark/clickbench
+        working-directory: benchmark
         run: |
           echo -e "## ClickBench Report\n" > /tmp/body
           shopt -s nullglob
+          uv sync
           for p in results/*; do
             [ -d "$p" ] || continue
             dataset=$(basename $p)
             aws s3 sync results/$dataset/ ${REPORT_S3_PREFIX}/ --include "*.json" --no-progress --checksum-algorithm=CRC32
             aws s3 sync "s3://benchmark/clickbench/release/${dataset}/latest/" ./results/${dataset}/ --exclude "*" --include "*.json" || true
-            ./update_results.py --dataset $dataset --pr ${{ inputs.source_id }}
+            uv run update_results.py --dataset $dataset --pr ${{ inputs.source_id }}
             aws s3 cp ./results/${dataset}.html ${REPORT_S3_PREFIX}/${dataset}.html --no-progress --checksum-algorithm=CRC32
             echo "* **${dataset}**: https://benchmark.databend.com/clickbench/pr/${{ inputs.source_id }}/${{ inputs.run_id }}/${dataset}.html" >> /tmp/body
           done
@@ -254,19 +252,17 @@ jobs:
           # - "internal"
     steps:
       - uses: actions/checkout@v4
-      - name: Install Dependencies
-        run: |
-          sudo apt-get update -yq
-          sudo apt-get install -yq python3-jinja2
+      - uses: astral-sh/setup-uv@v5
       - uses: actions/download-artifact@v4
         with:
-          path: benchmark/clickbench/results
+          path: benchmark/results
           pattern: benchmark-${{ matrix.dataset }}-*
           merge-multiple: true
       - name: Prepare results directory
-        working-directory: benchmark/clickbench
+        working-directory: benchmark
         run: |
           shopt -s nullglob
+          uv sync
           for result in results/*.json; do
             dataset=$(echo $result | sed -E 's/.*result-(\w+)-.*\.json/\1/')
             mkdir -p results/${dataset}/
@@ -279,7 +275,7 @@ jobs:
             mv $ndjson ndjsons/${dataset}/$(basename $ndjson)
           done
       - name: Generate report and upload to R2
-        working-directory: benchmark/clickbench
+        working-directory: benchmark
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
@@ -288,7 +284,7 @@ jobs:
         run: |
           aws s3 sync s3://benchmark/clickbench/release/${{ matrix.dataset }}/$(date --date='-1 month' -u +%Y)/$(date --date='-1 month' -u +%m)/ ./results/${{ matrix.dataset }}/
           aws s3 sync s3://benchmark/clickbench/release/${{ matrix.dataset }}/$(date -u +%Y)/$(date -u +%m)/ ./results/${{ matrix.dataset }}/
-          ./update_results.py --dataset ${{ matrix.dataset }} --release ${{ inputs.source_id }}
+          uv run update_results.py --dataset ${{ matrix.dataset }} --release ${{ inputs.source_id }}
 
           RESULT_PREFIX="s3://benchmark/clickbench/release/${{ matrix.dataset }}/$(date -u +%Y)/$(date -u +%m)/$(date -u +%Y-%m-%d)/${{ inputs.source_id }}"
           LATEST_PREFIX="s3://benchmark/clickbench/release/${{ matrix.dataset }}/latest/latest"
@@ -299,7 +295,7 @@ jobs:
 
           aws s3 cp ./results/${{ matrix.dataset }}.html s3://benchmark/clickbench/release/${{ matrix.dataset }}.html --no-progress --checksum-algorithm=CRC32
       - name: Upload NDJSON archives to R2
-        working-directory: benchmark/clickbench
+        working-directory: benchmark
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}

benchmark/clickbench/README.md renamed to benchmark/README.md

@@ -44,7 +44,6 @@ class ResultRecord:
     version: str
     warehouse: str
     machine: str
-    cluster_size: str
     tags: List[str]
     result: List[List[float]]
     values: Dict[str, List[float]]
@@ -103,7 +102,8 @@ def load_config() -> BenchmarkConfig:
     benchmark_id = os.environ.get("BENCHMARK_ID", str(int(time.time())))
     dataset = os.environ.get("BENCHMARK_DATASET", "hits")
     size = os.environ.get("BENCHMARK_SIZE", "Small")
-    cache_size = os.environ.get("BENCHMARK_CACHE_SIZE", "0")
+    raw_cache_size = os.environ.get("BENCHMARK_CACHE_SIZE", "")
+    cache_size = raw_cache_size.strip() or "0"
     version = os.environ.get("BENCHMARK_VERSION", "")
     database = os.environ.get("BENCHMARK_DATABASE", "default")
     tries_raw = os.environ.get("BENCHMARK_TRIES", "3")
@@ -162,9 +162,9 @@ def ensure_dependencies() -> None:
     logger.info("bendsql version: %s", subprocess.check_output(["bendsql", "--version"]).decode().strip())
 
 
-SIZE_MAPPING: Dict[str, Dict[str, str]] = {
-    "Small": {"cluster_size": "16", "machine": "Small"},
-    "Large": {"cluster_size": "64", "machine": "Large"},
+SIZE_MAPPING: Dict[str, str] = {
+    "Small": "Small",
+    "Large": "Large",
 }
 
 
@@ -282,8 +282,7 @@ def main() -> None:
 
     run_date = datetime.now(timezone.utc).strftime("%Y-%m-%d")
     tags = ["s3", f"cache-{config.cache_size}"]
-    cluster_size = SIZE_MAPPING[config.size]["cluster_size"]
-    machine = SIZE_MAPPING[config.size]["machine"]
+    machine = SIZE_MAPPING[config.size]
     system: Optional[str] = None
     comment: Optional[str] = None
     if config.source and config.source_id:
@@ -307,7 +306,6 @@ def main() -> None:
         version=config.version,
         warehouse=config.warehouse,
         machine=machine,
-        cluster_size=cluster_size,
         tags=tags,
         result=[],
         values={},

benchmark/clickbench/benchmark_cloud.py renamed to benchmark/benchmark_cloud.py

@@ -90,7 +90,6 @@ echo '{}' >result.json
 yq -i ".date = \"$(date -u +%Y-%m-%d)\"" -o json result.json
 yq -i ".load_time = ${load_time} | .data_size = ${data_size} | .result = []" -o json result.json
 yq -i ".machine = \"${instance_type}\"" -o json result.json
-yq -i '.cluster_size = 1' -o json result.json
 yq -i '.tags = ["gp3"]' -o json result.json
 
 echo "Running queries..."

benchmark/clickbench/benchmark_local.sh renamed to benchmark/benchmark_local.sh

@@ -79,7 +79,6 @@ echo '{}' >result.json
 yq -i ".date = \"$(date -u +%Y-%m-%d)\"" -o json result.json
 yq -i ".load_time = ${load_time} | .data_size = ${data_size} | .result = []" -o json result.json
 yq -i ".machine = \"${format_instance_type}\"" -o json result.json
-yq -i '.cluster_size = 1' -o json result.json
 yq -i '.tags = ["gp3"]' -o json result.json
 yq -i ".system = \"${1}\"" -o json result.json