diff --git a/.buildkite/nightly-release-pipeline.yaml b/.buildkite/nightly-release-pipeline.yaml new file mode 100644 index 00000000000..25c52ba3b45 --- /dev/null +++ b/.buildkite/nightly-release-pipeline.yaml @@ -0,0 +1,20 @@ +steps: + - label: "Build and upload wheel" + key: "build-wheel" + agents: + queue: cpu_queue_release + commands: + - "curl -LsSf https://astral.sh/uv/install.sh | sh" + - 'export PATH="$HOME/.local/bin:$PATH"' + - "uv venv --python=3.12 && source .venv/bin/activate" + - "uv pip install --upgrade build" + - "python3 -m build" + - "bash .buildkite/scripts/upload-nightly-wheels.sh" + + - label: "Generate and upload wheel indices" + depends_on: "build-wheel" + allow_dependency_failure: true + agents: + queue: small_cpu_queue_release + commands: + - "bash .buildkite/scripts/generate-and-upload-nightly-index.sh" diff --git a/.buildkite/scripts/generate-and-upload-nightly-index.sh b/.buildkite/scripts/generate-and-upload-nightly-index.sh new file mode 100755 index 00000000000..6624af32303 --- /dev/null +++ b/.buildkite/scripts/generate-and-upload-nightly-index.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash + +set -ex + +# Generate and upload wheel indices for all vllm-omni wheels in the commit directory. +# This script should run once after all wheels have been built and uploaded. +# All paths are under the omni/ prefix in the vllm-wheels S3 bucket. + +# ======== setup ======== + +BUCKET="vllm-wheels" +INDICES_OUTPUT_DIR="indices" +PYTHON="${PYTHON_PROG:-python3}" +SUBPATH="omni/$BUILDKITE_COMMIT" +S3_COMMIT_PREFIX="s3://$BUCKET/$SUBPATH/" + +# detect if python3.12+ is available +has_new_python=$($PYTHON -c "print(1 if __import__('sys').version_info >= (3,12) else 0)") +if [[ "$has_new_python" -eq 0 ]]; then + # use new python from docker + docker pull python:3-slim + PYTHON="docker run --rm -v $(pwd):/app -w /app python:3-slim python3" +fi + +echo "Using python interpreter: $PYTHON" +echo "Python version: $($PYTHON --version)" + +# ======== generate and upload indices ======== + +# list all wheels in the commit directory +echo "Existing wheels on S3:" +aws s3 ls "$S3_COMMIT_PREFIX" || echo "(no objects found)" +obj_json="objects.json" +aws s3api list-objects-v2 --bucket "$BUCKET" --prefix "$SUBPATH/" --delimiter / --output json > "$obj_json" +mkdir -p "$INDICES_OUTPUT_DIR" + +# HACK: we do not need regex module here, but it is required by pre-commit hook +# To avoid any external dependency, we simply replace it back to the stdlib re module +sed -i 's/import regex as re/import re/g' .buildkite/scripts/generate-nightly-index.py + +# Generate indices -- the version is just the commit hash (not omni/{commit}) +# because relative paths are computed between the index and wheel directories, +# both of which live under the omni/ prefix in S3. +$PYTHON .buildkite/scripts/generate-nightly-index.py \ + --version "$BUILDKITE_COMMIT" \ + --current-objects "$obj_json" \ + --output-dir "$INDICES_OUTPUT_DIR" \ + --comment "commit $BUILDKITE_COMMIT" + +# copy indices to /omni/{commit}/ unconditionally +echo "Uploading indices to $S3_COMMIT_PREFIX" +aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "$S3_COMMIT_PREFIX" + +# copy to /omni/nightly/ when NIGHTLY=1 +if [[ "${NIGHTLY:-}" == "1" ]]; then + echo "Uploading indices to overwrite /omni/nightly/" + aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "s3://$BUCKET/omni/nightly/" +fi + +# detect version from any wheel in the commit directory +first_wheel_key=$($PYTHON -c "import json; obj=json.load(open('$obj_json')); print(next((c['Key'] for c in obj.get('Contents', []) if c['Key'].endswith('.whl')), ''))") +if [[ -z "$first_wheel_key" ]]; then + echo "Error: No wheels found in $S3_COMMIT_PREFIX" + exit 1 +fi +first_wheel=$(basename "$first_wheel_key") +aws s3 cp "s3://$BUCKET/${first_wheel_key}" "/tmp/${first_wheel}" +version=$(unzip -p "/tmp/${first_wheel}" '**/METADATA' | grep '^Version: ' | cut -d' ' -f2) +rm -f "/tmp/${first_wheel}" +echo "Version in wheel: $version" +pure_version="${version%%+*}" +echo "Pure version (without variant): $pure_version" + +# re-generate and copy to /omni/{version}/ only if it does not have "dev" in the version +if [[ "$version" != *"dev"* ]]; then + echo "Re-generating indices for /omni/$pure_version/" + rm -rf "${INDICES_OUTPUT_DIR:?}" + mkdir -p "$INDICES_OUTPUT_DIR" + # wheel-dir is overridden to be the commit directory, so that the indices point to the correct wheel path + $PYTHON .buildkite/scripts/generate-nightly-index.py \ + --version "$pure_version" \ + --wheel-dir "$BUILDKITE_COMMIT" \ + --current-objects "$obj_json" \ + --output-dir "$INDICES_OUTPUT_DIR" \ + --comment "version $pure_version" + aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "s3://$BUCKET/omni/$pure_version/" +fi diff --git a/.buildkite/scripts/generate-nightly-index.py b/.buildkite/scripts/generate-nightly-index.py new file mode 100755 index 00000000000..c616c446b09 --- /dev/null +++ b/.buildkite/scripts/generate-nightly-index.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import argparse +import json +import sys +from dataclasses import asdict, dataclass +from datetime import datetime +from pathlib import Path +from typing import Any +from urllib.parse import quote + +import regex as re + + +def normalize_package_name(name: str) -> str: + """Normalize package name per PEP 503.""" + return re.sub(r"[-_.]+", "-", name).lower() + + +if not sys.version_info >= (3, 12): + raise RuntimeError("This script requires Python 3.12 or higher.") + +INDEX_HTML_TEMPLATE = """ + + + + +{items} + + +""" + + +@dataclass +class WheelFileInfo: + package_name: str + version: str + build_tag: str | None + python_tag: str + abi_tag: str + platform_tag: str + filename: str + + +def parse_from_filename(file: str) -> WheelFileInfo: + """ + Parse wheel filename per PEP 427: + {package_name}-{version}(-{build_tag})?-{python_tag}-{abi_tag}-{platform_tag}.whl + """ + wheel_file_re = re.compile( + r"^(?P.+)-(?P[^-]+?)(-(?P[^-]+))?-(?P[^-]+)-(?P[^-]+)-(?P[^-]+)\.whl$" + ) + match = wheel_file_re.match(file) + if not match: + raise ValueError(f"Invalid wheel file name: {file}") + + return WheelFileInfo( + package_name=match.group("package_name"), + version=match.group("version"), + build_tag=match.group("build_tag"), + python_tag=match.group("python_tag"), + abi_tag=match.group("abi_tag"), + platform_tag=match.group("platform_tag"), + filename=file, + ) + + +def generate_project_list(package_names: list[str], comment: str = "") -> str: + """Generate top-level PEP 503 project list HTML.""" + href_tags = [] + for name in sorted(package_names): + href_tags.append(f' {name}/
') + return INDEX_HTML_TEMPLATE.format(items="\n".join(href_tags), comment=comment) + + +def generate_package_index( + wheel_files: list[WheelFileInfo], + wheel_base_dir: Path, + index_base_dir: Path, + comment: str = "", +) -> tuple[str, str]: + """Generate package index HTML and metadata JSON linking to wheel files.""" + href_tags = [] + metadata = [] + for file in sorted(wheel_files, key=lambda x: x.filename): + relative_path = wheel_base_dir.relative_to(index_base_dir, walk_up=True) / file.filename + # handle '+' in URL; avoid double-encoding '/' and '%2B' (AWS S3 behavior) + file_path_quoted = quote(relative_path.as_posix(), safe=":%/") + href_tags.append(f' {file.filename}
') + file_meta = asdict(file) + file_meta["path"] = file_path_quoted + metadata.append(file_meta) + index_str = INDEX_HTML_TEMPLATE.format(items="\n".join(href_tags), comment=comment) + metadata_str = json.dumps(metadata, indent=2) + return index_str, metadata_str + + +def generate_index( + whl_files: list[str], + wheel_base_dir: Path, + index_base_dir: Path, + comment: str = "", +): + """ + Generate PEP 503 index for all wheel files. + + Output structure: + index_base_dir/ + index.html # project list linking to vllm-omni/ + vllm-omni/ + index.html # package index linking to wheel files + metadata.json # machine-readable metadata + """ + parsed_files = [parse_from_filename(f) for f in whl_files] + + if not parsed_files: + print("No wheel files found, skipping index generation.") + return + + comment_str = f" ({comment})" if comment else "" + comment_tmpl = f"Generated on {datetime.now().isoformat()}{comment_str}" + + # Group by normalized package name + packages: dict[str, list[WheelFileInfo]] = {} + for file in parsed_files: + name = normalize_package_name(file.package_name) + packages.setdefault(name, []).append(file) + + print(f"Found packages: {list(packages.keys())}") + + # Generate per-package index + for package, files in packages.items(): + package_dir = index_base_dir / package + package_dir.mkdir(parents=True, exist_ok=True) + index_str, metadata_str = generate_package_index(files, wheel_base_dir, package_dir, comment) + with open(package_dir / "index.html", "w") as f: + f.write(index_str) + with open(package_dir / "metadata.json", "w") as f: + f.write(metadata_str) + + # Generate top-level project list + project_list_str = generate_project_list(sorted(packages.keys()), comment_tmpl) + with open(index_base_dir / "index.html", "w") as f: + f.write(project_list_str) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Generate PEP 503 wheel index from S3 object listing.") + parser.add_argument("--version", type=str, required=True, help="Version string (e.g., commit hash)") + parser.add_argument("--current-objects", type=str, required=True, help="Path to JSON from S3 list-objects-v2") + parser.add_argument("--output-dir", type=str, required=True, help="Directory to write index files") + parser.add_argument("--wheel-dir", type=str, default=None, help="Wheel directory (defaults to --version)") + parser.add_argument("--comment", type=str, default="", help="Comment for generated HTML") + + args = parser.parse_args() + + version = args.version + if "\\" in version or "/" in version: + raise ValueError("Version string must not contain slashes or backslashes.") + + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + with open(args.current_objects) as f: + current_objects: dict[str, list[dict[str, Any]]] = json.load(f) + + wheel_files = [ + item["Key"].split("/")[-1] for item in current_objects.get("Contents", []) if item["Key"].endswith(".whl") + ] + + print(f"Found {len(wheel_files)} wheel files for version {version}: {wheel_files}") + + # For release versions, filter to only matching non-dev wheels + PY_VERSION_RE = re.compile(r"^\d+\.\d+\.\d+([a-zA-Z0-9.+-]*)?$") + if PY_VERSION_RE.match(version): + wheel_files = [f for f in wheel_files if version in f and "dev" not in f] + print(f"Non-nightly version detected, wheel files used: {wheel_files}") + else: + print("Nightly version detected, keeping all wheel files.") + + wheel_dir = (args.wheel_dir or version).strip().rstrip("/") + wheel_base_dir = Path(output_dir).parent / wheel_dir + index_base_dir = Path(output_dir) + + generate_index( + whl_files=wheel_files, + wheel_base_dir=wheel_base_dir, + index_base_dir=index_base_dir, + comment=args.comment.strip(), + ) + print(f"Successfully generated index in {output_dir}") diff --git a/.buildkite/scripts/upload-nightly-wheels.sh b/.buildkite/scripts/upload-nightly-wheels.sh new file mode 100755 index 00000000000..d50da1deda1 --- /dev/null +++ b/.buildkite/scripts/upload-nightly-wheels.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +set -ex + +# Upload a single wheel to S3 under the omni/ prefix. +# Index generation is handled separately by generate-and-upload-nightly-index.sh. + +BUCKET="vllm-wheels" +SUBPATH="omni/$BUILDKITE_COMMIT" +S3_COMMIT_PREFIX="s3://$BUCKET/$SUBPATH/" + +# ========= collect & upload the wheel ========== + +# python3 -m build outputs to dist/ by default +wheel_files=(dist/*.whl) + +# Check that exactly one wheel is found +if [[ ${#wheel_files[@]} -ne 1 ]]; then + echo "Error: Expected exactly one wheel file in dist/, but found ${#wheel_files[@]}" + exit 1 +fi +wheel="${wheel_files[0]}" + +echo "Uploading wheel: $wheel" + +# Extract the version from the wheel +version=$(unzip -p "$wheel" '**/METADATA' | grep '^Version: ' | cut -d' ' -f2) +echo "Version in wheel: $version" + +# Upload wheel to S3 +aws s3 cp "$wheel" "$S3_COMMIT_PREFIX" + +echo "Wheel uploaded to $S3_COMMIT_PREFIX. Index generation is handled by a separate step."