diff --git a/Makefile b/Makefile index e34dec2fa5..00db8a4d4c 100644 --- a/Makefile +++ b/Makefile @@ -440,7 +440,7 @@ refresh-lock-files: @echo "===================================================================" @echo "🔁 Refreshing lock files using INDEX_MODE=$(INDEX_MODE)" @echo "===================================================================" - @cd $(ROOT_DIR) && bash scripts/pylocks_generator.sh $(INDEX_MODE) $(DIR) + @cd $(ROOT_DIR) && ./uv run scripts/pylocks_generator.py $(INDEX_MODE) $(DIR) # This is only for the workflow action # For running manually, set the required environment variables diff --git a/ci/generate_code.sh b/ci/generate_code.sh index fb7234c0c5..b89f10569d 100755 --- a/ci/generate_code.sh +++ b/ci/generate_code.sh @@ -8,4 +8,4 @@ uv --version || pip install "uv==0.10.6" "${REPO_ROOT}/uv" run scripts/dockerfile_fragments.py "${REPO_ROOT}/uv" run manifests/tools/generate_kustomization.py -bash scripts/pylocks_generator.sh +"${REPO_ROOT}/uv" run scripts/pylocks_generator.py diff --git a/pyproject.toml b/pyproject.toml index 4b92a1acfe..6d9788f0a6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,7 @@ dev = [ "podman", "kubernetes", "openshift-python-wrapper", + "typer", ] [tool.uv] diff --git a/scripts/lockfile-generators/create-requirements-lockfile.sh b/scripts/lockfile-generators/create-requirements-lockfile.sh index 279c07f8f7..38fea33b37 100755 --- a/scripts/lockfile-generators/create-requirements-lockfile.sh +++ b/scripts/lockfile-generators/create-requirements-lockfile.sh @@ -6,7 +6,7 @@ set -euo pipefail # Why this script exists # ---------------------- # Hermetic builds need every Python wheel prefetched. This script: -# 1. Delegates to pylocks_generator.sh to generate pylock..toml +# 1. Delegates to pylocks_generator.py to generate pylock..toml # (ensures consistency with CI's check-generated-code). # 2. Converts the pylock to a pip-compatible requirements..txt. # 3. (--download) Downloads every wheel into cachi2/output/deps/pip/. @@ -28,7 +28,7 @@ set -euo pipefail # --pyproject-toml codeserver/ubi9-python-3.12/pyproject.toml --flavor cuda SCRIPTS_PATH="scripts/lockfile-generators" -PYLOCKS_GENERATOR="scripts/pylocks_generator.sh" +PYLOCKS_GENERATOR="scripts/pylocks_generator.py" # --- Defaults --- PYPROJECT="" @@ -40,7 +40,7 @@ show_help() { cat << 'EOF' Usage: ./scripts/lockfile-generators/create-requirements-lockfile.sh [OPTIONS] -Generate pylock..toml (via pylocks_generator.sh) and convert it to +Generate pylock..toml (via pylocks_generator.py) and convert it to a pip-compatible requirements..txt with sha256 hashes. Options: @@ -54,7 +54,7 @@ Options: -h, --help Show this help message and exit Steps performed: - 1. pylocks_generator.sh → /uv.lock.d/pylock..toml + 1. pylocks_generator.py → /uv.lock.d/pylock..toml 2. Convert pylock..toml → /requirements..txt 3. (--download) Download all wheels from pylock..toml URLs EOF @@ -71,7 +71,7 @@ if [[ ! -d "$SCRIPTS_PATH" ]]; then error_exit "This script MUST be run from the repository root." fi if [[ ! -f "$PYLOCKS_GENERATOR" ]]; then - error_exit "pylocks_generator.sh not found at ${PYLOCKS_GENERATOR}" + error_exit "pylocks_generator.py not found at ${PYLOCKS_GENERATOR}" fi # --- Argument Parsing --- @@ -93,7 +93,7 @@ PROJECT_DIR="$(dirname "$PYPROJECT")" PYLOCK_FILE="${PROJECT_DIR}/uv.lock.d/pylock.${FLAVOR}.toml" REQUIREMENTS_FILE="${PROJECT_DIR}/requirements.${FLAVOR}.txt" -# Read build args from build-args/.conf (same source as pylocks_generator.sh) +# Read build args from build-args/.conf (same source as pylocks_generator.py) CONF_FILE="${PROJECT_DIR}/build-args/${FLAVOR}.conf" INDEX_URL="" if [[ -f "$CONF_FILE" ]]; then @@ -102,20 +102,20 @@ if [[ -f "$CONF_FILE" ]]; then fi # ========================================================================= -# Step 1: Generate pylock.toml via pylocks_generator.sh +# Step 1: Generate pylock.toml via pylocks_generator.py # # Delegates to the same script CI uses (ci/generate_code.sh), ensuring the # generated pylock.toml is identical to what check-generated-code expects. # ========================================================================= -echo "=== Step 1: Generating pylock via pylocks_generator.sh ===" +echo "=== Step 1: Generating pylock via pylocks_generator.py ===" echo " project dir : ${PROJECT_DIR}" echo " flavor : ${FLAVOR}" echo "" -bash "$PYLOCKS_GENERATOR" rh-index "$PROJECT_DIR" +./uv run "$PYLOCKS_GENERATOR" rh-index "$PROJECT_DIR" if [[ ! -f "$PYLOCK_FILE" ]]; then - error_exit "pylocks_generator.sh did not produce ${PYLOCK_FILE}" + error_exit "pylocks_generator.py did not produce ${PYLOCK_FILE}" fi echo "" diff --git a/scripts/pylocks_generator.py b/scripts/pylocks_generator.py new file mode 100644 index 0000000000..0ca5d134e0 --- /dev/null +++ b/scripts/pylocks_generator.py @@ -0,0 +1,411 @@ +#!/usr/bin/env python3 + +"""Generate Python dependency lock files (pylock.toml) using uv pip compile. + +This script generates Python dependency lock files (pylock.toml) for multiple +directories using either internal Red Hat wheel indexes or the public PyPI index. + +Features: + - Supports multiple Python project directories, detected by pyproject.toml. + - Detects available Dockerfile flavors (CPU, CUDA, ROCm) for rh-index mode. + - Validates Python version extracted from directory name (expects format .../ubi9-python-X.Y). + - Generates per-flavor locks in 'uv.lock.d/' for rh-index mode. + - Overwrites existing pylock.toml in-place for public PyPI index mode. + +Index Modes: + auto (default) -- Uses rh-index if uv.lock.d/ exists, public-index otherwise. + rh-index -- Uses internal Red Hat wheel indexes. Generates uv.lock.d/pylock..toml. + public-index -- Uses public PyPI index and updates pylock.toml in place. + +Fallback Index (RHAIENG-3071): + For CUDA and ROCm flavors, if CPU_INDEX_URL is defined in the build-args/*.conf file, + it will be added as a fallback index for packages not available in the specialized indexes. + +Usage: + 1. Lock using auto mode (default) for all projects in MAIN_DIRS:: + + python pylocks_generator.py + + 2. Lock using rh-index for a specific directory:: + + python pylocks_generator.py rh-index jupyter/minimal/ubi9-python-3.12 + + 3. Lock using public index for a specific directory:: + + python pylocks_generator.py public-index jupyter/minimal/ubi9-python-3.12 + + 4. Force upgrade all packages to latest versions:: + + FORCE_LOCKFILES_UPGRADE=1 python pylocks_generator.py + +Notes: + - If the script fails for a directory, it lists the failed directories at the end. + - Public index mode does not create uv.lock.d directories and keeps the old format. + - Python version extraction depends on directory naming convention; invalid formats are skipped. +""" + +from __future__ import annotations + +import os +import re +import subprocess +import sys +from enum import Enum +from pathlib import Path +from typing import Annotated + +import typer + +# ============================================================================= +# CONFIGURATION +# ============================================================================= + +ROOT_DIR = Path(__file__).resolve().parent.parent +UV = ROOT_DIR / "uv" +CVE_CONSTRAINTS_FILE = ROOT_DIR / "dependencies" / "cve-constraints.txt" +PUBLIC_INDEX = "--default-index=https://pypi.org/simple" +MAIN_DIRS = ("jupyter", "runtimes", "rstudio", "codeserver") +UV_MIN_VERSION = (0, 4, 0) + +NO_EMIT_PACKAGES = ( + "odh-notebooks-meta-llmcompressor-deps", + "odh-notebooks-meta-runtime-elyra-deps", + "odh-notebooks-meta-runtime-datascience-deps", + "odh-notebooks-meta-workbench-datascience-deps", +) + +FLAVORS = ("cpu", "cuda", "rocm") + + +class IndexMode(str, Enum): + auto = "auto" + rh_index = "rh-index" + public_index = "public-index" + + +# ============================================================================= +# HELPER FUNCTIONS +# ============================================================================= + +BLUE = "\033[1;34m" +YELLOW = "\033[1;33m" +RED = "\033[1;31m" +GREEN = "\033[1;32m" +RESET = "\033[0m" + + +def info(msg: str) -> None: + print(f"🔹 {BLUE}{msg}{RESET}") + + +def warn(msg: str) -> None: + print(f"⚠️ {YELLOW}{msg}{RESET}", file=sys.stderr) + + +def error(msg: str) -> None: + print(f"❌ {RED}{msg}{RESET}", file=sys.stderr) + + +def ok(msg: str) -> None: + print(f"✅ {GREEN}{msg}{RESET}", file=sys.stderr) + + +def read_conf_value(conf_file: Path, key: str) -> str | None: + """Read a key=value from a .conf file, skipping comments and blank lines.""" + for line in conf_file.read_text().splitlines(): + stripped = line.strip() + if stripped.startswith("#") or "=" not in stripped: + continue + k, _, v = stripped.partition("=") + if k.strip() == key: + return v.strip() + return None + + +# ============================================================================= +# PRE-FLIGHT CHECK +# ============================================================================= + + +def check_uv() -> None: + """Verify the uv wrapper exists and meets the minimum version requirement.""" + if not UV.is_file() or not os.access(UV, os.X_OK): + error(f"Expected uv wrapper at '{UV}' but it is missing or not executable.") + raise SystemExit(1) + + try: + result = subprocess.run( + [str(UV), "--version"], + capture_output=True, + text=True, + check=False, + ) + version_str = result.stdout.strip().split()[1] if result.stdout.strip() else "0.0.0" + except (IndexError, FileNotFoundError): + version_str = "0.0.0" + + version_tuple = tuple(int(x) for x in version_str.split(".")) + if version_tuple < UV_MIN_VERSION: + min_ver = ".".join(str(x) for x in UV_MIN_VERSION) + error(f"uv version {version_str} found, but >= {min_ver} is required.") + error("Please upgrade uv: https://github.com/astral-sh/uv") + raise SystemExit(1) + + +# ============================================================================= +# TARGET DIRECTORY DISCOVERY +# ============================================================================= + + +def find_target_dirs(target_dir: Path | None) -> list[Path]: + """Find directories containing pyproject.toml.""" + if target_dir is not None: + return [target_dir] + + info("Scanning main directories for Python projects...") + dirs: set[Path] = set() + for base_name in MAIN_DIRS: + base = ROOT_DIR / base_name + if base.is_dir(): + dirs.update(p.parent for p in base.rglob("pyproject.toml")) + return sorted(dirs) + + +# ============================================================================= +# FLAVOR DETECTION +# ============================================================================= + + +def detect_flavors(project_dir: Path) -> set[str]: + """Detect available Dockerfile flavors (cpu, cuda, rocm) in a directory.""" + return {f for f in FLAVORS if (project_dir / f"Dockerfile.{f}").is_file()} + + +def extract_python_version(project_dir: Path) -> str | None: + """Extract Python version from directory name suffix (e.g. ubi9-python-3.12 -> 3.12).""" + name = project_dir.resolve().name + # The version is everything after the last hyphen + version = name.rsplit("-", maxsplit=1)[-1] + if re.fullmatch(r"\d+\.\d+", version): + return version + return None + + +# ============================================================================= +# INDEX FLAGS +# ============================================================================= + + +def get_index_flags(project_dir: Path, flavor: str) -> list[str] | None: + """Build uv index flags from build-args/.conf. + + Returns None on failure (missing conf or INDEX_URL). + """ + conf_file = project_dir / "build-args" / f"{flavor}.conf" + if not conf_file.is_file(): + warn(f"Missing build-args config for {flavor}: {conf_file}") + return None + + index_url = read_conf_value(conf_file, "INDEX_URL") + if not index_url: + warn(f"INDEX_URL not found in {conf_file}") + return None + + flags = [f"--default-index={index_url}", f"--index={index_url}"] + + # For CUDA and ROCm flavors, add CPU index as fallback (RHAIENG-3071) + if flavor in ("cuda", "rocm"): + cpu_index_url = read_conf_value(conf_file, "CPU_INDEX_URL") + if cpu_index_url: + flags.append(f"--index={cpu_index_url}") + print(" 📎 Using CPU index as fallback", file=sys.stderr) + + return flags + + +# ============================================================================= +# LOCK FILE GENERATION +# ============================================================================= + + +def run_lock( + project_dir: Path, + flavor: str, + index_flags: list[str], + mode: IndexMode, + python_version: str, + upgrade: bool, +) -> bool: + """Run uv pip compile to generate a lock file. Returns True on success.""" + if mode == IndexMode.public_index: + output = "pylock.toml" + desc = "pylock.toml (public index)" + print("➡️ Generating pylock.toml from public PyPI index...") + else: + (project_dir / "uv.lock.d").mkdir(exist_ok=True) + output = f"uv.lock.d/pylock.{flavor}.toml" + desc = f"{flavor.upper()} lock file" + print(f"➡️ Generating {flavor.upper()} lock file...") + + # Tag filtering was added in uv 0.9.16 (https://github.com/astral-sh/uv/pull/16956) + # but bypassed in --universal mode. uv 0.10.5 (https://github.com/astral-sh/uv/pull/18081) + # now filters wheels by requires-python and marker disjointness even in --universal mode. + # Documentation at https://docs.astral.sh/uv/reference/cli/#uv-pip-compile--python-platform says that + # `--python-platform linux` is alias for `x86_64-unknown-linux-gnu`; we cannot use this to get a multiarch pylock + # Let's use --universal temporarily, and in the future we can switch to using uv.lock + # when https://github.com/astral-sh/uv/issues/6830 is resolved, or symlink `ln -s uv.lock.d/uv.${flavor}.lock uv.lock` + # Note: currently generating uv.lock.d/pylock.${flavor}.toml; future rename to uv.${flavor}.lock is planned + # See also --universal discussion with Gerard + # https://redhat-internal.slack.com/archives/C0961HQ858Q/p1757935641975969?thread_ts=1757542802.032519&cid=C0961HQ858Q + cmd: list[str] = [ + str(UV), + "pip", + "compile", + "pyproject.toml", + "--output-file", + output, + "--format", + "pylock.toml", + "--generate-hashes", + "--emit-index-url", + f"--python-version={python_version}", + "--universal", + "--no-annotate", + "--quiet", + ] + + for pkg in NO_EMIT_PACKAGES: + cmd.extend(["--no-emit-package", pkg]) + + if upgrade: + cmd.append("--upgrade") + + # Use relative path to avoid absolute paths in pylock.toml headers + if CVE_CONSTRAINTS_FILE.is_file(): + relative_constraints = os.path.relpath(CVE_CONSTRAINTS_FILE, project_dir) + cmd.extend(["--constraints", relative_constraints]) + + cmd.extend(index_flags) + + result = subprocess.run(cmd, cwd=project_dir, check=False) + + if result.returncode != 0: + warn(f"Failed to generate {desc} in {project_dir}") + output_path = project_dir / output + output_path.unlink(missing_ok=True) + return False + + ok(f"{desc} generated successfully.") + return True + + +# ============================================================================= +# MAIN +# ============================================================================= + +app = typer.Typer(add_completion=False) + + +@app.command() +def main( + index_mode: Annotated[ + IndexMode, typer.Argument(help="Index mode: auto, rh-index, or public-index") + ] = IndexMode.auto, + target_dir: Annotated[ + Path | None, typer.Argument(help="Specific project directory to process") + ] = None, +) -> None: + """Generate pylock.toml lock files for Python project directories.""" + # PRE-FLIGHT + check_uv() + + # UPGRADE FLAG + upgrade = os.environ.get("FORCE_LOCKFILES_UPGRADE", "0") == "1" + if upgrade: + info("FORCE_LOCKFILES_UPGRADE=1 detected. Will upgrade all packages to latest versions.") + + info(f"Using index mode: {index_mode.value}") + + # TARGET DIRECTORIES + target_dirs = find_target_dirs(target_dir) + if not target_dirs: + error("No directories containing pyproject.toml were found.") + raise SystemExit(1) + + # MAIN LOOP + success_dirs: list[Path] = [] + failed_dirs: list[Path] = [] + + for tdir in target_dirs: + print() + print("=" * 67) + info(f"Processing directory: {tdir}") + print("=" * 67) + + python_version = extract_python_version(tdir) + if python_version is None: + warn(f"Could not extract valid Python version from directory name: {tdir}") + warn("Expected directory format: .../ubi9-python-X.Y") + continue + + flavors = detect_flavors(tdir) + if not flavors: + warn(f"No Dockerfiles found in {tdir} (cpu/cuda/rocm). Skipping.") + continue + + print(f"📦 Python version: {python_version}") + print("🧩 Detected flavors:") + for f in sorted(flavors): + print(f" • {f.upper()}") + print() + + # Resolve effective mode + if index_mode == IndexMode.auto: + effective_mode = IndexMode.rh_index if (tdir / "uv.lock.d").is_dir() else IndexMode.public_index + else: + effective_mode = index_mode + info(f"Effective mode for this directory: {effective_mode.value}") + + dir_success = True + + if effective_mode == IndexMode.public_index: + if not run_lock(tdir, "cpu", [PUBLIC_INDEX], effective_mode, python_version, upgrade): + dir_success = False + else: + for flavor in ("cpu", "cuda", "rocm"): + if flavor not in flavors: + continue + flags = get_index_flags(tdir, flavor) + if flags is None: + dir_success = False + continue + if not run_lock(tdir, flavor, flags, effective_mode, python_version, upgrade): + dir_success = False + + if dir_success: + success_dirs.append(tdir) + else: + failed_dirs.append(tdir) + + # SUMMARY + print() + print("=" * 67) + ok("Lock generation complete.") + print("=" * 67) + + if success_dirs: + print("✅ Successfully generated locks for:") + for d in success_dirs: + print(f" • {d}") + + if failed_dirs: + print() + warn("Failed lock generation for:") + for d in failed_dirs: + print(f" • {d}") + print("Please comment out the missing package to continue and report the missing package to the RH index maintainers") + raise SystemExit(1) + + +if __name__ == "__main__": + app() diff --git a/scripts/pylocks_generator.sh b/scripts/pylocks_generator.sh deleted file mode 100755 index 91492b8398..0000000000 --- a/scripts/pylocks_generator.sh +++ /dev/null @@ -1,375 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# ============================================================================= -# pylocks_generator.sh -# -# This script generates Python dependency lock files (pylock.toml) for multiple -# directories using either internal Red Hat wheel indexes or the public PyPI index. -# -# Features: -# • Supports multiple Python project directories, detected by pyproject.toml. -# • Detects available Dockerfile flavors (CPU, CUDA, ROCm) for rh-index mode. -# • Validates Python version extracted from directory name (expects format .../ubi9-python-X.Y). -# • Generates per-flavor locks in 'uv.lock.d/' for rh-index mode. -# • Overwrites existing pylock.toml in-place for public PyPI index mode. -# -# Index Modes: -# • auto (default) -> Uses rh-index if uv.lock.d/ exists, public-index otherwise. -# • rh-index -> Uses internal Red Hat wheel indexes. Generates uv.lock.d/pylock..toml . -# • public-index -> Uses public PyPI index and updates pylock.toml in place. -# -# Fallback Index (RHAIENG-3071): -# For CUDA and ROCm flavors, if CPU_INDEX_URL is defined in the build-args/*.conf file, -# it will be added as a fallback index for packages not available in the specialized indexes. -# -# Usage: -# 1. Lock using auto mode (default) for all projects in MAIN_DIRS: -# bash pylocks_generator.sh -# -# 2. Lock using rh-index for a specific directory: -# bash pylocks_generator.sh rh-index jupyter/minimal/ubi9-python-3.12 -# -# 3. Lock using public index for a specific directory: -# bash pylocks_generator.sh public-index jupyter/minimal/ubi9-python-3.12 -# -# 4. Force upgrade all packages to latest versions: -# FORCE_LOCKFILES_UPGRADE=1 bash pylocks_generator.sh -# -# Notes: -# • If the script fails for a directory, it lists the failed directories at the end. -# • Public index mode does not create uv.lock.d directories and keeps the old format. -# • Python version extraction depends on directory naming convention; invalid formats are skipped. -# ============================================================================= - -# ---------------------------- -# CONFIGURATION -# ---------------------------- -PUBLIC_INDEX="--default-index=https://pypi.org/simple" - -MAIN_DIRS=("jupyter" "runtimes" "rstudio" "codeserver") - -# CVE constraints file - applied to all lock file generations -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -ROOT_DIR="$(dirname "$SCRIPT_DIR")" -UV="${ROOT_DIR}/uv" -CVE_CONSTRAINTS_FILE="$ROOT_DIR/dependencies/cve-constraints.txt" - -# ---------------------------- -# HELPER FUNCTIONS -# ---------------------------- -info() { echo -e "🔹 \033[1;34m$1\033[0m"; } -warn() { echo -e "⚠️ \033[1;33m$1\033[0m" >&2; } -error() { echo -e "❌ \033[1;31m$1\033[0m" >&2; } -ok() { echo -e "✅ \033[1;32m$1\033[0m" >&2; } - -uppercase() { - echo "$1" | tr '[:lower:]' '[:upper:]' -} - -read_conf_value() { - local conf_file="$1" - local key="$2" - - awk -F= -v key="$key" ' - /^[[:space:]]*#/ { next } - NF < 2 { next } - { - gsub(/^[[:space:]]+|[[:space:]]+$/, "", $1) - if ($1 == key) { - $1="" - sub(/^=/, "", $0) - gsub(/^[[:space:]]+|[[:space:]]+$/, "", $0) - print $0 - exit - } - } - ' "$conf_file" -} - -# ---------------------------- -# PRE-FLIGHT CHECK -# ---------------------------- -if [[ ! -x "$UV" ]]; then - error "Expected uv wrapper at '$UV' but it is missing or not executable." - exit 1 -fi - -if ! command -v uv &>/dev/null; then - error "uv command not found. Please install uv: https://github.com/astral-sh/uv" - exit 1 -fi - -UV_MIN_VERSION="0.4.0" -UV_VERSION=$("$UV" --version 2>/dev/null | awk '{print $2}' || echo "0.0.0") - -version_ge() { - [ "$(printf '%s\n' "$2" "$1" | sort -V | head -n1)" = "$2" ] -} - -if ! version_ge "$UV_VERSION" "$UV_MIN_VERSION"; then - error "uv version $UV_VERSION found, but >= $UV_MIN_VERSION is required." - error "Please upgrade uv: https://github.com/astral-sh/uv" - exit 1 -fi - -# ---------------------------- -# ARGUMENT PARSING -# ---------------------------- -# default to auto if not provided -INDEX_MODE="${1:-auto}" -TARGET_DIR_ARG="${2:-}" - -# Check for upgrade flag via environment variable -# Set FORCE_LOCKFILES_UPGRADE=1 to upgrade all packages to latest versions -UPGRADE_FLAG="" -if [[ "${FORCE_LOCKFILES_UPGRADE:-0}" == "1" ]]; then - UPGRADE_FLAG="--upgrade" - info "FORCE_LOCKFILES_UPGRADE=1 detected. Will upgrade all packages to latest versions." -fi - -# Validate mode -if [[ "$INDEX_MODE" != "auto" && "$INDEX_MODE" != "rh-index" && "$INDEX_MODE" != "public-index" ]]; then - error "Invalid mode '$INDEX_MODE'. Valid options: auto, rh-index, public-index" - exit 1 -fi -info "Using index mode: $INDEX_MODE" - -# ---------------------------- -# GET TARGET DIRECTORIES -# ---------------------------- -if [ -n "$TARGET_DIR_ARG" ]; then - TARGET_DIRS=("$TARGET_DIR_ARG") -else - info "Scanning main directories for Python projects..." - TARGET_DIRS=() - for base in "${MAIN_DIRS[@]}"; do - if [ -d "$base" ]; then - while IFS= read -r -d '' pyproj; do - TARGET_DIRS+=("$(dirname "$pyproj")") - done < <(find "$base" -type f -name "pyproject.toml" -print0) - fi - done -fi - -if [ ${#TARGET_DIRS[@]} -eq 0 ]; then - error "No directories containing pyproject.toml were found." - exit 1 -fi - -# ---------------------------- -# MAIN LOOP -# ---------------------------- -FAILED_DIRS=() -SUCCESS_DIRS=() - -for TARGET_DIR in "${TARGET_DIRS[@]}"; do - echo - echo "===================================================================" - info "Processing directory: $TARGET_DIR" - echo "===================================================================" - - cd "$TARGET_DIR" || continue - PYTHON_VERSION="${PWD##*-}" - - # Validate Python version extraction - if [[ ! "$PYTHON_VERSION" =~ ^[0-9]+\.[0-9]+$ ]]; then - warn "Could not extract valid Python version from directory name: $PWD" - warn "Expected directory format: .../ubi9-python-X.Y" - cd - >/dev/null - continue - fi - - # Detect available Dockerfiles (flavors) - HAS_CPU=false - HAS_CUDA=false - HAS_ROCM=false - [ -f "Dockerfile.cpu" ] && HAS_CPU=true - [ -f "Dockerfile.cuda" ] && HAS_CUDA=true - [ -f "Dockerfile.rocm" ] && HAS_ROCM=true - - if ! $HAS_CPU && ! $HAS_CUDA && ! $HAS_ROCM; then - warn "No Dockerfiles found in $TARGET_DIR (cpu/cuda/rocm). Skipping." - cd - >/dev/null - continue - fi - - echo "📦 Python version: $PYTHON_VERSION" - echo "🧩 Detected flavors:" - $HAS_CPU && echo " • CPU" - $HAS_CUDA && echo " • CUDA" - $HAS_ROCM && echo " • ROCm" - echo - - # Resolve effective mode for this directory - if [[ "$INDEX_MODE" == "auto" ]]; then - if [[ -d "uv.lock.d" ]]; then - EFFECTIVE_MODE="rh-index" - else - EFFECTIVE_MODE="public-index" - fi - else - EFFECTIVE_MODE="$INDEX_MODE" - fi - info "Effective mode for this directory: $EFFECTIVE_MODE" - - DIR_SUCCESS=true - CONF_DIR="build-args" - - get_index_flags() { - local flavor="$1" - local conf_file="${CONF_DIR}/${flavor}.conf" - local index_url - local cpu_index_url - local index_flags - - if [[ ! -f "$conf_file" ]]; then - warn "Missing build-args config for ${flavor}: $conf_file" - return 1 - fi - - index_url=$(read_conf_value "$conf_file" "INDEX_URL") - if [[ -z "$index_url" ]]; then - warn "INDEX_URL not found in $conf_file" - return 1 - fi - - index_flags="--default-index=${index_url} --index=${index_url}" - - # For CUDA and ROCm flavors, add CPU index as fallback for packages - # not available in the specialized indexes (RHAIENG-3071) - if [[ "$flavor" == "cuda" || "$flavor" == "rocm" ]]; then - cpu_index_url=$(read_conf_value "$conf_file" "CPU_INDEX_URL") - if [[ -n "$cpu_index_url" ]]; then - index_flags="${index_flags} --index=${cpu_index_url}" - echo " 📎 Using CPU index as fallback" >&2 - fi - fi - - echo "$index_flags" - } - - run_flavor_lock() { - local flavor="$1" - local index_flags - - if ! index_flags=$(get_index_flags "$flavor"); then - DIR_SUCCESS=false - return - fi - - run_lock "$flavor" "$index_flags" "$EFFECTIVE_MODE" - } - - run_lock() { - local flavor="$1" - local index="$2" - local mode="$3" - local output - local desc - - if [[ "$mode" == "public-index" ]]; then - output="pylock.toml" - desc="pylock.toml (public index)" - echo "➡️ Generating pylock.toml from public PyPI index..." - else - mkdir -p uv.lock.d - output="uv.lock.d/pylock.${flavor}.toml" - desc="$(uppercase "$flavor") lock file" - echo "➡️ Generating $(uppercase "$flavor") lock file..." - fi - - # Tag filtering was added in uv 0.9.16 (https://github.com/astral-sh/uv/pull/16956) - # but bypassed in --universal mode. uv 0.10.5 (https://github.com/astral-sh/uv/pull/18081) - # now filters wheels by requires-python and marker disjointness even in --universal mode. - # Documentation at https://docs.astral.sh/uv/reference/cli/#uv-pip-compile--python-platform says that - # `--python-platform linux` is alias for `x86_64-unknown-linux-gnu`; we cannot use this to get a multiarch pylock - # Let's use --universal temporarily, and in the future we can switch to using uv.lock - # when https://github.com/astral-sh/uv/issues/6830 is resolved, or symlink `ln -s uv.lock.d/uv.${flavor}.lock uv.lock` - # Note: currently generating uv.lock.d/pylock.${flavor}.toml; future rename to uv.${flavor}.lock is planned - # See also --universal discussion with Gerard - # https://redhat-internal.slack.com/archives/C0961HQ858Q/p1757935641975969?thread_ts=1757542802.032519&cid=C0961HQ858Q - - # Build constraints flag if CVE constraints file exists - # Use relative path to avoid absolute paths in pylock.toml headers - # (which would differ between CI and local environments) - local -a constraints_flag=() - if [[ -f "$CVE_CONSTRAINTS_FILE" ]]; then - local relative_constraints - # Use Python for cross-platform relative path computation (realpath --relative-to is GNU-only) - relative_constraints=$(python3 -c "import os; print(os.path.relpath('$CVE_CONSTRAINTS_FILE', '$PWD'))") - constraints_flag=(--constraints "$relative_constraints") - fi - - set +e - # shellcheck disable=SC2086 - "$UV" pip compile pyproject.toml \ - --output-file "$output" \ - --format pylock.toml \ - --generate-hashes \ - --emit-index-url \ - --python-version="$PYTHON_VERSION" \ - --universal \ - --no-annotate \ - --quiet \ - --no-emit-package odh-notebooks-meta-llmcompressor-deps \ - --no-emit-package odh-notebooks-meta-runtime-elyra-deps \ - --no-emit-package odh-notebooks-meta-runtime-datascience-deps \ - --no-emit-package odh-notebooks-meta-workbench-datascience-deps \ - $UPGRADE_FLAG \ - "${constraints_flag[@]}" \ - $index - local status=$? - set -e - - if [ $status -ne 0 ]; then - warn "Failed to generate $desc in $TARGET_DIR" - rm -f "$output" - DIR_SUCCESS=false - else - ok "$desc generated successfully." - fi - } - - # Run lock generation based on effective mode - if [[ "$EFFECTIVE_MODE" == "public-index" ]]; then - run_lock "cpu" "$PUBLIC_INDEX" "$EFFECTIVE_MODE" - else - $HAS_CPU && run_flavor_lock "cpu" - $HAS_CUDA && run_flavor_lock "cuda" - $HAS_ROCM && run_flavor_lock "rocm" - fi - - if $DIR_SUCCESS; then - SUCCESS_DIRS+=("$TARGET_DIR") - else - FAILED_DIRS+=("$TARGET_DIR") - fi - - cd - >/dev/null -done - -# ---------------------------- -# SUMMARY -# ---------------------------- -echo -echo "===================================================================" -ok "Lock generation complete." -echo "===================================================================" - -if [ ${#SUCCESS_DIRS[@]} -gt 0 ]; then - echo "✅ Successfully generated locks for:" - for d in "${SUCCESS_DIRS[@]}"; do - echo " • $d" - done -fi - -if [ ${#FAILED_DIRS[@]} -gt 0 ]; then - echo - warn "Failed lock generation for:" - for d in "${FAILED_DIRS[@]}"; do - echo " • $d" - echo "Please comment out the missing package to continue and report the missing package to the RH index maintainers" - done - exit 1 -fi diff --git a/uv.lock b/uv.lock index 65ef6a17b4..a7c04af42b 100644 --- a/uv.lock +++ b/uv.lock @@ -952,6 +952,7 @@ dev = [ { name = "ruamel-yaml", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "ruff", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "testcontainers", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, + { name = "typer", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, ] [package.metadata] @@ -974,6 +975,7 @@ dev = [ { name = "ruamel-yaml" }, { name = "ruff" }, { name = "testcontainers" }, + { name = "typer" }, ] [[package]]