[DRAFT] fix(ci): resolve pre-existing CI failures on rhoai-3.3

.github/actions/apt-install/action.yml

@@ -0,0 +1,56 @@
+---
+name: 'Install APT packages'
+description: 'Fast apt-get install with slow repos removed'
+inputs:
+  packages:
+    description: 'Space-separated list of packages to install'
+    required: true
+  update:
+    description: 'Run apt-get update before installing'
+    required: false
+    default: 'true'
+runs:
+  using: composite
+  steps:
+    - name: Configure apt for CI
+      shell: bash
+      run: |
+        # These are idempotent and fast — safe to run every time
+        sudo rm -f /var/lib/man-db/auto-update
+
+        # the Microsoft repo's kubelet does not provide /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
+        #  [Service]
+        #  EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env
+        #  ExecStart=/usr/bin/kubelet $KUBELET_KUBEADM_ARGS
+        # and otherwise fetching from microsoft-prod wastes time
+        sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list
+        sudo rm -f /etc/apt/sources.list.d/azure-cli.list
+
+        cat <<'EOF' | sudo tee /etc/dpkg/dpkg.cfg.d/01_nodoc > /dev/null
+        path-exclude /usr/share/doc/*
+        path-exclude /usr/share/man/*
+        path-exclude /usr/share/locale/*
+        path-include /usr/share/locale/en*
+        EOF
+
+        cat <<'EOF' | sudo tee /etc/apt/apt.conf.d/99-ci-optimizations > /dev/null
+        Acquire::Queue-Mode "access";
+        Acquire::Retries "3";
+        APT::Install-Recommends "false";
+        APT::Install-Suggests "false";
+        APT::Get::Assume-Yes "true";
+        APT::Get::Quiet "true";
+        APT::Get::Show-Upgraded "false";
+        APT::AutoRemove::SuggestsImportant "false";
+        DPkg::Options:: "--force-unsafe-io";
+        EOF
+
+    - name: Update package index
+      if: ${{ inputs.update == 'true' }}
+      shell: bash
+      run: |
+        sudo apt-get update
+
+    - name: Install packages
+      shell: bash
+      run: sudo apt-get install ${{ inputs.packages }}

.github/actions/playwright-test/action.yml

@@ -0,0 +1,126 @@
+---
+name: 'Playwright Browser Tests'
+description: |
+  Run Playwright browser tests for workbench images.
+
+  Features:
+  - Runs Playwright tests inside the official Microsoft Playwright container
+  - Uses Podman to run the test container with proper isolation
+  - Supports testcontainers for container orchestration
+  - Uploads test reports as artifacts on pull requests
+
+  Requirements:
+  - Podman must be installed and running
+  - The workbench image must be available in the local Podman storage
+  - tests/browser directory must contain the Playwright test configuration
+
+inputs:
+  test-target:
+    description: 'The workbench image to test (full image reference).'
+    required: true
+  podman-socket:
+    description: 'Path to Podman socket'
+    required: false
+    default: '/var/run/podman/podman.sock'
+  playwright-version:
+    description: 'Version of the Playwright container to use (e.g., v1.58.1-noble). If not provided, extracted from package.json5'
+    required: false
+    default: ''
+  working-directory:
+    description: 'Directory containing the Playwright tests'
+    required: false
+    default: 'tests/browser'
+  upload-report:
+    description: 'Whether to upload the Playwright report as an artifact'
+    required: false
+    default: 'false'
+  artifact-name:
+    description: 'Name for the uploaded artifact (required if upload-report is true)'
+    required: false
+    default: 'playwright-report'
+  artifact-retention-days:
+    description: 'Number of days to retain the artifact'
+    required: false
+    default: '30'
+  grep-pattern:
+    description: 'Grep pattern passed to --grep to filter tests (e.g. "@codeserver", "@smoke"). Empty string runs all tests.'
+    required: false
+    default: ''
+
+outputs:
+  outcome:
+    description: 'The outcome of the Playwright tests (success or failure)'
+    value: ${{ steps.playwright.outcome }}
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Determine Playwright version
+      id: playwright-version
+      shell: bash
+      run: |
+        if [[ -n "${INPUT_PLAYWRIGHT_VERSION}" ]]; then
+          echo "version=${INPUT_PLAYWRIGHT_VERSION}" >> "$GITHUB_OUTPUT"
+        else
+          # Extract version from package.json5 (single source of truth)
+          # package.json5 has: "@playwright/test": "=1.59.1"
+          # Container tag format is: v1.59.1-noble
+          PKG_VERSION=$(grep -oP '"@playwright/test":\s*"=?\K[0-9.]+' "${WORKING_DIRECTORY}/package.json5")
+          if [[ -z "$PKG_VERSION" ]]; then
+            echo "::error::Failed to extract Playwright version from package.json5"
+            exit 1
+          fi
+          echo "version=v${PKG_VERSION}-noble" >> "$GITHUB_OUTPUT"
+          echo "Extracted Playwright version: v${PKG_VERSION}-noble"
+        fi
+      env:
+        INPUT_PLAYWRIGHT_VERSION: ${{ inputs.playwright-version }}
+        WORKING_DIRECTORY: ${{ inputs.working-directory }}
+
+    - name: Run Playwright tests
+      id: playwright
+      shell: bash
+      # --ipc=host because Microsoft says so in Playwright docs
+      # --net=host because testcontainers connects to the Reaper container's exposed port
+      # we need to pass through the relevant environment variables
+      #  DEBUG configures Node.js debuggers, sets different verbosity as needed
+      #  CI=true is set on every CI nowadays
+      #  PODMAN_SOCK should be mounted to /var/run/docker.sock, other likely mounting locations may not exist (mkdir -p)
+      #  TEST_TARGET is the workbench image the test will run
+      # --volume(s) let us access docker socket and not clobber host's node_modules
+      run: |
+        podman run \
+          --interactive --rm \
+          --ipc=host \
+          --net=host \
+          --env "CI=true" \
+          --env "NPM_CONFIG_fund=false" \
+          --env "DEBUG=testcontainers:*" \
+          --env "PODMAN_SOCK=/var/run/docker.sock" \
+          --env "TEST_TARGET" \
+          --volume "${PODMAN_SOCKET}":/var/run/docker.sock \
+          --volume "${PWD}":/mnt \
+          --volume /mnt/node_modules \
+          "mcr.microsoft.com/playwright:${PLAYWRIGHT_VERSION}" \
+          /bin/bash <<EOF
+            set -Eeuxo pipefail
+            cd /mnt
+            npm install -g pnpm@10.33.0 && pnpm install --frozen-lockfile
+            pnpm exec playwright test ${GREP_PATTERN:+--grep "$GREP_PATTERN"}
+            exit 0
+        EOF
+      working-directory: ${{ inputs.working-directory }}
+      env:
+        # Only set TEST_TARGET if provided, otherwise playwright.config.ts uses its default
+        TEST_TARGET: ${{ inputs.test-target || '' }}
+        PODMAN_SOCKET: ${{ inputs.podman-socket }}
+        PLAYWRIGHT_VERSION: ${{ steps.playwright-version.outputs.version }}
+        GREP_PATTERN: ${{ inputs.grep-pattern }}
+
+    - name: Upload Playwright report
+      if: ${{ !cancelled() && inputs.upload-report == 'true' }}
+      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
+      with:
+        name: ${{ inputs.artifact-name }}
+        path: ${{ inputs.working-directory }}/playwright-report/
+        retention-days: ${{ inputs.artifact-retention-days }}

.github/actions/provision-k8s/action.yml

@@ -4,22 +4,17 @@ description: 'Installs cri-o and provisions a single-node Kubernetes cluster usi
 runs:
   using: "composite"
   steps:
+    - name: Install prerequisites
+      uses: ./.github/actions/apt-install
+      with:
+        packages: software-properties-common curl
+
     - name: Install cri-o
       id: install-crio
       shell: bash
       run: |
         set -Eeuxo pipefail
 
-        # the Microsoft repo's kubelet does not provide /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
-        #  [Service]
-        #  EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env
-        #  ExecStart=/usr/bin/kubelet $KUBELET_KUBEADM_ARGS
-        sudo ls /etc/apt/sources.list.d/
-        sudo rm /etc/apt/sources.list.d/microsoft-prod.list
-
-        sudo apt-get update
-        sudo apt-get install -y software-properties-common curl
-
         # https://github.com/cri-o/packaging?tab=readme-ov-file#distributions-using-deb-packages
 
         curl -fsSL https://pkgs.k8s.io/core:/stable:/v${KUBERNETES_VERSION}/deb/Release.key | \

.github/actions/setup-uv/action.yml

@@ -0,0 +1,23 @@
+---
+name: 'Setup uv and Python'
+description: 'Install uv (version from pyproject.toml) and Python (version from .python-version) with caching'
+runs:
+  using: composite
+  steps:
+    - name: Read Python version from .python-version
+      id: python-version
+      shell: python3 {0}
+      run: |
+        import os, pathlib
+        version = pathlib.Path(".python-version").read_text().strip().split("\n")[0]
+        with open(os.environ["GITHUB_OUTPUT"], "a") as f:
+            print(f"version={version}", file=f)
+
+    # https://github.com/astral-sh/setup-uv
+    - name: Install uv and Python
+      uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b  # v8.1.0
+      with:
+        version-file: pyproject.toml
+        python-version: ${{ steps.python-version.outputs.version }}
+        enable-cache: true
+        cache-dependency-glob: "uv.lock"

.pre-commit-config.yaml

@@ -2,10 +2,16 @@
 # https://github.com/pre-commit/pre-commit-hooks?tab=readme-ov-file#hooks-available
 repos:
   # https://docs.astral.sh/uv/guides/integration/pre-commit/
-  - repo: https://github.com/astral-sh/uv-pre-commit
-    rev: 0.9.18
+  # Using a local hook instead of uv-pre-commit so it uses the system uv
+  # installed by setup-uv, avoiding version mismatch with required-version.
+  - repo: local
     hooks:
       - id: uv-lock
+        name: uv-lock
+        entry: uv lock --locked
+        language: system
+        files: '(^uv\.lock$|^pyproject\.toml$|^uv\.toml$)'
+        pass_filenames: false
   # https://github.com/astral-sh/ruff-pre-commit
   - repo: https://github.com/astral-sh/ruff-pre-commit
     rev: v0.14.10

.tekton/odh-pipeline-runtime-datascience-cpu-py312-v3-0-push.yaml

@@ -41,7 +41,7 @@ spec:
   - name: dockerfile
     value: runtimes/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu
   - name: build-args-file
-    value: runtimes/datascience/ubi9-python-3.12/build-args/cpu.conf
+    value: runtimes/datascience/ubi9-python-3.12/build-args/konflux.cpu.conf
   - name: path-context
     value: .
   - name: hermetic

.tekton/odh-pipeline-runtime-minimal-cpu-py312-v3-0-push.yaml

@@ -40,7 +40,7 @@ spec:
   - name: dockerfile
     value: runtimes/minimal/ubi9-python-3.12/Dockerfile.konflux.cpu
   - name: build-args-file
-    value: runtimes/minimal/ubi9-python-3.12/build-args/cpu.conf
+    value: runtimes/minimal/ubi9-python-3.12/build-args/konflux.cpu.conf
   - name: path-context
     value: .
   - name: hermetic

.tekton/odh-pipeline-runtime-pytorch-cuda-py312-v3-0-push.yaml

@@ -40,7 +40,7 @@ spec:
   - name: dockerfile
     value: runtimes/pytorch/ubi9-python-3.12/Dockerfile.konflux.cuda
   - name: build-args-file
-    value: runtimes/pytorch/ubi9-python-3.12/build-args/cuda.conf
+    value: runtimes/pytorch/ubi9-python-3.12/build-args/konflux.cuda.conf
   - name: path-context
     value: .
   - name: hermetic

.tekton/odh-pipeline-runtime-pytorch-llmcompressor-cuda-py312-v3-0-push.yaml

@@ -39,7 +39,7 @@ spec:
   - name: dockerfile
     value: runtimes/pytorch+llmcompressor/ubi9-python-3.12/Dockerfile.konflux.cuda
   - name: build-args-file
-    value: runtimes/pytorch+llmcompressor/ubi9-python-3.12/build-args/cuda.conf
+    value: runtimes/pytorch+llmcompressor/ubi9-python-3.12/build-args/konflux.cuda.conf
   - name: path-context
     value: .
   - name: hermetic

.tekton/odh-pipeline-runtime-pytorch-rocm-py312-v3-0-push.yaml

@@ -39,7 +39,7 @@ spec:
   - name: dockerfile
     value: runtimes/rocm-pytorch/ubi9-python-3.12/Dockerfile.konflux.rocm
   - name: build-args-file
-    value: runtimes/rocm-pytorch/ubi9-python-3.12/build-args/rocm.conf
+    value: runtimes/rocm-pytorch/ubi9-python-3.12/build-args/konflux.rocm.conf
   - name: path-context
     value: .
   - name: hermetic

.tekton/odh-pipeline-runtime-tensorflow-cuda-py312-v3-0-push.yaml

@@ -40,7 +40,7 @@ spec:
   - name: dockerfile
     value: runtimes/tensorflow/ubi9-python-3.12/Dockerfile.konflux.cuda
   - name: build-args-file
-    value: runtimes/tensorflow/ubi9-python-3.12/build-args/cuda.conf
+    value: runtimes/tensorflow/ubi9-python-3.12/build-args/konflux.cuda.conf
   - name: path-context
     value: .
   - name: hermetic

.tekton/odh-pipeline-runtime-tensorflow-rocm-py312-v3-0-push.yaml

@@ -38,7 +38,7 @@ spec:
   - name: dockerfile
     value: runtimes/rocm-tensorflow/ubi9-python-3.12/Dockerfile.konflux.rocm
   - name: build-args-file
-    value: runtimes/rocm-tensorflow/ubi9-python-3.12/build-args/rocm.conf
+    value: runtimes/rocm-tensorflow/ubi9-python-3.12/build-args/konflux.rocm.conf
   - name: path-context
     value: .
   - name: hermetic

.tekton/odh-workbench-jupyter-datascience-cpu-py312-v3-0-push.yaml

@@ -40,7 +40,7 @@ spec:
   - name: dockerfile
     value: jupyter/datascience/ubi9-python-3.12/Dockerfile.konflux.cpu
   - name: build-args-file
-    value: jupyter/datascience/ubi9-python-3.12/build-args/cpu.conf
+    value: jupyter/datascience/ubi9-python-3.12/build-args/konflux.cpu.conf
   - name: path-context
     value: .
   - name: hermetic

.tekton/odh-workbench-jupyter-minimal-cpu-py312-v3-0-push.yaml

@@ -39,7 +39,7 @@ spec:
   - name: dockerfile
     value: jupyter/minimal/ubi9-python-3.12/Dockerfile.konflux.cpu
   - name: build-args-file
-    value: jupyter/minimal/ubi9-python-3.12/build-args/cpu.conf
+    value: jupyter/minimal/ubi9-python-3.12/build-args/konflux.cpu.conf
   - name: path-context
     value: .
   - name: hermetic

.tekton/odh-workbench-jupyter-minimal-cuda-py312-v3-0-push.yaml

@@ -40,7 +40,7 @@ spec:
   - name: dockerfile
     value: jupyter/minimal/ubi9-python-3.12/Dockerfile.konflux.cuda
   - name: build-args-file
-    value: jupyter/minimal/ubi9-python-3.12/build-args/cuda.conf
+    value: jupyter/minimal/ubi9-python-3.12/build-args/konflux.cuda.conf
   - name: path-context
     value: .
   - name: hermetic

.tekton/odh-workbench-jupyter-minimal-rocm-py312-v3-0-push.yaml

@@ -39,7 +39,7 @@ spec:
   - name: dockerfile
     value: jupyter/minimal/ubi9-python-3.12/Dockerfile.konflux.rocm
   - name: build-args-file
-    value: jupyter/minimal/ubi9-python-3.12/build-args/rocm.conf
+    value: jupyter/minimal/ubi9-python-3.12/build-args/konflux.rocm.conf
   - name: path-context
     value: .
   - name: hermetic

.tekton/odh-workbench-jupyter-pytorch-cuda-py312-v3-0-push.yaml

@@ -44,7 +44,7 @@ spec:
   - name: dockerfile
     value: jupyter/pytorch/ubi9-python-3.12/Dockerfile.konflux.cuda
   - name: build-args-file
-    value: jupyter/pytorch/ubi9-python-3.12/build-args/cuda.conf
+    value: jupyter/pytorch/ubi9-python-3.12/build-args/konflux.cuda.conf
   - name: path-context
     value: .
   - name: hermetic

.tekton/odh-workbench-jupyter-pytorch-llmcompressor-cuda-py312-v3-0-push.yaml

@@ -45,7 +45,7 @@ spec:
   - name: dockerfile
     value: jupyter/pytorch+llmcompressor/ubi9-python-3.12/Dockerfile.konflux.cuda
   - name: build-args-file
-    value: jupyter/pytorch+llmcompressor/ubi9-python-3.12/build-args/cuda.conf
+    value: jupyter/pytorch+llmcompressor/ubi9-python-3.12/build-args/konflux.cuda.conf
   - name: path-context
     value: .
   - name: hermetic

.tekton/odh-workbench-jupyter-pytorch-rocm-py312-v3-0-push.yaml

@@ -43,7 +43,7 @@ spec:
   - name: dockerfile
     value: jupyter/rocm/pytorch/ubi9-python-3.12/Dockerfile.konflux.rocm
   - name: build-args-file
-    value: jupyter/rocm/pytorch/ubi9-python-3.12/build-args/rocm.conf
+    value: jupyter/rocm/pytorch/ubi9-python-3.12/build-args/konflux.rocm.conf
   - name: path-context
     value: .
   - name: hermetic

.tekton/odh-workbench-jupyter-tensorflow-cuda-py312-v3-0-push.yaml

@@ -43,7 +43,7 @@ spec:
   - name: dockerfile
     value: jupyter/tensorflow/ubi9-python-3.12/Dockerfile.konflux.cuda
   - name: build-args-file
-    value: jupyter/tensorflow/ubi9-python-3.12/build-args/cuda.conf
+    value: jupyter/tensorflow/ubi9-python-3.12/build-args/konflux.cuda.conf
   - name: path-context
     value: .
   - name: hermetic

.tekton/odh-workbench-jupyter-tensorflow-rocm-py312-v3-0-push.yaml

@@ -43,7 +43,7 @@ spec:
   - name: dockerfile
     value: jupyter/rocm/tensorflow/ubi9-python-3.12/Dockerfile.konflux.rocm
   - name: build-args-file
-    value: jupyter/rocm/tensorflow/ubi9-python-3.12/build-args/rocm.conf
+    value: jupyter/rocm/tensorflow/ubi9-python-3.12/build-args/konflux.rocm.conf
   - name: path-context
     value: .
   - name: hermetic