diff --git a/agents/opencode/deployment/Containerfile.mlflow b/agents/opencode/deployment/Containerfile.mlflow new file mode 100644 index 00000000..e18ea76d --- /dev/null +++ b/agents/opencode/deployment/Containerfile.mlflow @@ -0,0 +1,29 @@ +# OpenCode with MLflow tracing support +# +# Extends the pre-built OpenCode image with: +# - mlflow[kubernetes] for experiment creation and SA token auth +# - Pre-built @mlflow/core from v3.14.0 (workspace header fix) + +FROM quay.io/opendatahub/odh-opencode-rhel9:1.4.4 + +USER 0 + +RUN pip3 install --no-cache-dir 'mlflow[kubernetes]==3.14.*' + +# Build @mlflow/core and @mlflow/opencode from v3.14.0. +# The workspace header fix (mlflow#23927) is in the mlflow 3.14 pip package +# but the npm plugin bundle (@mlflow/opencode) has not been re-released +# with this fix yet. Building from the release tag includes it. +# The base image already has nodejs 16 — just need npm. +RUN microdnf install -y --nodocs npm \ + && git clone --depth 1 --branch v3.14.0 https://github.com/mlflow/mlflow.git /tmp/mlflow \ + && cd /tmp/mlflow/libs/typescript && npm install && npm run build \ + && mkdir -p /opt/mlflow-plugin/core /opt/mlflow-plugin/opencode \ + && cp -r /tmp/mlflow/libs/typescript/core/dist /opt/mlflow-plugin/core/ \ + && cp /tmp/mlflow/libs/typescript/core/package.json /opt/mlflow-plugin/core/ \ + && cp -r /tmp/mlflow/libs/typescript/integrations/opencode/dist /opt/mlflow-plugin/opencode/ \ + && cp /tmp/mlflow/libs/typescript/integrations/opencode/package.json /opt/mlflow-plugin/opencode/ \ + && rm -rf /tmp/mlflow \ + && microdnf clean all + +USER 1001 diff --git a/agents/opencode/deployment/docs/mlflow-tracing-setup.md b/agents/opencode/deployment/docs/mlflow-tracing-setup.md new file mode 100644 index 00000000..f71ec1eb --- /dev/null +++ b/agents/opencode/deployment/docs/mlflow-tracing-setup.md @@ -0,0 +1,94 @@ +# MLflow Tracing Setup for OpenCode on RHOAI + +Enable MLflow tracing for OpenCode on Red Hat OpenShift AI. Works in both web and CLI modes. + +For trace schema, backend comparisons, and latency benchmarks, see [mlflow-tracing.md](mlflow-tracing.md). + +## Prerequisites + +- OpenCode deployment manifests from [aicatalyst-team/opencode-openshift](https://github.com/aicatalyst-team/opencode-openshift) +- MLflow instance running on your RHOAI cluster with a workspace matching your namespace + +## Setup + +### 1. Build the image + +Follow the [aicatalyst-team/opencode-openshift](https://github.com/aicatalyst-team/opencode-openshift) deployment guide, but use [`Containerfile.mlflow`](../Containerfile.mlflow) instead of the base Containerfile. It extends the base OpenCode image with `mlflow[kubernetes]` and the pre-built `@mlflow/opencode` plugin. + +### 2. Grant RBAC + +Grant the `edit` role to the service account used by the deployment. Check your deployment's `serviceAccountName` — it may be `default` or a named account like `opencode-web`: + +```bash +oc adm policy add-role-to-user edit -z -n +``` + +### 3. Configure MLflow env vars + +Find your MLflow tracking URI: + +```bash +oc get svc -A | grep mlflow +# Use the service name, namespace, and port to construct: +# https://..svc:/mlflow +``` + +Set these env vars on the deployment: + +| Env var | What to set | +|---|---| +| `MLFLOW_TRACKING_URI` | The URI from the command above | +| `MLFLOW_WORKSPACE` | Your namespace / project name | +| `MLFLOW_EXPERIMENT_NAME` | Name for your experiment (e.g., `opencode-traces`) | +| `MLFLOW_TRACKING_INSECURE_TLS` | `true` for dev/test, remove for production with proper TLS | +| `NODE_TLS_REJECT_UNAUTHORIZED` | `0` for dev/test, remove for production with proper TLS | +| `OPENCODE_MODE` | `web` (default) or `cli` for terminal mode | + +### 4. Deploy + +**If deploying for the first time:** Add the env vars above and the MLflow entrypoint block from [`entrypoint-patch.yaml`](../overlays/mlflow-tracing/entrypoint-patch.yaml) to your deployment manifests before applying. + +**If OpenCode is already deployed:** Make sure the deployment is using the image built from [`Containerfile.mlflow`](../Containerfile.mlflow). Then clone the [opencode-openshift](https://github.com/aicatalyst-team/opencode-openshift) repo and apply the [`overlays/mlflow-tracing/`](../overlays/mlflow-tracing/) kustomize overlay: + +```bash +# Clone the base manifests (if not already cloned) +git clone https://github.com/aicatalyst-team/opencode-openshift.git + +# Edit kustomization.yaml — set namespace and adjust the manifests path +# Edit deployment-patch.yaml — set the env vars above +oc apply -k overlays/mlflow-tracing/ +oc rollout restart deployment/ +``` + +## Verify + +```bash +# Check startup logs +oc logs deployment/ | grep -i mlflow + +# CLI mode +oc exec deployment/ -- bash -c ' + source $HOME/.mlflow-env 2>/dev/null + cd /opt/app-root/workspace + opencode run "What is 2+2?" +' + +# Web mode: send a message in the browser, then check MLflow +``` + +## View traces + +```bash +oc get consolelink mlflow -o jsonpath='{.spec.href}' +``` + +Navigate to your workspace and experiment name to see traces. + +## Key notes + +- The image must be built with [`Containerfile.mlflow`](../Containerfile.mlflow) +- `MLFLOW_TRACKING_TOKEN` is read from the pod's SA token automatically +- The entrypoint handles plugin install, experiment creation, and auth — no manual plugin setup needed +- Until the next `@mlflow/opencode` npm release includes the workspace header fix, the entrypoint replaces the cached plugin with a build from the v3.14.0 release tag + +> **TLS note:** `NODE_TLS_REJECT_UNAUTHORIZED=0` disables TLS verification for all Node.js connections in the container, not just MLflow. For production, use `NODE_EXTRA_CA_CERTS` instead. Upstream work is in progress to support `MLFLOW_TRACKING_INSECURE_TLS` and `MLFLOW_TRACKING_SERVER_CERT_PATH` scoped to MLflow connections only ([mlflow#24140](https://github.com/mlflow/mlflow/issues/24140)). Kubernetes-native auth (`MLFLOW_TRACKING_AUTH`) is also being added to the TypeScript SDK ([mlflow#24141](https://github.com/mlflow/mlflow/issues/24141)). diff --git a/agents/opencode/deployment/docs/mlflow-tracing.md b/agents/opencode/deployment/docs/mlflow-tracing.md new file mode 100644 index 00000000..802951ec --- /dev/null +++ b/agents/opencode/deployment/docs/mlflow-tracing.md @@ -0,0 +1,73 @@ +# MLflow Tracing for OpenCode on RHOAI + +> **Setup instructions:** See [mlflow-tracing-setup.md](mlflow-tracing-setup.md). + +MLflow tracing for OpenCode uses the `@mlflow/opencode` plugin, which hooks into OpenCode's `session.idle` events and exports traces to MLflow. It works in both web and CLI modes with zero impact on agent response times. + +To validate the tracing stack, the prompt **"build me a tetris game in html and css"** was run through two backends: vLLM directly and OGX routing to vLLM. Both produced the same trace schema. + +--- + +## Trace Schema + +```text +opencode_conversation (root) ++-- llm_call -- LLM inference (model, messages, tokens) ++-- tool_write -- file write (path, content) ++-- llm_call -- next inference turn ++-- tool_bash -- shell command (command, output) ++-- tool_write -- another file write ++-- llm_call -- final response +``` + +Each span captures: tool name, input parameters, output/result, and per-span latency. + +| Field | Captured | +|---|---| +| Session ID | Yes | +| Total duration | Yes | +| Total tokens | Yes | +| Tool call sequence (waterfall) | Yes | +| Model | Yes | +| Status | Yes | + +> **Privacy note:** Traces capture full prompt and response text, which may contain secrets, PII, or proprietary code. Treat the MLflow experiment store as sensitive data. + +--- + +## Results: "Build me a Tetris game" across Both Backends + +### vLLM direct (`gpt-oss-120b`) + +| Metric | Value | +|---|---| +| Tokens | 33,623 | +| Latency | 19.28s | +| Spans | 8 | + +![vLLM trace](screenshots/vllm-tetris-trace.png) + +--- + +### OGX to vLLM (`vllm/gpt-oss-120b`) + +| Metric | Value | +|---|---| +| Tokens | 62,623 | +| Latency | 29.35s | +| Spans | 12 | + +![OGX trace](screenshots/ogx-tetris-trace.png) + +--- + +## Latency Comparison: vLLM Direct vs OGX to vLLM + +The prompt **"What is the capital of France? One word only."** was run 5 times through each path. The first run was excluded as a cold start warmup. + +| Path | Run 2 | Run 3 | Run 4 | Run 5 | Avg (runs 2 to 5) | +|---|---|---|---|---|---| +| **vLLM direct** | 0.363s | 0.352s | 0.347s | 0.361s | **0.356s** | +| **OGX to vLLM** | 0.189s | 0.428s | 0.389s | 0.365s | **0.343s** | + +No measurable latency difference. OGX acts as a thin passthrough with no transformation overhead. diff --git a/agents/opencode/deployment/docs/screenshots/ogx-tetris-trace.png b/agents/opencode/deployment/docs/screenshots/ogx-tetris-trace.png new file mode 100644 index 00000000..ea02cccb Binary files /dev/null and b/agents/opencode/deployment/docs/screenshots/ogx-tetris-trace.png differ diff --git a/agents/opencode/deployment/docs/screenshots/vllm-tetris-trace.png b/agents/opencode/deployment/docs/screenshots/vllm-tetris-trace.png new file mode 100644 index 00000000..44d2ff7b Binary files /dev/null and b/agents/opencode/deployment/docs/screenshots/vllm-tetris-trace.png differ diff --git a/agents/opencode/deployment/overlays/mlflow-tracing/deployment-patch.yaml b/agents/opencode/deployment/overlays/mlflow-tracing/deployment-patch.yaml new file mode 100644 index 00000000..cdb2455e --- /dev/null +++ b/agents/opencode/deployment/overlays/mlflow-tracing/deployment-patch.yaml @@ -0,0 +1,25 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: opencode-web +spec: + template: + spec: + containers: + - name: opencode-web + env: + - name: MLFLOW_TRACKING_URI + value: "https://mlflow..svc:8443/mlflow" + - name: MLFLOW_WORKSPACE + value: "YOUR-NAMESPACE" + - name: MLFLOW_EXPERIMENT_NAME + value: "opencode-traces" + - name: MLFLOW_TRACKING_INSECURE_TLS + value: "true" # dev/test only + # TLS: uncomment ONE of the following for clusters with non-public certs. + # Option A (production): mount the cluster CA bundle + # - name: NODE_EXTRA_CA_CERTS + # value: "/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt" + # Option B (dev/test only): disable TLS for ALL Node.js connections + # - name: NODE_TLS_REJECT_UNAUTHORIZED + # value: "0" diff --git a/agents/opencode/deployment/overlays/mlflow-tracing/entrypoint-patch.yaml b/agents/opencode/deployment/overlays/mlflow-tracing/entrypoint-patch.yaml new file mode 100644 index 00000000..3414d262 --- /dev/null +++ b/agents/opencode/deployment/overlays/mlflow-tracing/entrypoint-patch.yaml @@ -0,0 +1,87 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: opencode-web-entrypoint +data: + entrypoint.sh: | + #!/bin/bash + set -euo pipefail + + git config --global init.defaultBranch main + git config --global user.email "opencode@openshift.local" + git config --global user.name "OpenCode" + git config --global --add safe.directory /opt/app-root/workspace + + cd /opt/app-root/workspace + if [ ! -d .git ]; then + git init + git commit --allow-empty -m "init" + fi + + CONFIG=$(cat /config-template/config-template.json) + CONFIG=${CONFIG//\$\{BASE_URL\}/$BASE_URL} + CONFIG=${CONFIG//\$\{API_KEY\}/$API_KEY} + CONFIG=${CONFIG//\$\{MODEL_NAME\}/$MODEL_NAME} + + if [ -f /mcp-config/mcp-servers.json ]; then + MCP_SERVERS=$(cat /mcp-config/mcp-servers.json) + CONFIG=$(echo "$CONFIG" | jq --argjson mcp "$MCP_SERVERS" '. + {mcp: $mcp}') + fi + + # =========================================== + # MLflow Tracing (enabled when MLFLOW_TRACKING_URI is set) + # =========================================== + if [ -n "${MLFLOW_TRACKING_URI:-}" ]; then + echo "[entrypoint] Configuring MLflow tracing" + + CONFIG=$(echo "$CONFIG" | jq '. + {plugin: ["@mlflow/opencode"]}') + + opencode plugin @mlflow/opencode 2>&1 \ + && echo "[entrypoint] MLflow plugin installed" \ + || echo "[entrypoint] WARN: MLflow plugin install failed" + + CACHED_CORE=$(find "$HOME/.cache/opencode/packages" -path "*/@mlflow/core/dist" -type d 2>/dev/null | head -1) + CACHED_OC=$(find "$HOME/.cache/opencode/packages" -path "*/@mlflow/opencode/dist" -type d 2>/dev/null | head -1) + [ -d "/opt/mlflow-plugin/core/dist" ] && [ -n "${CACHED_CORE:-}" ] && cp -r /opt/mlflow-plugin/core/dist/* "$CACHED_CORE/" + [ -d "/opt/mlflow-plugin/opencode/dist" ] && [ -n "${CACHED_OC:-}" ] && cp -r /opt/mlflow-plugin/opencode/dist/* "$CACHED_OC/" + + SA_TOKEN_PATH="/var/run/secrets/kubernetes.io/serviceaccount/token" + [ -f "$SA_TOKEN_PATH" ] && export MLFLOW_TRACKING_TOKEN=$(cat "$SA_TOKEN_PATH") + + if [ -n "${MLFLOW_EXPERIMENT_NAME:-}" ]; then + EXPERIMENT_ID=$(python3 -c ' + import warnings; warnings.filterwarnings("ignore") + import os, mlflow + os.environ.setdefault("MLFLOW_TRACKING_INSECURE_TLS", "false") + mlflow.set_tracking_uri(os.environ["MLFLOW_TRACKING_URI"]) + name = os.environ["MLFLOW_EXPERIMENT_NAME"] + exp = mlflow.get_experiment_by_name(name) + print(exp.experiment_id if exp else mlflow.create_experiment(name)) + ' 2>/dev/null) && export MLFLOW_EXPERIMENT_ID="$EXPERIMENT_ID" \ + && echo "[entrypoint] MLflow experiment: ${MLFLOW_EXPERIMENT_NAME} (ID: ${MLFLOW_EXPERIMENT_ID})" \ + || echo "[entrypoint] WARN: Could not resolve MLflow experiment ID" + fi + + cat > "$HOME/.mlflow-env" </dev/null) + export MLFLOW_EXPERIMENT_ID="${MLFLOW_EXPERIMENT_ID:-}" + MLENV + echo 'source $HOME/.mlflow-env 2>/dev/null' >> "$HOME/.bashrc" 2>/dev/null || true + fi + + export OPENCODE_CONFIG_CONTENT="$CONFIG" + + MODE="${OPENCODE_MODE:-web}" + case "$MODE" in + web) + exec opencode web --hostname 0.0.0.0 --port 8003 + ;; + cli) + echo "[entrypoint] CLI mode — attach with: oc exec -it deployment/ -- bash -c 'source \$HOME/.mlflow-env && cd /opt/app-root/workspace && opencode'" + exec sleep infinity + ;; + *) + echo "[entrypoint] Unknown mode: $MODE" + exit 1 + ;; + esac diff --git a/agents/opencode/deployment/overlays/mlflow-tracing/kustomization.yaml b/agents/opencode/deployment/overlays/mlflow-tracing/kustomization.yaml new file mode 100644 index 00000000..78615716 --- /dev/null +++ b/agents/opencode/deployment/overlays/mlflow-tracing/kustomization.yaml @@ -0,0 +1,21 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# Base OpenCode manifests + MLflow tracing +# Clone https://github.com/aicatalyst-team/opencode-openshift and adjust +# the path below to point to its manifests/ directory. +resources: + - ../../manifests # adjust to your local opencode-openshift/manifests path + +namespace: YOUR-NAMESPACE + +patches: + - target: + kind: Deployment + name: opencode-web + path: deployment-patch.yaml + + - target: + kind: ConfigMap + name: opencode-web-entrypoint + path: entrypoint-patch.yaml