From 77dd2eda54b104b6585222ab4b86454c443fad5b Mon Sep 17 00:00:00 2001 From: Lee Yarwood Date: Mon, 5 Jan 2026 14:32:30 +0000 Subject: [PATCH 1/2] build: Add KubeVirt integration for local development environment Add comprehensive KubeVirt support to enable local testing of VM management tools with a complete Kind + KubeVirt setup. New Makefile Targets: - local-env-setup-kubevirt: Complete environment setup including KubeVirt - kubevirt-install: Install KubeVirt operator and CDI - kubevirt-uninstall: Remove KubeVirt and CDI from cluster - kubevirt-status: Display KubeVirt, CDI, and VM status KubeVirt Installation (build/kubevirt.mk): - Installs KubeVirt v1.7.0 operator and custom resource - Installs CDI (Containerized Data Importer) v1.64.0 for disk management - Waits for components to be ready before proceeding - Provides status checking for all KubeVirt resources The local-env-setup-kubevirt target orchestrates a complete setup: 1. Creates Kind cluster with ingress and cert-manager 2. Installs KubeVirt and CDI 3. Builds the MCP server binary This enables developers to test VM lifecycle management tools (start, stop, restart, create) in a local Kubernetes environment. Assisted-By: Claude Signed-off-by: Lee Yarwood --- Makefile | 23 ++++++++++++++ build/kubevirt.mk | 78 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+) create mode 100644 build/kubevirt.mk diff --git a/Makefile b/Makefile index c6709b6e1..97e0b09e9 100644 --- a/Makefile +++ b/Makefile @@ -112,6 +112,29 @@ local-env-setup: ## Setup complete local development environment with Kind clust @echo "Or run with MCP inspector:" @echo " npx @modelcontextprotocol/inspector@latest \$$(pwd)/$(BINARY_NAME) --config _output/config.toml" +.PHONY: local-env-setup-kubevirt +local-env-setup-kubevirt: ## Setup complete local development environment with Kind cluster and KubeVirt + @echo "=========================================" + @echo "Kubernetes MCP Server - Local Setup" + @echo " with KubeVirt" + @echo "=========================================" + $(MAKE) kind-create-cluster + $(MAKE) kubevirt-install + $(MAKE) build + @echo "" + @echo "=========================================" + @echo "Local environment ready!" + @echo "=========================================" + @echo "" + @echo "Run the MCP server with:" + @echo " ./$(BINARY_NAME)" + @echo "" + @echo "Or run with MCP inspector:" + @echo " npx @modelcontextprotocol/inspector@latest \$$(pwd)/$(BINARY_NAME)" + @echo "" + @echo "KubeVirt is now available!" + @echo "Check status with: make kubevirt-status" + .PHONY: local-env-teardown local-env-teardown: ## Tear down the local Kind cluster $(MAKE) kind-delete-cluster diff --git a/build/kubevirt.mk b/build/kubevirt.mk new file mode 100644 index 000000000..663c46453 --- /dev/null +++ b/build/kubevirt.mk @@ -0,0 +1,78 @@ +# KubeVirt installation and management + +# KubeVirt version configuration +KUBEVIRT_VERSION ?= v1.7.0 +CDI_VERSION ?= v1.64.0 + +# Detect if we're using a released version or main/latest +KUBEVIRT_RELEASE_URL = https://github.com/kubevirt/kubevirt/releases/download/$(KUBEVIRT_VERSION) +CDI_RELEASE_URL = https://github.com/kubevirt/containerized-data-importer/releases/download/$(CDI_VERSION) + +##@ KubeVirt + +.PHONY: kubevirt-install +kubevirt-install: ## Install KubeVirt and CDI on the cluster + @echo "=========================================" + @echo "Installing KubeVirt $(KUBEVIRT_VERSION)" + @echo "=========================================" + @echo "" + @echo "Installing KubeVirt operator..." + @kubectl apply -f $(KUBEVIRT_RELEASE_URL)/kubevirt-operator.yaml + @echo "" + @echo "Installing KubeVirt CR..." + @kubectl apply -f $(KUBEVIRT_RELEASE_URL)/kubevirt-cr.yaml + @echo "" + @echo "Waiting for KubeVirt to become ready (this can take a few minutes)..." + @kubectl -n kubevirt wait kv kubevirt --for condition=Available --timeout=15m + @echo "✅ KubeVirt is ready" + @echo "" + @echo "Installing CDI (Containerized Data Importer) $(CDI_VERSION)..." + @kubectl apply -f $(CDI_RELEASE_URL)/cdi-operator.yaml + @kubectl apply -f $(CDI_RELEASE_URL)/cdi-cr.yaml + @echo "" + @echo "Waiting for CDI to become ready..." + @kubectl wait --for=condition=Available cdi/cdi -n cdi --timeout=5m + @echo "✅ CDI is ready" + @echo "" + @echo "=========================================" + @echo "KubeVirt Installation Complete" + @echo "=========================================" + @echo "" + @echo "KubeVirt version: $(KUBEVIRT_VERSION)" + @echo "CDI version: $(CDI_VERSION)" + @echo "" + @echo "Verify installation with:" + @echo " kubectl get kubevirt -n kubevirt" + @echo " kubectl get cdi -n cdi" + @echo "" + +.PHONY: kubevirt-uninstall +kubevirt-uninstall: ## Uninstall KubeVirt and CDI from the cluster + @echo "Uninstalling KubeVirt and CDI..." + @kubectl delete -f $(KUBEVIRT_RELEASE_URL)/kubevirt-cr.yaml --ignore-not-found + @kubectl delete -f $(KUBEVIRT_RELEASE_URL)/kubevirt-operator.yaml --ignore-not-found + @kubectl delete -f $(CDI_RELEASE_URL)/cdi-cr.yaml --ignore-not-found + @kubectl delete -f $(CDI_RELEASE_URL)/cdi-operator.yaml --ignore-not-found + @echo "✅ KubeVirt and CDI uninstalled" + +.PHONY: kubevirt-status +kubevirt-status: ## Show KubeVirt and CDI status + @echo "=========================================" + @echo "KubeVirt Status" + @echo "=========================================" + @echo "" + @echo "KubeVirt:" + @kubectl get kubevirt -n kubevirt -o wide || { echo "KubeVirt not installed"; exit 1; } + @echo "" + @echo "CDI:" + @kubectl get cdi -n cdi -o wide || { echo "CDI not installed"; exit 1; } + @echo "" + @echo "KubeVirt Pods:" + @kubectl get pods -n kubevirt + @echo "" + @echo "CDI Pods:" + @kubectl get pods -n cdi + @echo "" + @echo "VirtualMachines (all namespaces):" + @kubectl get virtualmachines --all-namespaces || echo "No VirtualMachines found" + @echo "" From 43600cf74e4aebfbe4a88db59e9e2fbed3dabe79 Mon Sep 17 00:00:00 2001 From: Lee Yarwood Date: Mon, 12 Jan 2026 20:09:52 +0000 Subject: [PATCH 2/2] kubevirt: Add eval tasks for VM creation and lifecycle Add evaluation tasks for testing KubeVirt virtual machine operations through the kubernetes-mcp-server's vm_create and vm_lifecycle tools. Tasks added: - create-vm-basic: Create a basic Fedora VM - create-vm-ubuntu: Create an Ubuntu VM - create-vm-with-instancetype: Create VM with specific instance type - create-vm-with-size: Create VM with size hint - delete-vm: Delete a running VM - pause-vm: Pause and unpause a VM - update-vm-resources: Modify CPU and memory of existing VM Infrastructure: - Shared verification helpers for VM state validation - Test functions for instancetype, resources, and networking - Support for both container disk and DataSource-based VMs These tasks validate that AI agents can correctly use the KubeVirt MCP tools to manage virtual machine lifecycles, including proper use of modern runStrategy field instead of deprecated 'running' field. Assisted-By: Claude Signed-off-by: Lee Yarwood --- build/kubevirt.mk | 3 + evals/tasks/README.md | 6 +- evals/tasks/kubevirt/README.md | 60 +++++ .../tasks/kubevirt/create-vm-basic/task.yaml | 37 +++ .../tasks/kubevirt/create-vm-ubuntu/task.yaml | 37 +++ .../create-vm-with-instancetype/task.yaml | 43 +++ .../kubevirt/create-vm-with-size/task.yaml | 43 +++ evals/tasks/kubevirt/delete-vm/task.yaml | 53 ++++ evals/tasks/kubevirt/helpers/README.md | 219 +++++++++++++++ evals/tasks/kubevirt/helpers/verify-vm.sh | 254 ++++++++++++++++++ evals/tasks/kubevirt/pause-vm/task.yaml | 54 ++++ .../kubevirt/update-vm-resources/task.yaml | 72 +++++ 12 files changed, 878 insertions(+), 3 deletions(-) create mode 100644 evals/tasks/kubevirt/README.md create mode 100644 evals/tasks/kubevirt/create-vm-basic/task.yaml create mode 100644 evals/tasks/kubevirt/create-vm-ubuntu/task.yaml create mode 100644 evals/tasks/kubevirt/create-vm-with-instancetype/task.yaml create mode 100644 evals/tasks/kubevirt/create-vm-with-size/task.yaml create mode 100644 evals/tasks/kubevirt/delete-vm/task.yaml create mode 100644 evals/tasks/kubevirt/helpers/README.md create mode 100644 evals/tasks/kubevirt/helpers/verify-vm.sh create mode 100644 evals/tasks/kubevirt/pause-vm/task.yaml create mode 100644 evals/tasks/kubevirt/update-vm-resources/task.yaml diff --git a/build/kubevirt.mk b/build/kubevirt.mk index 663c46453..caa100b89 100644 --- a/build/kubevirt.mk +++ b/build/kubevirt.mk @@ -76,3 +76,6 @@ kubevirt-status: ## Show KubeVirt and CDI status @echo "VirtualMachines (all namespaces):" @kubectl get virtualmachines --all-namespaces || echo "No VirtualMachines found" @echo "" + @echo "VirtualMachineInstances (all namespaces):" + @kubectl get virtualmachineinstances --all-namespaces || echo "No VirtualMachineInstances found" + @echo "" diff --git a/evals/tasks/README.md b/evals/tasks/README.md index f7d259a25..faba13959 100644 --- a/evals/tasks/README.md +++ b/evals/tasks/README.md @@ -5,11 +5,12 @@ This directory hosts the reusable task scenarios that power MCP evaluations for ## Task Families - [Kubernetes tasks](kubernetes/) – core cluster workflows such as creating pods, fixing deployments, managing RBAC, or debugging state issues. -- [Kiali tasks](kiali/) – service-mesh and observability workflows that exercise the Kiali MCP toolset (Istio config, topology, mesh health, tracing). +- [Kiali tasks](kiali/) – service-mesh and observability workflows that exercise the Kiali MCP toolset (Istio config, topology, mesh health, tracing). +- [KubeVirt tasks](kubevirt/) – virtual machine management workflows that exercise the KubeVirt MCP toolset (VM creation, lifecycle management, resource updates). ## Anatomy of a Task -Every subdirectory under `kubernetes/` or `kiali/` defines a single scenario: +Every subdirectory under `kubernetes/`, `kiali/`, or `kubevirt/` defines a single scenario: 1. `*.yaml` – declarative description consumed by the evaluation harness (prompts, success criteria, required tools). 2. `setup.sh` / `verify.sh` / `cleanup.sh` – shell hooks (optional) that prime the cluster, assert post-conditions, and reset resources so tasks stay idempotent. @@ -33,4 +34,3 @@ When a new MCP toolset lands , keep its evaluations isolated by creating a sibli 3. Any shared fixtures the stack needs (place them in a `shared/` subdirectory if multiple scenarios reuse them). This structure keeps task stacks discoverable and lets eval harnesses target toolset-specific workflows without mixing concerns from the core Kubernetes or Kiali libraries. - diff --git a/evals/tasks/kubevirt/README.md b/evals/tasks/kubevirt/README.md new file mode 100644 index 000000000..ba8699ad9 --- /dev/null +++ b/evals/tasks/kubevirt/README.md @@ -0,0 +1,60 @@ +# KubeVirt Task Stack + +KubeVirt-focused MCP tasks live here. Each folder under this directory represents a self-contained scenario that exercises the KubeVirt toolset (virtual machine creation, lifecycle management, troubleshooting). + +## Adding a New Task + +1. Create a new subdirectory (e.g., `create-vm-foo/`) and place the scenario YAML plus any helper scripts or artifacts inside it. +2. Make sure the YAML's `metadata` block includes `name` and `difficulty` so it shows up correctly in the catalog below. +3. Keep prompts concise and action-oriented; verification commands should rely on KubeVirt resources and helper functions whenever possible. + +## Tasks Defined + +### VM Creation + +- **[easy] create-vm-basic** - Create a basic Fedora virtual machine + - **Prompt:** *Please create a Fedora virtual machine named test-vm in the vm-test namespace.* + +- **[easy] create-vm-ubuntu** - Create an Ubuntu virtual machine + - **Prompt:** *Create an Ubuntu virtual machine named ubuntu-vm in the vm-test namespace.* + +- **[easy] create-vm-with-instancetype** - Create a VM using VirtualMachineInstancetype + - **Prompt:** *Create a Fedora virtual machine with specific instance types and preferences.* + +- **[easy] create-vm-with-size** - Create a VM with specific size requirements + - **Prompt:** *Create a virtual machine with custom CPU and memory specifications.* + +### VM Lifecycle Management + +- **[medium] pause-vm** - Pause a running virtual machine + - **Prompt:** *Please pause the virtual machine named paused-vm in the vm-test namespace.* + +- **[medium] delete-vm** - Delete a virtual machine + - **Prompt:** *Please delete the virtual machine named deleted-vm in the vm-test namespace.* + +### VM Modification + +- **[hard] update-vm-resources** - Update VM CPU and memory resources + - **Prompt:** *A VirtualMachine named test-vm-update exists in the vm-test namespace. It currently has 1 vCPU and 2Gi of memory. Please update the VirtualMachine to add an additional vCPU (making it 2 vCPUs total) and increase the memory to at least 3Gi.* + +## Helper Scripts + +Many tasks rely on helper scripts located in `evals/tasks/kubevirt/helpers/`: + +- `verify-vm.sh` - Common VM verification functions used across multiple test scenarios + +## Running Tasks + +These tasks are designed to be used with the gevals evaluation framework. Each task includes: + +- **setup** - Prepares the test environment (creates namespace, sets up initial VM state) +- **verify** - Validates the expected outcome after the agent completes the task +- **cleanup** - Removes resources created during the test +- **prompt** - The instruction given to the AI agent + +Example workflow: + +1. Setup creates the initial state +2. Agent receives the prompt and executes actions using MCP tools +3. Verify checks if the agent accomplished the goal +4. Cleanup removes test resources diff --git a/evals/tasks/kubevirt/create-vm-basic/task.yaml b/evals/tasks/kubevirt/create-vm-basic/task.yaml new file mode 100644 index 000000000..8be170e83 --- /dev/null +++ b/evals/tasks/kubevirt/create-vm-basic/task.yaml @@ -0,0 +1,37 @@ +kind: Task +metadata: + name: "create-basic-vm" + difficulty: easy +steps: + setup: + inline: |- + #!/usr/bin/env bash + NS="${EVAL_NAMESPACE:-vm-test}" + kubectl delete namespace "$NS" --ignore-not-found + kubectl create namespace "$NS" + verify: + inline: |- + #!/usr/bin/env bash + source evals/tasks/kubevirt/helpers/verify-vm.sh + NS="${EVAL_NAMESPACE:-vm-test}" + + # Wait for VirtualMachine to be created + verify_vm_exists "test-vm" "$NS" || exit 1 + + # Verify container disk is Fedora + verify_container_disk "test-vm" "$NS" "fedora" || exit 1 + + # Verify runStrategy is set and deprecated 'running' field is not used + verify_run_strategy "test-vm" "$NS" || exit 1 + verify_no_deprecated_running_field "test-vm" "$NS" || exit 1 + + echo "All validations passed" + exit 0 + cleanup: + inline: |- + #!/usr/bin/env bash + NS="${EVAL_NAMESPACE:-vm-test}" + kubectl delete virtualmachine test-vm -n "$NS" --ignore-not-found + kubectl delete namespace "$NS" --ignore-not-found + prompt: + inline: Create a Fedora virtual machine named test-vm in the ${EVAL_NAMESPACE:-vm-test} namespace. diff --git a/evals/tasks/kubevirt/create-vm-ubuntu/task.yaml b/evals/tasks/kubevirt/create-vm-ubuntu/task.yaml new file mode 100644 index 000000000..6f6b35ec7 --- /dev/null +++ b/evals/tasks/kubevirt/create-vm-ubuntu/task.yaml @@ -0,0 +1,37 @@ +kind: Task +metadata: + name: "create-ubuntu-vm" + difficulty: easy +steps: + setup: + inline: |- + #!/usr/bin/env bash + NS="${EVAL_NAMESPACE:-vm-test}" + kubectl delete namespace "$NS" --ignore-not-found + kubectl create namespace "$NS" + verify: + inline: |- + #!/usr/bin/env bash + source evals/tasks/kubevirt/helpers/verify-vm.sh + NS="${EVAL_NAMESPACE:-vm-test}" + + # Wait for VirtualMachine to be created + verify_vm_exists "ubuntu-vm" "$NS" || exit 1 + + # Verify container disk is Ubuntu + verify_container_disk "ubuntu-vm" "$NS" "ubuntu" || exit 1 + + # Verify runStrategy is set and deprecated 'running' field is not used + verify_run_strategy "ubuntu-vm" "$NS" || exit 1 + verify_no_deprecated_running_field "ubuntu-vm" "$NS" || exit 1 + + echo "All validations passed" + exit 0 + cleanup: + inline: |- + #!/usr/bin/env bash + NS="${EVAL_NAMESPACE:-vm-test}" + kubectl delete virtualmachine ubuntu-vm -n "$NS" --ignore-not-found + kubectl delete namespace "$NS" --ignore-not-found + prompt: + inline: Create an Ubuntu virtual machine named ubuntu-vm in the ${EVAL_NAMESPACE:-vm-test} namespace. diff --git a/evals/tasks/kubevirt/create-vm-with-instancetype/task.yaml b/evals/tasks/kubevirt/create-vm-with-instancetype/task.yaml new file mode 100644 index 000000000..4682ef4ed --- /dev/null +++ b/evals/tasks/kubevirt/create-vm-with-instancetype/task.yaml @@ -0,0 +1,43 @@ +kind: Task +metadata: + name: "create-vm-with-instancetype" + difficulty: easy +steps: + setup: + inline: |- + #!/usr/bin/env bash + NS="${EVAL_NAMESPACE:-vm-test}" + kubectl delete namespace "$NS" --ignore-not-found + kubectl create namespace "$NS" + verify: + inline: |- + #!/usr/bin/env bash + source evals/tasks/kubevirt/helpers/verify-vm.sh + NS="${EVAL_NAMESPACE:-vm-test}" + + # Wait for VirtualMachine to be created + verify_vm_exists "test-vm-instancetype" "$NS" || exit 1 + + # Verify that it has the specific instancetype reference (u1.medium) + verify_instancetype "test-vm-instancetype" "$NS" "u1.medium" || exit 1 + + # Verify runStrategy is set and deprecated 'running' field is not used + verify_run_strategy "test-vm-instancetype" "$NS" || exit 1 + verify_no_deprecated_running_field "test-vm-instancetype" "$NS" || exit 1 + + # Verify no direct resource specification (should use instancetype) + verify_no_direct_resources "test-vm-instancetype" "$NS" + + # Verify container disk is Fedora (default workload) + verify_container_disk "test-vm-instancetype" "$NS" "fedora" + + echo "All validations passed" + exit 0 + cleanup: + inline: |- + #!/usr/bin/env bash + NS="${EVAL_NAMESPACE:-vm-test}" + kubectl delete virtualmachine test-vm-instancetype -n "$NS" --ignore-not-found + kubectl delete namespace "$NS" --ignore-not-found + prompt: + inline: Create a Fedora virtual machine named test-vm-instancetype in the ${EVAL_NAMESPACE:-vm-test} namespace with instancetype 'u1.medium'. diff --git a/evals/tasks/kubevirt/create-vm-with-size/task.yaml b/evals/tasks/kubevirt/create-vm-with-size/task.yaml new file mode 100644 index 000000000..746b0e61b --- /dev/null +++ b/evals/tasks/kubevirt/create-vm-with-size/task.yaml @@ -0,0 +1,43 @@ +kind: Task +metadata: + name: "create-vm-with-size" + difficulty: easy +steps: + setup: + inline: |- + #!/usr/bin/env bash + NS="${EVAL_NAMESPACE:-vm-test}" + kubectl delete namespace "$NS" --ignore-not-found + kubectl create namespace "$NS" + verify: + inline: |- + #!/usr/bin/env bash + source evals/tasks/kubevirt/helpers/verify-vm.sh + NS="${EVAL_NAMESPACE:-vm-test}" + + # Wait for VirtualMachine to be created + verify_vm_exists "test-vm-size" "$NS" || exit 1 + + # Verify that it has an instancetype or direct resources + verify_has_resources_or_instancetype "test-vm-size" "$NS" || exit 1 + + # Check if instancetype contains 'large' (matching the requested size) + verify_instancetype_contains "test-vm-size" "$NS" "large" "requested size 'large'" + + # Verify runStrategy is set and deprecated 'running' field is not used + verify_run_strategy "test-vm-size" "$NS" || exit 1 + verify_no_deprecated_running_field "test-vm-size" "$NS" || exit 1 + + # Verify container disk is Fedora (default workload) + verify_container_disk "test-vm-size" "$NS" "fedora" + + echo "All validations passed" + exit 0 + cleanup: + inline: |- + #!/usr/bin/env bash + NS="${EVAL_NAMESPACE:-vm-test}" + kubectl delete virtualmachine test-vm-size -n "$NS" --ignore-not-found + kubectl delete namespace "$NS" --ignore-not-found + prompt: + inline: Create a Fedora virtual machine named test-vm-size in the ${EVAL_NAMESPACE:-vm-test} namespace with size 'large' diff --git a/evals/tasks/kubevirt/delete-vm/task.yaml b/evals/tasks/kubevirt/delete-vm/task.yaml new file mode 100644 index 000000000..7d350772b --- /dev/null +++ b/evals/tasks/kubevirt/delete-vm/task.yaml @@ -0,0 +1,53 @@ +kind: Task +metadata: + name: "delete-vm" + difficulty: medium +steps: + setup: + inline: |- + #!/usr/bin/env bash + NS="${EVAL_NAMESPACE:-vm-test}" + kubectl delete namespace "$NS" --ignore-not-found + kubectl create namespace "$NS" + + # Create a running VM + kubectl apply -f - < [timeout]` + +**Example:** +```bash +verify_vm_exists "my-vm" "vm-test" "30s" || exit 1 +``` + +**Default timeout:** 30s + +--- + +### verify_container_disk +Verifies that a VM uses a specific container disk OS (checks all volumes). + +**Usage:** `verify_container_disk ` + +**Example:** +```bash +verify_container_disk "my-vm" "vm-test" "fedora" || exit 1 +verify_container_disk "ubuntu-vm" "vm-test" "ubuntu" || exit 1 +``` + +--- + +### verify_run_strategy +Verifies that runStrategy is set (checks both spec and status). + +**Usage:** `verify_run_strategy ` + +**Example:** +```bash +verify_run_strategy "my-vm" "vm-test" || exit 1 +``` + +**Note:** This function accepts runStrategy in either `spec.runStrategy` or `status.runStrategy` to accommodate VMs created with the deprecated `running` field. + +--- + +### verify_no_deprecated_running_field +Verifies that the deprecated `running` field is NOT set in the VirtualMachine spec. + +**Usage:** `verify_no_deprecated_running_field ` + +**Example:** +```bash +verify_no_deprecated_running_field "my-vm" "vm-test" || exit 1 +``` + +**Note:** The `running` field is deprecated in KubeVirt. VirtualMachines should use `runStrategy` instead. This function ensures compliance with current best practices. + +--- + +### verify_instancetype +Verifies that a VM has an instancetype reference with optional exact match. + +**Usage:** `verify_instancetype [expected-instancetype] [expected-kind]` + +**Examples:** +```bash +# Just verify instancetype exists +verify_instancetype "my-vm" "vm-test" || exit 1 + +# Verify specific instancetype +verify_instancetype "my-vm" "vm-test" "u1.medium" || exit 1 + +# Verify instancetype and kind +verify_instancetype "my-vm" "vm-test" "u1.medium" "VirtualMachineClusterInstancetype" || exit 1 +``` + +**Default kind:** VirtualMachineClusterInstancetype + +--- + +### verify_instancetype_contains +Verifies that instancetype name contains a substring (e.g., size like "large"). + +**Usage:** `verify_instancetype_contains [description]` + +**Example:** +```bash +verify_instancetype_contains "my-vm" "vm-test" "large" "requested size 'large'" +verify_instancetype_contains "my-vm" "vm-test" "medium" +``` + +**Note:** Returns success even if substring not found (prints warning only). + +--- + +### verify_instancetype_prefix +Verifies that instancetype starts with a specific prefix (e.g., performance family like "c1"). + +**Usage:** `verify_instancetype_prefix [description]` + +**Example:** +```bash +verify_instancetype_prefix "my-vm" "vm-test" "c1" "compute-optimized" +verify_instancetype_prefix "my-vm" "vm-test" "u1" "general-purpose" +``` + +**Note:** Returns success even if prefix doesn't match (prints warning only). + +--- + +### verify_no_direct_resources +Verifies that VM uses instancetype for resources (no direct memory specification). + +**Usage:** `verify_no_direct_resources ` + +**Example:** +```bash +verify_no_direct_resources "my-vm" "vm-test" +``` + +**Note:** Returns success even if direct resources found (prints warning only). + +--- + +### verify_has_resources_or_instancetype +Verifies that VM has either an instancetype or direct resource specification. + +**Usage:** `verify_has_resources_or_instancetype ` + +**Example:** +```bash +verify_has_resources_or_instancetype "my-vm" "vm-test" || exit 1 +``` + +**Note:** Fails only if neither instancetype nor direct resources are present. + +--- + +### verify_cpu_cores +Verifies that a VM has the expected number of CPU cores. + +**Usage:** `verify_cpu_cores ` + +**Example:** +```bash +verify_cpu_cores "my-vm" "vm-test" 2 || exit 1 +verify_cpu_cores "my-vm" "vm-test" 4 || exit 1 +``` + +**Note:** This function checks the direct CPU specification in `spec.template.spec.domain.cpu.cores`. For VMs using instancetypes, CPU is defined by the instancetype itself. + +--- + +### verify_memory_increased +Verifies that VM memory is greater than the original value. + +**Usage:** `verify_memory_increased ` + +**Examples:** +```bash +verify_memory_increased "my-vm" "vm-test" "2Gi" || exit 1 +verify_memory_increased "my-vm" "vm-test" "4096Mi" || exit 1 +``` + +**Note:** This function compares memory values in bytes, supporting Gi, Mi, Ki, G, M, K suffixes. It fails if current memory is not greater than the original value. + +## Design Principles + +1. **Flexible matching**: Functions use pattern matching instead of exact volume names to handle different VM creation approaches. + +2. **Clear output**: Each function prints clear success (✓) or failure (✗) messages. + +3. **Warning vs Error**: Some functions print warnings (⚠) for non-critical mismatches but still return success. + +4. **Return codes**: Functions return 0 for success, 1 for failure. Always check return codes with `|| exit 1` for critical validations. + +## Example Test Verification + +```bash +#!/usr/bin/env bash +source "$(dirname "${BASH_SOURCE[0]}")/../../helpers/verify-vm.sh" + +# Wait for VM to exist +verify_vm_exists "test-vm" "vm-test" || exit 1 + +# Verify container disk +verify_container_disk "test-vm" "vm-test" "fedora" || exit 1 + +# Verify runStrategy is used (not deprecated 'running' field) +verify_run_strategy "test-vm" "vm-test" || exit 1 +verify_no_deprecated_running_field "test-vm" "vm-test" || exit 1 + +# Verify instancetype with size +verify_instancetype "test-vm" "vm-test" || exit 1 +verify_instancetype_contains "test-vm" "vm-test" "large" + +# Verify no direct resources +verify_no_direct_resources "test-vm" "vm-test" + +echo "All validations passed" +exit 0 +``` diff --git a/evals/tasks/kubevirt/helpers/verify-vm.sh b/evals/tasks/kubevirt/helpers/verify-vm.sh new file mode 100644 index 000000000..32240240f --- /dev/null +++ b/evals/tasks/kubevirt/helpers/verify-vm.sh @@ -0,0 +1,254 @@ +#!/usr/bin/env bash +# Shared verification helper functions for VirtualMachine tests + +# verify_vm_exists: Waits for a VirtualMachine to be created +# Usage: verify_vm_exists [timeout] +verify_vm_exists() { + local vm_name="$1" + local namespace="$2" + local timeout="${3:-30s}" + + if ! kubectl wait --for=jsonpath='{.metadata.name}'="$vm_name" virtualmachine/"$vm_name" -n "$namespace" --timeout="$timeout" 2>/dev/null; then + echo "VirtualMachine $vm_name not found in namespace $namespace" + kubectl get virtualmachines -n "$namespace" + return 1 + fi + echo "VirtualMachine $vm_name created successfully" + return 0 +} + +# verify_container_disk: Verifies that a VM uses a specific container disk OS +# Usage: verify_container_disk +# Example: verify_container_disk test-vm vm-test fedora +verify_container_disk() { + local vm_name="$1" + local namespace="$2" + local os_name="$3" + + # Get all container disk images from all volumes + local container_disks + container_disks=$(kubectl get virtualmachine "$vm_name" -n "$namespace" -o jsonpath='{.spec.template.spec.volumes[*].containerDisk.image}') + + if [[ "$container_disks" =~ $os_name ]]; then + echo "✓ VirtualMachine uses $os_name container disk" + return 0 + else + echo "✗ Expected $os_name container disk, found volumes with images: $container_disks" + kubectl get virtualmachine "$vm_name" -n "$namespace" -o yaml + return 1 + fi +} + +# verify_run_strategy: Verifies that runStrategy is set (in spec or status) +# Usage: verify_run_strategy +verify_run_strategy() { + local vm_name="$1" + local namespace="$2" + + local spec_run_strategy + local status_run_strategy + spec_run_strategy=$(kubectl get virtualmachine "$vm_name" -n "$namespace" -o jsonpath='{.spec.runStrategy}') + status_run_strategy=$(kubectl get virtualmachine "$vm_name" -n "$namespace" -o jsonpath='{.status.runStrategy}') + + if [[ -n "$spec_run_strategy" ]]; then + echo "✓ VirtualMachine uses runStrategy in spec: $spec_run_strategy" + return 0 + elif [[ -n "$status_run_strategy" ]]; then + echo "✓ VirtualMachine has runStrategy in status: $status_run_strategy" + echo " Note: VM may have been created with deprecated 'running' field, but runStrategy is set in status" + return 0 + else + echo "✗ VirtualMachine missing runStrategy field in both spec and status" + return 1 + fi +} + +# verify_no_deprecated_running_field: Verifies that deprecated 'running' field is NOT set +# Usage: verify_no_deprecated_running_field +verify_no_deprecated_running_field() { + local vm_name="$1" + local namespace="$2" + + local running_field + running_field=$(kubectl get virtualmachine "$vm_name" -n "$namespace" -o jsonpath='{.spec.running}') + + if [[ -z "$running_field" ]]; then + echo "✓ VirtualMachine does not use deprecated 'running' field" + return 0 + else + echo "✗ VirtualMachine uses deprecated 'running' field with value: $running_field" + echo " Please use 'runStrategy' instead of 'running'" + kubectl get virtualmachine "$vm_name" -n "$namespace" -o yaml + return 1 + fi +} + +# verify_instancetype: Verifies that a VM has an instancetype reference +# Usage: verify_instancetype [expected-instancetype] [expected-kind] +verify_instancetype() { + local vm_name="$1" + local namespace="$2" + local expected_instancetype="$3" + local expected_kind="${4:-VirtualMachineClusterInstancetype}" + + local instancetype + instancetype=$(kubectl get virtualmachine "$vm_name" -n "$namespace" -o jsonpath='{.spec.instancetype.name}') + + if [[ -z "$instancetype" ]]; then + echo "✗ VirtualMachine has no instancetype reference" + return 1 + fi + + echo "✓ VirtualMachine has instancetype reference: $instancetype" + + # Check expected instancetype if provided + if [[ -n "$expected_instancetype" ]]; then + if [[ "$instancetype" == "$expected_instancetype" ]]; then + echo "✓ Instancetype matches expected value: $expected_instancetype" + else + echo "✗ Expected instancetype '$expected_instancetype', found: $instancetype" + return 1 + fi + fi + + # Verify instancetype kind + local instancetype_kind + instancetype_kind=$(kubectl get virtualmachine "$vm_name" -n "$namespace" -o jsonpath='{.spec.instancetype.kind}') + if [[ "$instancetype_kind" == "$expected_kind" ]]; then + echo "✓ Instancetype kind is $expected_kind" + else + echo "⚠ Instancetype kind is: $instancetype_kind (expected: $expected_kind)" + fi + + return 0 +} + +# verify_instancetype_contains: Verifies that instancetype name contains a string +# Usage: verify_instancetype_contains [description] +verify_instancetype_contains() { + local vm_name="$1" + local namespace="$2" + local substring="$3" + local description="${4:-$substring}" + + local instancetype + instancetype=$(kubectl get virtualmachine "$vm_name" -n "$namespace" -o jsonpath='{.spec.instancetype.name}') + + if [[ -z "$instancetype" ]]; then + echo "✗ VirtualMachine has no instancetype reference" + return 1 + fi + + if [[ "$instancetype" =~ $substring ]]; then + echo "✓ Instancetype matches $description: $instancetype" + return 0 + else + echo "⚠ Instancetype '$instancetype' doesn't match $description" + return 0 # Return success for warnings + fi +} + +# verify_no_direct_resources: Verifies VM uses instancetype (no direct memory spec) +# Usage: verify_no_direct_resources +verify_no_direct_resources() { + local vm_name="$1" + local namespace="$2" + + local guest_memory + guest_memory=$(kubectl get virtualmachine "$vm_name" -n "$namespace" -o jsonpath='{.spec.template.spec.domain.memory.guest}') + + if [[ -z "$guest_memory" ]]; then + echo "✓ VirtualMachine uses instancetype for resources (no direct memory spec)" + return 0 + else + echo "⚠ VirtualMachine has direct memory specification: $guest_memory" + return 0 # Return success for warnings + fi +} + +# verify_has_resources_or_instancetype: Verifies VM has either instancetype or direct resources +# Usage: verify_has_resources_or_instancetype +verify_has_resources_or_instancetype() { + local vm_name="$1" + local namespace="$2" + + local instancetype + instancetype=$(kubectl get virtualmachine "$vm_name" -n "$namespace" -o jsonpath='{.spec.instancetype.name}') + + if [[ -n "$instancetype" ]]; then + echo "✓ VirtualMachine has instancetype reference: $instancetype" + return 0 + fi + + # Check for direct resource specification + local guest_memory + guest_memory=$(kubectl get virtualmachine "$vm_name" -n "$namespace" -o jsonpath='{.spec.template.spec.domain.memory.guest}') + + if [[ -n "$guest_memory" ]]; then + echo "⚠ No instancetype set, but VM has direct memory specification: $guest_memory" + return 0 + else + echo "✗ VirtualMachine has no instancetype and no direct resource specification" + kubectl get virtualmachine "$vm_name" -n "$namespace" -o yaml + return 1 + fi +} + +# verify_cpu_cores: Verifies that a VM has the expected number of CPU cores +# Usage: verify_cpu_cores +verify_cpu_cores() { + local vm_name="$1" + local namespace="$2" + local expected_cores="$3" + + local cpu_cores + cpu_cores=$(kubectl get virtualmachine "$vm_name" -n "$namespace" -o jsonpath='{.spec.template.spec.domain.cpu.cores}') + + if [[ -z "$cpu_cores" ]]; then + echo "✗ VirtualMachine has no CPU cores specification" + kubectl get virtualmachine "$vm_name" -n "$namespace" -o yaml | grep -A 5 "cpu:" + return 1 + fi + + if [[ "$cpu_cores" -eq "$expected_cores" ]]; then + echo "✓ VirtualMachine has expected CPU cores: $cpu_cores" + return 0 + else + echo "✗ Expected $expected_cores CPU cores, found: $cpu_cores" + return 1 + fi +} + +# verify_memory_increased: Verifies that VM memory is greater than the original value +# Usage: verify_memory_increased +# Example: verify_memory_increased test-vm vm-test "2Gi" +verify_memory_increased() { + local vm_name="$1" + local namespace="$2" + local original_memory="$3" + + local current_memory + current_memory=$(kubectl get virtualmachine "$vm_name" -n "$namespace" -o jsonpath='{.spec.template.spec.domain.memory.guest}') + + if [[ -z "$current_memory" ]]; then + echo "✗ VirtualMachine has no memory specification" + kubectl get virtualmachine "$vm_name" -n "$namespace" -o yaml | grep -A 5 "memory:" + return 1 + fi + + # Convert memory values to bytes for comparison + # This is a simple comparison that handles Gi, Mi, Ki suffixes + local original_bytes + local current_bytes + + original_bytes=$(echo "$original_memory" | sed 's/Gi/*1073741824/;s/Mi/*1048576/;s/Ki/*1024/;s/G/*1000000000/;s/M/*1000000/;s/K/*1000/' | bc 2>/dev/null || echo "0") + current_bytes=$(echo "$current_memory" | sed 's/Gi/*1073741824/;s/Mi/*1048576/;s/Ki/*1024/;s/G/*1000000000/;s/M/*1000000/;s/K/*1000/' | bc 2>/dev/null || echo "0") + + if [[ "$current_bytes" -gt "$original_bytes" ]]; then + echo "✓ VirtualMachine memory increased from $original_memory to $current_memory" + return 0 + else + echo "✗ Expected memory greater than $original_memory, found: $current_memory" + return 1 + fi +} diff --git a/evals/tasks/kubevirt/pause-vm/task.yaml b/evals/tasks/kubevirt/pause-vm/task.yaml new file mode 100644 index 000000000..3e8b2c5bf --- /dev/null +++ b/evals/tasks/kubevirt/pause-vm/task.yaml @@ -0,0 +1,54 @@ +kind: Task +metadata: + name: "pause-vm" + difficulty: medium +steps: + setup: + inline: |- + #!/usr/bin/env bash + NS="${EVAL_NAMESPACE:-vm-test}" + kubectl delete namespace "$NS" --ignore-not-found + kubectl create namespace "$NS" + + # Create a running VM + kubectl apply -f - <