OSAC-1561: add Metal3 backend E2E lifecycle tests

.gitignore

@@ -22,3 +22,6 @@ reports/
 
 # Infrastructure backend outputs
 infra/*/.env.infra
+
+# Workflow artifacts
+.artifacts/

Containerfile

@@ -4,7 +4,7 @@ ARG GRPCURL_VERSION=1.9.1
 ARG OSAC_VERSION=""
 ARG OSAC_CLI_BIN=""
 
-RUN dnf install -y python3.11 python3.11-pip make jq && dnf clean all
+RUN dnf install -y python3.11 python3.11-pip make jq openssh-clients && dnf clean all
 
 RUN curl -Lsf "https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/stable/openshift-client-linux.tar.gz" \
     | tar xz --no-same-owner -C /usr/local/bin oc kubectl

Makefile

@@ -1,6 +1,6 @@
 REPORTS_DIR ?= reports
 
-.PHONY: test lint format test-vmaas test-caas test-storage
+.PHONY: test lint format test-vmaas test-caas test-storage test-bmaas
 
 test:
 	mkdir -p $(REPORTS_DIR)
@@ -25,6 +25,10 @@ test-storage:
 	mkdir -p $(REPORTS_DIR)
 	pytest tests/storage/ -v $(if $(TEST),-k "$(TEST)") --junitxml=$(REPORTS_DIR)/storage.xml
 
+test-bmaas:
+	mkdir -p $(REPORTS_DIR)
+	pytest tests/bmaas/ -v $(if $(TEST),-k "$(TEST)") --junitxml=$(REPORTS_DIR)/bmaas.xml
+
 # ─── Infrastructure orchestration ───────────────────────────────────
 
 INFRA       ?= netris

tests/bmaas/conftest.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+import subprocess
+import tempfile
+import uuid
+from collections.abc import Generator
+from pathlib import Path
+
+import pytest
+
+from tests.core.grpc_client import GRPCClient
+from tests.core.runner import env
+
+
+@pytest.fixture(scope="session")
+def bmi_template() -> str:
+    return env("OSAC_BMI_TEMPLATE", "osac.templates.bm_host_provisioning")
+
+
+@pytest.fixture(scope="session")
+def bmh_namespace() -> str:
+    return env("OSAC_BMH_NAMESPACE", "host-inventory")
+
+
+@pytest.fixture(scope="session")
+def test_run_id() -> str:
+    return str(uuid.uuid4())[:8]
+
+
+@pytest.fixture(scope="session")
+def ssh_public_key() -> Generator[str, None, None]:
+    with tempfile.TemporaryDirectory() as tmpdir:
+        key_path = Path(tmpdir) / "bmi-test-key"
+        subprocess.run(
+            ["ssh-keygen", "-t", "ed25519", "-f", str(key_path), "-N", "", "-C", "bmi-e2e-test"],
+            capture_output=True,
+            check=True,
+        )
+        yield (key_path.with_suffix(".pub")).read_text().strip()
+
+
+@pytest.fixture(scope="session")
+def catalog_item(private_grpc: GRPCClient, bmi_template: str, test_run_id: str) -> Generator[str, None, None]:
+    name = f"e2e-bmaas-{test_run_id}"
+    print(f"\nCreating BareMetalInstanceCatalogItem: {name}")
+    item_id: str = private_grpc.create_baremetal_instance_catalog_item(
+        name=name,
+        title=f"E2E BMaaS Test ({test_run_id})",
+        description="Temporary catalog item for BMaaS E2E tests",
+        template=bmi_template,
+        field_definitions=[{"path": "ssh_public_key", "display_name": "SSH Public Key", "editable": True}],
+    )
+    print(f"CatalogItem created: {item_id}")
+
+    yield item_id
+
+    try:
+        print(f"\nDeleting BareMetalInstanceCatalogItem {item_id}...")
+        private_grpc.delete_baremetal_instance_catalog_item(item_id=item_id)
+        print(f"CatalogItem {item_id} deleted")
+    except Exception as e:
+        print(f"WARNING: Failed to delete catalog item {item_id}: {e}")

tests/bmaas/test_baremetal_instance_lifecycle.py

@@ -0,0 +1,98 @@
+from __future__ import annotations
+
+from tests.core.grpc_client import GRPCClient
+from tests.core.helpers import (
+    wait_for_bmh_available,
+    wait_for_bmh_provisioned,
+    wait_for_bmi_cr,
+    wait_for_bmi_deletion,
+    wait_for_bmi_grpc_removal,
+    wait_for_bmi_running,
+)
+from tests.core.k8s_client import K8sClient
+from tests.core.osac_cli import OsacCLI
+from tests.core.runner import poll_until
+
+
+def test_baremetal_instance_lifecycle(
+    cli: OsacCLI,
+    grpc: GRPCClient,
+    k8s_hub_client: K8sClient,
+    catalog_item: str,
+    bmh_namespace: str,
+    test_run_id: str,
+    ssh_public_key: str,
+) -> None:
+    name = f"e2e-bmi-{test_run_id}"
+    bmi_id: str = cli.create_baremetal_instance(name=name, catalog_item=catalog_item, ssh_key=ssh_public_key)
+
+    try:
+        assert bmi_id in grpc.list_baremetal_instance_ids()
+
+        bmi_cr_name: str = wait_for_bmi_cr(k8s=k8s_hub_client, uuid=bmi_id)
+        wait_for_bmi_running(grpc=grpc, bmi_id=bmi_id)
+
+        external_host_id: str = k8s_hub_client.get_baremetal_instance_external_host_id(name=bmi_cr_name)
+        assert "/" in external_host_id, f"Expected namespace/name format, got: {external_host_id}"
+        bmh_ns, bmh_name = external_host_id.split("/", 1)
+        assert bmh_ns == bmh_namespace, f"BMH landed in {bmh_ns}, expected {bmh_namespace}"
+
+        # Verify provisioning
+        wait_for_bmh_provisioned(k8s=k8s_hub_client, name=bmh_name, bmh_namespace=bmh_ns)
+
+        image_url: str = k8s_hub_client.get_bmh_image_url(name=bmh_name, bmh_namespace=bmh_ns)
+        assert image_url != "", f"BMH {bmh_name} has no image URL after provisioning"
+
+        consumer_ref: str = k8s_hub_client.get_bmh_consumer_ref(name=bmh_name, bmh_namespace=bmh_ns)
+        assert consumer_ref != "", f"BMH {bmh_name} has no consumerRef after allocation"
+
+        online: str = k8s_hub_client.get_bmh_online(name=bmh_name, bmh_namespace=bmh_ns)
+        assert online == "true", f"BMH {bmh_name} should be online after provisioning, got: {online}"
+
+        # Power off
+        halted = "BARE_METAL_INSTANCE_RUN_STRATEGY_HALTED"
+        grpc.update_baremetal_instance_run_strategy(bmi_id=bmi_id, run_strategy=halted)
+
+        poll_until(
+            fn=lambda: k8s_hub_client.get_bmh_powered_on(name=bmh_name, bmh_namespace=bmh_ns),
+            until=lambda v: v == "false",
+            retries=60,
+            delay=5,
+            description=f"{bmh_name} powered off",
+        )
+
+        # Power on
+        grpc.update_baremetal_instance_run_strategy(
+            bmi_id=bmi_id, run_strategy="BARE_METAL_INSTANCE_RUN_STRATEGY_ALWAYS"
+        )
+
+        poll_until(
+            fn=lambda: k8s_hub_client.get_bmh_powered_on(name=bmh_name, bmh_namespace=bmh_ns),
+            until=lambda v: v == "true",
+            retries=60,
+            delay=5,
+            description=f"{bmh_name} powered on",
+        )
+
+        # Deprovision
+        cli.delete_baremetal_instance(uuid=bmi_id)
+        wait_for_bmi_deletion(k8s=k8s_hub_client, name=bmi_cr_name)
+        wait_for_bmi_grpc_removal(grpc=grpc, uuid=bmi_id)
+
+        wait_for_bmh_available(k8s=k8s_hub_client, name=bmh_name, bmh_namespace=bmh_ns)
+
+        image_url_after: str = k8s_hub_client.get_bmh_image_url(name=bmh_name, bmh_namespace=bmh_ns)
+        assert image_url_after == "", f"BMH {bmh_name} image not cleared after deprovision: {image_url_after}"
+
+        consumer_ref_after: str = k8s_hub_client.get_bmh_consumer_ref(name=bmh_name, bmh_namespace=bmh_ns)
+        assert consumer_ref_after == "", f"BMH {bmh_name} consumerRef not cleared: {consumer_ref_after}"
+    except BaseException:
+        bmi_cr: str = k8s_hub_client.get_baremetal_instance_name(uuid=bmi_id, checked=False)
+        if bmi_cr:
+            try:
+                cli.delete_baremetal_instance(uuid=bmi_id)
+                wait_for_bmi_deletion(k8s=k8s_hub_client, name=bmi_cr)
+                wait_for_bmi_grpc_removal(grpc=grpc, uuid=bmi_id)
+            except Exception:
+                pass
+        raise

tests/core/grpc_client.py

@@ -337,3 +337,56 @@ def update_instance_type(self, *, name: str, state: str) -> dict[str, Any]:
 
     def delete_instance_type(self, *, name: str) -> None:
         self.call(service=f"{PRIVATE_API}.InstanceTypes/Delete", data={"id": name})
+
+    # BareMetalInstance operations (public API)
+
+    def list_baremetal_instance_ids(self) -> list[str]:
+        response: dict[str, Any] = self.call(service=f"{PUBLIC_API}.BareMetalInstances/List")
+        return [item["id"] for item in response.get("items", [])]
+
+    def get_baremetal_instance(self, *, bmi_id: str) -> dict[str, Any]:
+        return self.call(service=f"{PUBLIC_API}.BareMetalInstances/Get", data={"id": bmi_id})
+
+    def get_baremetal_instance_state(self, *, bmi_id: str) -> str:
+        response: dict[str, Any] = self.get_baremetal_instance(bmi_id=bmi_id)
+        return response.get("object", {}).get("status", {}).get("state", "")
+
+    def update_baremetal_instance_run_strategy(self, *, bmi_id: str, run_strategy: str) -> dict[str, Any]:
+        return self.call(
+            service=f"{PUBLIC_API}.BareMetalInstances/Update",
+            data={
+                "object": {"id": bmi_id, "spec": {"run_strategy": run_strategy}},
+                "updateMask": {"paths": ["spec.run_strategy"]},
+            },
+        )
+
+    def delete_baremetal_instance(self, *, bmi_id: str) -> None:
+        self.call(service=f"{PUBLIC_API}.BareMetalInstances/Delete", data={"id": bmi_id})
+
+    # BareMetalInstanceCatalogItem operations (private API for admin setup)
+
+    def create_baremetal_instance_catalog_item(
+        self,
+        *,
+        name: str,
+        title: str,
+        description: str,
+        template: str,
+        field_definitions: list[dict[str, Any]] | None = None,
+    ) -> str:
+        obj: dict[str, Any] = {
+            "metadata": {"name": name},
+            "title": title,
+            "description": description,
+            "template": template,
+            "published": True,
+        }
+        if field_definitions is not None:
+            obj["field_definitions"] = field_definitions
+        response: dict[str, Any] = self.call(
+            service=f"{PRIVATE_API}.BareMetalInstanceCatalogItems/Create", data={"object": obj}
+        )
+        return response["object"]["id"]
+
+    def delete_baremetal_instance_catalog_item(self, *, item_id: str) -> None:
+        self.call(service=f"{PRIVATE_API}.BareMetalInstanceCatalogItems/Delete", data={"id": item_id})

tests/core/helpers.py

@@ -559,3 +559,81 @@ def wait_for_secrets_removed(*, k8s: K8sClient, tenant_name: str, namespace: str
         delay=5,
         description=f"Secrets for tenant {tenant_name} in {namespace} removed",
     )
+
+
+# BareMetalInstance helpers
+
+
+def wait_for_bmi_cr(*, k8s: K8sClient, uuid: str) -> str:
+    return poll_until(
+        fn=lambda: k8s.get_baremetal_instance_name(uuid=uuid, checked=False),
+        until=lambda v: v != "",
+        retries=30,
+        delay=2,
+        description=f"BareMetalInstance CR for {uuid}",
+    )
+
+
+def wait_for_bmi_running(*, grpc: GRPCClient, bmi_id: str) -> None:
+    def _check_state() -> str:
+        state: str = grpc.get_baremetal_instance_state(bmi_id=bmi_id)
+        assert "FAILED" not in state, f"BareMetalInstance {bmi_id} entered {state}"
+        return state
+
+    poll_until(
+        fn=_check_state,
+        until=lambda v: v == "BARE_METAL_INSTANCE_STATE_RUNNING",
+        retries=120,
+        delay=10,
+        description=f"{bmi_id} RUNNING",
+    )
+
+
+def wait_for_bmi_deletion(*, k8s: K8sClient, name: str) -> None:
+    poll_until(
+        fn=lambda: not k8s.is_present(resource="baremetalinstance", name=name),
+        until=lambda v: v is True,
+        retries=120,
+        delay=10,
+        description=f"{name} BareMetalInstance deletion",
+    )
+
+
+def wait_for_bmi_grpc_removal(*, grpc: GRPCClient, uuid: str) -> None:
+    poll_until(
+        fn=lambda: uuid not in grpc.list_baremetal_instance_ids(),
+        until=lambda v: v is True,
+        retries=60,
+        delay=5,
+        description=f"{uuid} removed from gRPC BareMetalInstance list",
+    )
+
+
+def wait_for_bmh_provisioned(*, k8s: K8sClient, name: str, bmh_namespace: str) -> None:
+    def _check() -> str:
+        state: str = k8s.get_bmh_provisioning_state(name=name, bmh_namespace=bmh_namespace)
+        assert state != "error", f"BMH {name} entered error state"
+        return state
+
+    poll_until(
+        fn=_check,
+        until=lambda v: v == "provisioned",
+        retries=120,
+        delay=10,
+        description=f"{name} BMH provisioned",
+    )
+
+
+def wait_for_bmh_available(*, k8s: K8sClient, name: str, bmh_namespace: str) -> None:
+    def _check() -> str:
+        state: str = k8s.get_bmh_provisioning_state(name=name, bmh_namespace=bmh_namespace)
+        assert state != "error", f"BMH {name} entered error state"
+        return state
+
+    poll_until(
+        fn=_check,
+        until=lambda v: v in ("available", "ready"),
+        retries=120,
+        delay=10,
+        description=f"{name} BMH available",
+    )

tests/core/k8s_client.py

@@ -439,3 +439,61 @@ def get_security_group_phase(self, *, name: str, checked: bool = True) -> str:
             "get", "securitygroup", name, "-n", self.namespace, "-o", "jsonpath={.status.phase}", checked=checked
         )
         return output if rc == 0 else ""
+
+    # BareMetalInstance queries
+
+    def get_baremetal_instance_name(self, *, uuid: str, checked: bool = True) -> str:
+        output, rc = self._get(
+            "get",
+            "baremetalinstance",
+            "-n",
+            self.namespace,
+            "-l",
+            f"osac.openshift.io/baremetalinstance-uuid={uuid}",
+            "-o",
+            "jsonpath={.items[0].metadata.name}",
+            checked=checked,
+        )
+        return output if rc == 0 else ""
+
+    def get_baremetal_instance_external_host_id(self, *, name: str) -> str:
+        return self.get_jsonpath(resource="baremetalinstance", name=name, jsonpath="{.spec.externalHostID}")
+
+    # BareMetalHost queries (explicit namespace — BMHs live in a different namespace)
+
+    def get_bmh_provisioning_state(self, *, name: str, bmh_namespace: str) -> str:
+        output, rc = self._get(
+            "get",
+            "baremetalhost",
+            name,
+            "-n",
+            bmh_namespace,
+            "-o",
+            "jsonpath={.status.provisioning.state}",
+            checked=False,
+        )
+        return output if rc == 0 else ""
+
+    def get_bmh_image_url(self, *, name: str, bmh_namespace: str) -> str:
+        output, rc = self._get(
+            "get", "baremetalhost", name, "-n", bmh_namespace, "-o", "jsonpath={.spec.image.url}", checked=False
+        )
+        return output if rc == 0 else ""
+
+    def get_bmh_consumer_ref(self, *, name: str, bmh_namespace: str) -> str:
+        output, rc = self._get(
+            "get", "baremetalhost", name, "-n", bmh_namespace, "-o", "jsonpath={.spec.consumerRef.name}", checked=False
+        )
+        return output if rc == 0 else ""
+
+    def get_bmh_online(self, *, name: str, bmh_namespace: str) -> str:
+        output, rc = self._get(
+            "get", "baremetalhost", name, "-n", bmh_namespace, "-o", "jsonpath={.spec.online}", checked=False
+        )
+        return output if rc == 0 else ""
+
+    def get_bmh_powered_on(self, *, name: str, bmh_namespace: str) -> str:
+        output, rc = self._get(
+            "get", "baremetalhost", name, "-n", bmh_namespace, "-o", "jsonpath={.status.poweredOn}", checked=False
+        )
+        return output if rc == 0 else ""