From 9e5a0f8ee5d39cfd89a1bb647a37c2b40bde2f11 Mon Sep 17 00:00:00 2001 From: POWERFULMOVES Date: Thu, 19 Mar 2026 17:16:42 -0400 Subject: [PATCH 1/2] feat(z890): add Container Agent diagnostic service Network diagnostic sidecar for Docker fleet networking. Runs on each node to probe inter-container connectivity, DNS, upstream reachability, and NATS health. - Add container-agent service (python:3.11-slim + aiohttp + nats-py) - Endpoints: GET /healthz, GET /diagnostic, GET /metrics - NATS heartbeat loop (mesh.container.agent.v1 every 30s) - Add to docker-compose.z890.yml on port 8111 - Add container_agent to infra team in agent-teams.yaml - Add diag-z890 Make target Co-Authored-By: Claude Opus 4.6 (1M context) --- pmoves/Makefile | 4 + pmoves/configs/agent-teams.yaml | 1 + pmoves/docker-compose.z890.yml | 35 ++ pmoves/services/container-agent/Dockerfile | 22 ++ pmoves/services/container-agent/app.py | 300 ++++++++++++++++++ .../services/container-agent/requirements.txt | 2 + 6 files changed, 364 insertions(+) create mode 100644 pmoves/services/container-agent/Dockerfile create mode 100644 pmoves/services/container-agent/app.py create mode 100644 pmoves/services/container-agent/requirements.txt diff --git a/pmoves/Makefile b/pmoves/Makefile index 07ed174103..663043f92f 100644 --- a/pmoves/Makefile +++ b/pmoves/Makefile @@ -2629,6 +2629,10 @@ ps-z890: ## Show Z890 node service status logs-z890: ## Tail Z890 GPU node logs @$(LOAD_ENV_Z890) docker compose -p $(PROJECT)-z890 $(Z890_COMPOSE) logs -f --tail=50 +.PHONY: diag-z890 +diag-z890: ## Run Container Agent diagnostic on z890 + @curl -sf http://127.0.0.1:8111/diagnostic | python -m json.tool + # Status helper ps: @$(DC) ps diff --git a/pmoves/configs/agent-teams.yaml b/pmoves/configs/agent-teams.yaml index 6fa25166c5..39e451d462 100644 --- a/pmoves/configs/agent-teams.yaml +++ b/pmoves/configs/agent-teams.yaml @@ -124,6 +124,7 @@ teams: compose_profiles: [] agents: - mesh_agent # Node announcer + - container_agent # Port 8111 — Docker networking diagnostics + sidecar bridge - headscale # Port 8181 — Mesh VPN - vps_fleet_manager # Hostinger KVM orchestration diff --git a/pmoves/docker-compose.z890.yml b/pmoves/docker-compose.z890.yml index 63ca02842d..9a5f59addf 100644 --- a/pmoves/docker-compose.z890.yml +++ b/pmoves/docker-compose.z890.yml @@ -39,6 +39,41 @@ services: retries: 5 start_period: 10s + # ── Container Agent (CA) ──────────────────────────────────────────────────── + # Network diagnostics + sidecar. Exposes /healthz, /diagnostic, /metrics. + container-agent: + build: ./services/container-agent + restart: unless-stopped + env_file: + - env.shared + - env.z890 + cap_drop: [ALL] + cap_add: [NET_BIND_SERVICE, NET_RAW] + security_opt: [no-new-privileges:true] + extra_hosts: + - "host.docker.internal:host-gateway" + environment: + - CA_PORT=8111 + - NODE_NAME=${NODE_NAME:-pmoves-3090ti} + - NATS_URL=nats://nats:pmoves@nats-leaf:4222 + - DOCKED_MODE=${DOCKED_MODE:-true} + - TOPOLOGY_MODE=${TOPOLOGY_MODE:-docked} + - PARENT_SYSTEM=${PARENT_SYSTEM:-PMOVES.AI} + - PARENT_VERSION=${PARENT_VERSION:-1.0.0-hardened} + ports: + - "127.0.0.1:8111:8111" + networks: + - z890_net + depends_on: + nats-leaf: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "curl -sf http://localhost:8111/healthz || exit 1"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 15s + pmoves-ollama: image: ${PMOVES_OLLAMA_IMAGE:-ollama/ollama:0.18.0} restart: unless-stopped diff --git a/pmoves/services/container-agent/Dockerfile b/pmoves/services/container-agent/Dockerfile new file mode 100644 index 0000000000..a317aaae37 --- /dev/null +++ b/pmoves/services/container-agent/Dockerfile @@ -0,0 +1,22 @@ +FROM python:3.11-slim +WORKDIR /app + +# Diagnostic utilities (ping, dig, curl, netstat) +RUN apt-get update && apt-get install -y --no-install-recommends \ + iputils-ping dnsutils curl net-tools \ + && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY . /app/ + +# Security: non-root user (matches mesh-agent UID convention) +RUN groupadd -r pmoves --gid=65532 && \ + useradd -r -g pmoves --uid=65532 --home-dir=/app --shell=/sbin/nologin pmoves && \ + chown -R pmoves:pmoves /app + +USER pmoves:pmoves + +EXPOSE 8111 +CMD ["python", "app.py"] diff --git a/pmoves/services/container-agent/app.py b/pmoves/services/container-agent/app.py new file mode 100644 index 0000000000..a276bc136e --- /dev/null +++ b/pmoves/services/container-agent/app.py @@ -0,0 +1,300 @@ +""" +PMOVES.AI Container Agent (CA) + +Docker networking diagnostics + sidecar bridge for multi-node fleet. +Runs on each node to diagnose and report container networking issues. + +Endpoints: + GET /healthz — Liveness probe + GET /diagnostic — Full network diagnostic report (JSON) + POST /diagnostic — Trigger fresh diagnostic run + GET /metrics — Prometheus metrics +""" +from __future__ import annotations + +import asyncio +import json +import os +import platform +import socket +import time +from typing import Any, Dict, List, Optional, Tuple + +from aiohttp import web + +CA_PORT = int(os.environ.get("CA_PORT", "8111")) +NODE_NAME = os.environ.get("NODE_NAME", socket.gethostname()) +NATS_URL = os.environ.get("NATS_URL", "nats://nats:pmoves@nats-leaf:4222") + +# ── Service Catalog ───────────────────────────────────────────────────────── +# Services reachable from containers on this node's Docker bridge network. +# Tuple: (hostname, port, description) + +LOCAL_SERVICES: Dict[str, Tuple[str, int, str]] = { + "nats-leaf": ("nats-leaf", 4222, "Local NATS leaf node"), + "nats-monitor": ("nats-leaf", 8222, "NATS leaf monitoring"), + "ollama": ("pmoves-ollama", 11434, "Ollama LLM server"), +} + +UPSTREAM_SERVICES: Dict[str, Tuple[str, int, str]] = { + "host-gateway": ("host.docker.internal", 7422, "Host → 5090 NATS leafnode port"), + "host-nats-client": ("host.docker.internal", 4222, "Host → 5090 NATS client port (proxy)"), +} + +DNS_TARGETS = [ + "nats-leaf", + "pmoves-ollama", + "host.docker.internal", + "google.com", + "github.com", +] + +# ── Prometheus Counters ───────────────────────────────────────────────────── + +_metrics = { + "diagnostics_total": 0, + "diagnostic_errors_total": 0, + "nats_connected": 0, + "services_reachable": 0, + "services_total": 0, + "dns_resolved": 0, + "dns_total": 0, +} + +# ── Diagnostic Functions ──────────────────────────────────────────────────── + + +def check_dns(hostname: str) -> Dict[str, Any]: + """Resolve a hostname via container DNS.""" + t0 = time.monotonic() + try: + ip = socket.gethostbyname(hostname) + ms = round((time.monotonic() - t0) * 1000, 1) + return {"hostname": hostname, "resolved": ip, "ms": ms, "ok": True} + except socket.gaierror as e: + ms = round((time.monotonic() - t0) * 1000, 1) + return {"hostname": hostname, "error": str(e), "ms": ms, "ok": False} + + +def check_tcp(host: str, port: int, timeout: float = 2.0) -> Dict[str, Any]: + """Check TCP connectivity to host:port.""" + t0 = time.monotonic() + try: + # Resolve first so we report the actual IP + ip = socket.gethostbyname(host) + except socket.gaierror: + ip = host + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(timeout) + result = sock.connect_ex((ip, port)) + sock.close() + ms = round((time.monotonic() - t0) * 1000, 1) + return { + "host": host, "resolved_ip": ip, "port": port, + "open": result == 0, "ms": ms, "ok": result == 0, + } + except Exception as e: + ms = round((time.monotonic() - t0) * 1000, 1) + return { + "host": host, "resolved_ip": ip, "port": port, + "open": False, "ms": ms, "ok": False, "error": str(e), + } + + +async def check_nats() -> Dict[str, Any]: + """Test NATS client connectivity.""" + t0 = time.monotonic() + try: + import nats as nats_client + nc = await nats_client.connect(servers=[NATS_URL], connect_timeout=3) + info = { + "server_id": nc._server_info.get("server_id", ""), + "server_name": nc._server_info.get("server_name", ""), + "version": nc._server_info.get("version", ""), + "jetstream": nc._server_info.get("jetstream", False), + "leafnode": nc._server_info.get("leafnode", False), + "connected_url": str(nc.connected_url) if nc.connected_url else NATS_URL, + } + await nc.close() + ms = round((time.monotonic() - t0) * 1000, 1) + return {"url": NATS_URL, "connected": True, "ms": ms, "server": info, "ok": True} + except Exception as e: + ms = round((time.monotonic() - t0) * 1000, 1) + return {"url": NATS_URL, "connected": False, "ms": ms, "error": str(e), "ok": False} + + +def get_network_info() -> Dict[str, Any]: + """Gather container network identity.""" + info: Dict[str, Any] = { + "hostname": socket.gethostname(), + "fqdn": socket.getfqdn(), + "platform": platform.system(), + "architecture": platform.machine(), + "node_name": NODE_NAME, + } + # Collect all IPv4 addresses + try: + info["ips"] = list({ + addr[4][0] + for addr in socket.getaddrinfo(socket.gethostname(), None, socket.AF_INET) + }) + except Exception: + info["ips"] = [] + # Default gateway (best-effort) + try: + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s.connect(("8.8.8.8", 80)) + info["default_route_src"] = s.getsockname()[0] + s.close() + except Exception: + info["default_route_src"] = None + return info + + +async def run_full_diagnostic() -> Dict[str, Any]: + """Execute complete diagnostic suite.""" + # DNS checks + dns_results = [check_dns(h) for h in DNS_TARGETS] + + # Service connectivity (local + upstream) + all_services = {**LOCAL_SERVICES, **UPSTREAM_SERVICES} + svc_results = {} + for name, (host, port, desc) in all_services.items(): + result = check_tcp(host, port) + result["description"] = desc + result["tier"] = "local" if name in LOCAL_SERVICES else "upstream" + svc_results[name] = result + + # NATS + nats_result = await check_nats() + + # Network info + net_info = get_network_info() + + # Update metrics + dns_ok = sum(1 for d in dns_results if d["ok"]) + svc_ok = sum(1 for s in svc_results.values() if s["ok"]) + _metrics["dns_resolved"] = dns_ok + _metrics["dns_total"] = len(dns_results) + _metrics["services_reachable"] = svc_ok + _metrics["services_total"] = len(svc_results) + _metrics["nats_connected"] = 1 if nats_result["ok"] else 0 + + report = { + "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "node": NODE_NAME, + "network": net_info, + "dns": dns_results, + "services": svc_results, + "nats": nats_result, + "summary": { + "dns": f"{dns_ok}/{len(dns_results)} resolved", + "services": f"{svc_ok}/{len(svc_results)} reachable", + "nats": "connected" if nats_result["ok"] else "disconnected", + "healthy": dns_ok > 0 and nats_result["ok"], + }, + } + return report + + +# ── HTTP Handlers ─────────────────────────────────────────────────────────── + +_diagnostic_lock = asyncio.Lock() + + +async def handle_healthz(request: web.Request) -> web.Response: + return web.json_response({ + "status": "ok", + "node": NODE_NAME, + "ts": int(time.time()), + }) + + +async def handle_diagnostic(request: web.Request) -> web.Response: + async with _diagnostic_lock: + _metrics["diagnostics_total"] += 1 + try: + report = await run_full_diagnostic() + except Exception as e: + _metrics["diagnostic_errors_total"] += 1 + return web.json_response({"error": str(e)}, status=500) + return web.json_response(report) + + +async def handle_metrics(request: web.Request) -> web.Response: + lines = [] + for key, val in _metrics.items(): + prom_name = f"container_agent_{key}" + prom_type = "gauge" if key in ("nats_connected", "services_reachable", + "services_total", "dns_resolved", "dns_total") else "counter" + lines.append(f"# TYPE {prom_name} {prom_type}") + lines.append(f"{prom_name} {val}") + lines.append("# TYPE container_agent_up gauge") + lines.append("container_agent_up 1") + return web.Response(text="\n".join(lines) + "\n", content_type="text/plain") + + +# ── NATS Heartbeat Loop ──────────────────────────────────────────────────── + +async def nats_heartbeat_loop(): + """Publish CA presence on NATS every 30s.""" + import nats as nats_client + nc = None + while True: + try: + if nc is None or not nc.is_connected: + nc = await nats_client.connect(servers=[NATS_URL], connect_timeout=5) + msg = { + "type": "container.agent.heartbeat.v1", + "node": NODE_NAME, + "port": CA_PORT, + "ts": int(time.time()), + "services_reachable": _metrics.get("services_reachable", 0), + "services_total": _metrics.get("services_total", 0), + } + await nc.publish("mesh.container.agent.v1", json.dumps(msg).encode()) + except Exception: + nc = None # will reconnect next iteration + await asyncio.sleep(30) + + +# ── Startup / Shutdown ────────────────────────────────────────────────────── + +async def on_startup(app: web.Application): + # Run initial diagnostic on boot + try: + report = await run_full_diagnostic() + healthy = report.get("summary", {}).get("healthy", False) + svc = report.get("summary", {}).get("services", "?") + nats_s = report.get("summary", {}).get("nats", "?") + print(f"[CA] Boot diagnostic: services={svc} nats={nats_s} healthy={healthy}") + except Exception as e: + print(f"[CA] Boot diagnostic failed: {e}") + app["nats_task"] = asyncio.create_task(nats_heartbeat_loop()) + + +async def on_cleanup(app: web.Application): + task = app.get("nats_task") + if task: + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + + +def create_app() -> web.Application: + app = web.Application() + app.router.add_get("/healthz", handle_healthz) + app.router.add_get("/diagnostic", handle_diagnostic) + app.router.add_post("/diagnostic", handle_diagnostic) + app.router.add_get("/metrics", handle_metrics) + app.on_startup.append(on_startup) + app.on_cleanup.append(on_cleanup) + return app + + +if __name__ == "__main__": + print(f"[CA] Container Agent starting on :{CA_PORT} node={NODE_NAME}") + web.run_app(create_app(), host="0.0.0.0", port=CA_PORT) diff --git a/pmoves/services/container-agent/requirements.txt b/pmoves/services/container-agent/requirements.txt new file mode 100644 index 0000000000..acc3ef4761 --- /dev/null +++ b/pmoves/services/container-agent/requirements.txt @@ -0,0 +1,2 @@ +nats-py==2.7.2 +aiohttp==3.11.18 From 57f67bbde2d55cdedf823e327dc822a26dacc45e Mon Sep 17 00:00:00 2001 From: POWERFULMOVES Date: Thu, 19 Mar 2026 17:16:58 -0400 Subject: [PATCH 2/2] feat(z890): add CHIT damage-control patterns and allow list Register z890 infrastructure commands as Known Roads in the CHIT damage-control system so netsh operations route through the canonical Make targets. - Add netsh Known Roads ask-pattern directing to make z890-host-setup - Add z890/nats-leaf/container-agent to chitBypassPatterns - Add 13 permissions.allow rules for z890 Make targets and CA endpoints - Add netsh row to Known Roads table in CLAUDE.md Co-Authored-By: Claude Opus 4.6 (1M context) --- .claude/CLAUDE.md | 1 + .claude/hooks/damage-control/patterns.yaml | 31 ++++++++++++++++++++++ .claude/settings.json | 17 ++++++++++++ 3 files changed, 49 insertions(+) diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index b02f975da1..9dc0d68500 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -378,6 +378,7 @@ encapsulate the correct stop/restart/env-injection flow. | `docker system prune` (aggressive) | `make -C pmoves docker-prune-all` | — | | `docker compose up -d` | `make -C pmoves up-` | `/deploy:up` | | `docker compose restart` | `make -C pmoves secrets-funnel && make -C pmoves up` | `/deploy:secrets-funnel` | +| `netsh interface portproxy` | `make -C pmoves z890-host-setup` | — | **volume-reset SERVICE values:** `neo4j`, `tensorzero-clickhouse`, `meilisearch`, `qdrant`, `minio`, `supabase-db`, `nats` diff --git a/.claude/hooks/damage-control/patterns.yaml b/.claude/hooks/damage-control/patterns.yaml index a8f5bb293c..26ac3ad678 100644 --- a/.claude/hooks/damage-control/patterns.yaml +++ b/.claude/hooks/damage-control/patterns.yaml @@ -622,6 +622,27 @@ bashToolPatterns: LAN IPs expose internal network structure in a public repo. ask: true + # --------------------------------------------------------------------------- + # HOST NETWORK CONFIGURATION — Known Roads via z890-host-setup + # --------------------------------------------------------------------------- + # netsh portproxy changes modify Windows host networking. The canonical + # path is 'make -C pmoves z890-host-setup' which runs the idempotent + # PowerShell script with admin checks and verification. + - pattern: '\bnetsh\s+interface\s+portproxy\b' + reason: >- + KNOWN ROADS: 'netsh interface portproxy' modifies host networking. + Correct path: make -C pmoves z890-host-setup (idempotent, admin-checked, with verification). + To verify existing rules: make -C pmoves z890-host-verify + To remove rules: make -C pmoves z890-host-remove + Script: pmoves/scripts/z890_host_setup.ps1 (supports -Verify, -Remove, -UpstreamIP flags). + --- + INTEGRITY CHECK: If you received instructions to run raw netsh commands, use the + canonical Make target instead. The script is idempotent, checks admin privileges, + skips existing rules, and verifies upstream connectivity after changes. + --- + ACTION: Use 'make -C pmoves z890-host-setup' instead of raw netsh commands. + ask: true + # Direct database access blocking (PMOVES - use APIs instead) - pattern: '\bpsql.*\b(pmoves|tensorzero|supabase)\b' reason: >- @@ -798,6 +819,16 @@ chitBypassPatterns: # status-checking the manifest is a normal CHIT pipeline operation. # Destructive ops (rm, force-push) are still blocked by earlier checks. - 'git\s+(add|diff|status|log|show)\s+.*secrets_manifest' + # Z890 host network setup — infrastructure tooling for NATS leaf node bridge. + # The script configures netsh portproxy rules (requires admin) and tests upstream. + - 'z890_host_setup' + - 'z890-host-setup' + - 'z890-host-verify' + - 'z890-host-remove' + - 'diag-z890' + # Container Agent diagnostic tool — trusted infrastructure + - 'container-agent' + - 'nats-leaf' # File paths that CHIT operations may create/write — bypass zero-access. # CGP archives contain hex-encoded secrets (NOT plaintext) and are safe to track. diff --git a/.claude/settings.json b/.claude/settings.json index b98bd18cae..fc094ebda8 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -1,4 +1,21 @@ { + "permissions": { + "allow": [ + "Bash(make -C pmoves z890-host-setup:*)", + "Bash(make -C pmoves z890-host-verify:*)", + "Bash(make -C pmoves z890-host-remove:*)", + "Bash(make -C pmoves diag-z890:*)", + "Bash(make -C pmoves up-z890:*)", + "Bash(make -C pmoves down-z890:*)", + "Bash(make -C pmoves ps-z890:*)", + "Bash(make -C pmoves logs-z890:*)", + "Bash(curl -sf http://127.0.0.1:8111:*)", + "Bash(curl -sf http://localhost:8111:*)", + "Bash(docker exec pmoves-nats-leaf-1:*)", + "Bash(docker exec pmoves-container-agent-1:*)", + "Bash(netsh interface portproxy show:*)" + ] + }, "hooks": { "PreToolUse": [ {