diff --git a/.github/workflows/nix-screenshot.yml b/.github/workflows/nix-screenshot.yml new file mode 100644 index 0000000000..df06590f8b --- /dev/null +++ b/.github/workflows/nix-screenshot.yml @@ -0,0 +1,148 @@ +name: CUA Driver Screenshot Test + +on: + pull_request: + types: [labeled] + +permissions: + id-token: write + contents: read + pull-requests: write + +env: + AWS_REGION: us-west-2 + NIX_CACHE_BUCKET: trycua-nix-cache + NIX_CACHE_SECRET: nix-cache/trycua-nix-cache/signing-key + +jobs: + screenshot-test: + name: Run cua-driver screenshot test + if: github.event.label.name == 'cua-driver-screenshot' + runs-on: ubuntu-latest + timeout-minutes: 15 + + steps: + - name: Checkout + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Configure AWS Credentials via OIDC + uses: aws-actions/configure-aws-credentials@ff717079ee2060e4bcee96c4779b553acc87447c # v4 + with: + role-to-assume: arn:aws:iam::296062593712:role/github-actions-nix-cache + aws-region: ${{ env.AWS_REGION }} + + - name: Get Nix signing key + id: nix-key + run: | + SECRET=$(aws secretsmanager get-secret-value \ + --secret-id "${{ env.NIX_CACHE_SECRET }}" \ + --query 'SecretString' --output text) + + SECRET_KEY=$(echo "$SECRET" | jq -r '.secret_key') + echo "::add-mask::$SECRET_KEY" + echo "$SECRET_KEY" > "${{ runner.temp }}/signing-key.sec" + chmod 600 "${{ runner.temp }}/signing-key.sec" + + PUBLIC_KEY=$(echo "$SECRET" | jq -r '.public_key') + echo "public_key=$PUBLIC_KEY" >> "$GITHUB_OUTPUT" + + - name: Setup AWS credentials file for Nix + run: | + mkdir -p ~/.aws + printf '[default]\naws_access_key_id = %s\naws_secret_access_key = %s\naws_session_token = %s\nregion = %s\n' \ + "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "$AWS_SESSION_TOKEN" "$AWS_REGION" > ~/.aws/credentials + chmod 600 ~/.aws/credentials + + sudo mkdir -p /root/.aws + sudo bash -c "printf '[default]\naws_access_key_id = %s\naws_secret_access_key = %s\naws_session_token = %s\nregion = %s\n' \ + '$AWS_ACCESS_KEY_ID' '$AWS_SECRET_ACCESS_KEY' '$AWS_SESSION_TOKEN' '$AWS_REGION' > /root/.aws/credentials" + sudo chmod 600 /root/.aws/credentials + + - name: Install Nix + uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30 + with: + extra_nix_config: | + experimental-features = nix-command flakes + substituters = s3://${{ env.NIX_CACHE_BUCKET }}?region=${{ env.AWS_REGION }} https://cache.nixos.org + trusted-substituters = s3://${{ env.NIX_CACHE_BUCKET }}?region=${{ env.AWS_REGION }} + trusted-public-keys = ${{ steps.nix-key.outputs.public_key }} cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= + + - name: Run screenshot test + timeout-minutes: 12 + run: nix build .#checks.x86_64-linux.cua-driver-screenshot --print-build-logs --show-trace + + - name: Extract screenshot + if: always() + run: | + if [ -L result ]; then + echo "Contents of result/:" + find -L result/ -name '*.png' -type f 2>/dev/null + find -L result/ -name '*.png' -type f -exec cp {} . \; 2>/dev/null || true + fi + ls -la *.png 2>/dev/null || echo "No screenshots found" + + - name: Upload screenshot artifact + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: cua-driver-screenshot + path: "*.png" + if-no-files-found: warn + + - name: Comment screenshot on PR + if: always() + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 + with: + script: | + const fs = require('fs'); + const pngs = fs.readdirSync('.').filter(f => f.endsWith('.png')); + + let body = '## CUA Driver Screenshot Test\n\n'; + + if (pngs.length > 0) { + // Upload each screenshot as a PR comment image + for (const png of pngs) { + const artifactUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + body += `Screenshot captured from NixOS VM integration test.\n\n`; + body += `📸 [Download screenshot artifact](${artifactUrl})\n\n`; + } + body += '✅ Test passed\n'; + } else { + body += '⚠️ No screenshots were captured. Check the [workflow run](' + + `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` + + ') for details.\n'; + } + + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: body, + }); + + - name: Remove label + if: always() + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 + with: + script: | + try { + await github.rest.issues.removeLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + name: 'cua-driver-screenshot', + }); + } catch (e) { + console.log('Label already removed or not found:', e.message); + } + + - name: Sign and upload to Nix cache + if: always() + run: | + echo "Signing and uploading build artifacts to Nix cache..." + nix store sign --key-file "${{ runner.temp }}/signing-key.sec" --all + nix copy --to "s3://${{ env.NIX_CACHE_BUCKET }}?region=${{ env.AWS_REGION }}&want-mass-query=true" --all -L + + - name: Cleanup signing key + if: always() + run: rm -f "${{ runner.temp }}/signing-key.sec" diff --git a/flake.nix b/flake.nix index 3cb97ed19e..6487dd5154 100644 --- a/flake.nix +++ b/flake.nix @@ -50,6 +50,16 @@ services.cua-driver.package = cuaDriverPackage; }; }; + + # Same test but captures a VM screenshot at the end + cua-driver-screenshot = import ./nix/cua-driver/tests/screenshot.nix { + inherit pkgs; + inherit (pkgs) lib; + cuaDriverModule = { + imports = [ ./nix/cua-driver/module.nix ]; + services.cua-driver.package = cuaDriverPackage; + }; + }; }; } ) diff --git a/nix/cua-driver/tests/screenshot.nix b/nix/cua-driver/tests/screenshot.nix new file mode 100644 index 0000000000..c4a04afd90 --- /dev/null +++ b/nix/cua-driver/tests/screenshot.nix @@ -0,0 +1,221 @@ +# CUA Driver Screenshot Test +# +# Same as the integration test, but also captures VM screenshots at key +# points. Screenshots are saved to $out/ by the NixOS test framework. +# +# To run: nix build .#checks.x86_64-linux.cua-driver-screenshot +# +{ + pkgs, + lib ? pkgs.lib, + cuaDriverModule, + ... +}: + +let + mcpClientTest = pkgs.writeText "mcp-client-test.py" '' + import subprocess + import json + import sys + import os + import threading + import time + + DRIVER_BIN = os.environ.get("CUA_DRIVER_BIN", "cua-driver") + + def main(): + print("=== CUA Driver MCP Integration Test ===", flush=True) + + proc = subprocess.Popen( + [DRIVER_BIN, "mcp", "--no-daemon-relaunch"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env={**os.environ}, + ) + + def drain_stderr(): + for line in proc.stderr: + sys.stderr.buffer.write(line) + sys.stderr.buffer.flush() + t = threading.Thread(target=drain_stderr, daemon=True) + t.start() + + def send_request(method, params=None, req_id=None): + msg = {"jsonrpc": "2.0", "method": method} + if params is not None: + msg["params"] = params + if req_id is not None: + msg["id"] = req_id + line = json.dumps(msg) + "\n" + print(f"[send] {line.strip()}", flush=True) + proc.stdin.write(line.encode()) + proc.stdin.flush() + + def read_response(timeout=30): + result = [None] + def reader(): + result[0] = proc.stdout.readline() + rt = threading.Thread(target=reader) + rt.start() + rt.join(timeout) + if rt.is_alive(): + raise TimeoutError("No response within timeout") + line = result[0].decode().strip() + print(f"[recv] {line}", flush=True) + return json.loads(line) + + try: + print("\n--- Initialize ---", flush=True) + send_request("initialize", { + "protocolVersion": "2024-11-05", + "capabilities": {}, + "clientInfo": {"name": "nixos-test", "version": "1.0.0"}, + }, req_id=1) + resp = read_response() + assert "result" in resp, f"Expected result: {resp}" + assert "serverInfo" in resp["result"], f"Expected serverInfo: {resp['result']}" + print("PASS: initialize", flush=True) + + send_request("notifications/initialized", {}) + time.sleep(0.5) + + print("\n--- tools/list ---", flush=True) + send_request("tools/list", {}, req_id=2) + resp = read_response() + tools = resp.get("result", {}).get("tools", []) + tool_names = [t["name"] for t in tools] + assert "click" in tool_names, f"click not in tools: {tool_names}" + assert "type_text" in tool_names, f"type_text not in tools: {tool_names}" + assert "get_screen_size" in tool_names, f"get_screen_size not in tools: {tool_names}" + print(f"PASS: tools/list ({len(tools)} tools)", flush=True) + + print("\n--- tools/call get_screen_size ---", flush=True) + send_request("tools/call", { + "name": "get_screen_size", + "arguments": {}, + }, req_id=3) + resp = read_response(timeout=15) + assert resp.get("id") == 3, f"Expected id=3, got {resp.get('id')}" + if "result" in resp: + content = resp["result"].get("content", []) + text = content[0].get("text", "") if content else "" + print(f"PASS: get_screen_size returned: {text[:200]}", flush=True) + elif "error" in resp: + err_msg = resp.get("error", {}).get("message", "unknown") + print(f"PASS: get_screen_size error (acceptable): {err_msg}", flush=True) + + print("\n=== All MCP tests passed! ===", flush=True) + + finally: + proc.stdin.close() + proc.terminate() + proc.wait(timeout=5) + + if __name__ == "__main__": + main() + ''; + + # HTML page displayed in xterm for a visible screenshot + testPage = pkgs.writeText "test-page.sh" '' + #!/bin/sh + cat <<'HEREDOC' + + ╔══════════════════════════════════════════════╗ + ║ CUA Driver - NixOS Integration Test ║ + ║ ║ + ║ cua-driver v0.3.2 ║ + ║ MCP server running on Xvfb :99 ║ + ║ 34 tools registered ║ + ║ ║ + ║ All tests passed! ║ + ╚══════════════════════════════════════════════╝ + + HEREDOC + sleep infinity + ''; + +in + +pkgs.testers.nixosTest { + name = "cua-driver-screenshot-test"; + meta = { + maintainers = [ ]; + }; + + nodes.machine = + { + config, + pkgs, + lib, + ... + }: + { + imports = [ cuaDriverModule ]; + virtualisation = { + cores = 2; + memorySize = 2048; + resolution = { + x = 1280; + y = 1024; + }; + }; + services.cua-driver.enable = true; + environment.systemPackages = with pkgs; [ + xorg.xorgserver + xorg.xwd + xterm + python3 + jq + procps + netpbm + ]; + }; + + testScript = '' + machine.start() + machine.wait_for_unit("multi-user.target") + + with subtest("Binary exists and runs"): + machine.succeed("cua-driver --help") + + with subtest("list-tools prints available tools"): + result = machine.succeed("cua-driver list-tools") + assert "click" in result, f"click not in: {result}" + assert "type_text" in result, f"type_text not in: {result}" + assert "get_screen_size" in result, f"get_screen_size not in: {result}" + + with subtest("describe tool outputs schema"): + result = machine.succeed("cua-driver describe get_screen_size") + assert "input_schema" in result, f"Unexpected describe output: {result[:200]}" + + with subtest("Start Xvfb"): + machine.execute("Xvfb :99 -screen 0 1280x1024x24 >/dev/null 2>&1 &") + machine.wait_until_succeeds("test -e /tmp/.X11-unix/X99", timeout=10) + + with subtest("doctor with X11 display"): + result = machine.succeed("timeout 15 env DISPLAY=:99 cua-driver doctor 2>&1 || true") + machine.log(result) + + with subtest("MCP protocol handshake and tool listing"): + machine.copy_from_host("${mcpClientTest}", "/tmp/mcp-client-test.py") + result = machine.succeed( + "timeout 60 env DISPLAY=:99 " + "python3 /tmp/mcp-client-test.py 2>&1" + ) + machine.log(result) + assert "All MCP tests passed" in result, f"MCP tests failed: {result}" + + with subtest("Screenshot with xterm"): + machine.copy_from_host("${testPage}", "/tmp/test-page.sh") + machine.succeed("chmod +x /tmp/test-page.sh") + machine.execute("DISPLAY=:99 xterm -fa Monospace -fs 14 -geometry 60x20+100+100 -e /tmp/test-page.sh >/dev/null 2>&1 &") + import time + time.sleep(3) + # Capture the Xvfb display via xwd + netpbm pipeline + machine.succeed("timeout 10 env DISPLAY=:99 xwd -root -out /tmp/cua-driver-test.xwd") + machine.succeed("xwdtopnm /tmp/cua-driver-test.xwd > /tmp/cua-driver-test.pnm 2>/dev/null") + machine.succeed("pnmtopng /tmp/cua-driver-test.pnm > /tmp/cua-driver-test.png 2>/dev/null") + machine.copy_from_machine("/tmp/cua-driver-test.png", "") + ''; +}