Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 38 additions & 62 deletions cli/src/commands/hatch.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { spawn } from "child_process";
import { randomBytes } from "crypto";
import { existsSync, readFileSync, unlinkSync, writeFileSync } from "fs";
import { homedir, tmpdir, userInfo } from "os";
import { existsSync, unlinkSync, writeFileSync } from "fs";
import { tmpdir, userInfo } from "os";
import { join } from "path";

import { buildOpenclawStartupScript } from "../adapters/openclaw";
Expand Down Expand Up @@ -197,14 +197,15 @@ async function pollInstance(
instanceName: string,
project: string,
zone: string,
account?: string,
): Promise<PollResult> {
try {
const remoteCmd =
"L=$(tail -1 /var/log/startup-script.log 2>/dev/null || true); " +
"S=$(systemctl is-active google-startup-scripts.service 2>/dev/null || true); " +
"E=$(cat /var/log/startup-error 2>/dev/null || true); " +
'printf "%s\\n===HATCH_SEP===\\n%s\\n===HATCH_ERR===\\n%s" "$L" "$S" "$E"';
const output = await execOutput("gcloud", [
const args = [
"compute",
"ssh",
instanceName,
Expand All @@ -216,7 +217,9 @@ async function pollInstance(
"--ssh-flag=-o ConnectTimeout=10",
"--ssh-flag=-o LogLevel=ERROR",
`--command=${remoteCmd}`,
]);
];
if (account) args.push(`--account=${account}`);
const output = await execOutput("gcloud", args);
const sepIdx = output.indexOf("===HATCH_SEP===");
if (sepIdx === -1) {
return { lastLine: output.trim() || null, done: false, failed: false };
Expand Down Expand Up @@ -258,9 +261,10 @@ async function checkCurlFailure(
instanceName: string,
project: string,
zone: string,
account?: string,
): Promise<boolean> {
try {
const output = await execOutput("gcloud", [
const args = [
"compute",
"ssh",
instanceName,
Expand All @@ -272,7 +276,9 @@ async function checkCurlFailure(
"--ssh-flag=-o ConnectTimeout=10",
"--ssh-flag=-o LogLevel=ERROR",
`--command=test -s ${INSTALL_SCRIPT_REMOTE_PATH} && echo EXISTS || echo MISSING`,
]);
];
if (account) args.push(`--account=${account}`);
const output = await execOutput("gcloud", args);
return output.trim() === "MISSING";
} catch {
return false;
Expand All @@ -284,30 +290,35 @@ async function recoverFromCurlFailure(
project: string,
zone: string,
sshUser: string,
account?: string,
): Promise<void> {
if (!existsSync(INSTALL_SCRIPT_PATH)) {
throw new Error(`Install script not found at ${INSTALL_SCRIPT_PATH}`);
}

console.log("📋 Uploading install script to instance...");
await exec("gcloud", [
const scpArgs = [
"compute",
"scp",
INSTALL_SCRIPT_PATH,
`${instanceName}:${INSTALL_SCRIPT_REMOTE_PATH}`,
`--zone=${zone}`,
`--project=${project}`,
]);
];
if (account) scpArgs.push(`--account=${account}`);
console.log("📋 Uploading install script to instance...");
await exec("gcloud", scpArgs);

console.log("🔧 Running install script on instance...");
await exec("gcloud", [
const sshArgs = [
"compute",
"ssh",
`${sshUser}@${instanceName}`,
`--zone=${zone}`,
`--project=${project}`,
`--command=source ${INSTALL_SCRIPT_REMOTE_PATH}`,
]);
];
if (account) sshArgs.push(`--account=${account}`);
console.log("🔧 Running install script on instance...");
await exec("gcloud", sshArgs);
}

export async function watchHatching(
Expand Down Expand Up @@ -418,54 +429,15 @@ export async function watchHatching(
});
}

interface CloudCredentials {
provider: string;
projectId?: string;
serviceAccountKey?: string;
}

interface WorkspaceConfig {
cloudCredentials?: CloudCredentials;
}

async function activateGcpCredentialsFromConfig(): Promise<void> {
const configPath = join(homedir(), ".vellum", "workspace", "config.json");
let config: WorkspaceConfig;
try {
config = JSON.parse(readFileSync(configPath, "utf8")) as WorkspaceConfig;
} catch {
return;
}

const creds = config.cloudCredentials;
if (!creds || creds.provider !== "gcp" || !creds.serviceAccountKey || !creds.projectId) {
return;
}

const keyPath = join(tmpdir(), `vellum-sa-key-${Date.now()}.json`);
writeFileSync(keyPath, creds.serviceAccountKey);
try {
await exec("gcloud", [
"auth",
"activate-service-account",
`--key-file=${keyPath}`,
]);
await exec("gcloud", ["config", "set", "project", creds.projectId]);
} finally {
try {
unlinkSync(keyPath);
} catch {}
}
}

async function hatchGcp(
species: Species,
detached: boolean,
name: string | null,
): Promise<void> {
const startTime = Date.now();
const account = process.env.GCP_ACCOUNT_EMAIL;
try {
await activateGcpCredentialsFromConfig();
const project = process.env.GCP_PROJECT ?? (await getActiveProject());
let instanceName: string;

Expand All @@ -491,14 +463,14 @@ async function hatchGcp(
console.log("");

if (name) {
if (await instanceExists(name, project, zone)) {
if (await instanceExists(name, project, zone, account)) {
console.error(
`Error: Instance name '${name}' is already taken. Please choose a different name.`,
);
process.exit(1);
}
} else {
while (await instanceExists(instanceName, project, zone)) {
while (await instanceExists(instanceName, project, zone, account)) {
console.log(`⚠️ Instance name ${instanceName} already exists, generating a new name...`);
const suffix = generateRandomSuffix();
instanceName = `${species}-${suffix}`;
Expand All @@ -518,7 +490,7 @@ async function hatchGcp(

console.log("🔨 Creating instance with startup script...");
try {
await exec("gcloud", [
const createArgs = [
"compute",
"instances",
"create",
Expand All @@ -533,29 +505,33 @@ async function hatchGcp(
`--metadata-from-file=startup-script=${startupScriptPath}`,
`--labels=species=${species},vellum-assistant=true`,
"--tags=vellum-assistant",
]);
];
if (account) createArgs.push(`--account=${account}`);
await exec("gcloud", createArgs);
} finally {
try {
unlinkSync(startupScriptPath);
} catch {}
}

console.log("🔒 Syncing firewall rules...");
await syncFirewallRules(DESIRED_FIREWALL_RULES, project, FIREWALL_TAG);
await syncFirewallRules(DESIRED_FIREWALL_RULES, project, FIREWALL_TAG, account);

console.log(`✅ Instance ${instanceName} created successfully\n`);

let externalIp: string | null = null;
try {
const ipOutput = await execOutput("gcloud", [
const describeArgs = [
"compute",
"instances",
"describe",
instanceName,
`--project=${project}`,
`--zone=${zone}`,
"--format=get(networkInterfaces[0].accessConfigs[0].natIP)",
]);
];
if (account) describeArgs.push(`--account=${account}`);
const ipOutput = await execOutput("gcloud", describeArgs);
externalIp = ipOutput.trim() || null;
} catch {
console.log("⚠️ Could not retrieve external IP yet (instance may still be starting)");
Expand Down Expand Up @@ -594,7 +570,7 @@ async function hatchGcp(
console.log("");

const success = await watchHatching(
() => pollInstance(instanceName, project, zone),
() => pollInstance(instanceName, project, zone, account),
instanceName,
startTime,
species,
Expand All @@ -603,11 +579,11 @@ async function hatchGcp(
if (!success) {
if (
species === "vellum" &&
(await checkCurlFailure(instanceName, project, zone))
(await checkCurlFailure(instanceName, project, zone, account))
) {
console.log("");
console.log("🔄 Detected install script curl failure, attempting recovery...");
await recoverFromCurlFailure(instanceName, project, zone, sshUser);
await recoverFromCurlFailure(instanceName, project, zone, sshUser, account);
console.log("✅ Recovery successful!");
} else {
console.log("");
Expand Down
42 changes: 27 additions & 15 deletions cli/src/lib/gcp.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,16 +41,19 @@ interface FirewallRuleState {
async function describeFirewallRule(
ruleName: string,
project: string,
account?: string,
): Promise<FirewallRuleState | null> {
try {
const output = await execOutput("gcloud", [
const args = [
"compute",
"firewall-rules",
"describe",
ruleName,
`--project=${project}`,
"--format=json(name,direction,allowed,sourceRanges,destinationRanges,targetTags,description)",
]);
];
if (account) args.push(`--account=${account}`);
const output = await execOutput("gcloud", args);
const parsed = JSON.parse(output);
const allowed = (parsed.allowed ?? [])
.map((a: { IPProtocol: string; ports?: string[] }) => {
Expand Down Expand Up @@ -87,7 +90,7 @@ function ruleNeedsUpdate(spec: FirewallRuleSpec, state: FirewallRuleState): bool
);
}

async function createFirewallRule(spec: FirewallRuleSpec, project: string): Promise<void> {
async function createFirewallRule(spec: FirewallRuleSpec, project: string, account?: string): Promise<void> {
const args = [
"compute",
"firewall-rules",
Expand All @@ -106,35 +109,41 @@ async function createFirewallRule(spec: FirewallRuleSpec, project: string): Prom
if (spec.destinationRanges) {
args.push(`--destination-ranges=${spec.destinationRanges}`);
}
if (account) args.push(`--account=${account}`);
await exec("gcloud", args);
}

async function deleteFirewallRule(ruleName: string, project: string): Promise<void> {
await exec("gcloud", [
async function deleteFirewallRule(ruleName: string, project: string, account?: string): Promise<void> {
const args = [
"compute",
"firewall-rules",
"delete",
ruleName,
`--project=${project}`,
"--quiet",
]);
];
if (account) args.push(`--account=${account}`);
await exec("gcloud", args);
}

export async function syncFirewallRules(
desiredRules: FirewallRuleSpec[],
project: string,
tag: string,
account?: string,
): Promise<void> {
let existingNames: string[];
try {
const output = await execOutput("gcloud", [
const listArgs = [
"compute",
"firewall-rules",
"list",
`--project=${project}`,
`--filter=targetTags:${tag}`,
"--format=value(name)",
]);
];
if (account) listArgs.push(`--account=${account}`);
const output = await execOutput("gcloud", listArgs);
existingNames = output
.split("\n")
.map((s) => s.trim())
Expand All @@ -148,23 +157,23 @@ export async function syncFirewallRules(
for (const existingName of existingNames) {
if (!desiredNames.has(existingName)) {
console.log(` 🗑️ Deleting stale firewall rule: ${existingName}`);
await deleteFirewallRule(existingName, project);
await deleteFirewallRule(existingName, project, account);
}
}

for (const spec of desiredRules) {
const state = await describeFirewallRule(spec.name, project);
const state = await describeFirewallRule(spec.name, project, account);

if (!state) {
console.log(` ➕ Creating firewall rule: ${spec.name}`);
await createFirewallRule(spec, project);
await createFirewallRule(spec, project, account);
continue;
}

if (ruleNeedsUpdate(spec, state)) {
console.log(` 🔄 Updating firewall rule: ${spec.name}`);
await deleteFirewallRule(spec.name, project);
await createFirewallRule(spec, project);
await deleteFirewallRule(spec.name, project, account);
await createFirewallRule(spec, project, account);
continue;
}

Expand Down Expand Up @@ -224,17 +233,20 @@ export async function instanceExists(
instanceName: string,
project: string,
zone: string,
account?: string,
): Promise<boolean> {
try {
await execOutput("gcloud", [
const args = [
"compute",
"instances",
"describe",
instanceName,
`--project=${project}`,
`--zone=${zone}`,
"--format=get(name)",
]);
];
if (account) args.push(`--account=${account}`);
await execOutput("gcloud", args);
return true;
} catch {
return false;
Expand Down
Loading
Loading