diff --git a/packages/opencode/migration/20260530170000_automation_persistence/migration.sql b/packages/opencode/migration/20260530170000_automation_persistence/migration.sql new file mode 100644 index 000000000..7f77fcaa5 --- /dev/null +++ b/packages/opencode/migration/20260530170000_automation_persistence/migration.sql @@ -0,0 +1,26 @@ +CREATE TABLE `automation_definition` ( + `id` text PRIMARY KEY, + `project_id` text NOT NULL, + `owner_directory` text NOT NULL, + `time_created` integer NOT NULL, + `time_updated` integer NOT NULL, + `data` text NOT NULL, + CONSTRAINT `fk_automation_definition_project_id_project_id_fk` FOREIGN KEY (`project_id`) REFERENCES `project`(`id`) ON DELETE CASCADE +); +--> statement-breakpoint +CREATE INDEX `automation_definition_project_owner_updated_idx` ON `automation_definition` (`project_id`,`owner_directory`,`time_updated`,`id`);--> statement-breakpoint +CREATE TABLE `automation_run` ( + `id` text PRIMARY KEY, + `automation_id` text NOT NULL, + `project_id` text NOT NULL, + `owner_directory` text NOT NULL, + `triggered_at` integer NOT NULL, + `data` text NOT NULL, + `time_created` integer NOT NULL, + `time_updated` integer NOT NULL, + CONSTRAINT `fk_automation_run_automation_id_automation_definition_id_fk` FOREIGN KEY (`automation_id`) REFERENCES `automation_definition`(`id`) ON DELETE CASCADE, + CONSTRAINT `fk_automation_run_project_id_project_id_fk` FOREIGN KEY (`project_id`) REFERENCES `project`(`id`) ON DELETE CASCADE +); +--> statement-breakpoint +CREATE INDEX `automation_run_automation_triggered_idx` ON `automation_run` (`automation_id`,`triggered_at`,`id`);--> statement-breakpoint +CREATE INDEX `automation_run_project_owner_idx` ON `automation_run` (`project_id`,`owner_directory`); diff --git a/packages/opencode/package.json b/packages/opencode/package.json index 88eb3d9cb..1c477e4a9 100644 --- a/packages/opencode/package.json +++ b/packages/opencode/package.json @@ -58,6 +58,7 @@ "@types/babel__core": "7.20.5", "@types/bun": "catalog:", "@types/cross-spawn": "catalog:", + "@types/luxon": "catalog:", "@types/mime-types": "3.0.1", "@types/npm-package-arg": "6.1.4", "@types/npmcli__arborist": "6.3.3", @@ -143,6 +144,7 @@ "hono-openapi": "catalog:", "ignore": "7.0.5", "jsonc-parser": "3.3.1", + "luxon": "catalog:", "mime-types": "3.0.2", "minimatch": "10.2.3", "npm-package-arg": "13.0.2", diff --git a/packages/opencode/src/automation/automation.sql.ts b/packages/opencode/src/automation/automation.sql.ts new file mode 100644 index 000000000..2c478cea0 --- /dev/null +++ b/packages/opencode/src/automation/automation.sql.ts @@ -0,0 +1,45 @@ +import { index, integer, sqliteTable, text } from "drizzle-orm/sqlite-core" +import type { Automation } from "." +import { ProjectTable } from "@/project/project.sql" +import type { ProjectID } from "@/project/schema" +import { Timestamps } from "@/storage/schema.sql" + +export const AutomationDefinitionTable = sqliteTable( + "automation_definition", + { + id: text().primaryKey().$type(), + project_id: text() + .$type() + .notNull() + .references(() => ProjectTable.id, { onDelete: "cascade" }), + owner_directory: text().notNull(), + time_created: integer().notNull(), + time_updated: integer().notNull(), + data: text({ mode: "json" }).notNull().$type(), + }, + (table) => [ + index("automation_definition_project_owner_updated_idx").on(table.project_id, table.owner_directory, table.time_updated, table.id), + ], +) + +export const AutomationRunTable = sqliteTable( + "automation_run", + { + id: text().primaryKey().$type(), + automation_id: text() + .notNull() + .references(() => AutomationDefinitionTable.id, { onDelete: "cascade" }), + project_id: text() + .$type() + .notNull() + .references(() => ProjectTable.id, { onDelete: "cascade" }), + owner_directory: text().notNull(), + triggered_at: integer().notNull(), + data: text({ mode: "json" }).notNull().$type(), + ...Timestamps, + }, + (table) => [ + index("automation_run_automation_triggered_idx").on(table.automation_id, table.triggered_at, table.id), + index("automation_run_project_owner_idx").on(table.project_id, table.owner_directory), + ], +) diff --git a/packages/opencode/src/automation/index.ts b/packages/opencode/src/automation/index.ts index 74fab2967..8068127b7 100644 --- a/packages/opencode/src/automation/index.ts +++ b/packages/opencode/src/automation/index.ts @@ -6,8 +6,10 @@ import { Instance } from "@/project/instance" import { ProjectID } from "@/project/schema" import { PermissionID } from "@/permission/schema" import { SessionID } from "@/session/schema" -import { NotFoundError } from "@/storage/db" +import { and, Database, desc, eq, gte, inArray, lt, NotFoundError, or, sql } from "@/storage/db" +import { Flock } from "@/util/flock" import type { AutomationRunAttendance, AutomationRunBlocker } from "./run-context" +import { AutomationDefinitionTable, AutomationRunTable } from "./automation.sql" export const AutomationID = { Definition: { @@ -45,6 +47,14 @@ export namespace Automation { .object({ error: z.literal("invalid_automation"), details: z.array(ValidationErrorDetail) }) .strict() .meta({ ref: "AutomationValidationError" }) + export const ConflictErrorResponse = z + .object({ error: z.literal("automation_conflict"), message: z.string() }) + .strict() + .meta({ ref: "AutomationConflictError" }) + export const ActiveRunStillRunningErrorResponse = z + .object({ error: z.literal("active_run_still_running"), runID: RunID }) + .strict() + .meta({ ref: "AutomationActiveRunStillRunningError" }) export const Stop = z .discriminatedUnion("kind", [ z.object({ kind: z.literal("count"), count: z.number().int().positive() }).strict(), @@ -231,14 +241,10 @@ export namespace Automation { } type State = { - definitions: Map - runs: Map activeWriters: Set activeRuns: Map } const state = Instance.state(() => ({ - definitions: new Map(), - runs: new Map(), activeWriters: new Set(), activeRuns: new Map(), })) @@ -321,6 +327,44 @@ export namespace Automation { }) } + function cronFieldValues(field: string, min: number, max: number) { + const values = new Set() + for (const item of field.split(",")) { + const [base, stepRaw] = item.split("/") + const step = stepRaw === undefined ? 1 : Number(stepRaw) + const range = base === "*" ? [min, max] : base.split("-").map(Number) + const start = range[0] + const end = base === "*" || (range.length === 1 && stepRaw !== undefined) ? max : range.length === 1 ? range[0] : range[1] + for (let value = start; value <= end; value += step) values.add(value) + } + return values + } + + function hasPossibleCronDayMonth(dayField: string, monthField: string) { + const maxDays = new Map([ + [1, 31], + [2, 29], + [3, 31], + [4, 30], + [5, 31], + [6, 30], + [7, 31], + [8, 31], + [9, 30], + [10, 31], + [11, 30], + [12, 31], + ]) + for (const month of cronFieldValues(monthField, 1, 12)) { + const maxDay = maxDays.get(month) + if (maxDay === undefined) continue + for (const day of cronFieldValues(dayField, 1, 31)) { + if (day <= maxDay) return true + } + } + return false + } + export function isValidCronExpression(expression: string) { const fields = expression.trim().split(/\s+/) if (fields.length !== 5) return false @@ -329,7 +373,8 @@ export namespace Automation { isValidCronField(fields[1], 0, 23) && isValidCronField(fields[2], 1, 31) && isValidCronField(fields[3], 1, 12) && - isValidCronField(fields[4], 0, 7) + isValidCronField(fields[4], 0, 7) && + (fields[4] !== "*" || hasPossibleCronDayMonth(fields[2], fields[3])) ) } @@ -417,22 +462,122 @@ export namespace Automation { nextFires: [], failureStreak: 0, } - state().definitions.set(definition.id, definition) + writeDefinition(definition) return definition } export function list(): Definition[] { - return [...state().definitions.values()].sort((a, b) => b.updatedAt - a.updatedAt || b.id.localeCompare(a.id)) + const projectID = Instance.project.id + const ownerDirectory = Instance.directory + return Database.use((db) => + db + .select() + .from(AutomationDefinitionTable) + .where( + and( + eq(AutomationDefinitionTable.project_id, projectID), + eq(AutomationDefinitionTable.owner_directory, ownerDirectory), + ), + ) + .orderBy(desc(AutomationDefinitionTable.time_updated), desc(AutomationDefinitionTable.id)) + .all() + .map((row) => Definition.parse(row.data)), + ) } export function get(id: string): Definition { - const definition = state().definitions.get(id) + const definition = getOptional(id) if (!definition) throw new NotFoundError({ message: `Automation not found: ${id}` }) return definition } function getOptional(id: string): Definition | undefined { - return state().definitions.get(id) + const projectID = Instance.project.id + const row = Database.use((db) => + db + .select() + .from(AutomationDefinitionTable) + .where(eq(AutomationDefinitionTable.id, id)) + .get(), + ) + if (!row || row.project_id !== projectID) return undefined + if (row.owner_directory !== Instance.directory) return undefined + return Definition.parse(row.data) + } + + function writeDefinition(definition: Definition) { + Database.use((db) => + db + .insert(AutomationDefinitionTable) + .values({ + id: definition.id, + project_id: definition.where.projectID, + owner_directory: Instance.directory, + time_created: definition.createdAt, + time_updated: definition.updatedAt, + data: definition, + }) + .run(), + ) + } + + function replaceDefinition(previous: Definition, next: Definition) { + return Database.transaction( + (db) => { + const row = db.select().from(AutomationDefinitionTable).where(eq(AutomationDefinitionTable.id, previous.id)).get() + if (!row || row.project_id !== previous.where.projectID || row.owner_directory !== Instance.directory) { + throw new NotFoundError({ message: `Automation not found: ${previous.id}` }) + } + const current = Definition.parse(row.data) + if (current.revision !== previous.revision) throw new ConflictError(previous.id) + db.update(AutomationDefinitionTable) + .set({ + project_id: next.where.projectID, + owner_directory: Instance.directory, + time_updated: next.updatedAt, + data: next, + }) + .where( + and( + eq(AutomationDefinitionTable.id, previous.id), + sql`json_extract(${AutomationDefinitionTable.data}, '$.revision') = ${previous.revision}`, + ), + ) + .run() + return next + }, + { behavior: "immediate" }, + ) + } + + function writeRun(run: Run) { + const definition = getOptional(run.automationID) + if (!definition) throw new NotFoundError({ message: `Automation not found: ${run.automationID}` }) + const now = Date.now() + Database.use((db) => + db + .insert(AutomationRunTable) + .values({ + id: run.id, + automation_id: run.automationID, + project_id: definition.where.projectID, + owner_directory: Instance.directory, + triggered_at: run.triggeredAt, + data: run, + time_created: now, + time_updated: now, + }) + .run(), + ) + } + + function getRun(runID: string): Run | undefined { + const projectID = Instance.project.id + const row = Database.use((db) => + db.select().from(AutomationRunTable).where(eq(AutomationRunTable.id, runID)).get(), + ) + if (!row || row.project_id !== projectID || row.owner_directory !== Instance.directory) return undefined + return Run.parse(row.data) } function isSameValue(left: unknown, right: unknown): boolean { @@ -468,25 +613,46 @@ export namespace Automation { }) const details = validateCreateInput(next) if (details.length) throw new ValidationError(details) - state().definitions.set(id, next) - return next + return replaceDefinition(previous, next) } - export function remove(id: string): { tombstone: Tombstone; stoppedRun?: Run } { + export async function remove(id: string): Promise<{ tombstone: Tombstone; stoppedRun?: Run }> { const previous = get(id) const stoppedRun = stopActiveRun(id) - state().definitions.delete(id) - if (!stoppedRun) state().runs.delete(id) + const liveRun = await getLiveActiveRun(id) + if (liveRun) throw new ActiveRunStillRunningError(liveRun.id) + Database.use((db) => db.delete(AutomationDefinitionTable).where(eq(AutomationDefinitionTable.id, id)).run()) return { tombstone: { id: previous.id, deleted: true, revision: previous.revision + 1 }, stoppedRun } } - function replaceRun(run: Run) { - const current = state().runs.get(run.automationID) ?? [] - state().runs.set( - run.automationID, - current.map((item) => (item.id === run.id ? run : item)), + function replaceRun(previous: Run, next: Run) { + return Database.transaction( + (db) => { + const row = db.select().from(AutomationRunTable).where(eq(AutomationRunTable.id, previous.id)).get() + if (!row || row.project_id !== Instance.project.id || row.owner_directory !== Instance.directory) return previous + const current = Run.parse(row.data) + if (current.revision !== previous.revision) return current + const now = Date.now() + db.update(AutomationRunTable) + .set({ + automation_id: next.automationID, + project_id: row.project_id, + owner_directory: row.owner_directory, + triggered_at: next.triggeredAt, + data: next, + time_updated: now, + }) + .where( + and( + eq(AutomationRunTable.id, previous.id), + sql`json_extract(${AutomationRunTable.data}, '$.revision') = ${previous.revision}`, + ), + ) + .run() + return next + }, + { behavior: "immediate" }, ) - return run } function reviseRun(run: Run, patch: Record): Run { @@ -500,7 +666,7 @@ export namespace Automation { for (const [key, value] of Object.entries(next)) { if (value === undefined) delete (next as Record)[key] } - return replaceRun(Run.parse(next)) + return replaceRun(run, Run.parse(next)) } function stopRun( @@ -522,24 +688,46 @@ export namespace Automation { const active = state().activeRuns.get(automationID) if (!active) return undefined active.controller.abort() - const current = state().runs.get(automationID)?.find((run) => run.id === active.runID) + const current = getRun(active.runID) return current ? stopRun(current, "cancelled") : undefined } + async function getLiveActiveRun(automationID: string) { + get(automationID) + const projectID = Instance.project.id + const ownerDirectory = Instance.directory + const rows = Database.use((db) => + db + .select() + .from(AutomationRunTable) + .where( + and( + eq(AutomationRunTable.automation_id, automationID), + eq(AutomationRunTable.project_id, projectID), + eq(AutomationRunTable.owner_directory, ownerDirectory), + sql`json_extract(${AutomationRunTable.data}, '$.state') in ('scheduled', 'running', 'awaiting_input')`, + ), + ) + .all(), + ) + for (const row of rows) { + const run = Run.parse(row.data) + if (!isActiveRun(run)) continue + if (await hasLiveRunLease(run.id)) return run + } + } + export function stopRunByID( runID: string, stopReason: Extract["stopReason"], options?: { now?: number }, ): Run | undefined { - for (const runs of state().runs.values()) { - const run = runs.find((item) => item.id === runID) - if (!run) continue - const active = state().activeRuns.get(run.automationID) - if (active?.runID === runID) active.controller.abort() - const stopped = stopRun(run, stopReason, options) - return stopped === run ? undefined : stopped - } - return undefined + const run = getRun(runID) + if (!run) return undefined + const active = state().activeRuns.get(run.automationID) + if (active?.runID === runID) active.controller.abort() + const stopped = stopRun(run, stopReason, options) + return stopped === run ? undefined : stopped } export function markRunStarted(run: Run, sessionID: SessionID, options?: { now?: number }): Run { @@ -554,15 +742,26 @@ export namespace Automation { } function setDefinitionAutomationSession(definition: Definition, sessionID: SessionID) { - if (definition.automationSessionID === sessionID) return definition - const next = Definition.parse({ - ...definition, - automationSessionID: sessionID, - revision: definition.revision + 1, - updatedAt: Date.now(), - }) - state().definitions.set(definition.id, next) - return next + let current = definition + if (current.automationSessionID === sessionID) return current + const buildNext = (source: Definition) => + Definition.parse({ + ...source, + automationSessionID: sessionID, + revision: source.revision + 1, + updatedAt: Date.now(), + }) + for (let attempt = 0; attempt < 2; attempt++) { + try { + return replaceDefinition(current, buildNext(current)) + } catch (error) { + if (!(error instanceof ConflictError)) throw error + const latest = getOptional(definition.id) + if (!latest || latest.context !== "continue" || latest.automationSessionID === sessionID) return latest ?? definition + current = latest + } + } + return current } export function markRunBlocked(run: Run, blocker: AutomationRunBlocker): Run { @@ -581,11 +780,10 @@ export namespace Automation { }) } - export function runNow(id: string, options?: { now?: number }): Run { + export function runNow(id: string, options?: { now?: number; runID?: string }): Run { const definition = get(id) - const current = state().runs.get(id) ?? [] const run = Run.parse({ - id: AutomationID.Run.ascending(), + id: options?.runID ?? AutomationID.Run.ascending(), automationID: id, revision: 1, definitionRevision: definition.revision, @@ -598,25 +796,193 @@ export namespace Automation { error: null, cost: null, }) - state().runs.set(id, [run, ...current]) + writeRun(run) return run } export function hasActiveRun(automationID: string): boolean { if (state().activeRuns.has(automationID)) return true - return (state().runs.get(automationID) ?? []).some( - (run) => run.state === "scheduled" || run.state === "running" || run.state === "awaiting_input", + get(automationID) + const projectID = Instance.project.id + const ownerDirectory = Instance.directory + return Boolean( + Database.use((db) => + db + .select({ id: AutomationRunTable.id }) + .from(AutomationRunTable) + .where( + and( + eq(AutomationRunTable.automation_id, automationID), + eq(AutomationRunTable.project_id, projectID), + eq(AutomationRunTable.owner_directory, ownerDirectory), + sql`json_extract(${AutomationRunTable.data}, '$.state') in ('scheduled', 'running', 'awaiting_input')`, + ), + ) + .limit(1) + .get(), + ), + ) + } + + function isActiveRun(run: Run) { + return run.state === "scheduled" || run.state === "running" || run.state === "awaiting_input" + } + + function runLeaseKey(runID: string) { + return `automation-run:${Instance.directory}:${runID}` + } + + async function hasLiveRunLease(runID: string) { + const lease = await Flock.tryAcquire(runLeaseKey(runID)) + if (!lease) return true + await lease.release().catch(() => undefined) + return false + } + + function hasDurableActiveWriter(run: Run, writerKey: string) { + const definition = get(run.automationID) + const projectID = definition.where.projectID + const ownerDirectory = Instance.directory + return Database.transaction( + (db) => { + const rows = db + .select() + .from(AutomationRunTable) + .where( + and( + eq(AutomationRunTable.project_id, projectID), + eq(AutomationRunTable.owner_directory, ownerDirectory), + sql`json_extract(${AutomationRunTable.data}, '$.state') in ('scheduled', 'running', 'awaiting_input')`, + ), + ) + .all() + const automationIDs = [...new Set(rows.map((row) => row.automation_id))] + const definitions = automationIDs.length + ? db + .select() + .from(AutomationDefinitionTable) + .where( + and( + eq(AutomationDefinitionTable.project_id, projectID), + eq(AutomationDefinitionTable.owner_directory, ownerDirectory), + inArray(AutomationDefinitionTable.id, automationIDs), + ), + ) + .all() + : [] + const writerKeys = new Map( + definitions.map((row) => { + const item = Definition.parse(row.data) + return [item.id, item.where.worktree ?? item.where.projectID] + }), + ) + return rows.some((row) => { + if (row.id === run.id) return false + const item = Run.parse(row.data) + if (!isActiveRun(item)) return false + return writerKeys.get(item.automationID) === writerKey + }) + }, + { behavior: "immediate" }, ) } export function hasRunTriggeredAtOrAfter(automationID: string, triggeredAt: number): boolean { - return (state().runs.get(automationID) ?? []).some((run) => run.triggeredAt >= triggeredAt) + get(automationID) + const projectID = Instance.project.id + const ownerDirectory = Instance.directory + return Boolean( + Database.use((db) => + db + .select({ id: AutomationRunTable.id }) + .from(AutomationRunTable) + .where( + and( + eq(AutomationRunTable.automation_id, automationID), + eq(AutomationRunTable.project_id, projectID), + eq(AutomationRunTable.owner_directory, ownerDirectory), + gte(AutomationRunTable.triggered_at, triggeredAt), + ), + ) + .limit(1) + .get(), + ), + ) } export function completedRunCount(automationID: string): number { get(automationID) - const runs = state().runs.get(automationID) ?? [] - return runs.filter((run) => run.state === "succeeded" || run.state === "failed").length + const projectID = Instance.project.id + const ownerDirectory = Instance.directory + const row = Database.use((db) => + db + .select({ count: sql`count(*)` }) + .from(AutomationRunTable) + .where( + and( + eq(AutomationRunTable.automation_id, automationID), + eq(AutomationRunTable.project_id, projectID), + eq(AutomationRunTable.owner_directory, ownerDirectory), + sql`json_extract(${AutomationRunTable.data}, '$.state') in ('succeeded', 'failed')`, + ), + ) + .get(), + ) + return Number(row?.count ?? 0) + } + + export async function reconcileInterruptedRuns(options?: { now?: number }): Promise { + const projectID = Instance.project.id + const ownerDirectory = Instance.directory + const now = options?.now ?? Date.now() + const rows = Database.use((db) => + db + .select() + .from(AutomationRunTable) + .where( + and( + eq(AutomationRunTable.project_id, projectID), + eq(AutomationRunTable.owner_directory, ownerDirectory), + sql`json_extract(${AutomationRunTable.data}, '$.state') in ('scheduled', 'running', 'awaiting_input')`, + ), + ) + .all(), + ) + const stopped: Run[] = [] + for (const row of rows) { + const run = Run.parse(row.data) + if (!isActiveRun(run)) continue + const active = state().activeRuns.get(run.automationID) + if (active?.runID === run.id) continue + if (await hasLiveRunLease(run.id)) continue + const next = Database.transaction( + (db) => { + const currentRow = db.select().from(AutomationRunTable).where(eq(AutomationRunTable.id, run.id)).get() + if (!currentRow) return + const current = Run.parse(currentRow.data) + if (!isActiveRun(current)) return + const nextData: Record = { + ...current, + revision: current.revision + 1, + state: "stopped", + completedAt: now, + result: null, + error: null, + stopReason: current.state === "awaiting_input" ? "blocker_lost" : "expired", + } + delete nextData.blocker + const next = Run.parse(nextData) + db.update(AutomationRunTable) + .set({ data: next, time_updated: now }) + .where(eq(AutomationRunTable.id, next.id)) + .run() + return next + }, + { behavior: "immediate" }, + ) + if (next) stopped.push(next) + } + return stopped } export function recordStoppedRun( @@ -628,35 +994,45 @@ export namespace Automation { return stopRun(run, stopReason, options) } - export function runNowExecuting( + export async function runNowExecuting( id: string, options: { executor: RunExecutor; attendance?: AutomationRunAttendance; now?: number }, - ): Run { - const initial = runNow(id, { now: options.now }) - void executeRun(initial, options.executor, options.attendance ?? "attended") - return initial + ): Promise { + const runID = AutomationID.Run.ascending() + const lease = await Flock.acquire(runLeaseKey(runID)) + try { + const initial = runNow(id, { now: options.now, runID }) + queueMicrotask(() => void executeRun(initial, options.executor, options.attendance ?? "attended", lease)) + return initial + } catch (error) { + await lease.release().catch(() => undefined) + throw error + } } - async function executeRun(initial: Run, executor: RunExecutor, attendance: AutomationRunAttendance) { + async function executeRun(initial: Run, executor: RunExecutor, attendance: AutomationRunAttendance, lease: Flock.Lease) { const data = state() - const definition = get(initial.automationID) - const writerKey = definition.where.worktree ?? definition.where.projectID - if (data.activeWriters.has(writerKey)) { - const stopped = reviseRun(initial, { - state: "stopped", - completedAt: Date.now(), - stopReason: "previous_run_awaiting_input", - }) - await publishRunUpdated(stopped) - return - } - data.activeWriters.add(writerKey) const controller = new AbortController() - data.activeRuns.set(initial.automationID, { writerKey, controller, runID: initial.id }) + let writerKey: string | undefined let current = initial try { + const definition = get(initial.automationID) + writerKey = definition.where.worktree ?? definition.where.projectID + for (const run of await reconcileInterruptedRuns()) await publishRunUpdated(run) + if (data.activeWriters.has(writerKey) || hasDurableActiveWriter(initial, writerKey)) { + const stopped = reviseRun(initial, { + state: "stopped", + completedAt: Date.now(), + stopReason: "previous_run_awaiting_input", + }) + await publishRunUpdated(stopped) + return + } + data.activeWriters.add(writerKey) + data.activeRuns.set(initial.automationID, { writerKey, controller, runID: initial.id }) const prepared = await executor({ definition, run: initial, attendance, signal: controller.signal }) - const latest = state().runs.get(initial.automationID)?.find((item) => item.id === initial.id) ?? initial + const latest = getRun(initial.id) + if (!latest) return if (controller.signal.aborted) { const stopped = stopRun(latest, "cancelled") current = stopped @@ -681,7 +1057,9 @@ export namespace Automation { current = succeeded await publishRunUpdated(succeeded) } catch (error) { - current = state().runs.get(initial.automationID)?.find((item) => item.id === initial.id) ?? current + const latest = getRun(initial.id) + if (!latest) return + current = latest if (controller.signal.aborted) { const stopped = stopRun(current, "cancelled") if (stopped !== current) await publishRunUpdated(stopped) @@ -708,21 +1086,46 @@ export namespace Automation { await publishRunUpdated(failed) } finally { const active = data.activeRuns.get(initial.automationID) - if (active?.runID === initial.id) { + if (writerKey && active?.runID === initial.id) { data.activeRuns.delete(initial.automationID) data.activeWriters.delete(writerKey) } + await lease.release().catch(() => undefined) } } export function runs(input: { automationID: string; limit?: number; cursor?: string }) { - get(input.automationID) const limit = Math.min(Math.max(input.limit ?? 50, 1), 100) - const all = state().runs.get(input.automationID) ?? [] - const cursorIndex = input.cursor ? all.findIndex((run) => run.id === input.cursor) : -1 - const start = input.cursor ? (cursorIndex === -1 ? all.length : cursorIndex + 1) : 0 - const items = all.slice(start, start + limit) - return { items, nextCursor: start + limit < all.length ? items.at(-1)?.id ?? null : null } + get(input.automationID) + const projectID = Instance.project.id + const ownerDirectory = Instance.directory + const cursorRun = input.cursor ? getRun(input.cursor) : undefined + if (input.cursor && (!cursorRun || cursorRun.automationID !== input.automationID)) return { items: [], nextCursor: null } + const cursorPredicate = cursorRun + ? or( + lt(AutomationRunTable.triggered_at, cursorRun.triggeredAt), + and(eq(AutomationRunTable.triggered_at, cursorRun.triggeredAt), lt(AutomationRunTable.id, cursorRun.id)), + ) + : undefined + const page = Database.use((db) => + db + .select() + .from(AutomationRunTable) + .where( + and( + eq(AutomationRunTable.automation_id, input.automationID), + eq(AutomationRunTable.project_id, projectID), + eq(AutomationRunTable.owner_directory, ownerDirectory), + cursorPredicate, + ), + ) + .orderBy(desc(AutomationRunTable.triggered_at), desc(AutomationRunTable.id)) + .limit(limit + 1) + .all() + .map((row) => Run.parse(row.data)), + ) + const items = page.slice(0, limit) + return { items, nextCursor: page.length > limit ? items.at(-1)?.id ?? null : null } } export const publishDefinitionUpdated = (definition: Definition) => Bus.publish(Event.DefinitionUpdated, definition) @@ -736,3 +1139,17 @@ export class ValidationError extends Error { this.name = "AutomationValidationError" } } + +export class ConflictError extends Error { + constructor(readonly id: string) { + super(`Automation changed while updating: ${id}`) + this.name = "AutomationConflictError" + } +} + +export class ActiveRunStillRunningError extends Error { + constructor(readonly runID: string) { + super(`Automation run is still running: ${runID}`) + this.name = "AutomationActiveRunStillRunningError" + } +} diff --git a/packages/opencode/src/automation/scheduler.ts b/packages/opencode/src/automation/scheduler.ts index c3e7b96e5..13fa62c06 100644 --- a/packages/opencode/src/automation/scheduler.ts +++ b/packages/opencode/src/automation/scheduler.ts @@ -1,13 +1,18 @@ import { Context, Effect, Fiber, Layer } from "effect" +import { DateTime } from "luxon" +import { Log } from "@opencode-ai/core/util/log" import { Automation } from "." import { Bus } from "@/bus" import { Instance, type InstanceContext } from "@/project/instance" import { NotFoundError } from "@/storage/db" +import { Flock } from "@/util/flock" import { sessionPromptExecutor } from "./runner" export namespace AutomationScheduler { const MAX_TIMER_DELAY_MS = 2_147_483_647 const MISSED_SCHEDULE_GRACE_MS = 60_000 + const CRON_LOOKAHEAD_MINUTES = 527_040 * 5 + const log = Log.create({ service: "automation.scheduler" }) export interface Clock { now(): number @@ -25,6 +30,7 @@ export namespace AutomationScheduler { export interface Interface { stop(): void stopOwnedRuns(): void + settleOwner(): Promise reschedule(definition: Automation.Definition): void cancel(automationID: string): void computeNextFireAt(definition: Automation.Definition, from?: number): number | null @@ -34,6 +40,9 @@ export namespace AutomationScheduler { clock?: Clock executor?: Automation.RunExecutor runtime?: TaskRuntime + ownerKey?: string + ownerRetryMs?: number + ownerRescanMs?: number } type ScheduledTask = { @@ -43,6 +52,16 @@ export namespace AutomationScheduler { definition: Automation.Definition } + type CronSchedule = { + minutes: Set + hours: Set + days: Set + months: Set + weekdays: Set + dayRestricted: boolean + weekdayRestricted: boolean + } + export const liveClock: Clock = { now: () => Date.now(), sleep: (delayMs, signal) => @@ -97,10 +116,82 @@ export namespace AutomationScheduler { return definition.fireAt } if (!canScheduleRecurring(definition)) return null - if (definition.rhythm.kind !== "interval") return null + if (definition.rhythm.kind === "cron") return computeNextCronFireAt(definition, from) return from + definition.rhythm.everyMs } + function cronValues(field: string, min: number, max: number, options?: { sundayAlias?: boolean }) { + const values = new Set() + for (const item of field.split(",")) { + const [base, stepRaw] = item.split("/") + if (!base || item.split("/").length > 2) throw new Error(`Invalid cron field: ${item}`) + const step = stepRaw === undefined ? 1 : Number(stepRaw) + if (!Number.isInteger(step) || step <= 0) throw new Error(`Invalid cron step: ${item}`) + const range = base === "*" ? [min, max] : base.split("-").map(Number) + if (range.length === 0 || range.length > 2 || range.some((value) => !Number.isInteger(value))) { + throw new Error(`Invalid cron field: ${item}`) + } + const start = range[0] + const end = base === "*" || (range.length === 1 && stepRaw !== undefined) ? max : range.length === 1 ? range[0] : range[1] + if (start < min || end > max || start > end) throw new Error(`Invalid cron range: ${item}`) + for (let value = start; value <= end; value += step) { + values.add(options?.sundayAlias && value === 7 ? 0 : value) + } + } + return values + } + + function parseCronSchedule(expression: string): CronSchedule { + const fields = expression.trim().split(/\s+/) + if (fields.length !== 5) throw new Error(`Invalid cron expression: ${expression}`) + const [minuteField, hourField, dayField, monthField, weekdayField] = fields + return { + minutes: cronValues(minuteField, 0, 59), + hours: cronValues(hourField, 0, 23), + days: cronValues(dayField, 1, 31), + months: cronValues(monthField, 1, 12), + weekdays: cronValues(weekdayField, 0, 7, { sundayAlias: true }), + dayRestricted: dayField !== "*", + weekdayRestricted: weekdayField !== "*", + } + } + + function cronMatches(schedule: CronSchedule, time: DateTime) { + const weekday = time.weekday === 7 ? 0 : time.weekday + const dayMatches = schedule.days.has(time.day) + const weekdayMatches = schedule.weekdays.has(weekday) + const calendarMatches = + schedule.dayRestricted && schedule.weekdayRestricted ? dayMatches || weekdayMatches : dayMatches && weekdayMatches + return ( + schedule.minutes.has(time.minute) && + schedule.hours.has(time.hour) && + schedule.months.has(time.month) && + calendarMatches + ) + } + + function computeNextCronFireAt(definition: Extract, from: number) { + if (definition.rhythm.kind !== "cron") return null + const schedule = parseCronSchedule(definition.rhythm.expression) + let cursor = DateTime.fromMillis(from, { zone: definition.timezone }).plus({ minutes: 1 }).startOf("minute") + for (let attempts = 0; attempts < CRON_LOOKAHEAD_MINUTES; attempts++) { + if (cronMatches(schedule, cursor)) return cursor.toMillis() + cursor = cursor.plus({ minutes: 1 }) + } + return null + } + + function computePreviousCronFireAt(definition: Extract, from: number, until: number) { + if (definition.rhythm.kind !== "cron" || until < from) return null + const schedule = parseCronSchedule(definition.rhythm.expression) + let cursor = DateTime.fromMillis(until, { zone: definition.timezone }).startOf("minute") + for (let attempts = 0; attempts < CRON_LOOKAHEAD_MINUTES && cursor.toMillis() >= from; attempts++) { + if (cronMatches(schedule, cursor)) return cursor.toMillis() + cursor = cursor.minus({ minutes: 1 }) + } + return null + } + function canScheduleRecurring(definition: Extract) { if (definition.stop.kind === "never") return true if (definition.stop.kind === "count") return Automation.completedRunCount(definition.id) < definition.stop.count @@ -111,7 +202,11 @@ export namespace AutomationScheduler { if (left.kind !== right.kind || left.paused !== right.paused) return false if (left.kind === "oneshot" && right.kind === "oneshot") return left.fireAt === right.fireAt if (left.kind === "recurring" && right.kind === "recurring") { - return JSON.stringify(left.rhythm) === JSON.stringify(right.rhythm) && JSON.stringify(left.stop) === JSON.stringify(right.stop) + return ( + left.timezone === right.timezone && + JSON.stringify(left.rhythm) === JSON.stringify(right.rhythm) && + JSON.stringify(left.stop) === JSON.stringify(right.stop) + ) } return false } @@ -120,12 +215,22 @@ export namespace AutomationScheduler { const clock = options.clock ?? liveClock const executor = options.executor ?? sessionPromptExecutor const runtime = options.runtime ?? liveRuntime + const ownerKey = options.ownerKey + const ownerRetryMs = options.ownerRetryMs ?? 5_000 + const ownerRescanMs = options.ownerRescanMs ?? 5_000 const tasks = new Map() + const unschedulable = new Map() const ownedRuns = new Map() const schedulerStoppedRuns = new Set() + let ownsTimers = !ownerKey + let ownerLease: Flock.Lease | undefined + let ownerAttempt: Promise | undefined + let ownerRetryTimer: ReturnType | undefined + let ownerRescanTimer: ReturnType | undefined let running = true const cancel = (automationID: string) => { + unschedulable.delete(automationID) const entry = tasks.get(automationID) if (!entry) return tasks.delete(automationID) @@ -135,8 +240,11 @@ export namespace AutomationScheduler { const scheduleNextInterval = (automationID: string) => { try { const latest = Automation.get(automationID) - if (latest.kind === "recurring" && latest.rhythm.kind === "interval" && !latest.paused && canScheduleRecurring(latest)) { - schedule(latest, clock.now() + latest.rhythm.everyMs) + if (latest.kind === "recurring" && !latest.paused && canScheduleRecurring(latest)) { + const next = + latest.rhythm.kind === "interval" ? clock.now() + latest.rhythm.everyMs : computeNextFireAt(latest, clock.now()) + if (next === null) cancel(automationID) + else schedule(latest, next) } else { cancel(automationID) } @@ -146,14 +254,14 @@ export namespace AutomationScheduler { } } - const fire = (automationID: string, triggeredAt: number) => { + const fire = async (automationID: string, triggeredAt: number) => { tasks.delete(automationID) const firedAt = clock.now() try { const latest = Automation.get(automationID) if (latest.paused) return if (latest.kind === "oneshot" && latest.fireAt !== triggeredAt) return - if (latest.kind === "recurring" && (latest.rhythm.kind !== "interval" || !canScheduleRecurring(latest))) return + if (latest.kind === "recurring" && !canScheduleRecurring(latest)) return if (firedAt - triggeredAt > MISSED_SCHEDULE_GRACE_MS) { const stopped = Automation.recordStoppedRun(automationID, "missed_schedule", { now: firedAt, triggeredAt }) schedulerStoppedRuns.add(stopped.id) @@ -165,6 +273,7 @@ export namespace AutomationScheduler { if (!NotFoundError.isInstance(error)) throw error return } + for (const run of await Automation.reconcileInterruptedRuns({ now: firedAt })) void Automation.publishRunUpdated(run) if (Automation.hasActiveRun(automationID)) { const stopped = Automation.recordStoppedRun(automationID, "previous_run_awaiting_input", { now: triggeredAt }) schedulerStoppedRuns.add(stopped.id) @@ -173,12 +282,14 @@ export namespace AutomationScheduler { return } try { - const run = Automation.runNowExecuting(automationID, { + const run = await Automation.runNowExecuting(automationID, { executor, attendance: "unattended", now: triggeredAt, }) ownedRuns.set(run.id, automationID) + const latest = Automation.get(automationID) + if (latest.kind === "recurring" && latest.rhythm.kind === "cron") scheduleNextInterval(automationID) } catch (error) { if (!NotFoundError.isInstance(error)) throw error } @@ -197,12 +308,12 @@ export namespace AutomationScheduler { if (!isCurrentTask(automationID, fireAt, token, signal)) return } if (!isCurrentTask(automationID, fireAt, token, signal)) return - fire(automationID, fireAt) + yield* Effect.promise(() => fire(automationID, fireAt)) }) const schedule = (definition: Automation.Definition, fireAt: number) => { cancel(definition.id) - if (!running || definition.paused) return + if (!running || !ownsTimers || definition.paused) return const token = Symbol(definition.id) tasks.set(definition.id, { task: runtime.fork((signal) => waitUntil(definition.id, fireAt, token, signal)), @@ -219,6 +330,16 @@ export namespace AutomationScheduler { return true } + const preserveDueSchedule = (definition: Automation.Definition) => { + const current = tasks.get(definition.id) + if (!current || current.fireAt > clock.now() || !isSameSchedule(current.definition, definition)) return false + current.definition = definition + return true + } + + const isStableCronSchedule = (definition: Automation.Definition) => + definition.kind === "recurring" && definition.rhythm.kind === "cron" && definition.stop.kind === "never" + const hasSchedulerOwnedActiveRun = (automationID: string) => { for (const ownedAutomationID of ownedRuns.values()) { if (ownedAutomationID === automationID) return true @@ -227,8 +348,31 @@ export namespace AutomationScheduler { } const reschedule = (definition: Automation.Definition) => { + if (!ownsTimers) return + if (preserveDueSchedule(definition)) return + if (isStableCronSchedule(definition) && preservePendingSchedule(definition)) return + const cached = unschedulable.get(definition.id) + if (cached && isSameSchedule(cached, definition)) return + unschedulable.delete(definition.id) + if (definition.kind === "recurring" && definition.rhythm.kind === "cron" && canScheduleRecurring(definition)) { + const firstScheduled = computeNextCronFireAt(definition, definition.createdAt) + const missed = firstScheduled === null ? null : computePreviousCronFireAt(definition, firstScheduled, clock.now()) + if (missed !== null && !Automation.hasRunTriggeredAtOrAfter(definition.id, missed)) { + const stopped = Automation.recordStoppedRun(definition.id, "missed_schedule", { now: clock.now(), triggeredAt: missed }) + schedulerStoppedRuns.add(stopped.id) + void Automation.publishRunUpdated(stopped) + } + } const next = computeNextFireAt(definition, clock.now()) if (next === null) { + cancel(definition.id) + if (isStableCronSchedule(definition)) unschedulable.set(definition.id, definition) + return + } + if (definition.kind === "oneshot" && next <= clock.now()) { + const stopped = Automation.recordStoppedRun(definition.id, "missed_schedule", { now: clock.now(), triggeredAt: next }) + schedulerStoppedRuns.add(stopped.id) + void Automation.publishRunUpdated(stopped) cancel(definition.id) return } @@ -255,7 +399,48 @@ export namespace AutomationScheduler { cancel(event.properties.id) }) - for (const definition of Automation.list()) reschedule(definition) + const scan = async () => { + if (!running || !ownsTimers) return + for (const run of await Automation.reconcileInterruptedRuns({ now: clock.now() })) void Automation.publishRunUpdated(run) + for (const definition of Automation.list()) { + try { + reschedule(definition) + } catch (error) { + log.error("automation scheduler scan failed", { error, automationID: definition.id }) + } + } + } + + const becomeOwner = async () => { + if (!running || !ownerKey || ownerLease) return + const lease = await Flock.tryAcquire(ownerKey).catch(() => undefined) + if (!lease || !running || ownerLease) { + if (lease) await lease.release().catch(() => undefined) + return + } + ownerLease = lease + ownsTimers = true + ownerRescanTimer = setInterval(() => void scan(), ownerRescanMs) + ownerRescanTimer.unref?.() + for (const run of await Automation.reconcileInterruptedRuns({ now: clock.now() })) void Automation.publishRunUpdated(run) + void scan() + } + + const settleOwner = () => { + if (!ownerKey || ownerLease || !running) return Promise.resolve() + ownerAttempt ??= becomeOwner().finally(() => { + ownerAttempt = undefined + }) + return ownerAttempt + } + + if (ownerKey) { + void settleOwner() + ownerRetryTimer = setInterval(() => void settleOwner(), ownerRetryMs) + ownerRetryTimer.unref?.() + } else { + void scan() + } const stopOwnedRuns = () => { for (const runID of [...ownedRuns.keys()]) { @@ -268,13 +453,17 @@ export namespace AutomationScheduler { return { stop() { running = false + if (ownerRetryTimer) clearInterval(ownerRetryTimer) + if (ownerRescanTimer) clearInterval(ownerRescanTimer) unsubscribeRunUpdates() unsubscribeDefinitionUpdates() unsubscribeDefinitionDeletes() stopOwnedRuns() for (const automationID of [...tasks.keys()]) cancel(automationID) + if (ownerLease) void ownerLease.release().catch(() => undefined) }, stopOwnedRuns, + settleOwner, reschedule, cancel, computeNextFireAt(definition, from = clock.now()) { @@ -292,7 +481,7 @@ export namespace AutomationScheduler { const owner = Instance.state( () => { const context = Instance.current - const state = { context, scheduler: make() } + const state = { context, scheduler: make({ ownerKey: `automation-scheduler:${context.directory}` }) } owners.set(context.directory, state) return state }, diff --git a/packages/opencode/src/server/instance/automation.ts b/packages/opencode/src/server/instance/automation.ts index 96f562854..df39e924a 100644 --- a/packages/opencode/src/server/instance/automation.ts +++ b/packages/opencode/src/server/instance/automation.ts @@ -2,7 +2,7 @@ import { Hono } from "hono" import type { Context } from "hono" import { describeRoute, resolver, validator } from "hono-openapi" import z from "zod" -import { Automation, AutomationID, ValidationError } from "@/automation" +import { ActiveRunStillRunningError, Automation, AutomationID, ConflictError, ValidationError } from "@/automation" import { sessionPromptExecutor } from "@/automation/runner" import { AutomationScheduler } from "@/automation/scheduler" import { errors } from "../error" @@ -11,11 +11,26 @@ function validationError(error: ValidationError) { return Automation.ValidationErrorResponse.parse({ error: "invalid_automation", details: error.details }) } +function conflictError(error: ConflictError) { + return Automation.ConflictErrorResponse.parse({ error: "automation_conflict", message: error.message }) +} + +function activeRunStillRunningError(error: ActiveRunStillRunningError) { + return Automation.ActiveRunStillRunningErrorResponse.parse({ + error: "active_run_still_running", + runID: error.runID, + }) +} + async function publishIfChanged(previous: Automation.Definition, definition: Automation.Definition) { if (definition.revision === previous.revision) return await Automation.publishDefinitionUpdated(definition) } +async function settleAutomationScheduler() { + await AutomationScheduler.current().settleOwner() +} + function validationIssuePath(issue: unknown) { const path = typeof issue === "object" && issue !== null && "path" in issue ? issue.path : undefined if (!Array.isArray(path)) return "" @@ -78,7 +93,10 @@ export const AutomationRoutes = (): Hono => }, }, }), - (c) => c.json({ items: Automation.list() }), + async (c) => { + await settleAutomationScheduler() + return c.json({ items: Automation.list() }) + }, ) .post( "/", @@ -101,7 +119,7 @@ export const AutomationRoutes = (): Hono => validator("json", Automation.CreateInput, automationBodyValidationHook), async (c) => { try { - AutomationScheduler.current() + await settleAutomationScheduler() const definition = Automation.create(c.req.valid("json")) await Automation.publishDefinitionUpdated(definition) return c.json(definition) @@ -126,7 +144,10 @@ export const AutomationRoutes = (): Hono => }, }), validator("param", z.object({ automationID: AutomationID.Definition.zod })), - (c) => c.json(Automation.get(c.req.valid("param").automationID)), + async (c) => { + await settleAutomationScheduler() + return c.json(Automation.get(c.req.valid("param").automationID)) + }, ) .put( "/:automationID", @@ -143,6 +164,10 @@ export const AutomationRoutes = (): Hono => description: "Automation validation failed", content: { "application/json": { schema: resolver(Automation.ValidationErrorResponse) } }, }, + 409: { + description: "Automation update conflict", + content: { "application/json": { schema: resolver(Automation.ConflictErrorResponse) } }, + }, ...errors(400, 404), }, }), @@ -151,13 +176,14 @@ export const AutomationRoutes = (): Hono => async (c) => { try { const automationID = c.req.valid("param").automationID - AutomationScheduler.current() + await settleAutomationScheduler() const previous = Automation.get(automationID) const definition = Automation.update(automationID, c.req.valid("json")) await publishIfChanged(previous, definition) return c.json(definition) } catch (error) { if (error instanceof ValidationError) return c.json(validationError(error), 422) + if (error instanceof ConflictError) return c.json(conflictError(error), 409) throw error } }, @@ -173,17 +199,26 @@ export const AutomationRoutes = (): Hono => description: "Paused automation definition", content: { "application/json": { schema: resolver(Automation.Definition) } }, }, + 409: { + description: "Automation update conflict", + content: { "application/json": { schema: resolver(Automation.ConflictErrorResponse) } }, + }, ...errors(404), }, }), validator("param", z.object({ automationID: AutomationID.Definition.zod })), async (c) => { - const automationID = c.req.valid("param").automationID - AutomationScheduler.current() - const previous = Automation.get(automationID) - const definition = Automation.update(automationID, { paused: true }) - await publishIfChanged(previous, definition) - return c.json(definition) + try { + const automationID = c.req.valid("param").automationID + await settleAutomationScheduler() + const previous = Automation.get(automationID) + const definition = Automation.update(automationID, { paused: true }) + await publishIfChanged(previous, definition) + return c.json(definition) + } catch (error) { + if (error instanceof ConflictError) return c.json(conflictError(error), 409) + throw error + } }, ) .post( @@ -197,17 +232,26 @@ export const AutomationRoutes = (): Hono => description: "Resumed automation definition", content: { "application/json": { schema: resolver(Automation.Definition) } }, }, + 409: { + description: "Automation update conflict", + content: { "application/json": { schema: resolver(Automation.ConflictErrorResponse) } }, + }, ...errors(404), }, }), validator("param", z.object({ automationID: AutomationID.Definition.zod })), async (c) => { - const automationID = c.req.valid("param").automationID - AutomationScheduler.current() - const previous = Automation.get(automationID) - const definition = Automation.update(automationID, { paused: false }) - await publishIfChanged(previous, definition) - return c.json(definition) + try { + const automationID = c.req.valid("param").automationID + await settleAutomationScheduler() + const previous = Automation.get(automationID) + const definition = Automation.update(automationID, { paused: false }) + await publishIfChanged(previous, definition) + return c.json(definition) + } catch (error) { + if (error instanceof ConflictError) return c.json(conflictError(error), 409) + throw error + } }, ) .delete( @@ -215,23 +259,34 @@ export const AutomationRoutes = (): Hono => describeRoute({ summary: "Delete automation", description: - "Delete an automation definition and return a tombstone. If a run is active, stop it and publish the stopped run before publishing the tombstone.", + "Delete an automation definition and return a tombstone. If a run is active in this process, stop it and publish the stopped run before publishing the tombstone. If a live run is owned by another process, return 409 without deleting.", operationId: "automation.delete", responses: { 200: { description: "Automation deletion tombstone", content: { "application/json": { schema: resolver(Automation.Tombstone) } }, }, + 409: { + description: "Automation has a live run owned by another process", + content: { "application/json": { schema: resolver(Automation.ActiveRunStillRunningErrorResponse) } }, + }, ...errors(404), }, }), validator("param", z.object({ automationID: AutomationID.Definition.zod })), async (c) => { - const removed = Automation.remove(c.req.valid("param").automationID) - AutomationScheduler.current().cancel(removed.tombstone.id) - if (removed.stoppedRun) await Automation.publishRunUpdated(removed.stoppedRun) - await Automation.publishDefinitionDeleted(removed.tombstone) - return c.json(removed.tombstone) + try { + const scheduler = AutomationScheduler.current() + await scheduler.settleOwner() + const removed = await Automation.remove(c.req.valid("param").automationID) + scheduler.cancel(removed.tombstone.id) + if (removed.stoppedRun) await Automation.publishRunUpdated(removed.stoppedRun) + await Automation.publishDefinitionDeleted(removed.tombstone) + return c.json(removed.tombstone) + } catch (error) { + if (error instanceof ActiveRunStillRunningError) return c.json(activeRunStillRunningError(error), 409) + throw error + } }, ) .post( @@ -250,7 +305,8 @@ export const AutomationRoutes = (): Hono => }), validator("param", z.object({ automationID: AutomationID.Definition.zod })), async (c) => { - const run = Automation.runNowExecuting(c.req.valid("param").automationID, { + await settleAutomationScheduler() + const run = await Automation.runNowExecuting(c.req.valid("param").automationID, { executor: sessionPromptExecutor, }) await Automation.publishRunUpdated(run) @@ -279,5 +335,8 @@ export const AutomationRoutes = (): Hono => cursor: AutomationID.Run.zod.optional(), }), ), - (c) => c.json(Automation.runs({ automationID: c.req.valid("param").automationID, ...c.req.valid("query") })), + async (c) => { + await settleAutomationScheduler() + return c.json(Automation.runs({ automationID: c.req.valid("param").automationID, ...c.req.valid("query") })) + }, ) diff --git a/packages/opencode/src/storage/schema.ts b/packages/opencode/src/storage/schema.ts index b07a70a50..480a93111 100644 --- a/packages/opencode/src/storage/schema.ts +++ b/packages/opencode/src/storage/schema.ts @@ -1,4 +1,5 @@ export { AccountTable, AccountStateTable, ControlAccountTable } from "../account/account.sql" +export { AutomationDefinitionTable, AutomationRunTable } from "../automation/automation.sql" export { ProjectTable } from "../project/project.sql" export { SessionTable, diff --git a/packages/opencode/src/util/flock.ts b/packages/opencode/src/util/flock.ts index 74c7905eb..df3331157 100644 --- a/packages/opencode/src/util/flock.ts +++ b/packages/opencode/src/util/flock.ts @@ -325,6 +325,30 @@ export namespace Flock { } } + export async function tryAcquire(key: string, input: Omit = {}): Promise { + input.signal?.throwIfAborted() + const cfg: Opts = { + staleMs: input.staleMs ?? defaultOpts.staleMs, + timeoutMs: 0, + baseDelayMs: input.baseDelayMs ?? defaultOpts.baseDelayMs, + maxDelayMs: input.maxDelayMs ?? defaultOpts.maxDelayMs, + } + const dir = input.dir ?? root + + await mkdir(dir, { recursive: true }) + const lockfile = path.join(dir, Hash.fast(key) + ".lock") + const lock = await tryAcquireLockDir(lockfile, cfg) + if (!lock.acquired) return undefined + lock.startHeartbeat() + const release = () => lock.release() + return { + release, + [Symbol.asyncDispose]() { + return release() + }, + } + } + export async function withLock(key: string, fn: () => Promise, input: Options = {}) { await using _ = await acquire(key, input) input.signal?.throwIfAborted() diff --git a/packages/opencode/test/server/automation-routes.test.ts b/packages/opencode/test/server/automation-routes.test.ts index bf82be54e..b6a1b2729 100644 --- a/packages/opencode/test/server/automation-routes.test.ts +++ b/packages/opencode/test/server/automation-routes.test.ts @@ -10,6 +10,7 @@ import { ErrorMiddleware } from "../../src/server/middleware" import { AutomationRoutes } from "../../src/server/instance/automation" import { PermissionID } from "../../src/permission/schema" import { SessionID } from "../../src/session/schema" +import { Flock } from "../../src/util/flock" import { tmpdir } from "../fixture/fixture" void Log.init({ print: false }) @@ -45,6 +46,24 @@ async function waitForRunCount(automationID: string, count: number) { throw new Error(`Timed out waiting for automation run count: ${count}`) } +async function waitForRunState(automationID: string, state: Automation.Run["state"]) { + const deadline = Date.now() + 2_000 + while (Date.now() < deadline) { + const run = Automation.runs({ automationID }).items[0] + if (run?.state === state) return run + await Bun.sleep(5) + } + throw new Error(`Timed out waiting for automation run state: ${state}`) +} + +function deferred() { + let resolve!: (value: T | PromiseLike) => void + const promise = new Promise((done) => { + resolve = done + }) + return { promise, resolve } +} + type RecurringCreateInput = Extract type OneshotCreateInput = Extract @@ -94,12 +113,224 @@ function run(overrides: Record = {}) { } describe("automation routes", () => { + test("reloads definitions and runs from durable storage after instance restart", async () => { + await using tmp = await tmpdir({ git: true }) + let automationID: string | undefined + + await Instance.provide({ + directory: tmp.path, + fn: async () => { + const definition = Automation.create(recurringInput(Instance.project.id), { now: 100 }) + const run = Automation.runNow(definition.id, { now: 200 }) + automationID = definition.id + + expect(run.automationID).toBe(definition.id) + }, + }) + + await Instance.disposeAll() + + await Instance.provide({ + directory: tmp.path, + fn: async () => { + if (!automationID) throw new Error("expected automationID") + expect(Automation.list().map((item) => item.id)).toEqual([automationID]) + expect(Automation.runs({ automationID }).items.map((item) => item.triggeredAt)).toEqual([200]) + }, + }) + }) + + test("reconciles persisted active runs with stopped reasons after restart", async () => { + await using tmp = await tmpdir({ git: true }) + let automationID: string | undefined + + await Instance.provide({ + directory: tmp.path, + fn: async () => { + const definition = Automation.create(recurringInput(Instance.project.id), { now: 100 }) + const scheduled = Automation.runNow(definition.id, { now: 200 }) + const running = Automation.markRunStarted(scheduled, SessionID.descending(), { now: 300 }) + Automation.markRunBlocked(running, { kind: "question", callID: "call_1" }) + automationID = definition.id + }, + }) + + await Instance.disposeAll() + + await Instance.provide({ + directory: tmp.path, + fn: async () => { + if (!automationID) throw new Error("expected automationID") + const reconciled = await Automation.reconcileInterruptedRuns({ now: 400 }) + expect(reconciled).toHaveLength(1) + expect(reconciled[0]).toMatchObject({ state: "stopped", stopReason: "blocker_lost", completedAt: 400 }) + expect(Automation.runs({ automationID }).items[0]).toMatchObject({ + state: "stopped", + stopReason: "blocker_lost", + completedAt: 400, + }) + }, + }) + }) + + test("does not reconcile a persisted active run while another process holds its run lease", async () => { + await withAutomationApp(async ({ projectID }) => { + const definition = Automation.create(recurringInput(projectID), { now: 100 }) + const active = Automation.runNow(definition.id, { now: 200 }) + await using _ = await Flock.acquire(`automation-run:${Instance.directory}:${active.id}`) + + const reconciled = await Automation.reconcileInterruptedRuns({ now: 300 }) + + expect(reconciled).toEqual([]) + expect(Automation.runs({ automationID: definition.id }).items[0]).toMatchObject({ + id: active.id, + state: "scheduled", + }) + + const blockedDefinition = Automation.create(recurringInput(projectID, { title: "Blocked repo brief" }), { now: 100 }) + let started = false + await Automation.runNowExecuting(blockedDefinition.id, { + now: 400, + executor: async () => { + started = true + return { sessionID: SessionID.descending(), result: "done", cost: 0 } + }, + }) + const blocked = await waitForRunState(blockedDefinition.id, "stopped") + expect(started).toBe(false) + expect(blocked).toMatchObject({ state: "stopped", stopReason: "previous_run_awaiting_input" }) + }) + }) + + test("does not expose an executing run to reconcile before its run lease is held", async () => { + await withAutomationApp(async ({ projectID }) => { + const release = deferred<{ sessionID: SessionID; result: string | null; cost?: number | null }>() + const definition = Automation.create(recurringInput(projectID), { now: 100 }) + + const pending = Automation.runNowExecuting(definition.id, { + now: 200, + executor: async () => release.promise, + }) + + expect(Automation.runs({ automationID: definition.id }).items).toEqual([]) + expect(await Automation.reconcileInterruptedRuns({ now: 300 })).toEqual([]) + + const initial = await pending + expect(Automation.runs({ automationID: definition.id }).items[0]).toMatchObject({ + id: initial.id, + state: "scheduled", + }) + + release.resolve({ sessionID: SessionID.descending(), result: "done", cost: 0 }) + await waitForRunState(definition.id, "succeeded") + }) + }) + + test("does not reconcile a run that is active in the current process", async () => { + await withAutomationApp(async ({ projectID }) => { + const release = deferred() + const entered = deferred() + const definition = Automation.create(recurringInput(projectID), { now: 100 }) + const initial = await Automation.runNowExecuting(definition.id, { + now: 200, + executor: async () => { + entered.resolve() + await release.promise + return { sessionID: SessionID.descending(), result: "done", cost: 0 } + }, + }) + + await entered.promise + await expect(Automation.reconcileInterruptedRuns({ now: 300 })).resolves.toEqual([]) + expect(Automation.runs({ automationID: definition.id }).items[0]).toMatchObject({ + id: initial.id, + state: "scheduled", + }) + + release.resolve() + const run = await waitForRunState(definition.id, "succeeded") + expect(run).toMatchObject({ id: initial.id, state: "succeeded" }) + }) + }) + + test("list route waits for scheduler owner settle before returning persisted definitions", async () => { + await withAutomationApp(async ({ app, projectID }) => { + const gate = deferred() + let settled = false + let responseSettled = false + AutomationScheduler.install({ + stop: () => undefined, + stopOwnedRuns: () => undefined, + settleOwner: async () => { + settled = true + await gate.promise + }, + reschedule: () => undefined, + cancel: () => undefined, + computeNextFireAt: () => null, + }) + const definition = Automation.create(recurringInput(projectID), { now: 100 }) + + const responsePromise = json(app, "/automation").then((response) => { + responseSettled = true + return response + }) + await Bun.sleep(0) + + expect(settled).toBe(true) + expect(responseSettled).toBe(false) + gate.resolve() + const response = await responsePromise + expect(response.items.map((item: Automation.Definition) => item.id)).toEqual([definition.id]) + }) + }) + + test("runNow route reconciles stale persisted active runs before queuing a new run", async () => { + await withAutomationApp(async ({ app, projectID }) => { + const gate = deferred() + let settled = false + let responseSettled = false + AutomationScheduler.install({ + stop: () => undefined, + stopOwnedRuns: () => undefined, + settleOwner: async () => { + settled = true + await gate.promise + for (const run of await Automation.reconcileInterruptedRuns({ now: 300 })) await Automation.publishRunUpdated(run) + }, + reschedule: () => undefined, + cancel: () => undefined, + computeNextFireAt: () => null, + }) + const definition = Automation.create(recurringInput(projectID), { now: 100 }) + const stale = Automation.runNow(definition.id, { now: 200 }) + + const responsePromise = json(app, `/automation/${definition.id}/run`, { method: "POST" }).then((response) => { + responseSettled = true + return response + }) + await Bun.sleep(0) + + expect(settled).toBe(true) + expect(responseSettled).toBe(false) + gate.resolve() + const response = await responsePromise + const runs = Automation.runs({ automationID: definition.id }).items + + expect(response).toMatchObject({ automationID: definition.id, state: "scheduled" }) + expect(response.id).not.toBe(stale.id) + expect(runs.find((run) => run.id === stale.id)).toMatchObject({ state: "stopped", stopReason: "expired" }) + }) + }) + + test("route deletion cancels timers before publishing the tombstone", async () => { await withAutomationApp(async ({ app, projectID }) => { const cancelled: string[] = [] AutomationScheduler.install({ stop: () => undefined, stopOwnedRuns: () => undefined, + settleOwner: async () => undefined, reschedule: () => undefined, cancel: (automationID) => cancelled.push(automationID), computeNextFireAt: () => null, @@ -213,6 +444,11 @@ describe("automation routes", () => { recurringInput(projectID, { rhythm: { kind: "cron", expression: "not a cron" } }), [{ field: "rhythm.expression", message: "invalid_cron_expression" }], ], + [ + "impossible cron date without weekday fallback", + recurringInput(projectID, { rhythm: { kind: "cron", expression: "0 0 31 2 *" } }), + [{ field: "rhythm.expression", message: "invalid_cron_expression" }], + ], [ "invalid timezone", recurringInput(projectID, { timezone: "Mars/Olympus" }), @@ -306,6 +542,24 @@ describe("automation routes", () => { }) }) + test("delete rejects a live run owned by another process", async () => { + await withAutomationApp(async ({ app, projectID }) => { + const created = Automation.create(recurringInput(projectID), { now: 100 }) + const active = Automation.runNow(created.id, { now: 200 }) + await using _ = await Flock.acquire(`automation-run:${Instance.directory}:${active.id}`) + + const response = await app.request(`/automation/${created.id}`, { method: "DELETE" }) + + expect(response.status).toBe(409) + expect(await response.json()).toEqual({ error: "active_run_still_running", runID: active.id }) + expect(Automation.get(created.id).id).toBe(created.id) + expect(Automation.runs({ automationID: created.id }).items[0]).toMatchObject({ + id: active.id, + state: "scheduled", + }) + }) + }) + test("update accepts a deterministic timestamp", async () => { await withAutomationApp(async ({ projectID }) => { const definition = Automation.create(recurringInput(projectID), { now: 100 }) @@ -523,10 +777,20 @@ describe("automation routes", () => { const paths = spec.paths as Record const create422 = paths["/automation"].post.responses["422"].content["application/json"].schema const update422 = paths["/automation/{automationID}"].put.responses["422"].content["application/json"].schema + const update409 = paths["/automation/{automationID}"].put.responses["409"].content["application/json"].schema + const pause409 = paths["/automation/{automationID}/pause"].post.responses["409"].content["application/json"].schema + const resume409 = paths["/automation/{automationID}/resume"].post.responses["409"].content["application/json"].schema + const delete409 = paths["/automation/{automationID}"].delete.responses["409"].content["application/json"].schema expect(create422).toEqual({ $ref: "#/components/schemas/AutomationValidationError" }) expect(update422).toEqual({ $ref: "#/components/schemas/AutomationValidationError" }) + expect(update409).toEqual({ $ref: "#/components/schemas/AutomationConflictError" }) + expect(pause409).toEqual({ $ref: "#/components/schemas/AutomationConflictError" }) + expect(resume409).toEqual({ $ref: "#/components/schemas/AutomationConflictError" }) + expect(delete409).toEqual({ $ref: "#/components/schemas/AutomationActiveRunStillRunningError" }) expect(spec.components?.schemas).toHaveProperty("AutomationValidationError") + expect(spec.components?.schemas).toHaveProperty("AutomationConflictError") + expect(spec.components?.schemas).toHaveProperty("AutomationActiveRunStillRunningError") }) test("openapi describes delete active-run stop side effect", async () => { @@ -537,6 +801,7 @@ describe("automation routes", () => { expect(description).toContain("If a run is active") expect(description).toContain("publish the stopped run") + expect(description).toContain("live run is owned by another process") }) test("runNow returns the queued run before background execution updates it", async () => { diff --git a/packages/opencode/test/server/automation-runner.test.ts b/packages/opencode/test/server/automation-runner.test.ts index e356c1a1e..da703e9e1 100644 --- a/packages/opencode/test/server/automation-runner.test.ts +++ b/packages/opencode/test/server/automation-runner.test.ts @@ -3,12 +3,15 @@ import { afterEach, describe, expect, test } from "bun:test" import { Effect } from "effect" import { Automation } from "../../src/automation" import { sessionPromptExecutor } from "../../src/automation/runner" +import { AutomationRunTable } from "../../src/automation/automation.sql" import { Bus } from "../../src/bus" +import { Database, eq } from "../../src/storage/db" import { Instance } from "../../src/project/instance" import { ProjectID } from "../../src/project/schema" import { Session } from "../../src/session" import { SessionID } from "../../src/session/schema" import { AutomationRunContext, AutomationStepCapError } from "../../src/automation/run-context" +import { Flock } from "../../src/util/flock" import { tmpdir } from "../fixture/fixture" afterEach(async () => { @@ -110,7 +113,7 @@ describe("automation runNow execution", () => { const definition = Automation.create(input(projectID)) const sessionID = SessionID.descending() - const initial = Automation.runNowExecuting(definition.id, { + const initial = await Automation.runNowExecuting(definition.id, { executor: async () => ({ sessionID, result: "done", cost: 0 }), }) expect(initial.state).toBe("scheduled") @@ -135,7 +138,7 @@ describe("automation runNow execution", () => { if (event.properties.automationID === definition.id) runEvents.push(event.properties) }) - Automation.runNowExecuting(definition.id, { + await Automation.runNowExecuting(definition.id, { executor: async ({ run }) => { const started = Automation.markRunStarted(run, sessionID, { now: run.triggeredAt }) await Automation.publishRunUpdated(started) @@ -159,14 +162,14 @@ describe("automation runNow execution", () => { }) let entered = 0 - Automation.runNowExecuting(first.id, { + await Automation.runNowExecuting(first.id, { executor: async () => { entered++ await held return { sessionID: SessionID.descending(), result: "first", cost: 0 } }, }) - Automation.runNowExecuting(second.id, { + await Automation.runNowExecuting(second.id, { executor: async () => { entered++ return { sessionID: SessionID.descending(), result: "second", cost: 0 } @@ -183,6 +186,55 @@ describe("automation runNow execution", () => { }) }) + test("blocks a run when durable storage already has an active run for the project", async () => { + await withAutomation(async (projectID) => { + const first = Automation.create(input(projectID, { title: "First automation" })) + const second = Automation.create(input(projectID, { title: "Second automation" })) + const active = Automation.runNow(first.id, { now: 100 }) + await using _ = await Flock.acquire(`automation-run:${Instance.directory}:${active.id}`) + let entered = false + + await Automation.runNowExecuting(second.id, { + now: 200, + executor: async () => { + entered = true + return { sessionID: SessionID.descending(), result: "second", cost: 0 } + }, + }) + + const stopped = await waitForRun(second.id, "stopped") + if (stopped.state !== "stopped") throw new Error("expected stopped run") + expect(stopped.stopReason).toBe("previous_run_awaiting_input") + expect(entered).toBe(false) + }) + }) + + test("reconciles stale durable writers before executing a manual run", async () => { + await withAutomation(async (projectID) => { + const first = Automation.create(input(projectID, { title: "Stale writer" })) + const second = Automation.create(input(projectID, { title: "Manual run" })) + const stale = Automation.runNow(first.id, { now: 100 }) + let entered = false + + await Automation.runNowExecuting(second.id, { + now: 200, + executor: async () => { + entered = true + return { sessionID: SessionID.descending(), result: "second", cost: 0 } + }, + }) + + const succeeded = await waitForRun(second.id, "succeeded") + expect(entered).toBe(true) + expect(succeeded.result).toBe("second") + expect(Automation.runs({ automationID: first.id }).items[0]).toMatchObject({ + id: stale.id, + state: "stopped", + stopReason: "expired", + }) + }) + }) + test("records and clears blocker state on the run ledger", async () => { await withAutomation(async (projectID) => { const definition = Automation.create(input(projectID)) @@ -205,7 +257,7 @@ describe("automation runNow execution", () => { await withAutomation(async (projectID) => { const definition = Automation.create(input(projectID)) - Automation.runNowExecuting(definition.id, { + await Automation.runNowExecuting(definition.id, { executor: async ({ run }) => { const started = Automation.markRunStarted(run, SessionID.descending(), { now: run.triggeredAt }) Automation.markRunBlocked(started, { kind: "question", callID: "call_1" }) @@ -220,13 +272,13 @@ describe("automation runNow execution", () => { const held = new Promise((resolve) => { release = resolve }) - Automation.runNowExecuting(definition.id, { + await Automation.runNowExecuting(definition.id, { executor: async () => { await held return { sessionID: SessionID.descending(), result: "first", cost: 0 } }, }) - Automation.runNowExecuting(definition.id, { + await Automation.runNowExecuting(definition.id, { executor: async () => ({ sessionID: SessionID.descending(), result: "second", cost: 0 }), }) const stopped = await waitForRun(definition.id, "stopped") @@ -237,6 +289,49 @@ describe("automation runNow execution", () => { }) }) + test("does not let stale active snapshots overwrite terminal runs", async () => { + await withAutomation(async (projectID) => { + const definition = Automation.create(input(projectID)) + const scheduled = Automation.runNow(definition.id, { now: 100 }) + const stopped = Automation.stopRunByID(scheduled.id, "cancelled", { now: 200 }) + + expect(stopped).toMatchObject({ state: "stopped", revision: 2 }) + const staleStarted = Automation.markRunStarted(scheduled, SessionID.descending(), { now: 300 }) + + expect(staleStarted).toMatchObject({ state: "stopped", revision: 2, stopReason: "cancelled" }) + expect(Automation.runs({ automationID: definition.id }).items[0]).toMatchObject({ + state: "stopped", + revision: 2, + stopReason: "cancelled", + }) + }) + }) + + test("does not publish execution updates after the durable run row disappears", async () => { + await withAutomation(async (projectID) => { + const definition = Automation.create(input(projectID)) + const runEvents: Automation.Run[] = [] + const executorFinished = defer() + const unsubscribeRun = Bus.subscribe(Automation.Event.RunUpdated, (event) => { + if (event.properties.automationID === definition.id) runEvents.push(event.properties) + }) + + await Automation.runNowExecuting(definition.id, { + executor: async ({ run }) => { + Database.use((db) => db.delete(AutomationRunTable).where(eq(AutomationRunTable.id, run.id)).run()) + executorFinished.resolve() + return { sessionID: SessionID.descending(), result: "done", cost: 0 } + }, + }) + await executorFinished.promise + await Bun.sleep(50) + unsubscribeRun() + + expect(runEvents).toEqual([]) + expect(Automation.runs({ automationID: definition.id }).items).toEqual([]) + }) + }) + test("publishes continue-session definition updates from the latest definition", async () => { await withAutomation(async (projectID) => { const definition = Automation.create(input(projectID, { context: "continue" })) @@ -246,7 +341,7 @@ describe("automation runNow execution", () => { definitionEvents.push(event.properties) }) - Automation.runNowExecuting(definition.id, { + await Automation.runNowExecuting(definition.id, { executor: async () => { Automation.update(definition.id, { title: "Updated repo brief", prompt: "Use the latest prompt." }) return { sessionID, result: "done", cost: 0 } @@ -275,11 +370,11 @@ describe("automation runNow execution", () => { const unsubscribeDefinition = Bus.subscribe(Automation.Event.DefinitionUpdated, (event) => { definitionEvents.push(event.properties) }) - let removed!: ReturnType + let removed!: Awaited> - Automation.runNowExecuting(definition.id, { + await Automation.runNowExecuting(definition.id, { executor: async () => { - removed = Automation.remove(definition.id) + removed = await Automation.remove(definition.id) return { sessionID: SessionID.descending(), result: "done", cost: 0 } }, }) @@ -304,7 +399,7 @@ describe("automation runNow execution", () => { if (event.properties.automationID === definition.id) runEvents.push(event.properties) }) - Automation.runNowExecuting(definition.id, { + await Automation.runNowExecuting(definition.id, { executor: async ({ run, signal }) => { Automation.markRunStarted(run, sessionID, { now: run.triggeredAt }) signal.addEventListener("abort", () => { @@ -318,7 +413,7 @@ describe("automation runNow execution", () => { }) await started.promise - const removed = Automation.remove(definition.id) + const removed = await Automation.remove(definition.id) expect(sawAbort).toBe(true) expect(removed.stoppedRun).toMatchObject({ @@ -378,10 +473,10 @@ describe("automation runNow execution", () => { fn: async () => { const definition = Automation.create(input(Instance.project.id, { title: "Cancel real prompt" })) - Automation.runNowExecuting(definition.id, { executor: sessionPromptExecutor }) + await Automation.runNowExecuting(definition.id, { executor: sessionPromptExecutor }) await ready.promise - const removed = Automation.remove(definition.id) + const removed = await Automation.remove(definition.id) const stoppedRun = removed.stoppedRun expect(stoppedRun).toMatchObject({ state: "stopped", stopReason: "cancelled" }) if (!stoppedRun?.sessionID) throw new Error("expected stopped run to keep its sessionID") @@ -440,14 +535,14 @@ describe("automation runNow execution", () => { directory: tmp.path, fn: async () => { const definition = Automation.create(input(Instance.project.id, { title: "Cancel before runner busy" })) - const removed = Promise.withResolvers>() + const removed = Promise.withResolvers>>() const unsubscribe = Bus.subscribe(Automation.Event.RunUpdated, (event) => { if (event.properties.automationID !== definition.id || event.properties.state !== "running") return - removed.resolve(Automation.remove(definition.id)) + void Automation.remove(definition.id).then(removed.resolve, removed.reject) }) - Automation.runNowExecuting(definition.id, { executor: sessionPromptExecutor }) - let result: ReturnType + await Automation.runNowExecuting(definition.id, { executor: sessionPromptExecutor }) + let result: Awaited> try { result = await Promise.race([ removed.promise, @@ -490,7 +585,7 @@ describe("automation runNow execution", () => { await withAutomation(async (projectID) => { const definition = Automation.create(input(projectID)) - Automation.runNowExecuting(definition.id, { + await Automation.runNowExecuting(definition.id, { executor: async ({ run }) => { Automation.markRunStarted(run, SessionID.descending(), { now: run.triggeredAt }) throw new AutomationStepCapError(50) diff --git a/packages/opencode/test/server/automation-scheduler.test.ts b/packages/opencode/test/server/automation-scheduler.test.ts index 3fccbccc9..97b1e0ba8 100644 --- a/packages/opencode/test/server/automation-scheduler.test.ts +++ b/packages/opencode/test/server/automation-scheduler.test.ts @@ -8,6 +8,7 @@ import { trackActiveRun } from "../../src/session/lifecycle-provenance" import { MessageID, SessionID } from "../../src/session/schema" import { createAutomateDefinition } from "../../src/tool/automate" import { tmpdir } from "../fixture/fixture" +import { Flock } from "../../src/util/flock" afterEach(async () => { await Instance.disposeAll() @@ -62,6 +63,10 @@ class FakeClock implements AutomationScheduler.Clock { } this.current = target } + + jumpTo(current: number) { + this.current = current + } } class OversleepClock implements AutomationScheduler.Clock { @@ -148,6 +153,15 @@ function recurringInput(projectID: ProjectID, everyMs: number, overrides: Partia } } +function cronInput(projectID: ProjectID, expression: string, overrides: Partial = {}): RecurringInput { + return recurringInput(projectID, 60_000, { + rhythm: { kind: "cron", expression }, + timezone: "UTC", + stop: { kind: "never" }, + ...overrides, + }) +} + function deferred() { let resolve!: (value: T | PromiseLike) => void const promise = new Promise((done) => { @@ -157,25 +171,57 @@ function deferred() { } async function waitForRunStates(automationID: string, states: Automation.Run["state"][]) { - const deadline = Date.now() + 1_000 + const deadline = Date.now() + 3_000 + let latest: Automation.Run[] = [] while (Date.now() < deadline) { const items = Automation.runs({ automationID }).items + latest = items if (items.length >= states.length && states.every((state, index) => items[index]?.state === state)) return items await Bun.sleep(5) } - throw new Error(`Timed out waiting for automation run states: ${states.join(", ")}`) + throw new Error(`Timed out waiting for automation run states: ${states.join(", ")}; latest=${JSON.stringify(latest)}`) +} + +function allRuns(automationID: string) { + const items: Automation.Run[] = [] + let cursor: string | undefined + while (true) { + const page = Automation.runs({ automationID, limit: 100, cursor }) + items.push(...page.items) + if (!page.nextCursor) return items + cursor = page.nextCursor + } } async function waitForRunCount(automationID: string, count: number) { - const deadline = Date.now() + 1_000 + const deadline = Date.now() + 3_000 while (Date.now() < deadline) { - const items = Automation.runs({ automationID, limit: 100 }).items + const items = allRuns(automationID) if (items.length >= count) return items await Bun.sleep(5) } throw new Error(`Timed out waiting for automation run count: ${count}`) } +async function waitForStarts(starts: unknown[], count: number) { + const deadline = Date.now() + 3_000 + while (Date.now() < deadline) { + if (starts.length >= count) return + await Bun.sleep(5) + } + throw new Error(`Timed out waiting for scheduler starts: ${count}`) +} + +async function waitForSignal(input: () => AbortSignal | undefined) { + const deadline = Date.now() + 3_000 + while (Date.now() < deadline) { + const signal = input() + if (signal) return signal + await Bun.sleep(5) + } + throw new Error("Timed out waiting for run signal") +} + describe("automation scheduler", () => { test("fires a one-shot automation once with unattended execution", async () => { await withAutomation(async (projectID) => { @@ -221,6 +267,7 @@ describe("automation scheduler", () => { }) await clock.advance(1_000) + await waitForStarts(calls, 1) expect(calls).toEqual([1_000]) scheduler.stop() }) @@ -260,6 +307,217 @@ describe("automation scheduler", () => { }) }) + test("computes cron next fires on wall-clock time instead of interval completion time", async () => { + await withAutomation(async (projectID) => { + const scheduler = AutomationScheduler.make() + const definition = Automation.create(cronInput(projectID, "0 9 * * *"), { + now: Date.UTC(2026, 4, 30, 8, 30), + }) + + const next = scheduler.computeNextFireAt(definition, Date.UTC(2026, 4, 30, 8, 30)) + + expect(next).toBe(Date.UTC(2026, 4, 30, 9, 0)) + scheduler.stop() + }) + }) + + test("computes cron day-of-month and day-of-week as standard crontab OR semantics", async () => { + await withAutomation(async (projectID) => { + const scheduler = AutomationScheduler.make() + const definition = Automation.create(cronInput(projectID, "0 9 1 * 1"), { + now: Date.UTC(2026, 5, 2, 8, 0), + }) + + const next = scheduler.computeNextFireAt(definition, Date.UTC(2026, 5, 2, 8, 0)) + + expect(next).toBe(Date.UTC(2026, 5, 8, 9, 0)) + scheduler.stop() + }) + }) + + test("allows restricted weekdays to provide a fallback for impossible month days", async () => { + await withAutomation(async (projectID) => { + const scheduler = AutomationScheduler.make() + const definition = Automation.create(cronInput(projectID, "0 9 31 2 1"), { + now: Date.UTC(2026, 1, 1, 8, 0), + }) + + const next = scheduler.computeNextFireAt(definition, Date.UTC(2026, 1, 1, 8, 0)) + + expect(next).toBe(Date.UTC(2026, 1, 2, 9, 0)) + scheduler.stop() + }) + }) + + test("computes cron single-value step expressions from the single-value start", async () => { + await withAutomation(async (projectID) => { + const scheduler = AutomationScheduler.make() + const definition = Automation.create(cronInput(projectID, "5/15 9 * * *"), { + now: Date.UTC(2026, 4, 30, 9, 0), + }) + + const next = scheduler.computeNextFireAt(definition, Date.UTC(2026, 4, 30, 9, 0)) + + expect(next).toBe(Date.UTC(2026, 4, 30, 9, 5)) + scheduler.stop() + }) + }) + + test("computes cron next fires across a leap-year cycle", async () => { + await withAutomation(async (projectID) => { + const scheduler = AutomationScheduler.make() + const definition = Automation.create(cronInput(projectID, "0 0 29 2 *"), { + now: Date.UTC(2026, 2, 1, 0, 0), + }) + + const next = scheduler.computeNextFireAt(definition, Date.UTC(2026, 2, 1, 0, 0)) + + expect(next).toBe(Date.UTC(2028, 1, 29, 0, 0)) + scheduler.stop() + }) + }) + + test("reschedules pending cron timers when timezone changes", async () => { + await withAutomation(async (projectID) => { + const clock = new FakeClock(Date.UTC(2024, 4, 30, 8, 30)) + const starts: number[] = [] + const scheduler = AutomationScheduler.make({ + clock, + executor: async () => { + starts.push(clock.now()) + return { sessionID: SessionID.descending(), result: "done", cost: 0 } + }, + }) + const definition = Automation.create(cronInput(projectID, "0 9 * * *"), { + now: Date.UTC(2024, 4, 30, 8, 30), + }) + + scheduler.reschedule(definition) + const updated = Automation.update(definition.id, { timezone: "America/New_York" }, { now: clock.now() }) + scheduler.reschedule(updated) + + await clock.advance(30 * 60_000) + expect(starts).toEqual([]) + + await clock.advance(4 * 60 * 60_000) + await waitForRunStates(definition.id, ["succeeded"]) + expect(starts).toEqual([Date.UTC(2024, 4, 30, 13, 0)]) + scheduler.stop() + }) + }) + + test("records a missed cron fire after scheduler downtime", async () => { + await withAutomation(async (projectID) => { + const createdAt = Date.UTC(2026, 4, 30, 8, 30) + const missedAt = Date.UTC(2026, 4, 30, 9, 0) + const resumedAt = Date.UTC(2026, 4, 30, 9, 5) + const clock = new FakeClock(createdAt) + const starts: number[] = [] + const definition = Automation.create(cronInput(projectID, "0 9 * * *"), { now: createdAt }) + + clock.jumpTo(resumedAt) + const scheduler = AutomationScheduler.make({ + clock, + executor: async () => { + starts.push(clock.now()) + return { sessionID: SessionID.descending(), result: "done", cost: 0 } + }, + }) + const runs = await waitForRunCount(definition.id, 1) + + expect(starts).toEqual([]) + expect(runs[0]).toMatchObject({ + state: "stopped", + stopReason: "missed_schedule", + triggeredAt: missedAt, + completedAt: resumedAt, + }) + scheduler.stop() + }) + }) + + test("does not record missed cron fires after count stop is reached", async () => { + await withAutomation(async (projectID) => { + const createdAt = Date.UTC(2026, 4, 30, 8, 30) + const firstFire = Date.UTC(2026, 4, 30, 9, 0) + const afterSecondFire = Date.UTC(2026, 4, 31, 9, 5) + const clock = new FakeClock(createdAt) + const starts: number[] = [] + const scheduler = AutomationScheduler.make({ + clock, + executor: async () => { + starts.push(clock.now()) + return { sessionID: SessionID.descending(), result: "done", cost: 0 } + }, + }) + const definition = Automation.create(cronInput(projectID, "0 9 * * *", { stop: { kind: "count", count: 1 } }), { + now: createdAt, + }) + + scheduler.reschedule(definition) + await clock.advance(firstFire - createdAt) + await waitForRunStates(definition.id, ["succeeded"]) + + clock.jumpTo(afterSecondFire) + scheduler.reschedule(definition) + await clock.advance(0) + + expect(starts).toEqual([firstFire]) + expect(Automation.runs({ automationID: definition.id }).items).toHaveLength(1) + scheduler.stop() + }) + }) + + test("does not cancel a due cron fire during owner rescan", async () => { + await withAutomation(async (projectID) => { + const clock = new FakeClock(0) + const starts: number[] = [] + const scheduler = AutomationScheduler.make({ + clock, + executor: async () => { + starts.push(clock.now()) + return { sessionID: SessionID.descending(), result: "done", cost: 0 } + }, + }) + const definition = Automation.create(cronInput(projectID, "* * * * *"), { now: 0 }) + + scheduler.reschedule(definition) + clock.jumpTo(60_000) + scheduler.reschedule(definition) + await clock.advance(0) + await waitForRunStates(definition.id, ["succeeded"]) + + expect(starts).toEqual([60_000]) + scheduler.stop() + }) + }) + + test("does not run timers while another owner holds the durable scheduler lock", async () => { + await withAutomation(async (projectID) => { + const key = `automation-scheduler-test-${Date.now()}-${Math.random()}` + await using lease = await Flock.acquire(key) + const clock = new FakeClock(0) + const calls: number[] = [] + const scheduler = AutomationScheduler.make({ + clock, + ownerKey: key, + ownerRetryMs: 60_000, + executor: async () => { + calls.push(clock.now()) + return { sessionID: SessionID.descending(), result: "done", cost: 0 } + }, + }) + const definition = Automation.create(oneshotInput(projectID, 1_000), { now: 0 }) + + scheduler.reschedule(definition) + await clock.advance(1_000) + + expect(calls).toEqual([]) + expect(Automation.runs({ automationID: definition.id }).items).toEqual([]) + scheduler.stop() + }) + }) + test("does not reschedule a one-shot automation after its fire time has run", async () => { await withAutomation(async (projectID) => { const clock = new FakeClock(0) @@ -329,7 +587,7 @@ describe("automation scheduler", () => { const definition = Automation.create(oneshotInput(projectID, 1_000), { now: 0 }) scheduler.reschedule(definition) - Automation.remove(definition.id) + await Automation.remove(definition.id) await expect(clock.advance(1_000)).resolves.toBeUndefined() expect(calls).toEqual([]) @@ -350,7 +608,7 @@ describe("automation scheduler", () => { return { sessionID: SessionID.descending(), result: "scheduled", cost: 0 } }, }) - Automation.runNowExecuting(definition.id, { + await Automation.runNowExecuting(definition.id, { now: 0, executor: async () => releaseManual.promise, }) @@ -426,7 +684,8 @@ describe("automation scheduler", () => { scheduler.reschedule(definition) await clock.advance(60_000) - expect(runSignal?.aborted).toBe(false) + await waitForRunStates(definition.id, ["scheduled"]) + expect((await waitForSignal(() => runSignal)).aborted).toBe(false) await Instance.dispose({ mode: "force" }) @@ -454,7 +713,8 @@ describe("automation scheduler", () => { scheduler.reschedule(definition) await clock.advance(60_000) - expect(runSignal?.aborted).toBe(false) + await waitForRunStates(definition.id, ["scheduled"]) + expect((await waitForSignal(() => runSignal)).aborted).toBe(false) await Instance.dispose() @@ -480,6 +740,7 @@ describe("automation scheduler", () => { stopOwnedRuns: () => { throw new Error("pre-stop failed") }, + settleOwner: async () => undefined, reschedule: () => undefined, cancel: () => undefined, computeNextFireAt: () => null, @@ -502,7 +763,7 @@ describe("automation scheduler", () => { return { sessionID: SessionID.descending(), result: "scheduled", cost: 0 } }, }) - Automation.runNowExecuting(definition.id, { + await Automation.runNowExecuting(definition.id, { now: 0, executor: async () => releaseManual.promise, }) @@ -546,6 +807,7 @@ describe("automation scheduler", () => { scheduler.reschedule(definition) await clock.advance(60_000) + await waitForStarts(starts, 1) expect(starts).toEqual([60_000]) await clock.advance(60_000) @@ -608,6 +870,7 @@ describe("automation scheduler", () => { scheduler.reschedule(definition) await clock.advance(30_000) + await waitForStarts(starts, 1) expect(starts).toEqual([30_000]) await clock.advance(10_000) @@ -679,7 +942,7 @@ describe("automation scheduler", () => { ) scheduler.reschedule(definition) - Automation.runNowExecuting(definition.id, { + await Automation.runNowExecuting(definition.id, { now: 30_000, executor: async () => ({ sessionID: SessionID.descending(), result: "manual", cost: 0 }), }) @@ -693,7 +956,7 @@ describe("automation scheduler", () => { }) }) - test("stops scheduling recurring automation after count limit above default page size", async () => { + test("stops scheduling recurring automation after count limit above page size", async () => { await withAutomation(async (projectID) => { const clock = new FakeClock(0) const starts: number[] = [] @@ -705,19 +968,20 @@ describe("automation scheduler", () => { }, }) const definition = Automation.create( - recurringInput(projectID, 60_000, { stop: { kind: "count", count: 51 } }), + recurringInput(projectID, 60_000, { stop: { kind: "count", count: 101 } }), { now: 0 }, ) scheduler.reschedule(definition) - for (let runCount = 1; runCount <= 51; runCount++) { + for (let runCount = 1; runCount <= 101; runCount++) { await clock.advance(60_000) + await waitForStarts(starts, runCount) await waitForRunCount(definition.id, runCount) } await clock.advance(60_000) - expect(starts).toHaveLength(51) - expect(Automation.runs({ automationID: definition.id, limit: 100 }).items).toHaveLength(51) + expect(starts).toHaveLength(101) + expect(allRuns(definition.id)).toHaveLength(101) scheduler.stop() }) }) @@ -751,13 +1015,17 @@ describe("automation scheduler", () => { await withAutomation(async (projectID) => { const clock = new FakeClock(0) const releaseWriter = deferred<{ sessionID: SessionID; result: string | null; cost?: number | null }>() + const writerEntered = deferred() const starts: number[] = [] const blocker = Automation.create(oneshotInput(projectID, 10_000_000), { now: 0 }) - Automation.runNowExecuting(blocker.id, { + await Automation.runNowExecuting(blocker.id, { now: 0, - executor: async () => releaseWriter.promise, + executor: async () => { + writerEntered.resolve() + return releaseWriter.promise + }, }) - await waitForRunStates(blocker.id, ["scheduled"]) + await writerEntered.promise const scheduler = AutomationScheduler.make({ clock, executor: async () => { @@ -785,6 +1053,7 @@ describe("automation scheduler", () => { await withAutomation(async (projectID) => { const clock = new FakeClock(0) const releaseBlocker = deferred<{ sessionID: SessionID; result: string | null; cost?: number | null }>() + const blockerEntered = deferred() const scheduler = AutomationScheduler.make({ clock, executor: async () => ({ sessionID: SessionID.descending(), result: "scheduled", cost: 0 }), @@ -792,15 +1061,18 @@ describe("automation scheduler", () => { const blocker = Automation.create(oneshotInput(projectID, 10_000_000), { now: 0 }) const definition = Automation.create(recurringInput(projectID, 60_000), { now: 0 }) - Automation.runNowExecuting(blocker.id, { + await Automation.runNowExecuting(blocker.id, { now: 0, - executor: async () => releaseBlocker.promise, + executor: async () => { + blockerEntered.resolve() + return releaseBlocker.promise + }, }) - await waitForRunStates(blocker.id, ["scheduled"]) + await blockerEntered.promise scheduler.reschedule(definition) await clock.advance(30_000) - Automation.runNowExecuting(definition.id, { + await Automation.runNowExecuting(definition.id, { now: 30_000, executor: async () => ({ sessionID: SessionID.descending(), result: "manual", cost: 0 }), }) @@ -829,6 +1101,7 @@ describe("automation scheduler", () => { const definition = Automation.create(oneshotInput(projectID, 1_000), { now: 0 }) const active = Automation.runNow(definition.id, { now: 0 }) Automation.markRunStarted(active, SessionID.descending(), { now: 0 }) + await using _ = await Flock.acquire(`automation-run:${Instance.directory}:${active.id}`) scheduler.reschedule(definition) await clock.advance(1_000) @@ -845,6 +1118,36 @@ describe("automation scheduler", () => { }) }) + test("reconciles a stale project writer before firing another automation", async () => { + await withAutomation(async (projectID) => { + const clock = new FakeClock(0) + const calls: number[] = [] + const scheduler = AutomationScheduler.make({ + clock, + executor: async () => { + calls.push(clock.now()) + return { sessionID: SessionID.descending(), result: "done", cost: 0 } + }, + }) + const staleWriter = Automation.create(recurringInput(projectID, 60_000, { title: "Stale writer" }), { now: 0 }) + const stale = Automation.runNow(staleWriter.id, { now: 0 }) + const definition = Automation.create(oneshotInput(projectID, 1_000), { now: 0 }) + + scheduler.reschedule(definition) + await clock.advance(1_000) + const runs = await waitForRunStates(definition.id, ["succeeded"]) + + expect(calls).toEqual([1_000]) + expect(Automation.runs({ automationID: staleWriter.id }).items[0]).toMatchObject({ + id: stale.id, + state: "stopped", + stopReason: "expired", + }) + expect(runs[0]).toMatchObject({ state: "succeeded", triggeredAt: 1_000 }) + scheduler.stop() + }) + }) + test("does not fire long one-shot timers before the target time", async () => { await withAutomation(async (projectID) => { const clock = new FakeClock(0) @@ -924,6 +1227,32 @@ describe("automation scheduler", () => { }) }) + test("records a missed one-shot after restart instead of catching up within timer grace", async () => { + await withAutomation(async (projectID) => { + const clock = new OversleepClock(90_000, 90_000) + const calls: number[] = [] + const definition = Automation.create(oneshotInput(projectID, 60_000), { now: 0 }) + const scheduler = AutomationScheduler.make({ + clock, + executor: async () => { + calls.push(clock.now()) + return { sessionID: SessionID.descending(), result: "done", cost: 0 } + }, + }) + + const runs = await waitForRunCount(definition.id, 1) + + expect(calls).toEqual([]) + expect(runs[0]).toMatchObject({ + state: "stopped", + stopReason: "missed_schedule", + triggeredAt: 60_000, + completedAt: 90_000, + }) + scheduler.stop() + }) + }) + test("records missed schedules instead of catching up after an overslept timer", async () => { await withAutomation(async (projectID) => { const clock = new OversleepClock(0, 180_001) diff --git a/packages/sdk/js/src/v2/gen/sdk.gen.ts b/packages/sdk/js/src/v2/gen/sdk.gen.ts index 736e1813c..a5a4f1553 100644 --- a/packages/sdk/js/src/v2/gen/sdk.gen.ts +++ b/packages/sdk/js/src/v2/gen/sdk.gen.ts @@ -3535,7 +3535,7 @@ export class Automation extends HeyApiClient { /** * Delete automation * - * Delete an automation definition and return a tombstone. If a run is active, stop it and publish the stopped run before publishing the tombstone. + * Delete an automation definition and return a tombstone. If a run is active in this process, stop it and publish the stopped run before publishing the tombstone. If a live run is owned by another process, return 409 without deleting. */ public delete( parameters: { diff --git a/packages/sdk/js/src/v2/gen/types.gen.ts b/packages/sdk/js/src/v2/gen/types.gen.ts index d4c207c62..cbb75362d 100644 --- a/packages/sdk/js/src/v2/gen/types.gen.ts +++ b/packages/sdk/js/src/v2/gen/types.gen.ts @@ -2440,6 +2440,11 @@ export type AutomationCreateInput = stop: AutomationStop } +export type AutomationConflictError = { + error: "automation_conflict" + message: string +} + export type AutomationUpdateInput = { title?: string prompt?: string @@ -2452,6 +2457,11 @@ export type AutomationUpdateInput = { stop?: AutomationStop } +export type AutomationActiveRunStillRunningError = { + error: "active_run_still_running" + runID: string +} + export type AutomationRunsResponse = { items: Array nextCursor: string | null @@ -5808,6 +5818,10 @@ export type AutomationDeleteErrors = { * Not found */ 404: NotFoundError + /** + * Automation has a live run owned by another process + */ + 409: AutomationActiveRunStillRunningError } export type AutomationDeleteError = AutomationDeleteErrors[keyof AutomationDeleteErrors] @@ -5872,6 +5886,10 @@ export type AutomationUpdateErrors = { * Not found */ 404: NotFoundError + /** + * Automation update conflict + */ + 409: AutomationConflictError /** * Automation validation failed */ @@ -5906,6 +5924,10 @@ export type AutomationPauseErrors = { * Not found */ 404: NotFoundError + /** + * Automation update conflict + */ + 409: AutomationConflictError } export type AutomationPauseError = AutomationPauseErrors[keyof AutomationPauseErrors] @@ -5936,6 +5958,10 @@ export type AutomationResumeErrors = { * Not found */ 404: NotFoundError + /** + * Automation update conflict + */ + 409: AutomationConflictError } export type AutomationResumeError = AutomationResumeErrors[keyof AutomationResumeErrors]